romfast/ROMFASTSQL
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add UPS email notifications and automatic UPS shutdown

Browse Source 
- Add email notifications via PVE::Notify for all UPS events:
  - ONBATT: when UPS switches to battery
  - ONLINE: when power is restored
  - LOWBATT: critical battery level
  - SHUTDOWN_START/NODE/PRIMARY: during cluster shutdown
  - COMMBAD: communication lost with UPS

- Add automatic UPS shutdown command after cluster shutdown
  (protects against power surge when power returns)

- Update upssched.conf with ONLINE handler and immediate ONBATT notification

- Add notification templates for HTML and text emails

- Update documentation with new features and timer configuration

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Marius
2026-01-13 20:11:30 +02:00
parent e0f84298e9
commit ab6ac77d50
8 changed files with 619 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

159
proxmox/ups/scripts/ups-shutdown-cluster.sh
View File

@@ -1,44 +1,114 @@
#!/bin/bash
#
# Script de shutdown orchestrat pentru cluster Proxmox când UPS este pe baterie critică
# Autor: Generat automat
# Data: 2025-10-06
# Actualizat: 2025-10-06 - Adăugat suport LXC containers
# Script de shutdown orchestrat pentru cluster Proxmox cand UPS este pe baterie critica
# Trimite notificari email via PVE::Notify pentru fiecare pas
#
# Creat: 2025-10-06
# Actualizat: 2026-01-13 - Adaugat notificari email si UPS shutdown
LOGFILE=/var/log/ups-shutdown.log
NODES=(10.0.20.200 10.0.20.202) # pve1, pveelite (pvemini va fi ultimul)
NODES=("10.0.20.200" "10.0.20.202") # pve1, pveelite (pvemini va fi ultimul)
NODE_NAMES=("pve1" "pveelite") # Nume pentru notificari
UPS_NAME="nutdev1"
UPS_USER="admin"
UPS_PASS="parola99"
TEMPLATE_DIR="/etc/pve/notification-templates/default"
HOSTNAME=$(hostname)
FQDN=$(hostname -f 2>/dev/null || hostname)
log_message() {
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a $LOGFILE
logger -t ups-shutdown "$1"
}
# Obtine status UPS
get_ups_info() {
echo "Status: $(upsc $UPS_NAME ups.status 2>/dev/null || echo 'UNKNOWN')"
echo "Battery: $(upsc $UPS_NAME battery.charge 2>/dev/null || echo '?')%"
echo "Input: $(upsc $UPS_NAME input.voltage 2>/dev/null || echo '?')V"
}
# Trimite notificare email via PVE::Notify
send_notification() {
local EVENT_TYPE="$1"
local EVENT_TITLE="$2"
local EVENT_DESC="$3"
local SEVERITY="$4"
local UPS_STATUS=$(upsc $UPS_NAME ups.status 2>/dev/null || echo "UNKNOWN")
local BATTERY_CHARGE=$(upsc $UPS_NAME battery.charge 2>/dev/null || echo "0")
local INPUT_VOLTAGE=$(upsc $UPS_NAME input.voltage 2>/dev/null || echo "0")
local EVENT_DATE=$(date '+%Y-%m-%d %H:%M:%S')
log_message "Sending notification: $EVENT_TITLE"
perl -I/usr/share/perl5 << EOFPERL 2>&1 | tee -a $LOGFILE
use strict;
use warnings;
use PVE::Notify;
my \$template_data = {
'hostname' => '$FQDN',
'event_date' => '$EVENT_DATE',
'event_type' => '$EVENT_TYPE',
'event_title' => '$EVENT_TITLE',
'event_description' => '$EVENT_DESC',
'event_class' => 'shutdown',
'alert_type' => 'danger',
'ups_status' => '$UPS_STATUS',
'battery_charge' => '$BATTERY_CHARGE',
'input_voltage' => '$INPUT_VOLTAGE',
'action_taken' => 'Shutdown in curs',
'next_steps' => ''
};
my \$fields = {
'hostname' => '$HOSTNAME',
'type' => 'ups-power-event'
};
eval {
PVE::Notify::notify('$SEVERITY', 'ups-power-event', \$template_data, \$fields);
print "Notification sent\\n";
};
if (\$@) {
print STDERR "Notification failed: \$@\\n";
}
EOFPERL
}
log_message "========================================"
log_message "UPS SHUTDOWN ORCHESTRATION STARTED"
log_message "UPS Status: $(upsc nutdev1 ups.status 2>/dev/null || echo 'UNKNOWN')"
log_message "Battery Charge: $(upsc nutdev1 battery.charge 2>/dev/null || echo 'UNKNOWN')%"
log_message "$(get_ups_info)"
log_message "========================================"
# Verifică dacă UPS este într-adevăr pe baterie critică
UPS_STATUS=$(upsc nutdev1 ups.status 2>/dev/null)
# Verifica daca UPS este intr-adevar pe baterie critica
UPS_STATUS=$(upsc $UPS_NAME ups.status 2>/dev/null)
if [[ ! $UPS_STATUS =~ (OB|LB) ]]; then
log_message "WARNING: UPS status is $UPS_STATUS - not critical. Aborting shutdown."
exit 0
fi
# Email: START SHUTDOWN
send_notification \
"SHUTDOWN_START" \
"Shutdown cluster PORNIT" \
"UPS pe baterie critica. Se initiaza oprirea ordonata a cluster-ului Proxmox." \
"error"
log_message "Step 1: Oprire VM-uri pe toate nodurile..."
# Oprește VM-uri pe toate nodurile (inclusiv local)
# Opreste VM-uri pe toate nodurile (inclusiv local)
for node in ${NODES[@]} localhost; do
if [ "$node" == "localhost" ]; then
NODE_NAME="pvemini (local)"
else
NODE_NAME=$node
fi
log_message " - Oprire VM-uri pe $NODE_NAME..."
if [ "$node" == "localhost" ]; then
# Local - oprește VM-urile direct
for vmid in $(qm list | awk 'NR>1 {print $1}'); do
vm_status=$(qm status $vmid | awk '{print $2}')
if [ "$vm_status" == "running" ]; then
@@ -47,7 +117,6 @@ for node in ${NODES[@]} localhost; do
fi
done
else
# Remote - SSH către alt nod
ssh -o ConnectTimeout=5 root@$node "
for vmid in \$(qm list | awk 'NR>1 {print \$1}'); do
vm_status=\$(qm status \$vmid | awk '{print \$2}')
@@ -62,24 +131,22 @@ done
log_message "Step 2: Oprire containere LXC pe toate nodurile..."
# Oprește containere LXC pe toate nodurile
# Opreste containere LXC pe toate nodurile
for node in ${NODES[@]} localhost; do
if [ "$node" == "localhost" ]; then
NODE_NAME="pvemini (local)"
else
NODE_NAME=$node
fi
log_message " - Oprire LXC pe $NODE_NAME..."
if [ "$node" == "localhost" ]; then
# Local - oprește containerele direct
pct list 2>/dev/null | awk 'NR>1 && $2=="running" {print $1}' | while read ctid; do
log_message " * Oprire container $ctid pe pvemini..."
pct shutdown $ctid --timeout 60 &
done
else
# Remote - SSH către alt nod
ssh -o ConnectTimeout=5 root@$node "
pct list 2>/dev/null | awk 'NR>1 && \$2==\"running\" {print \$1}' | while read ctid; do
echo ' * Oprire container '\$ctid' pe $node...'
@@ -89,25 +156,61 @@ for node in ${NODES[@]} localhost; do
fi
done
log_message "Step 3: Așteptare 90 secunde pentru oprirea VM-urilor și LXC..."
log_message "Step 3: Asteptare 90 secunde pentru oprirea VM-urilor si LXC..."
sleep 90
log_message "Step 4: Oprire noduri secundare (pve1, pveelite)..."
for node in ${NODES[@]}; do
log_message " - Shutdown nod $node..."
ssh -o ConnectTimeout=5 root@$node "shutdown -h +1 'UPS on battery critical - shutting down'" 2>&1 | tee -a $LOGFILE &
log_message "Step 4: Oprire noduri secundare..."
# Opreste nodurile secundare si trimite notificare pentru fiecare
for i in "${!NODES[@]}"; do
node="${NODES[$i]}"
node_name="${NODE_NAMES[$i]}"
log_message " - Shutdown nod $node_name ($node)..."
# Email: SHUTDOWN NOD SECUNDAR
send_notification \
"SHUTDOWN_NODE" \
"Shutdown $node_name trimis" \
"Comanda shutdown a fost trimisa catre nodul $node_name ($node)." \
"error"
ssh -o ConnectTimeout=5 root@$node "shutdown -h +1 'UPS battery critical - shutting down'" 2>&1 | tee -a $LOGFILE &
done
log_message "Step 5: Așteptare 30 secunde pentru shutdown noduri secundare..."
sleep 30
log_message "Step 5: Asteptare 60 secunde pentru shutdown noduri secundare..."
sleep 60
log_message "Step 6: Oprire nod local (pvemini - primary)..."
# Email: SHUTDOWN NOD PRIMARY (ultimul email inainte de shutdown)
send_notification \
"SHUTDOWN_PRIMARY" \
"Shutdown pvemini (ULTIMUL NOD)" \
"Se opreste nodul primary pvemini. Acesta este ultimul nod din cluster. UPS-ul se va opri dupa shutdown." \
"error"
log_message "Step 7: Oprire UPS dupa shutdown..."
# Comanda UPS sa se opreasca dupa un delay (permite shutdown-ul sa se finalizeze)
# Verifica daca comanda este disponibila
if upscmd -l $UPS_NAME 2>/dev/null | grep -q "shutdown.stayoff"; then
log_message " - Comanda UPS shutdown.stayoff (oprire completa)..."
upscmd -u $UPS_USER -p $UPS_PASS $UPS_NAME shutdown.stayoff 2>&1 | tee -a $LOGFILE
elif upscmd -l $UPS_NAME 2>/dev/null | grep -q "shutdown.return"; then
log_message " - Comanda UPS shutdown.return (oprire cu restart la revenire curent)..."
upscmd -u $UPS_USER -p $UPS_PASS $UPS_NAME shutdown.return 2>&1 | tee -a $LOGFILE
else
log_message " - WARNING: Nu s-a gasit comanda UPS shutdown disponibila"
fi
log_message "========================================"
log_message "UPS SHUTDOWN ORCHESTRATION COMPLETED"
log_message "$(get_ups_info)"
log_message "Local node will shutdown in 1 minute"
log_message "========================================"
# Oprește nodul local (ultimul)
shutdown -h +1 "UPS on battery critical - primary node shutting down"
# Opreste nodul local (ultimul)
shutdown -h +1 "UPS battery critical - primary node shutting down"
exit 0

277
proxmox/ups/scripts/upssched-cmd
View File

@@ -1,30 +1,297 @@
#!/bin/bash
#
# Script apelat de upssched pentru a gestiona evenimentele UPS
# Trimite notificari email via PVE::Notify
#
# Creat: 2025-10-06
# Actualizat: 2026-01-13 - Adaugat notificari email
LOGFILE=/var/log/ups-events.log
UPS_NAME="nutdev1"
TEMPLATE_DIR="/etc/pve/notification-templates/default"
HOSTNAME=$(hostname)
FQDN=$(hostname -f 2>/dev/null || hostname)
log_event() {
echo "[2025-10-06 20:03:38] $1" >> $LOGFILE
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" >> $LOGFILE
}
# Obtine status UPS
get_ups_status() {
upsc $UPS_NAME ups.status 2>/dev/null || echo "UNKNOWN"
}
get_battery_charge() {
upsc $UPS_NAME battery.charge 2>/dev/null || echo "0"
}
get_input_voltage() {
upsc $UPS_NAME input.voltage 2>/dev/null || echo "0"
}
get_battery_runtime() {
local runtime=$(upsc $UPS_NAME battery.runtime 2>/dev/null)
if [ -n "$runtime" ] && [ "$runtime" != "0" ]; then
echo $((runtime / 60))
else
echo ""
fi
}
# Creeaza template-uri daca nu exista
create_templates() {
mkdir -p $TEMPLATE_DIR
cat > "$TEMPLATE_DIR/ups-power-event-subject.txt.hbs" << 'EOFTEMPLATE'
[{{ hostname }}] UPS {{ event_type }} - {{ event_title }}
EOFTEMPLATE
cat > "$TEMPLATE_DIR/ups-power-event-body.txt.hbs" << 'EOFTEMPLATE'
========================================
UPS POWER EVENT - {{ event_title }}
========================================
Hostname: {{ hostname }}
Date: {{ event_date }}
Event: {{ event_type }}
{{ event_description }}
UPS STATUS:
-----------
Status: {{ ups_status }}
Battery Charge: {{ battery_charge }}%
Input Voltage: {{ input_voltage }}V
{{#if battery_runtime}}
Battery Runtime: {{ battery_runtime }} min
{{/if}}
{{#if action_taken}}
ACTION: {{ action_taken }}
{{/if}}
{{#if next_steps}}
NEXT STEPS: {{ next_steps }}
{{/if}}
========================================
Log: /var/log/ups-events.log
========================================
EOFTEMPLATE
cat > "$TEMPLATE_DIR/ups-power-event-body.html.hbs" << 'EOFTEMPLATE'
<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8">
<style>
body { font-family: Arial, sans-serif; margin: 0; padding: 20px; background-color: #f5f5f5; }
.container { max-width: 600px; margin: 0 auto; background: white; padding: 25px; border-radius: 8px; box-shadow: 0 2px 8px rgba(0,0,0,0.1); }
h1 { margin-top: 0; padding-bottom: 15px; border-bottom: 3px solid #3498db; }
.status-onbatt { color: #e67e22; border-color: #e67e22; }
.status-online { color: #27ae60; border-color: #27ae60; }
.status-lowbatt { color: #c0392b; border-color: #c0392b; }
.status-shutdown { color: #8e44ad; border-color: #8e44ad; }
.alert-box { padding: 15px; border-radius: 5px; margin: 20px 0; border-left: 4px solid; }
.alert-warning { background: #fef9e7; border-color: #f39c12; }
.alert-danger { background: #fdedec; border-color: #e74c3c; }
.alert-success { background: #eafaf1; border-color: #2ecc71; }
.alert-info { background: #ebf5fb; border-color: #3498db; }
.metrics { display: grid; grid-template-columns: repeat(2, 1fr); gap: 10px; margin: 20px 0; }
.metric { background: #ecf0f1; padding: 15px; border-radius: 5px; text-align: center; }
.metric-label { font-size: 12px; color: #7f8c8d; text-transform: uppercase; }
.metric-value { font-size: 24px; font-weight: bold; color: #2c3e50; margin-top: 5px; }
.footer { margin-top: 25px; padding-top: 15px; border-top: 1px solid #ecf0f1; font-size: 12px; color: #7f8c8d; }
</style>
</head>
<body>
<div class="container">
<h1 class="status-{{ event_class }}">[UPS] {{ event_title }}</h1>
<p><strong>Hostname:</strong> {{ hostname }}<br>
<strong>Date:</strong> {{ event_date }}<br>
<strong>Event:</strong> {{ event_type }}</p>
<div class="alert-box alert-{{ alert_type }}">
<strong>{{ event_description }}</strong>
</div>
<h3>UPS Status</h3>
<div class="metrics">
<div class="metric">
<div class="metric-label">Status</div>
<div class="metric-value">{{ ups_status }}</div>
</div>
<div class="metric">
<div class="metric-label">Battery</div>
<div class="metric-value">{{ battery_charge }}%</div>
</div>
<div class="metric">
<div class="metric-label">Input Voltage</div>
<div class="metric-value">{{ input_voltage }}V</div>
</div>
</div>
{{#if action_taken}}
<p><strong>Action:</strong> {{ action_taken }}</p>
{{/if}}
{{#if next_steps}}
<p><strong>Next Steps:</strong> {{ next_steps }}</p>
{{/if}}
<div class="footer">
<p>Log: /var/log/ups-events.log<br>Proxmox UPS Monitoring</p>
</div>
</div>
</body>
</html>
EOFTEMPLATE
log_event "Templates created in $TEMPLATE_DIR/"
}
# Trimite notificare email via PVE::Notify
send_notification() {
local EVENT_TYPE="$1"
local EVENT_TITLE="$2"
local EVENT_DESC="$3"
local EVENT_CLASS="$4"
local ALERT_TYPE="$5"
local SEVERITY="$6"
local ACTION="$7"
local NEXT_STEPS="$8"
# Verifica si creeaza template-uri
if [ ! -f "$TEMPLATE_DIR/ups-power-event-subject.txt.hbs" ]; then
create_templates
fi
# Obtine status UPS
local UPS_STATUS=$(get_ups_status)
local BATTERY_CHARGE=$(get_battery_charge)
local INPUT_VOLTAGE=$(get_input_voltage)
local BATTERY_RUNTIME=$(get_battery_runtime)
local EVENT_DATE=$(date '+%Y-%m-%d %H:%M:%S')
log_event "Sending $SEVERITY notification: $EVENT_TITLE"
perl -I/usr/share/perl5 << EOFPERL 2>&1 | tee -a $LOGFILE
use strict;
use warnings;
use PVE::Notify;
my \$template_data = {
'hostname' => '$FQDN',
'event_date' => '$EVENT_DATE',
'event_type' => '$EVENT_TYPE',
'event_title' => '$EVENT_TITLE',
'event_description' => '$EVENT_DESC',
'event_class' => '$EVENT_CLASS',
'alert_type' => '$ALERT_TYPE',
'ups_status' => '$UPS_STATUS',
'battery_charge' => '$BATTERY_CHARGE',
'input_voltage' => '$INPUT_VOLTAGE',
'battery_runtime' => '$BATTERY_RUNTIME',
'action_taken' => '$ACTION',
'next_steps' => '$NEXT_STEPS'
};
my \$fields = {
'hostname' => '$HOSTNAME',
'type' => 'ups-power-event'
};
eval {
PVE::Notify::notify('$SEVERITY', 'ups-power-event', \$template_data, \$fields);
print "Email notification sent successfully\\n";
};
if (\$@) {
print STDERR "Failed to send notification: \$@\\n";
}
EOFPERL
}
# Handler principal
case $1 in
onbatt_start)
log_event "=========================================="
log_event "UPS EVENT: Trecere pe baterie - Timer 3 minute pornit"
logger -t upssched-cmd "UPS switched to battery - 3 minute timer started"
send_notification \
"ONBATT" \
"Trecere pe baterie" \
"UPS a trecut pe baterie! Daca curentul nu revine in 3 minute, se va initia shutdown-ul cluster-ului." \
"onbatt" \
"warning" \
"warning" \
"Timer 3 minute pornit" \
"Asteptare revenire curent sau shutdown automat"
;;
onbatt)
log_event "UPS EVENT: Pe baterie de 3 minute - Începe shutdown orchestrat"
log_event "=========================================="
log_event "UPS EVENT: Pe baterie de 3 minute - Incepe shutdown orchestrat"
logger -t upssched-cmd "UPS on battery for 3 minutes - starting orchestrated shutdown"
send_notification \
"ONBATT_TIMEOUT" \
"Pe baterie 3 min - SHUTDOWN" \
"UPS a fost pe baterie timp de 3 minute. Se initiaza shutdown-ul orchestrat al cluster-ului." \
"onbatt" \
"danger" \
"error" \
"Pornire shutdown orchestrat cluster" \
"Toate nodurile se vor opri in ordine"
/usr/local/bin/ups-shutdown-cluster.sh &
;;
online)
log_event "=========================================="
log_event "UPS EVENT: Curent revenit - UPS online"
logger -t upssched-cmd "Power restored - UPS back online"
send_notification \
"ONLINE" \
"Curent revenit - OK" \
"Curentul electric a revenit. UPS functioneaza normal pe linia AC." \
"online" \
"success" \
"info" \
"Sistem stabil" \
"Nicio actiune necesara"
;;
lowbatt)
log_event "UPS EVENT: BATERIE SCĂZUTĂ - Shutdown IMEDIAT"
log_event "=========================================="
log_event "UPS EVENT: BATERIE SCAZUTA - Shutdown IMEDIAT"
logger -t upssched-cmd "UPS LOW BATTERY - immediate shutdown"
send_notification \
"LOWBATT" \
"BATERIE CRITICA - SHUTDOWN IMEDIAT" \
"UPS raporteaza baterie critica! Shutdown imediat pentru protectia datelor." \
"lowbatt" \
"danger" \
"error" \
"Shutdown imediat cluster" \
"Sistemele se opresc ACUM"
/usr/local/bin/ups-shutdown-cluster.sh &
;;
commbad)
log_event "UPS EVENT: Comunicație pierdută cu UPS de 30 secunde"
log_event "=========================================="
log_event "UPS EVENT: Comunicatie pierduta cu UPS de 30 secunde"
logger -t upssched-cmd "Lost communication with UPS for 30 seconds"
# Nu facem shutdown automat pentru pierdere comunicație
send_notification \
"COMMBAD" \
"Comunicatie pierduta cu UPS" \
"Nu se poate comunica cu UPS-ul de 30 de secunde. Verificati conexiunea USB." \
"commbad" \
"warning" \
"warning" \
"Monitorizare activa" \
"Verificati conexiunea fizica USB a UPS-ului"
;;
*)
log_event "UPS EVENT: Eveniment necunoscut - $1"
logger -t upssched-cmd "Unknown UPS event: $1"