ROMFASTSQL/proxmox/cluster/scripts/verifica-pveelite.ps1

<#
.SYNOPSIS
    Verificare completa a nodului pveelite dupa revenirea din incidentul 2026-08-27.

.DESCRIPTION
    Nodul pveelite s-a resetat singur la 16:01 pe 27.08.2026 si a murit definitiv la
    17:26, pe o masina complet idle. Cauza nu s-a putut stabili de la distanta - nodul
    era mort la nivel L2. Scriptul asta se ruleaza IMEDIAT dupa ce nodul e pornit fizic
    la loc si raspunde la doua intrebari, in ordinea asta:

      1. DE CE a cazut - din log-urile boot-urilor anterioare, care acum sunt citibile.
      2. E nodul in stare buna acum - ZFS, cluster, replicare, HA, SMART, Tailscale.

    Verdictul de la pasul 1 conteaza mai mult decat restul. Daca log-urile boot-ului
    anterior sunt TACUTE inainte de ora caderii - fara MCE, fara eroare termica, fara
    panic - atunci cauza e alimentare/hardware, exact ca la pvemini pe 2026-04-20, iar
    nodul NU e de incredere ca tinta de failover pana la testarea sursei si a memoriei.
    "A pornit, deci e bine" nu e o concluzie valida aici.

    Context complet: ../incidents/2026-08-27-pveelite-down.md

    RULEAZA DE PE STATIA DE ADMIN. Motivul: SSH intre nodurile Proxmox trece prin
    Tailscale si cere autentificare interactiva, deci comenzile inlantuite de tip
    "ssh pvemini ssh pveelite" se blocheaza.

    Necesita clientul OpenSSH din Windows si o cheie SSH acceptata de root pe noduri.

.PARAMETER Wait
    Asteapta pana la N minute ca pveelite sa raspunda, in loc sa iasa imediat.
    Util daca pornesti scriptul inainte sa apesi butonul de power.

.EXAMPLE
    .\verifica-pveelite.ps1

.EXAMPLE
    .\verifica-pveelite.ps1 -Wait 10
#>
[CmdletBinding()]
param(
    [int]$Wait = 0
)

$ErrorActionPreference = 'Stop'

# ---------------------------------------------------------------- configurare

$NodeIp = [ordered]@{
    pve1     = '10.0.20.200'
    pvemini  = '10.0.20.201'
    pveelite = '10.0.20.202'
}

# Reperele incidentului 2026-08-27, folosite pentru a taia log-urile la fereastra
# relevanta si a nu ineca verdictul in zgomot.
$ResetSpontan  = '2026-08-27 16:01:48'   # reset care s-a auto-reparat in 89 s
$CadereFinala  = '2026-08-27 17:26:44'   # caderea definitiva
$IncidentStart = '2026-08-27 13:20'      # boot-ul de dupa oprirea planificata

# Tipare de eroare hardware in jurnalul kernel. Daca NICIUNUL nu apare, verdictul
# inclina spre alimentare (masina nu apuca sa scrie nimic inainte sa dispara).
$TipareHw = 'mce|machine check|hardware error|thermal|temperature above|critical temp|' +
            'nmi|kernel panic|oops|BUG:|edac|pcieport.*error|ATA bus error|I/O error'

$ReplJobs = 9   # 9 job-uri de replicare au pveelite ca tinta

# ------------------------------------------------------------------- utilitare

function Write-Step { param([string]$Text) Write-Host ""; Write-Host "== $Text" -ForegroundColor Green }
function Write-Ok   { param([string]$Text) Write-Host "  [ok] $Text" -ForegroundColor Green }
function Write-Warn { param([string]$Text) Write-Host "  [!]  $Text" -ForegroundColor Yellow }
function Write-Bad  { param([string]$Text) Write-Host "  [X]  $Text" -ForegroundColor Red }
function Write-Info { param([string]$Text) Write-Host "       $Text" -ForegroundColor DarkGray }

function Test-Alive {
    param([string]$Address, [int]$TimeoutMs = 2000)
    try {
        $p = New-Object System.Net.NetworkInformation.Ping
        return ($p.Send($Address, $TimeoutMs).Status -eq 'Success')
    } catch {
        return $false
    }
}

function Invoke-Node {
    param([string]$Node, [string]$Command, [int]$Timeout = 15)
    # stderr e capturat separat: pe Windows PowerShell 5.1, 2>&1 pe un executabil
    # nativ transforma fiecare linie de stderr in ErrorRecord si strica $?.
    $out = ssh -o BatchMode=yes -o ConnectTimeout=$Timeout `
               -o StrictHostKeyChecking=accept-new -o LogLevel=ERROR `
               "root@$($NodeIp[$Node])" $Command
    return $out
}

# Colectorul de concluzii: fiecare pas adauga aici, la final se tipareste bilantul.
$Rezultate = [System.Collections.ArrayList]::new()
function Add-Rezultat {
    param([string]$Nume, [ValidateSet('ok','warn','bad')][string]$Stare, [string]$Detaliu)
    [void]$Rezultate.Add([pscustomobject]@{ Nume = $Nume; Stare = $Stare; Detaliu = $Detaliu })
}

# =========================================================== 0. nodul raspunde

Write-Host ""
Write-Host "VERIFICARE pveelite dupa incidentul din 2026-08-27" -ForegroundColor Cyan
Write-Host "Referinta: proxmox/cluster/incidents/2026-08-27-pveelite-down.md" -ForegroundColor DarkGray

Write-Step "0. Nodul raspunde?"

$deadline = (Get-Date).AddMinutes($Wait)
$viu = Test-Alive $NodeIp.pveelite
while (-not $viu -and (Get-Date) -lt $deadline) {
    Write-Info "inca nu raspunde, reincerc... (pana la $($deadline.ToString('HH:mm')))"
    Start-Sleep -Seconds 15
    $viu = Test-Alive $NodeIp.pveelite
}

if (-not $viu) {
    Write-Bad "pveelite ($($NodeIp.pveelite)) nu raspunde la ping."
    Write-Info "Porneste nodul fizic, apoi reia. Sau ruleaza cu -Wait 10 ca sa astepte."
    exit 1
}
Write-Ok "ping OK"

try {
    $hostn = Invoke-Node -Node pveelite -Command 'hostname'
    Write-Ok "SSH OK - $hostn"
} catch {
    Write-Bad "raspunde la ping dar nu la SSH. Verifica pe consola fizica."
    exit 1
}

$uptime = Invoke-Node -Node pveelite -Command 'uptime -p; uptime -s'
Write-Info ($uptime -join ' | ')

# ================================================= 1. DE CE a cazut (esentialul)

Write-Step "1. De ce a cazut - log-urile boot-urilor anterioare"

Write-Info "boot-uri inregistrate:"
$boots = Invoke-Node -Node pveelite -Command 'journalctl --list-boots --no-pager | tail -5'
$boots | ForEach-Object { Write-Info "  $_" }

# Boot -1 = fereastra 16:03 -> 17:26 (caderea definitiva).
# Boot -2 = fereastra 13:20 -> 16:01 (resetul spontan).
foreach ($b in @(
    @{ Idx = -1; Cand = $CadereFinala; Ce = 'caderea definitiva de la 17:26' },
    @{ Idx = -2; Cand = $ResetSpontan; Ce = 'resetul spontan de la 16:01'   }
)) {
    Write-Host ""
    Write-Info "--- boot $($b.Idx): ultimele linii inainte de $($b.Ce) ---"
    $tail = Invoke-Node -Node pveelite -Command "journalctl -b $($b.Idx) --no-pager | tail -15"
    if ($tail) { $tail | ForEach-Object { Write-Host "       $_" -ForegroundColor DarkGray } }
    else       { Write-Warn "boot $($b.Idx) nu mai exista in jurnal (rotit sau volatile)" }

    $err = Invoke-Node -Node pveelite -Command "journalctl -b $($b.Idx) -p err --no-pager | tail -20"
    if ($err) {
        Write-Warn "erori in boot $($b.Idx):"
        $err | ForEach-Object { Write-Host "       $_" -ForegroundColor Yellow }
    } else {
        Write-Info "fara erori de prioritate >= err in boot $($b.Idx)"
    }
}

# Cautarea tintita de semnaturi hardware, peste toate boot-urile de ieri.
Write-Host ""
Write-Info "--- semnaturi hardware in jurnal (MCE / termic / panic / EDAC) ---"
$hw = Invoke-Node -Node pveelite -Command `
    "journalctl --since '$IncidentStart' --no-pager | grep -iE '$TipareHw' | tail -25"

$verdictHw = $false
if ($hw) {
    Write-Bad "GASIT - nodul a lasat urme de eroare hardware:"
    $hw | ForEach-Object { Write-Host "       $_" -ForegroundColor Red }
    Add-Rezultat 'Cauza caderii' 'bad' 'semnaturi hardware in jurnal - vezi liniile de mai sus'
    $verdictHw = $true
} else {
    Write-Info "niciun MCE, eroare termica, panic sau EDAC in jurnal"
}

# mcelog / edac, daca sunt instalate
$mce = Invoke-Node -Node pveelite -Command `
    "which mcelog >/dev/null 2>&1 && mcelog --client 2>/dev/null | tail -10; ls /sys/devices/system/edac/mc/mc0/*_count 2>/dev/null | head -3"
if ($mce) { $mce | ForEach-Object { Write-Info "  $_" } }

# Cate reseturi necurate a inregistrat masina
$necurate = Invoke-Node -Node pveelite -Command `
    "journalctl --list-boots --no-pager | wc -l; last -x reboot shutdown 2>/dev/null | head -8"
Write-Info "istoric reboot/shutdown:"
$necurate | Select-Object -Skip 1 | ForEach-Object { Write-Info "  $_" }

if (-not $verdictHw) {
    Write-Host ""
    Write-Bad "VERDICT: jurnalul e TACUT inainte de ambele caderi."
    Write-Info "Masina a disparut fara sa apuce sa scrie nimic - exact tiparul pvemini"
    Write-Info "din 2026-04-20. Cauza e alimentare sau hardware (sursa / RAM / termic),"
    Write-Info "NU software. Nodul nu e de incredere ca tinta de failover pana cand:"
    Write-Info "  - sursa e testata sau inlocuita"
    Write-Info "  - memoria e testata (memtest86+, minim o trecere completa)"
    Write-Info "  - se verifica praful si ventilatia (vezi temperaturile de la pasul 6)"
    Add-Rezultat 'Cauza caderii' 'bad' 'jurnal tacut => alimentare/hardware, nod nesigur'
}

# ============================================================ 2. integritate ZFS

Write-Step "2. Integritate ZFS (dupa doua opriri necurate)"

$zpool = Invoke-Node -Node pveelite -Command 'zpool status -v'
$zpool | ForEach-Object { Write-Info $_ }

if ($zpool -match 'state: ONLINE' -and $zpool -match 'No known data errors') {
    if ($zpool -match 'resilver|scan: resilvered') {
        Write-Warn "pool ONLINE, dar a rulat un resilver - normal dupa oprire necurata"
        Add-Rezultat 'ZFS' 'warn' 'ONLINE, resilver efectuat, fara erori de date'
    } else {
        Write-Ok "pool ONLINE, fara erori de date"
        Add-Rezultat 'ZFS' 'ok' 'ONLINE, fara erori'
    }
} else {
    Write-Bad "pool NU e curat - citeste iesirea de mai sus inainte de orice altceva"
    Add-Rezultat 'ZFS' 'bad' 'pool degradat sau cu erori de date'
}

# ============================================================ 3. inapoi in cluster

Write-Step "3. Revenirea in cluster"

$pvecm = Invoke-Node -Node pvemini -Command 'pvecm status; echo "---"; cat /etc/pve/.members'
$pvecm | ForEach-Object { Write-Info $_ }

$m = $pvecm | Select-String 'Total votes:\s*(\d+)' | Select-Object -First 1
$totalVotes = if ($m) { $m.Matches[0].Groups[1].Value } else { '?' }
$eliteOnline = [bool]($pvecm -match '"pveelite".*"online": 1')

if ($totalVotes -eq '3' -and $eliteOnline) {
    Write-Ok "cluster complet: 3 din 3 voturi, pveelite online"
    Add-Rezultat 'Cluster' 'ok' '3/3 voturi, marja de quorum refacuta'
} else {
    Write-Bad "cluster inca incomplet: Total votes = $totalVotes"
    Add-Rezultat 'Cluster' 'bad' "Total votes = $totalVotes, pveelite inca in afara"
}

# ================================================================ 4. replicarea

Write-Step "4. Replicarea catre pveelite"

Write-Info "job-urile se reiau singure la urmatorul interval - nu e nevoie de interventie."
Write-Info "Daca vrei sa nu astepti: pvesr run --id <job> pe pvemini."

$repl = Invoke-Node -Node pvemini -Command "pvesr status" -Timeout 30
$repl | ForEach-Object { Write-Info $_ }

$esecuri = @($repl | Select-String 'pveelite' | Where-Object { $_ -notmatch '\s0\s+OK' -and $_ -match 'failed' })
if ($esecuri.Count -eq 0) {
    Write-Ok "toate job-urile catre pveelite sunt OK, FailCount 0"
    Add-Rezultat 'Replicare' 'ok' "toate cele $ReplJobs job-uri recuperate"
} else {
    Write-Warn "$($esecuri.Count) job-uri inca in eroare - normal daca nodul tocmai a pornit"
    Write-Info "reverifica peste 15-20 min sau forteaza cu: pvesr run --id <job>"
    Add-Rezultat 'Replicare' 'warn' "$($esecuri.Count) job-uri inca in eroare, de reverificat"
}

# ======================================================================= 5. HA

Write-Step "5. HA - LRM-ul lui pveelite"

$ha = Invoke-Node -Node pvemini -Command 'ha-manager status'
$ha | ForEach-Object { Write-Info $_ }

if ($ha -match 'lrm pveelite \(active') {
    Write-Ok "LRM pveelite activ"
    Add-Rezultat 'HA' 'ok' 'LRM activ'
} elseif ($ha -match 'lrm pveelite.*old timestamp') {
    Write-Warn "LRM pveelite inca 'dead' - poate dura un minut dupa boot; reverifica"
    Add-Rezultat 'HA' 'warn' "LRM inca 'old timestamp'"
} else {
    Write-Warn "starea LRM neclara, citeste iesirea de mai sus"
    Add-Rezultat 'HA' 'warn' 'stare LRM neclara'
}

# Atentie deliberata: vm:109 a fost recuperat pe pvemini in timpul incidentului.
if ($ha -match 'service vm:109.*pveelite') {
    Write-Warn "vm:109 a revenit pe pveelite - verifica intentionat, a fost mutat pe pvemini"
}

# ===================================================================== 6. SMART

Write-Step "6. SMART si temperaturi (precedent: incidentul Kingston din aprilie)"

$smart = Invoke-Node -Node pveelite -Command @'
for d in $(lsblk -dno NAME,TYPE | awk '$2=="disk"{print $1}'); do
  echo "--- /dev/$d ---"
  smartctl -H -A /dev/$d 2>/dev/null | grep -iE 'overall-health|Temperature|Reallocated|Media_Wearout|Percentage Used|Power_On|Unsafe_Shutdown|CRC' | head -10
done
'@ -Timeout 30
$smart | ForEach-Object { Write-Info $_ }

if ($smart -match 'FAILED') {
    Write-Bad "SMART raporteaza FAILED pe cel putin un disc"
    Add-Rezultat 'SMART' 'bad' 'un disc raporteaza FAILED'
} else {
    Write-Ok "SMART: fara FAILED (citeste totusi temperaturile de mai sus)"
    Add-Rezultat 'SMART' 'ok' 'fara FAILED'
}

# Senzorii placii, daca exista - relevant pentru ipoteza termica
$sens = Invoke-Node -Node pveelite -Command "which sensors >/dev/null 2>&1 && sensors 2>/dev/null | grep -iE 'Core|Package|temp' | head -12"
if ($sens) {
    Write-Info "senzori:"
    $sens | ForEach-Object { Write-Info "  $_" }
} else {
    Write-Info "lm-sensors nu e instalat (apt install lm-sensors) - fara citiri termice live"
}

# ================================================================= 7. Tailscale

Write-Step "7. Tailscale - accesul out-of-band"

Write-Info "In incident, tailscaled era mort de 27 de zile: fara el, orice cadere"
Write-Info "viitoare a nodului cere din nou deplasare fizica."

$ts = Invoke-Node -Node pveelite -Command 'systemctl is-active tailscaled 2>/dev/null; tailscale status 2>&1 | head -3'
$ts | ForEach-Object { Write-Info $_ }

if ($ts -match '^active') {
    Write-Ok "tailscaled activ"
    Add-Rezultat 'Tailscale' 'ok' 'activ'
} else {
    Write-Warn "tailscaled NU e activ - reactiveaza-l acum:"
    Write-Info "  ssh root@$($NodeIp.pveelite) 'systemctl enable --now tailscaled && tailscale up'"
    Add-Rezultat 'Tailscale' 'warn' 'inactiv - fara acces out-of-band'
}

# ==================================================================== bilantul

Write-Host ""
Write-Host "================ BILANT ================" -ForegroundColor Cyan
foreach ($r in $Rezultate) {
    switch ($r.Stare) {
        'ok'   { Write-Host ("  [ok] {0,-16} {1}" -f $r.Nume, $r.Detaliu) -ForegroundColor Green }
        'warn' { Write-Host ("  [!]  {0,-16} {1}" -f $r.Nume, $r.Detaliu) -ForegroundColor Yellow }
        'bad'  { Write-Host ("  [X]  {0,-16} {1}" -f $r.Nume, $r.Detaliu) -ForegroundColor Red }
    }
}
Write-Host ""

$cauza = $Rezultate | Where-Object { $_.Nume -eq 'Cauza caderii' }
if ($cauza -and $cauza.Stare -eq 'bad') {
    Write-Host "Concluzia care conteaza: cauza caderii NU e lamurita ca fiind software." -ForegroundColor Red
    Write-Host "Trateaza pveelite ca nod nesigur si consemneaza rezultatul in" -ForegroundColor Red
    Write-Host "proxmox/cluster/incidents/2026-08-27-pveelite-down.md inainte sa inchizi incidentul." -ForegroundColor Red
    Write-Host ""
}

if ($Rezultate | Where-Object { $_.Stare -eq 'bad' }) { exit 1 } else { exit 0 }