ROMFASTSQL/proxmox/cluster/scripts/cluster-startup.ps1

<#
.SYNOPSIS
    Repornirea clusterului Proxmox `romfast` dupa o oprire planificata.

.DESCRIPTION
    Asteapta nodurile, asteapta quorumul, porneste guest-urile in ordinea corecta
    de dependente (Oracle primul), restaureaza cron-urile de monitorizare.

    Porneste DOAR guest-urile care rulau inainte de oprire, citite din fisierul de
    stare scris de cluster-shutdown. Fara fisier de stare, foloseste lista
    implicita, minus guest-urile din NeverAutostart.

    RULEAZA DE PE STATIA DE ADMIN, nu de pe un nod.

    NODURILE TREBUIE PORNITE FIZIC INTAI. Scriptul le asteapta.
    Porneste-le pe TOATE TREI aproximativ simultan: un singur nod pornit nu are
    quorum, /etc/pve ramane read-only si nu se poate porni niciun guest.

.PARAMETER Yes
    Nu cere confirmari.

.PARAMETER DryRun
    Arata ce ar face, fara sa execute nimic care modifica starea.

.PARAMETER All
    Ignora fisierul de stare, foloseste lista implicita.

.EXAMPLE
    .\cluster-startup.ps1 -DryRun
.EXAMPLE
    .\cluster-startup.ps1
#>
[CmdletBinding()]
param(
    [switch]$Yes,
    [switch]$DryRun,
    [switch]$All
)

$ErrorActionPreference = 'Stop'

# ---------------------------------------------------------------- configurare

$NodeIp = [ordered]@{
    pve1     = '10.0.20.200'
    pvemini  = '10.0.20.201'
    pveelite = '10.0.20.202'
}

# Ordinea de PORNIRE: baza de date intai, consumatorii dupa.
# Inversul ordinii de oprire din cluster-shutdown.
$GuestStartOrder = @(
    108,   # central-oracle       - PRIMUL, restul depind de el
    201,   # roacentral           - IIS reverse proxy, consumator Oracle
    104,   # flowise              - consumator Oracle
    100,   # portainer
    102,   # docker.romfast.ro
    103,   # dokploy
    106,   # gitea
    171,   # claude-agent
    101,   # minecraft
    110,   # moltbot
    303,   # Win11-Adina
    304,   # Win11-Marius
    302,   # oracle-test-302
    109,   # oracle-dr-windows
    301,   # docker-portainer-template
    310    # Win11-Template
)

# Guest-uri care NU se pornesc automat cand LIPSESTE fisierul de stare.
# Daca fisierul de stare exista, el decide - daca erau pornite, se repornesc.
#   109 - VM DR. Pornirea lui nedorita a declansat bucla OOM din incidentul
#         2026-04-20; in mod normal sta oprit si e pornit doar de testul DR.
#   301, 310 - template-uri, nu servicii.
$NeverAutostart = @(109, 301, 310)

# Reteaua dedicata de cluster ("insula"), existenta din 2026-08-29. De ea atarna
# replicarea ZFS (migration: network=10.10.10.0/24 in datacenter.cfg) si storage-ul
# NFS backup-pvemini. Pe pve1 si pveelite IP-ul sta pe dongle-ul USB, adus la boot
# de un "auto enx..." din /etc/network/interfaces: daca nucleul enumereaza USB-ul
# DUPA ce networking.service a terminat, interfata ramane fara IP, iar nodul pare
# perfect sanatos. Detalii: ../docs/switch-2.5g-dedicat-noduri.md
$IslandIp = [ordered]@{
    pve1     = '10.10.10.200'
    pvemini  = '10.10.10.201'
    pveelite = '10.10.10.202'
}
$IslandNfsMount  = '/mnt/pve/backup-pvemini'
$IslandNfsServer = '10.10.10.201'
$IslandNfsNodes  = @('pve1', 'pveelite')

$OracleCt     = 108
$OracleDocker = 'oracle-xe'
# Masurat pe 2026-08-29, la pornire de la rece dupa oprirea completa a clusterului:
# instanta a avut nevoie de ~10 minute ca sa ajunga OPEN. 600s erau prea strimti.
$OracleWait   = 1200     # secunde asteptate pana baza raspunde la interogari

$NodeWait   = 1800       # secunde asteptate pana apar toate nodurile
$QuorumWait = 300
$GuestWait  = 300
$CronBackup = '/root/crontab.backup-mentenanta.txt'

$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
$StateFile = if ($env:CLUSTER_STATE_FILE) { $env:CLUSTER_STATE_FILE } else { Join-Path $ScriptDir '.cluster-state.txt' }

# ------------------------------------------------------------------- utilitare

function Write-Step { param([string]$Text) Write-Host ""; Write-Host "== $Text" -ForegroundColor Green }
function Write-Ok   { param([string]$Text) Write-Host "  [ok] $Text" -ForegroundColor Green }
function Write-Warn { param([string]$Text) Write-Host "  [!]  $Text" -ForegroundColor Yellow }
function Write-Log  { param([string]$Text) Write-Host "[$(Get-Date -Format HH:mm:ss)] $Text" -ForegroundColor DarkGray }
function Stop-WithError { param([string]$Text) Write-Host "  [X] $Text" -ForegroundColor Red; exit 1 }

function Test-Alive {
    param([string]$Address, [int]$TimeoutMs = 2000)
    try {
        $p = New-Object System.Net.NetworkInformation.Ping
        return ($p.Send($Address, $TimeoutMs).Status -eq 'Success')
    } catch {
        return $false
    }
}

function Invoke-Node {
    param([string]$Node, [string]$Command)
    $out = ssh -o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new -o LogLevel=ERROR "root@$($NodeIp[$Node])" $Command
    return $out
}

function Invoke-NodeChange {
    param([string]$Node, [string]$Command)
    if ($DryRun) {
        Write-Host "  [dry-run] ${Node}: $Command" -ForegroundColor DarkGray
        return @()
    }
    return Invoke-Node -Node $Node -Command $Command
}

# Comenzi cu ghilimele imbricate NU se trimit prin Invoke-Node.
#
# PowerShell 5.1 pierde ghilimelele duble cand paseaza un argument catre un
# executabil nativ: `bash -lc "printf '...' | sqlplus"` ajunge pe nod ca
# `bash -lc printf '...' | sqlplus`, adica printf fara format, iar sqlplus rulat
# pe NOD, unde nu exista. Esecul e usor de citit gresit ca "lipseste sqlplus".
# In loc sa ne luptam cu nivelurile de citare, trimitem scriptul codificat.
# Verificat pe 2026-08-29, dupa ce varianta directa a esuat pe cluster live.
function Invoke-NodeScript {
    param([string]$Node, [string]$Script)
    $lf  = $Script -replace "`r`n", "`n"
    $b64 = [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($lf))
    return Invoke-Node -Node $Node -Command "echo $b64 | base64 -d | bash"
}

function Confirm-Step {
    param([string]$Message)
    if ($Yes -or $DryRun) { return }
    Write-Host ""
    Write-Host $Message -ForegroundColor Yellow
    $reply = Read-Host "Scrie DA ca sa continui"
    if ($reply -ne 'DA') { Stop-WithError "Anulat de utilizator." }
}

function Get-NodeGuests {
    param([string]$Node)
    $result = @()

    foreach ($line in ((Invoke-Node -Node $Node -Command 'pct list') | Select-Object -Skip 1)) {
        $f = ($line -split '\s+') | Where-Object { $_ -ne '' }
        if ($f.Count -ge 2) {
            $result += [pscustomobject]@{ Type='ct'; Node=$Node; Id=[int]$f[0]; Status=$f[1] }
        }
    }
    foreach ($line in ((Invoke-Node -Node $Node -Command 'qm list') | Select-Object -Skip 1)) {
        $f = ($line -split '\s+') | Where-Object { $_ -ne '' }
        if ($f.Count -ge 3) {
            $result += [pscustomobject]@{ Type='vm'; Node=$Node; Id=[int]$f[0]; Status=$f[2] }
        }
    }
    return $result
}

function Get-GuestStatus {
    param([string]$Node, [string]$Type, [int]$Id)
    $cmd = if ($Type -eq 'ct') { "pct status $Id" } else { "qm status $Id" }
    $out = Invoke-Node -Node $Node -Command $cmd
    if ($out -match 'status:\s*(\w+)') { return $Matches[1] }
    return 'unknown'
}

# ------------------------------------------------- pas 1: asteapta nodurile

Write-Step "Astept nodurile"
Write-Warn "Nodurile trebuie pornite FIZIC. Porneste-le pe toate trei aproximativ simultan."

$waited = 0
$missing = @()
while ($waited -lt $NodeWait) {
    $missing = @()
    foreach ($node in $NodeIp.Keys) {
        if (-not (Test-Alive $NodeIp[$node])) { $missing += $node }
    }
    if ($missing.Count -eq 0) { break }
    Write-Host "`r  astept: $($missing -join ', ') (${waited}s)          " -NoNewline -ForegroundColor DarkGray
    Start-Sleep -Seconds 10
    $waited += 10
}
Write-Host "`r                                                            `r" -NoNewline
if ($missing.Count -ne 0) { Stop-WithError "Nodurile $($missing -join ', ') nu au aparut in ${NodeWait}s." }

foreach ($node in $NodeIp.Keys) {
    $w = 0
    while ($true) {
        $h = Invoke-Node -Node $node -Command 'uptime -p'
        if ($LASTEXITCODE -eq 0 -and $h) { break }
        $w += 10
        if ($w -gt 300) { Stop-WithError "$node raspunde la ping dar nu la SSH." }
        Start-Sleep -Seconds 10
    }
    Write-Ok "$node - up, SSH OK ($h)"
}

# --------------------------------------------------- pas 2: asteapta quorum

Write-Step "Astept quorumul"

$waited = 0
while ($true) {
    $q = Invoke-Node -Node 'pvemini' -Command 'pvecm status'
    if (($q -join "`n") -match 'Quorate:\s*Yes') { break }
    if ($waited -ge $QuorumWait) { Stop-WithError "Clusterul nu a atins quorumul in ${QuorumWait}s." }
    Write-Host "`r  astept quorum (${waited}s)...   " -NoNewline -ForegroundColor DarkGray
    Start-Sleep -Seconds 10
    $waited += 10
}
Write-Host "`r                                    `r" -NoNewline

$votes = 0
if (($q -join "`n") -match 'Total votes:\s*(\d+)') { $votes = [int]$Matches[1] }
if ($votes -ne 3) { Write-Warn "Quorum atins, dar doar $votes/3 voturi. Un nod lipseste din cluster." }
Write-Ok "quorum OK, $votes/3 voturi"

Invoke-Node -Node 'pvemini' -Command 'ha-manager status' | Select-Object -First 4 | Write-Host

# ------------------------------------ pas 3: reteaua dedicata de cluster (insula)

# Se verifica INAINTE de pornirea guest-urilor: daca insula e jos, replicarea si
# backup-ul NFS tac fara nicio alerta, iar simptomul apare abia peste ore.
# Nu blocheaza pornirea - la revenirea curentului, Oracle sus conteaza mai mult
# decat replicarea - dar nu te lasa sa treci pe langa problema fara sa o vezi.

Write-Step "Verific reteaua de cluster (10.10.10.0/24)"

if ($DryRun) {
    Write-Warn "dry-run - sar peste verificarea insulei"
} else {
    $islandBad = @()

    foreach ($node in $NodeIp.Keys) {
        $want = $IslandIp[$node]
        $have = (Invoke-Node -Node $node -Command "ip -4 -o addr show | grep -c ' $want/' || true") -join ''

        if ($have.Trim() -eq '0') {
            # Cazul asteptat de esec: dongle-ul USB enumerat tarziu la boot.
            # ifreload -a aplica doar diferentele fata de /etc/network/interfaces,
            # deci nu atinge interfetele tap/veth ale guest-urilor.
            Write-Warn "$node - lipseste $want, incerc 'ifreload -a'"
            Invoke-Node -Node $node -Command 'PATH=/usr/sbin:/sbin:/usr/bin:/bin ifreload -a' | Out-Null
            Start-Sleep -Seconds 8
            $have = (Invoke-Node -Node $node -Command "ip -4 -o addr show | grep -c ' $want/' || true") -join ''
        }

        if ($have.Trim() -eq '0') {
            $islandBad += $node
            Write-Warn "$node - TOT fara $want dupa ifreload"
        } else {
            Write-Ok "$node - $want prezent"
        }
    }

    # Un IP configurat local nu garanteaza si cablul/switch-ul. pvemini e sursa
    # replicarii si serverul NFS, deci se verifica din el spre celelalte doua.
    foreach ($node in @('pve1', 'pveelite')) {
        $ip = $IslandIp[$node]
        $p = (Invoke-Node -Node 'pvemini' -Command "ping -c1 -W2 $ip >/dev/null 2>&1 && echo OK || echo FAIL") -join ''
        if ($p.Trim() -eq 'OK') {
            Write-Ok "pvemini -> $node ($ip) - ping OK"
        } else {
            if ($islandBad -notcontains $node) { $islandBad += $node }
            Write-Warn "pvemini -> $node ($ip) - NU raspunde (cablu? switch?)"
        }
    }

    $nfsBad = @()
    foreach ($node in $IslandNfsNodes) {
        $src = (Invoke-Node -Node $node -Command "findmnt -no SOURCE $IslandNfsMount 2>/dev/null || true") -join ''
        if ($src -match [regex]::Escape($IslandNfsServer)) {
            Write-Ok "$node - NFS $IslandNfsMount montat de pe $IslandNfsServer"
        } elseif ($src.Trim()) {
            # Prins pe 2026-08-29 la primul boot la rece: o linie ramasa in /etc/fstab
            # monta storage-ul de pe IP-ul de productie INAINTE ca pvestatd sa apuce,
            # deci backup-ul mergea tacut pe reteaua gresita, cu storage.cfg corect.
            $nfsBad += $node
            Write-Warn "$node - NFS montat de pe ALT server: $src (asteptat $IslandNfsServer)"
            Write-Warn "  verifica: ssh root@$($NodeIp[$node]) 'grep nfs /etc/fstab'"
        } else {
            # pvestatd remonteaza singur in cateva zeci de secunde daca reteaua e sus.
            Write-Warn "$node - NFS $IslandNfsMount NEMONTAT (pvestatd il reia daca insula urca)"
        }
    }

    if ($nfsBad.Count -gt 0) {
        Write-Warn "Backup-ul NFS trece pe alta retea decat cea proiectata pe: $($nfsBad -join ', ')"
    }

    if ($islandBad.Count -gt 0) {
        Write-Warn ""
        Write-Warn "INSULA E JOS pe: $($islandBad -join ', ')"
        Write-Warn "Consecinta: replicarea ZFS si storage-ul NFS backup-pvemini NU vor functiona."
        Write-Warn "Guest-urile pornesc si merg normal - problema e tacuta, se vede abia in pvesr status."
        Write-Warn "Verifica: ssh root@<nod> `"ip -br a; dmesg | tail -20`"  (dongle USB enumerat tarziu?)"
        Confirm-Step "Continui cu pornirea guest-urilor, cu replicarea jos?"
    } elseif ($nfsBad.Count -eq 0) {
        Write-Ok "insula completa - replicarea si NFS-ul au calea libera"
    } else {
        Write-Warn "insula e sus, dar NFS-ul nu o foloseste - vezi mai sus"
    }
}

# ------------------------------------------------- pas 4: ce trebuie pornit

Write-Step "Ce se porneste"

$toStart = @()

if ((-not $All) -and (Test-Path $StateFile)) {
    foreach ($line in (Get-Content $StateFile)) {
        if ($line -match '^\s*#' -or $line -match '^\s*$') { continue }
        $f = ($line -split '\s+') | Where-Object { $_ -ne '' }
        if ($f.Count -ge 5) {
            $toStart += [pscustomobject]@{ Type=$f[0]; Node=$f[1]; Id=[int]$f[2]; Status=$f[3]; Ha=$f[4] }
        }
    }
    Write-Ok "citit din $StateFile ($($toStart.Count) guest-uri)"
} else {
    if (-not $All) { Write-Warn "Nu exista $StateFile - folosesc lista implicita." }

    $haSids = @()
    foreach ($line in (Invoke-Node -Node 'pvemini' -Command 'ha-manager config')) {
        if ($line -match '^(ct|vm):(\d+)') { $haSids += "$($Matches[1]):$($Matches[2])" }
    }

    $allGuests = @()
    foreach ($node in $NodeIp.Keys) {
        foreach ($g in (Get-NodeGuests -Node $node)) {
            $sid = "$($g.Type):$($g.Id)"
            $g | Add-Member -NotePropertyName Ha -NotePropertyValue $(if ($haSids -contains $sid) { 'ha' } else { 'noha' })
            $allGuests += $g
        }
    }

    # Fara fisier de stare pornim doar ce e explicit in lista de ordine SI nu e
    # in NeverAutostart. Altfel am porni DR-ul si template-urile, care erau
    # oprite intentionat.
    foreach ($id in $GuestStartOrder) {
        if ($NeverAutostart -contains $id) {
            Write-Warn "$id sarit (NeverAutostart) - porneste-l manual daca chiar il vrei"
            continue
        }
        $g = $allGuests | Where-Object { $_.Id -eq $id } | Select-Object -First 1
        if ($g) { $toStart += $g }
    }
}

if ($toStart.Count -eq 0) { Stop-WithError "Nimic de pornit." }

foreach ($g in $toStart) {
    "    {0,-3} {1,-9} {2,-4}  {3}" -f $g.Type, $g.Node, $g.Id, $g.Ha | Write-Host
}

Confirm-Step "Se pornesc guest-urile de mai sus, Oracle primul."

# ------------------------------------------------- pas 5: pornire guest-uri

Write-Step "Pornire guest-uri"

$failed = @()

function Start-Guest {
    param([pscustomobject]$Guest)

    $sid = "$($Guest.Type):$($Guest.Id)"
    $cmd = if ($Guest.Type -eq 'ct') { 'pct' } else { 'qm' }

    if ((Get-GuestStatus -Node $Guest.Node -Type $Guest.Type -Id $Guest.Id) -eq 'running') {
        Write-Ok "$sid deja pornit"
        return $true
    }

    if ($Guest.Ha -eq 'ha') {
        # Pentru resursele HA, --state started repune resursa sub controlul CRM.
        Invoke-NodeChange -Node $Guest.Node -Command "ha-manager set $sid --state started" | Out-Null
    } else {
        Invoke-NodeChange -Node $Guest.Node -Command "$cmd start $($Guest.Id)" | Out-Null
    }

    if ($DryRun) { return $true }

    $waited = 0
    while ($waited -lt $GuestWait) {
        if ((Get-GuestStatus -Node $Guest.Node -Type $Guest.Type -Id $Guest.Id) -eq 'running') {
            Write-Ok "$sid pornit (${waited}s)"
            return $true
        }
        Start-Sleep -Seconds 5
        $waited += 5
    }
    Write-Warn "$sid nu a pornit in ${GuestWait}s - verifica manual"
    return $false
}

function Wait-Oracle {
    param([string]$Node)
    Write-Log "astept ca instanta Oracle sa accepte interogari..."
    if ($DryRun) { return }

    # 'bash -lc', NU 'bash -c': fara shell de login nu exista sqlplus in PATH, iar
    # asteptarea ar expira dupa $OracleWait secunde chiar cu baza perfect deschisa.
    #
    # 'timeout 30' e OBLIGATORIU, nu o precautie. Pe 2026-08-29, la pornirea de la
    # rece, sqlplus lansat catre o instanta inca in startup s-a blocat definitiv:
    # docker exec a mostenit blocajul, ssh a asteptat, iar bucla de mai jos n-a
    # apucat sa faca nici macar o iteratie. Scriptul a stat agatat 14 minute fara
    # niciun mesaj, cu propriul prag de asteptare nefolosit. Sonda trebuie sa
    # raspunda mereu, fie si cu esec.
    $sql = @"
timeout 30 pct exec $OracleCt -- docker exec $OracleDocker bash -lc "printf 'set pagesize 0 feedback off\nselect status from v\`$instance;\nexit\n' | sqlplus -s / as sysdba" 2>&1
"@
    $waited = 0
    while ($waited -lt $OracleWait) {
        $out = Invoke-NodeScript -Node $Node -Script $sql
        # 'OPEN' e raspunsul instantei deschise; orice altceva (ORA-, container
        # inca in pornire) inseamna ca mai asteptam.
        if (($out -join '') -match 'OPEN') {
            Write-Host "`r                                                  `r" -NoNewline
            Write-Ok "Oracle deschis si interogabil (${waited}s)"
            return
        }
        Write-Host "`r  Oracle inca nu raspunde (${waited}s)...   " -NoNewline -ForegroundColor DarkGray
        Start-Sleep -Seconds 15
        $waited += 15
    }
    Write-Host "`r                                                  `r" -NoNewline
    Write-Warn "Oracle nu a raspuns in ${OracleWait}s. Continui, dar VERIFICA MANUAL."
    Write-Warn "  ssh root@$($NodeIp[$Node]) `"pct exec $OracleCt -- docker logs --tail 50 $OracleDocker`""
}

foreach ($id in $GuestStartOrder) {
    $g = $toStart | Where-Object { $_.Id -eq $id } | Select-Object -First 1
    if (-not $g) { continue }

    Write-Log "pornesc $($g.Type):$($g.Id) pe $($g.Node) ($($g.Ha))"
    if (-not (Start-Guest -Guest $g)) { $failed += "$($g.Type):$($g.Id)" }

    # Oracle e blocant: nimic care depinde de el nu porneste pana nu e deschis.
    if ($g.Id -eq $OracleCt) { Wait-Oracle -Node $g.Node }
}

# orice a mai ramas in lista si nu era in ordinea definita
foreach ($g in $toStart) {
    if ($GuestStartOrder -contains $g.Id) { continue }
    Write-Warn "guest neplanificat: $($g.Type):$($g.Id) pe $($g.Node) - il pornesc"
    if (-not (Start-Guest -Guest $g)) { $failed += "$($g.Type):$($g.Id)" }
}

# ------------------------------------------------ pas 6: restaurare cron-uri

Write-Step "Restaurare cron-uri de monitorizare"

foreach ($node in $NodeIp.Keys) {
    $exists = Invoke-Node -Node $node -Command "test -f $CronBackup; echo `$?"
    if (($exists -join '') -notmatch '^0') {
        Write-Warn "$node - nu exista $CronBackup, sar peste"
        continue
    }
    Invoke-NodeChange -Node $node -Command "crontab $CronBackup" | Out-Null
    if (-not $DryRun) {
        $left = Invoke-Node -Node $node -Command "crontab -l | grep -c '^#MENTENANTA' || true"
        if (($left -join '').Trim() -eq '0') {
            Write-Ok "$node - cron-uri restaurate"
        } else {
            Write-Warn "$node - au ramas $left linii #MENTENANTA, verifica manual"
        }
    }
}

# ------------------------------------------------------- pas 7: verificare

Write-Step "Verificare finala"

if ($DryRun) {
    Write-Warn "dry-run - sar peste verificare"
} else {
    Invoke-Node -Node 'pvemini' -Command 'ha-manager status' | Write-Host
    Write-Host ""
    $rep = Invoke-Node -Node 'pvemini' -Command 'pvesr status'
    $rep | Write-Host
    Write-Host ""

    $badRep = @($rep | Select-Object -Skip 1 | Where-Object { $_ -notmatch 'OK\s*$' -and $_.Trim() -ne '' })
    if ($badRep.Count -eq 0) {
        Write-Ok "replicare: toate job-urile OK"
    } else {
        Write-Warn "job-uri de replicare cu probleme:"
        $badRep | Write-Host
    }
}

Write-Step "Cluster pornit"

if ($failed.Count -gt 0) {
    Write-Warn "Guest-uri care NU au pornit: $($failed -join ', ')"
    Write-Warn "Verifica-le manual inainte de a considera repornirea terminata."
} else {
    Write-Ok "toate guest-urile au pornit"
}

Write-Host ""
Write-Host "  De verificat manual:"
Write-Host "    - datacenter.cfg are inca AMBELE linii: 'ha: shutdown_policy=freeze'"
Write-Host "      si 'migration: network=10.10.10.0/24,type=insecure'"
Write-Host "    - daca lipsesc date recente, compara cu ultima replicare din pvesr status"
Write-Host "    - daca a fost prima repornire dupa 2026-08-29: noteaza in"
Write-Host "      docs/switch-2.5g-dedicat-noduri.md daca insula a urcat singura la boot"
Write-Host ""