ROMFASTSQL/proxmox/cluster/scripts/cluster-startup.ps1

<#
.SYNOPSIS
    Repornirea clusterului Proxmox `romfast` dupa o oprire planificata.

.DESCRIPTION
    Asteapta nodurile, asteapta quorumul, porneste guest-urile in ordinea corecta
    de dependente (Oracle primul), restaureaza cron-urile de monitorizare.

    Porneste DOAR guest-urile care rulau inainte de oprire, citite din fisierul de
    stare scris de cluster-shutdown. Fara fisier de stare, foloseste lista
    implicita, minus guest-urile din NeverAutostart.

    RULEAZA DE PE STATIA DE ADMIN, nu de pe un nod.

    NODURILE TREBUIE PORNITE FIZIC INTAI. Scriptul le asteapta.
    Porneste-le pe TOATE TREI aproximativ simultan: un singur nod pornit nu are
    quorum, /etc/pve ramane read-only si nu se poate porni niciun guest.

.PARAMETER Yes
    Nu cere confirmari.

.PARAMETER DryRun
    Arata ce ar face, fara sa execute nimic care modifica starea.

.PARAMETER All
    Ignora fisierul de stare, foloseste lista implicita.

.EXAMPLE
    .\cluster-startup.ps1 -DryRun
.EXAMPLE
    .\cluster-startup.ps1
#>
[CmdletBinding()]
param(
    [switch]$Yes,
    [switch]$DryRun,
    [switch]$All
)

$ErrorActionPreference = 'Stop'

# ---------------------------------------------------------------- configurare

$NodeIp = [ordered]@{
    pve1     = '10.0.20.200'
    pvemini  = '10.0.20.201'
    pveelite = '10.0.20.202'
}

# Ordinea de PORNIRE: baza de date intai, consumatorii dupa.
# Inversul ordinii de oprire din cluster-shutdown.
$GuestStartOrder = @(
    108,   # central-oracle       - PRIMUL, restul depind de el
    201,   # roacentral           - IIS reverse proxy, consumator Oracle
    104,   # flowise              - consumator Oracle
    100,   # portainer
    102,   # docker.romfast.ro
    103,   # dokploy
    106,   # gitea
    171,   # claude-agent
    101,   # minecraft
    110,   # moltbot
    303,   # Win11-Adina
    302,   # oracle-test-302
    109,   # oracle-dr-windows
    301,   # docker-portainer-template
    310    # Win11-Template
)

# Guest-uri care NU se pornesc automat cand LIPSESTE fisierul de stare.
# Daca fisierul de stare exista, el decide - daca erau pornite, se repornesc.
#   109 - VM DR. Pornirea lui nedorita a declansat bucla OOM din incidentul
#         2026-04-20; in mod normal sta oprit si e pornit doar de testul DR.
#   301, 310 - template-uri, nu servicii.
$NeverAutostart = @(109, 301, 310)

$OracleCt     = 108
$OracleDocker = 'oracle-xe'
$OracleWait   = 600      # secunde asteptate pana baza raspunde la interogari

$NodeWait   = 1800       # secunde asteptate pana apar toate nodurile
$QuorumWait = 300
$GuestWait  = 300
$CronBackup = '/root/crontab.backup-mentenanta.txt'

$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
$StateFile = if ($env:CLUSTER_STATE_FILE) { $env:CLUSTER_STATE_FILE } else { Join-Path $ScriptDir '.cluster-state.txt' }

# ------------------------------------------------------------------- utilitare

function Write-Step { param([string]$Text) Write-Host ""; Write-Host "== $Text" -ForegroundColor Green }
function Write-Ok   { param([string]$Text) Write-Host "  [ok] $Text" -ForegroundColor Green }
function Write-Warn { param([string]$Text) Write-Host "  [!]  $Text" -ForegroundColor Yellow }
function Write-Log  { param([string]$Text) Write-Host "[$(Get-Date -Format HH:mm:ss)] $Text" -ForegroundColor DarkGray }
function Stop-WithError { param([string]$Text) Write-Host "  [X] $Text" -ForegroundColor Red; exit 1 }

function Test-Alive {
    param([string]$Address, [int]$TimeoutMs = 2000)
    try {
        $p = New-Object System.Net.NetworkInformation.Ping
        return ($p.Send($Address, $TimeoutMs).Status -eq 'Success')
    } catch {
        return $false
    }
}

function Invoke-Node {
    param([string]$Node, [string]$Command)
    $out = ssh -o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new -o LogLevel=ERROR "root@$($NodeIp[$Node])" $Command
    return $out
}

function Invoke-NodeChange {
    param([string]$Node, [string]$Command)
    if ($DryRun) {
        Write-Host "  [dry-run] ${Node}: $Command" -ForegroundColor DarkGray
        return @()
    }
    return Invoke-Node -Node $Node -Command $Command
}

function Confirm-Step {
    param([string]$Message)
    if ($Yes -or $DryRun) { return }
    Write-Host ""
    Write-Host $Message -ForegroundColor Yellow
    $reply = Read-Host "Scrie DA ca sa continui"
    if ($reply -ne 'DA') { Stop-WithError "Anulat de utilizator." }
}

function Get-NodeGuests {
    param([string]$Node)
    $result = @()

    foreach ($line in ((Invoke-Node -Node $Node -Command 'pct list') | Select-Object -Skip 1)) {
        $f = ($line -split '\s+') | Where-Object { $_ -ne '' }
        if ($f.Count -ge 2) {
            $result += [pscustomobject]@{ Type='ct'; Node=$Node; Id=[int]$f[0]; Status=$f[1] }
        }
    }
    foreach ($line in ((Invoke-Node -Node $Node -Command 'qm list') | Select-Object -Skip 1)) {
        $f = ($line -split '\s+') | Where-Object { $_ -ne '' }
        if ($f.Count -ge 3) {
            $result += [pscustomobject]@{ Type='vm'; Node=$Node; Id=[int]$f[0]; Status=$f[2] }
        }
    }
    return $result
}

function Get-GuestStatus {
    param([string]$Node, [string]$Type, [int]$Id)
    $cmd = if ($Type -eq 'ct') { "pct status $Id" } else { "qm status $Id" }
    $out = Invoke-Node -Node $Node -Command $cmd
    if ($out -match 'status:\s*(\w+)') { return $Matches[1] }
    return 'unknown'
}

# ------------------------------------------------- pas 1: asteapta nodurile

Write-Step "Astept nodurile"
Write-Warn "Nodurile trebuie pornite FIZIC. Porneste-le pe toate trei aproximativ simultan."

$waited = 0
$missing = @()
while ($waited -lt $NodeWait) {
    $missing = @()
    foreach ($node in $NodeIp.Keys) {
        if (-not (Test-Alive $NodeIp[$node])) { $missing += $node }
    }
    if ($missing.Count -eq 0) { break }
    Write-Host "`r  astept: $($missing -join ', ') (${waited}s)          " -NoNewline -ForegroundColor DarkGray
    Start-Sleep -Seconds 10
    $waited += 10
}
Write-Host "`r                                                            `r" -NoNewline
if ($missing.Count -ne 0) { Stop-WithError "Nodurile $($missing -join ', ') nu au aparut in ${NodeWait}s." }

foreach ($node in $NodeIp.Keys) {
    $w = 0
    while ($true) {
        $h = Invoke-Node -Node $node -Command 'uptime -p'
        if ($LASTEXITCODE -eq 0 -and $h) { break }
        $w += 10
        if ($w -gt 300) { Stop-WithError "$node raspunde la ping dar nu la SSH." }
        Start-Sleep -Seconds 10
    }
    Write-Ok "$node - up, SSH OK ($h)"
}

# --------------------------------------------------- pas 2: asteapta quorum

Write-Step "Astept quorumul"

$waited = 0
while ($true) {
    $q = Invoke-Node -Node 'pvemini' -Command 'pvecm status'
    if (($q -join "`n") -match 'Quorate:\s*Yes') { break }
    if ($waited -ge $QuorumWait) { Stop-WithError "Clusterul nu a atins quorumul in ${QuorumWait}s." }
    Write-Host "`r  astept quorum (${waited}s)...   " -NoNewline -ForegroundColor DarkGray
    Start-Sleep -Seconds 10
    $waited += 10
}
Write-Host "`r                                    `r" -NoNewline

$votes = 0
if (($q -join "`n") -match 'Total votes:\s*(\d+)') { $votes = [int]$Matches[1] }
if ($votes -ne 3) { Write-Warn "Quorum atins, dar doar $votes/3 voturi. Un nod lipseste din cluster." }
Write-Ok "quorum OK, $votes/3 voturi"

Invoke-Node -Node 'pvemini' -Command 'ha-manager status' | Select-Object -First 4 | Write-Host

# ------------------------------------------------- pas 3: ce trebuie pornit

Write-Step "Ce se porneste"

$toStart = @()

if ((-not $All) -and (Test-Path $StateFile)) {
    foreach ($line in (Get-Content $StateFile)) {
        if ($line -match '^\s*#' -or $line -match '^\s*$') { continue }
        $f = ($line -split '\s+') | Where-Object { $_ -ne '' }
        if ($f.Count -ge 5) {
            $toStart += [pscustomobject]@{ Type=$f[0]; Node=$f[1]; Id=[int]$f[2]; Status=$f[3]; Ha=$f[4] }
        }
    }
    Write-Ok "citit din $StateFile ($($toStart.Count) guest-uri)"
} else {
    if (-not $All) { Write-Warn "Nu exista $StateFile - folosesc lista implicita." }

    $haSids = @()
    foreach ($line in (Invoke-Node -Node 'pvemini' -Command 'ha-manager config')) {
        if ($line -match '^(ct|vm):(\d+)') { $haSids += "$($Matches[1]):$($Matches[2])" }
    }

    $allGuests = @()
    foreach ($node in $NodeIp.Keys) {
        foreach ($g in (Get-NodeGuests -Node $node)) {
            $sid = "$($g.Type):$($g.Id)"
            $g | Add-Member -NotePropertyName Ha -NotePropertyValue $(if ($haSids -contains $sid) { 'ha' } else { 'noha' })
            $allGuests += $g
        }
    }

    # Fara fisier de stare pornim doar ce e explicit in lista de ordine SI nu e
    # in NeverAutostart. Altfel am porni DR-ul si template-urile, care erau
    # oprite intentionat.
    foreach ($id in $GuestStartOrder) {
        if ($NeverAutostart -contains $id) {
            Write-Warn "$id sarit (NeverAutostart) - porneste-l manual daca chiar il vrei"
            continue
        }
        $g = $allGuests | Where-Object { $_.Id -eq $id } | Select-Object -First 1
        if ($g) { $toStart += $g }
    }
}

if ($toStart.Count -eq 0) { Stop-WithError "Nimic de pornit." }

foreach ($g in $toStart) {
    "    {0,-3} {1,-9} {2,-4}  {3}" -f $g.Type, $g.Node, $g.Id, $g.Ha | Write-Host
}

Confirm-Step "Se pornesc guest-urile de mai sus, Oracle primul."

# ------------------------------------------------- pas 4: pornire guest-uri

Write-Step "Pornire guest-uri"

$failed = @()

function Start-Guest {
    param([pscustomobject]$Guest)

    $sid = "$($Guest.Type):$($Guest.Id)"
    $cmd = if ($Guest.Type -eq 'ct') { 'pct' } else { 'qm' }

    if ((Get-GuestStatus -Node $Guest.Node -Type $Guest.Type -Id $Guest.Id) -eq 'running') {
        Write-Ok "$sid deja pornit"
        return $true
    }

    if ($Guest.Ha -eq 'ha') {
        # Pentru resursele HA, --state started repune resursa sub controlul CRM.
        Invoke-NodeChange -Node $Guest.Node -Command "ha-manager set $sid --state started" | Out-Null
    } else {
        Invoke-NodeChange -Node $Guest.Node -Command "$cmd start $($Guest.Id)" | Out-Null
    }

    if ($DryRun) { return $true }

    $waited = 0
    while ($waited -lt $GuestWait) {
        if ((Get-GuestStatus -Node $Guest.Node -Type $Guest.Type -Id $Guest.Id) -eq 'running') {
            Write-Ok "$sid pornit (${waited}s)"
            return $true
        }
        Start-Sleep -Seconds 5
        $waited += 5
    }
    Write-Warn "$sid nu a pornit in ${GuestWait}s - verifica manual"
    return $false
}

function Wait-Oracle {
    param([string]$Node)
    Write-Log "astept ca instanta Oracle sa accepte interogari..."
    if ($DryRun) { return }

    $sql = 'pct exec ' + $OracleCt + ' -- docker exec ' + $OracleDocker + ' bash -c "printf ''set pagesize 0 feedback off\nselect global_name from global_name;\nexit\n'' | sqlplus -s / as sysdba"'
    $waited = 0
    while ($waited -lt $OracleWait) {
        $out = Invoke-Node -Node $Node -Command $sql
        if ($LASTEXITCODE -eq 0 -and (($out -join '') -match '[A-Z]')) {
            Write-Host "`r                                                  `r" -NoNewline
            Write-Ok "Oracle deschis si interogabil (${waited}s)"
            return
        }
        Write-Host "`r  Oracle inca nu raspunde (${waited}s)...   " -NoNewline -ForegroundColor DarkGray
        Start-Sleep -Seconds 15
        $waited += 15
    }
    Write-Host "`r                                                  `r" -NoNewline
    Write-Warn "Oracle nu a raspuns in ${OracleWait}s. Continui, dar VERIFICA MANUAL."
    Write-Warn "  ssh root@$($NodeIp[$Node]) `"pct exec $OracleCt -- docker logs --tail 50 $OracleDocker`""
}

foreach ($id in $GuestStartOrder) {
    $g = $toStart | Where-Object { $_.Id -eq $id } | Select-Object -First 1
    if (-not $g) { continue }

    Write-Log "pornesc $($g.Type):$($g.Id) pe $($g.Node) ($($g.Ha))"
    if (-not (Start-Guest -Guest $g)) { $failed += "$($g.Type):$($g.Id)" }

    # Oracle e blocant: nimic care depinde de el nu porneste pana nu e deschis.
    if ($g.Id -eq $OracleCt) { Wait-Oracle -Node $g.Node }
}

# orice a mai ramas in lista si nu era in ordinea definita
foreach ($g in $toStart) {
    if ($GuestStartOrder -contains $g.Id) { continue }
    Write-Warn "guest neplanificat: $($g.Type):$($g.Id) pe $($g.Node) - il pornesc"
    if (-not (Start-Guest -Guest $g)) { $failed += "$($g.Type):$($g.Id)" }
}

# ------------------------------------------------ pas 5: restaurare cron-uri

Write-Step "Restaurare cron-uri de monitorizare"

foreach ($node in $NodeIp.Keys) {
    $exists = Invoke-Node -Node $node -Command "test -f $CronBackup; echo `$?"
    if (($exists -join '') -notmatch '^0') {
        Write-Warn "$node - nu exista $CronBackup, sar peste"
        continue
    }
    Invoke-NodeChange -Node $node -Command "crontab $CronBackup" | Out-Null
    if (-not $DryRun) {
        $left = Invoke-Node -Node $node -Command "crontab -l | grep -c '^#MENTENANTA' || true"
        if (($left -join '').Trim() -eq '0') {
            Write-Ok "$node - cron-uri restaurate"
        } else {
            Write-Warn "$node - au ramas $left linii #MENTENANTA, verifica manual"
        }
    }
}

# ------------------------------------------------------- pas 6: verificare

Write-Step "Verificare finala"

if ($DryRun) {
    Write-Warn "dry-run - sar peste verificare"
} else {
    Invoke-Node -Node 'pvemini' -Command 'ha-manager status' | Write-Host
    Write-Host ""
    $rep = Invoke-Node -Node 'pvemini' -Command 'pvesr status'
    $rep | Write-Host
    Write-Host ""

    $badRep = @($rep | Select-Object -Skip 1 | Where-Object { $_ -notmatch 'OK\s*$' -and $_.Trim() -ne '' })
    if ($badRep.Count -eq 0) {
        Write-Ok "replicare: toate job-urile OK"
    } else {
        Write-Warn "job-uri de replicare cu probleme:"
        $badRep | Write-Host
    }
}

Write-Step "Cluster pornit"

if ($failed.Count -gt 0) {
    Write-Warn "Guest-uri care NU au pornit: $($failed -join ', ')"
    Write-Warn "Verifica-le manual inainte de a considera repornirea terminata."
} else {
    Write-Ok "toate guest-urile au pornit"
}

Write-Host ""
Write-Host "  De verificat manual:"
Write-Host "    - shutdown_policy e inca 'freeze' (recomandat sa ramana asa)"
Write-Host "    - daca lipsesc date recente, compara cu ultima replicare din pvesr status"
Write-Host ""