diff --git a/.gitattributes b/.gitattributes index d063bd4..83f73cf 100644 --- a/.gitattributes +++ b/.gitattributes @@ -13,3 +13,8 @@ proxmox/cluster/ups/config/battery-install-date text eol=lf *.ps1 text eol=crlf *.bat text eol=crlf *.cmd text eol=crlf + +# Regula udev și unitatea systemd pentru hotplug-ul adaptoarelor USB LAN ajung în +# /etc/udev/rules.d și /etc/systemd/system pe noduri. Un CRLF într-o regulă udev +# o face să nu se mai potrivească niciodată, tăcut. +proxmox/cluster/config/usb-lan-hotplug/* text eol=lf diff --git a/proxmox/cluster/config/usb-lan-hotplug/99-usb-lan-hotplug.rules b/proxmox/cluster/config/usb-lan-hotplug/99-usb-lan-hotplug.rules new file mode 100644 index 0000000..e18eca6 --- /dev/null +++ b/proxmox/cluster/config/usb-lan-hotplug/99-usb-lan-hotplug.rules @@ -0,0 +1,22 @@ +# Reatasarea adaptoarelor USB LAN dupa un reset pe magistrala USB. +# +# De ce exista: adaptoarele Realtek RTL8156 (driver r8152) folosite ca placa de +# retea principala pe pve1 si pveelite se reseteaza singure pe xHCI. Kernelul +# sterge interfata si o recreeaza, dar nimeni NU o readauga in vmbr0 si nu o +# ridica - interfata e declarata "inet manual", fara "auto" si fara +# "allow-hotplug", deci se ridica doar ca efect secundar la boot. +# +# Rezultatul: nodul merge perfect si e invizibil in retea, pe termen nelimitat. +# Pe pveelite asta a insemnat 16 ore de tacere in noaptea de 27 spre 28 august +# 2026 - vezi ../../incidents/2026-08-27-pveelite-down.md +# +# Regula prinde momentul in care interfata reapare si reaplica configuratia. +# Se declanseaza si la boot; e inofensiv, ifreload -a e idempotent. + +ACTION=="add", SUBSYSTEM=="net", ENV{ID_NET_DRIVER}=="r8152", RUN+="/usr/bin/systemctl --no-block start usb-lan-hotplug@$env{INTERFACE}.service" + +# De ce prefixul 99 si nu 70: ID_NET_DRIVER e populat abia de +# /usr/lib/udev/rules.d/80-net-setup-link.rules. La 70- variabila e inca goala, +# regula nu se potriveste si nu se intampla nimic - tacut. Verificat pe pveelite +# la 2026-08-28: cu 70-, "udevadm test" citea fisierul dar RUN-ul nu aparea in +# lista finala de comenzi. diff --git a/proxmox/cluster/config/usb-lan-hotplug/usb-lan-hotplug@.service b/proxmox/cluster/config/usb-lan-hotplug/usb-lan-hotplug@.service new file mode 100644 index 0000000..6298706 --- /dev/null +++ b/proxmox/cluster/config/usb-lan-hotplug/usb-lan-hotplug@.service @@ -0,0 +1,26 @@ +[Unit] +Description=Reataseaza adaptorul USB LAN %I in punte dupa un reset USB +Documentation=file:///etc/udev/rules.d/70-usb-lan-hotplug.rules +After=network.target + +[Service] +Type=oneshot + +# Dispozitivul tocmai a fost re-enumerat: driverul r8152 se incarca, interfata e +# redenumita din ethN in enxMAC, iar linkul negociaza. Reaplicarea configuratiei +# inainte ca astea sa se aseze prinde interfata intr-o stare intermediara. +ExecStartPre=/bin/sleep 3 + +# ifreload -a, nu ifup: interfata e "inet manual" si e port de punte, iar doar +# ifreload reface legatura cu vmbr0. E si comanda verificata in teren - exact ea +# a readus pveelite in retea pe 28.08 la 09:49, de la consola fizica. +# Aplica numai diferentele fata de /etc/network/interfaces, deci nu atinge +# interfetele tap/veth ale guest-urilor. +ExecStart=/usr/sbin/ifreload -a + +# Reteaua e deja rupta cand ajungem aici; o incercare esuata nu trebuie sa lase +# unitatea in "failed" si sa blocheze urmatoarea reaparitie a interfetei. +SuccessExitStatus=0 1 + +[Install] +WantedBy=multi-user.target diff --git a/proxmox/cluster/scripts/deploy-usb-lan-hotplug.ps1 b/proxmox/cluster/scripts/deploy-usb-lan-hotplug.ps1 new file mode 100644 index 0000000..dadd517 --- /dev/null +++ b/proxmox/cluster/scripts/deploy-usb-lan-hotplug.ps1 @@ -0,0 +1,244 @@ +<# +.SYNOPSIS + Instaleaza reatasarea automata a adaptoarelor USB LAN dupa un reset USB. + +.DESCRIPTION + Adaptoarele Realtek RTL8156 (driver r8152) folosite ca placa de retea + principala pe pve1 si pveelite se reseteaza singure pe magistrala USB. + Kernelul sterge interfata si o recreeaza, dar nimeni nu o readauga in vmbr0: + in /etc/network/interfaces ea e declarata "inet manual", fara "auto" si fara + "allow-hotplug", deci se ridica doar ca efect secundar, la boot. + + Consecinta e un nod care merge perfect si e invizibil in retea, pe termen + nelimitat. Pe pveelite asta a insemnat 16 ore de tacere in noaptea de 27 spre + 28 august 2026, plus inca un reset a doua zi la 10:31. + + Scriptul instaleaza o regula udev care prinde reaparitia interfetei si o + unitate systemd care reaplica configuratia de retea. Detecteaza singur ce + noduri au adaptor r8152 si le sare pe celelalte - pvemini are placa Intel + igc pe PCIe si nu are nevoie. + + Context: ../incidents/2026-08-27-pveelite-down.md + + RULEAZA DE PE STATIA DE ADMIN. SSH intre nodurile Proxmox trece prin + Tailscale si cere autentificare interactiva. + +.PARAMETER Node + Instaleaza doar pe nodul dat (pve1 / pvemini / pveelite). Implicit: toate. + +.PARAMETER DryRun + Arata ce ar face, fara sa modifice nimic. + +.PARAMETER Test + Dupa instalare, forteaza un reset USB real ca sa verifice ca regula chiar + functioneaza. PERICULOS FARA PLASA - vezi nota de mai jos. Scriptul isi pune + singur o plasa de siguranta: un "ifreload -a" programat peste 90 de secunde, + care repara nodul chiar daca regula nu s-a declansat. Nu folosi pe pvemini. + +.EXAMPLE + .\deploy-usb-lan-hotplug.ps1 -DryRun + +.EXAMPLE + .\deploy-usb-lan-hotplug.ps1 + +.EXAMPLE + .\deploy-usb-lan-hotplug.ps1 -Node pveelite -Test +#> +[CmdletBinding()] +param( + [ValidateSet('pve1','pvemini','pveelite')] + [string]$Node, + [switch]$DryRun, + [switch]$Test +) + +$ErrorActionPreference = 'Stop' + +$NodeIp = [ordered]@{ + pve1 = '10.0.20.200' + pvemini = '10.0.20.201' + pveelite = '10.0.20.202' +} + +$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path +$CfgDir = Join-Path (Split-Path -Parent $ScriptDir) 'config\usb-lan-hotplug' +$RuleSrc = Join-Path $CfgDir '99-usb-lan-hotplug.rules' +$UnitSrc = Join-Path $CfgDir 'usb-lan-hotplug@.service' + +$RuleDst = '/etc/udev/rules.d/99-usb-lan-hotplug.rules' +$UnitDst = '/etc/systemd/system/usb-lan-hotplug@.service' + +function Write-Step { param([string]$Text) Write-Host ""; Write-Host "== $Text" -ForegroundColor Green } +function Write-Ok { param([string]$Text) Write-Host " [ok] $Text" -ForegroundColor Green } +function Write-Warn { param([string]$Text) Write-Host " [!] $Text" -ForegroundColor Yellow } +function Write-Bad { param([string]$Text) Write-Host " [X] $Text" -ForegroundColor Red } +function Write-Info { param([string]$Text) Write-Host " $Text" -ForegroundColor DarkGray } + +function Test-Alive { + param([string]$Address, [int]$TimeoutMs = 2000) + try { + $p = New-Object System.Net.NetworkInformation.Ping + return ($p.Send($Address, $TimeoutMs).Status -eq 'Success') + } catch { return $false } +} + +function Invoke-Node { + param([string]$Ip, [string]$Command, [int]$Timeout = 15) + return ssh -o BatchMode=yes -o ConnectTimeout=$Timeout ` + -o StrictHostKeyChecking=accept-new -o LogLevel=ERROR ` + "root@$Ip" $Command +} + +function Invoke-NodeScript { + # PowerShell inghite ghilimelele cand paseaza argumente catre ssh, iar shell-ul + # de pe nod primeste [ = r8152 ] in loc de [ "$d" = r8152 ]. In loc sa ne luptam + # cu nivelurile de citare, trimitem scriptul codificat si il decodam acolo. + param([string]$Ip, [string]$Script, [int]$Timeout = 20) + $lf = $Script -replace "`r`n", "`n" + $b64 = [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($lf)) + return Invoke-Node -Ip $Ip -Command "echo $b64 | base64 -d | sh" -Timeout $Timeout +} + +foreach ($f in @($RuleSrc, $UnitSrc)) { + if (-not (Test-Path $f)) { Write-Bad "lipseste $f"; exit 1 } + # Un CRLF intr-o regula udev o face sa nu se potriveasca niciodata, tacut. + # .gitattributes forteaza LF pe directorul asta, dar verificam oricum. + if ([IO.File]::ReadAllText($f) -match "`r") { + Write-Bad "$f are CRLF - trebuie LF. Verifica .gitattributes." + exit 1 + } +} +Write-Host "" +Write-Host "Hotplug pentru adaptoarele USB LAN (r8152)" -ForegroundColor Cyan +Write-Info "sursa: $CfgDir" + +$tinte = if ($Node) { @($Node) } else { $NodeIp.Keys } +$instalate = @() + +foreach ($n in $tinte) { + $ip = $NodeIp[$n] + Write-Step "$n ($ip)" + + if (-not (Test-Alive $ip)) { + Write-Warn "nu raspunde la ping - sarit. Reia cu -Node $n cand revine." + continue + } + + # Nodurile fara adaptor USB nu au nevoie de regula. pvemini are Intel igc. + $ifs = Invoke-NodeScript -Ip $ip -Script @' +for n in /sys/class/net/*; do + i=${n##*/} + d=$(readlink -f "$n/device/driver" 2>/dev/null) + case "$d" in *r8152) echo "$i" ;; esac +done +'@ + if (-not $ifs) { + Write-Ok "nu are adaptor USB r8152 - nu are nevoie de regula, sarit" + continue + } + Write-Info "adaptoare r8152: $($ifs -join ', ')" + + # Interfata conteaza doar daca e chiar portul puntii; altfel regula e inutila. + $port = Invoke-NodeScript -Ip $ip -Script @' +sed -n 's/^[[:space:]]*bridge-ports[[:space:]]*//p' /etc/network/interfaces +'@ + Write-Info "bridge-ports vmbr0: $port" + if ($ifs -notcontains $port) { + Write-Warn "adaptorul USB nu e portul puntii - instalez oricum, dar verifica manual" + } + + if ($DryRun) { + Write-Info "[dry-run] scp $RuleSrc -> ${ip}:$RuleDst" + Write-Info "[dry-run] scp $UnitSrc -> ${ip}:$UnitDst" + Write-Info "[dry-run] udevadm control --reload && systemctl daemon-reload" + continue + } + + scp -o BatchMode=yes -o LogLevel=ERROR $RuleSrc "root@${ip}:$RuleDst" | Out-Null + scp -o BatchMode=yes -o LogLevel=ERROR $UnitSrc "root@${ip}:$UnitDst" | Out-Null + Invoke-Node -Ip $ip -Command "udevadm control --reload-rules; systemctl daemon-reload" | Out-Null + + # Verificare reala: fisierele exista, unitatea e vazuta de systemd si regula + # e sintactic valida. udevadm test nu modifica nimic. + # Backtick-ul escapeaza variabilele de shell, ca sa nu le interpoleze PowerShell. + $ver = Invoke-NodeScript -Ip $ip -Script @" +if [ -f $RuleDst ]; then echo 'regula: prezenta'; else echo 'regula: LIPSA'; fi +if [ -f $UnitDst ]; then echo 'unitate: prezenta'; else echo 'unitate: LIPSA'; fi +cr=`$(tr -d '\n' < $RuleDst | tr -d '[:print:]' | wc -c) +echo "neprintabile in regula: `$cr" +if systemctl cat usb-lan-hotplug@test.service >/dev/null 2>&1; then echo 'systemd: vede unitatea'; else echo 'systemd: NU vede unitatea'; fi +"@ + $ver | ForEach-Object { Write-Info $_ } + + if ($ver -match 'LIPSA' -or $ver -match 'neprintabile in regula: [1-9]' -or $ver -match 'NU vede') { + Write-Bad "instalare incompleta pe $n" + } else { + Write-Ok "instalat" + $instalate += $n + } +} + +# ------------------------------------------------------- testul cu reset real + +if ($Test -and -not $DryRun) { + if (-not $Node) { Write-Bad "-Test cere -Node explicit"; exit 1 } + if ($Node -eq 'pvemini') { Write-Bad "refuz -Test pe pvemini: acolo ruleaza productia"; exit 1 } + if ($instalate -notcontains $Node) { Write-Bad "$Node nu a fost instalat cu succes - nu testez"; exit 1 } + + $ip = $NodeIp[$Node] + Write-Step "Test cu reset USB real pe $Node" + Write-Warn "Nodul isi pierde reteaua pentru cateva secunde." + Write-Info "Plasa de siguranta: 'ifreload -a' programat peste 90 s, care repara" + Write-Info "nodul si daca regula nu se declanseaza - deci fara drum pana la el." + + $usbPath = Invoke-NodeScript -Ip $ip -Script @' +for d in /sys/bus/usb/devices/*; do + [ -e "$d/idVendor" ] || continue + [ "$(cat "$d/idVendor")" = "0bda" ] || continue + ls "$d"/*/net >/dev/null 2>&1 || continue + echo "${d##*/}" + break +done +'@ + if (-not $usbPath) { Write-Bad "nu gasesc dispozitivul USB Realtek"; exit 1 } + Write-Info "dispozitiv USB: $usbPath" + + # Plasa se armeaza INAINTE de reset, cu nohup, ca sa supravietuiasca inchiderii + # sesiunii SSH odata cu caderea retelei. + Invoke-NodeScript -Ip $ip -Script @' +nohup sh -c 'sleep 90; /usr/sbin/ifreload -a' >/dev/null 2>&1 & +'@ | Out-Null + Write-Ok "plasa de siguranta armata (90 s)" + + # Reset real: dezautorizeaza si reautorizeaza dispozitivul, exact ce face + # controllerul xHCI cand cedeaza singur. + Invoke-NodeScript -Ip $ip -Script @" +nohup sh -c 'echo 0 > /sys/bus/usb/devices/$usbPath/authorized; sleep 2; echo 1 > /sys/bus/usb/devices/$usbPath/authorized' >/dev/null 2>&1 & +"@ | Out-Null + Write-Info "reset trimis, astept revenirea..." + + $revenit = $false + foreach ($i in 1..30) { + Start-Sleep -Seconds 4 + if (Test-Alive $ip) { $revenit = $true; break } + } + + if ($revenit) { + Write-Ok "nodul a revenit dupa ~$($i * 4) s" + if ($i * 4 -lt 60) { + Write-Ok "sub 60 s => regula udev a lucrat, nu plasa de siguranta" + } else { + Write-Warn "peste 60 s => posibil sa fi lucrat plasa. Verifica jurnalul:" + Write-Info " ssh root@$ip 'journalctl -u usb-lan-hotplug@* --since \"-5 min\"'" + } + $j = Invoke-Node -Ip $ip -Command "journalctl -u 'usb-lan-hotplug@*' --since '-5 min' --no-pager | tail -12" + $j | ForEach-Object { Write-Info $_ } + } else { + Write-Bad "nodul NU a revenit in 2 minute - verifica la consola fizica" + exit 1 + } +} + +Write-Host "" +if ($instalate) { Write-Ok "instalat pe: $($instalate -join ', ')" } +else { Write-Warn "niciun nod modificat" }