From dc61c43b3271acb7364eb3bef58d8b7b5abe7fa0 Mon Sep 17 00:00:00 2001 From: Claude Agent Date: Sun, 13 Sep 2026 10:18:12 +0000 Subject: [PATCH] feat(discord-bridge): registru infra_actions (stare cluster + actiuni de mentenanta) Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01KddsXCqEbKMhdFJDYbAsx8 --- .../discord-bridge/infra_actions.py | 839 ++++++++++++++++++ .../discord-bridge/security/infra | 3 + .../discord-bridge/tests/test_infra.py | 7 +- .../tests/test_infra_actions.py | 342 +++++++ 4 files changed, 1190 insertions(+), 1 deletion(-) create mode 100644 proxmox/lxc171-claude-agent/discord-bridge/infra_actions.py create mode 100644 proxmox/lxc171-claude-agent/discord-bridge/tests/test_infra_actions.py diff --git a/proxmox/lxc171-claude-agent/discord-bridge/infra_actions.py b/proxmox/lxc171-claude-agent/discord-bridge/infra_actions.py new file mode 100644 index 0000000..83ab441 --- /dev/null +++ b/proxmox/lxc171-claude-agent/discord-bridge/infra_actions.py @@ -0,0 +1,839 @@ +#!/usr/bin/env python3 +"""infra_actions.py -- nucleul tabului Infrastructura (dashboard + /infra Discord). + +Interfata comuna (vezi planul de implementare, WP4): + + ACTIONS: dict[str, dict] registrul actiunilor + status(fresh=False) -> dict stare cluster, cache 30s + check(action_id, st, dry_run) -> (bool, str) precondi\u021bii, functie pura + run(action_id, dry_run) -> dict executa o actiune + stop(action_id) -> dict opreste un job systemd-run + log_tail(action_id, n=60) -> dict ultimele linii din jurnal + maintenance_pending(st) -> bool cluster sus, dar in mentenanta + post_startup_instructions() -> str | None mesaj Discord + pin, la oprire + +Nu se ating nodurile decat prin `security/infra` (jurnalizat) pentru actiuni, si +direct prin ssh read-only pentru status() (nu umple infra.log la fiecare 30s). + +Nimic din acest modul nu ruleaza vreo comanda distructiva in mod implicit: toate +actiunile reale trec prin check() inainte, iar cele care opresc/pornesc ceva cer +`--yes`/`confirm` mai sus, in dashboard/bot. +""" + +from __future__ import annotations + +import concurrent.futures +import json +import re +import shlex +import subprocess +import sys +import threading +import time +import urllib.error +import urllib.request +from pathlib import Path + +_HERE = Path(__file__).resolve().parent +if str(_HERE) not in sys.path: + sys.path.insert(0, str(_HERE)) + +import config # noqa: E402 + +INFRA_BIN = str(_HERE / "security" / "infra") + +# Cei 3 noduri Proxmox, adrese fixe (acelasi tabel ca in security/infra). +NODES: dict[str, str] = { + "pve1": "10.0.20.200", + "pvemini": "10.0.20.201", + "pveelite": "10.0.20.202", +} + +SSH_STATUS_OPTS = [ + "-o", "BatchMode=yes", + "-o", "StrictHostKeyChecking=accept-new", + "-o", "ConnectTimeout=5", +] + +_STATUS_TIMEOUT_S = 20 +_CACHE_TTL_S = 30.0 + +# ---------------------------------------------------------------- registrul + +ACTIONS: dict[str, dict] = { + "stare": { + "label": "Stare cluster", "desc": "Cvorum, noduri, HA, UPS, backup, SSL.", + "host": "local", "cmd": None, "dry": None, "job": False, "confirm": False, + "pre": None, + }, + "sarcini": { + "label": "Sarcini active", "desc": "Job-urile systemd-run pornite din tab.", + "host": "local", "cmd": None, "dry": None, "job": False, "confirm": False, + "pre": None, + }, + "oprire": { + "label": "Oprire cluster", + "desc": "Salveaza starea, opreste guest-urile (mai putin CT 171), trimite instructiunile de pornire, opreste nodurile.", + "host": "pvemini", + "cmd": "/opt/scripts/cluster-shutdown.sh --allow-on-node --yes", + "dry": "/opt/scripts/cluster-shutdown.sh --allow-on-node --dry-run", + "job": True, "confirm": True, + "pre": "real: 3/3 noduri online, cvorum, niciun job activ; dry: pvemini online", + }, + "pornire": { + "label": "Pornire cluster", + "desc": "Restaureaza crontab-ul si porneste guest-urile care erau active.", + "host": "pvemini", + "cmd": "/opt/scripts/cluster-startup.sh --allow-on-node --yes", + "dry": "/opt/scripts/cluster-startup.sh --allow-on-node --dry-run", + "job": True, "confirm": True, + "pre": "pvemini online, cvorum, niciun job de oprire/pornire activ", + }, + "wol": { + "label": "Trimite Wake-on-LAN", + "desc": "Ruleaza C:\\wolcluster.bat de pe statia de administrare (trezeste cele 3 noduri).", + "host": "oracle-prod-admin", + "cmd": "cmd /c C:\\wolcluster.bat", + "dry": None, "job": False, "confirm": False, + "pre": "cel putin un nod offline", + }, + "ups-test": { + "label": "Test lunar UPS", + "desc": "Descarcare controlata a bateriei UPS + raport.", + "host": "pvemini", "cmd": "/opt/scripts/ups-monthly-test.sh", "dry": None, + "job": True, "confirm": True, + "pre": "UPS OL, incarcare >= 90%, niciun job UPS activ", + }, + "ups-simulare": { + "label": "Simulare oprire UPS", + "desc": "Ruleaza scriptul de oprire la panica UPS in --dry-run --force (nimic real).", + "host": "pvemini", + "cmd": "runuser -u nut -- sudo -n /usr/local/bin/ups-shutdown-cluster.sh --dry-run --force", + "dry": None, "job": True, "confirm": False, + "pre": "pvemini online", + }, + "ups-istoric": { + "label": "Istoric baterie UPS", + "desc": "Ultimele randuri din trendul de baterie.", + "host": "pvemini", "cmd": "tail -n 13 /var/log/ups-battery-trend.csv", + "dry": None, "job": False, "confirm": False, + "pre": "pvemini online", + }, + "dr-test": { + "label": "Test DR saptamanal (VM 109)", + "desc": "Porneste VM 109, verifica restore-ul, il opreste la final.", + "host": "vm109", + "cmd": ('touch /var/run/vm109-debug.flag; /opt/scripts/weekly-dr-test-proxmox.sh; ' + 'rc=$?; rm -f /var/run/vm109-debug.flag; exit $rc'), + "dry": None, "job": True, "confirm": True, + "pre": "VM 109 oprit, niciun job dr-test/dr-patch/vm109 activ", + }, + "dr-patch": { + "label": "Fereastra de patch VM 109", + "desc": "Porneste VM 109 in fereastra de update Windows.", + "host": "vm109", "cmd": "/opt/scripts/vm109-patch-window.sh --now", + "dry": None, "job": True, "confirm": True, + "pre": "VM 109 oprit, niciun job dr-test/dr-patch/vm109 activ", + }, + "vm109-pornire": { + "label": "Porneste VM 109", + "desc": "Pornire manuala, in afara ferestrelor programate.", + "host": "vm109", "cmd": "touch /var/run/vm109-debug.flag && qm start 109", + "dry": None, "job": False, "confirm": True, + "pre": "VM 109 oprit, niciun job dr activ", + }, + "vm109-oprire": { + "label": "Opreste VM 109", + "desc": "Oprire manuala.", + "host": "vm109", "cmd": "qm stop 109; rm -f /var/run/vm109-debug.flag", + "dry": None, "job": False, "confirm": True, + "pre": "VM 109 pornit, niciun job dr-test/dr-patch activ", + }, + "backup-pe-pvemini": { + "label": "Failover backup -> pvemini", + "desc": "pveelite e jos: pvemini preia rolul de tinta de backup Oracle.", + "host": "pvemini", "cmd": "/opt/scripts/failover-dr-to-pvemini.sh", + "dry": None, "job": True, "confirm": True, + "pre": "pveelite offline, pvemini online, failover-ul nu e deja activ", + }, + "backup-pe-pveelite": { + "label": "Failback backup -> pveelite", + "desc": "Revine la topologia normala dupa ce pveelite e iar online.", + "host": "pvemini", "cmd": "/opt/scripts/failback-dr-to-pveelite.sh", + "dry": None, "job": True, "confirm": True, + "pre": "pveelite online, failover-ul e activ", + }, +} + +_VALID_ID = re.compile(r"^[a-z0-9-]+$") +for _id, _spec in ACTIONS.items(): + assert _VALID_ID.match(_id), _id + assert _spec["host"] in ("local", "pvemini", "pveelite", "vm109", "oracle-prod-admin"), _id + +STARTUP_TEXT = ( + "Cluster oprit. Pornire:\n" + "1. JuiceSSH (prin Tailscale) -> 10.0.20.36 port 22122\n" + "2. C:\\wolcluster.bat (trezeste pve1, pvemini, pveelite prin Wake-on-LAN)\n" + "3. Asteapta mesajul \u201eCluster sus\u201c aici, in Discord\n" + "Rezerva daca WoL nu merge: ssh root@10.0.20.201 " + "/opt/scripts/cluster-startup.sh --allow-on-node --yes" +) + +# ------------------------------------------------------------------- status + +_STATUS_SH = r""" +set -o pipefail +echo @@cluster +pvesh get /cluster/resources --output-format json 2>&1 +echo @@quorum +pvecm status 2>&1 +echo @@ha +ha-manager status 2>&1 +echo @@repl +pvesh get /nodes/$(hostname)/replication --output-format json 2>&1 +echo @@jobs +systemctl list-units --plain --no-legend --all 'infra-*' 2>&1 +ls -t /var/log/infra-actions 2>/dev/null | head -20 +if command -v upsc >/dev/null 2>&1 && upsc nutdev1 >/tmp/.infra-ups 2>&1; then + echo @@ups + cat /tmp/.infra-ups + rm -f /tmp/.infra-ups +fi +if command -v zfs >/dev/null 2>&1 && zfs list rpool/oracle-backups >/dev/null 2>&1; then + echo @@zfs + zfs get -H -o value readonly rpool/oracle-backups 2>&1 +fi +echo @@maint +crontab -l 2>/dev/null | grep -c '^#MENTENANTA' +if [ -f /opt/scripts/monitor-ssl-certificates.sh ]; then + echo @@ssl + for d in $(sed -n '/^DOMAINS=(/,/^)/p' /opt/scripts/monitor-ssl-certificates.sh | grep -oE '"[^"]+"' | tr -d '"'); do + end=$(echo | timeout 5 openssl s_client -connect "$d:443" -servername "$d" 2>/dev/null \ + | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2) + if [ -n "$end" ]; then + days=$(( ($(date -d "$end" +%s) - $(date +%s)) / 86400 )) + echo "$d $days" + fi + done +fi +if [ -d /mnt/pve/oracle-backups/ROA/autobackup ]; then + echo @@backup + find /mnt/pve/oracle-backups/ROA/autobackup -maxdepth 1 -type f \ + \( -name '*FULL*.BKP' -o -name 'L0_*.BKP' \) -printf '%T@ %p\n' 2>/dev/null | sort -nr | head -1 + find /mnt/pve/oracle-backups/ROA/autobackup -maxdepth 1 -type f \ + \( -name '*INCR*.BKP' -o -name '*INCREMENTAL*.BKP' -o -name '*CUMULATIVE*.BKP' -o -name 'L1_*.BKP' \) \ + -printf '%T@ %p\n' 2>/dev/null | sort -nr | head -1 +fi +""" + + +def _split_sections(text: str) -> dict[str, str]: + sections: dict[str, str] = {} + cur: str | None = None + buf: list[str] = [] + for line in text.splitlines(): + if line.startswith("@@"): + if cur is not None: + sections[cur] = "\n".join(buf) + cur = line[2:].strip() + buf = [] + elif cur is not None: + buf.append(line) + if cur is not None: + sections[cur] = "\n".join(buf) + return sections + + +def _ssh_status_one(ip: str) -> str: + """Read-only: ssh direct, ocoleste security/infra (nu jurnalizeaza).""" + proc = subprocess.run( + ["ssh", *SSH_STATUS_OPTS, f"root@{ip}", "bash", "-s"], + input=_STATUS_SH, capture_output=True, text=True, timeout=_STATUS_TIMEOUT_S, + ) + return proc.stdout + + +def _collect() -> dict[str, "str | Exception"]: + outputs: dict[str, "str | Exception"] = {} + with concurrent.futures.ThreadPoolExecutor(max_workers=len(NODES)) as pool: + futures = {pool.submit(_ssh_status_one, ip): name for name, ip in NODES.items()} + for fut in concurrent.futures.as_completed(futures, timeout=_STATUS_TIMEOUT_S + 5): + name = futures[fut] + try: + outputs[name] = fut.result() + except Exception as exc: # timeout, connection refused, etc. + outputs[name] = exc + return outputs + + +def parse_status(outputs: dict[str, "str | Exception"]) -> dict: + """Pura: transforma iesirile brute per nod in forma din contractul status().""" + errors: dict[str, str] = {} + sections: dict[str, dict[str, str]] = {} + for host, out in outputs.items(): + if isinstance(out, Exception): + errors[host] = str(out) + continue + sections[host] = _split_sections(out) + + # --- cluster resources (de la primul nod care raspunde) --- + cluster_json: list[dict] | None = None + for host in ("pvemini", "pve1", "pveelite"): + txt = sections.get(host, {}).get("cluster", "").strip() + if not txt: + continue + try: + cluster_json = json.loads(txt) + break + except (ValueError, TypeError): + continue + + nodes: list[dict] = [] + guests_running = guests_total = 0 + vm109: dict | None = None + if cluster_json: + for name in NODES: + entry = next((e for e in cluster_json if e.get("type") == "node" and e.get("node") == name), None) + nodes.append({ + "name": name, + "ip": NODES[name], + "online": name in sections, + "cpu": (entry or {}).get("cpu", 0.0), + "mem": (entry or {}).get("mem", 0), + "maxmem": (entry or {}).get("maxmem", 0), + "uptime": (entry or {}).get("uptime", 0), + "guests": sum(1 for e in cluster_json if e.get("type") in ("qemu", "lxc") and e.get("node") == name), + }) + for e in cluster_json: + if e.get("type") in ("qemu", "lxc"): + guests_total += 1 + if e.get("status") == "running": + guests_running += 1 + if e.get("type") == "qemu" and str(e.get("vmid")) == "109": + vm109 = {"node": e.get("node"), "status": e.get("status")} + else: + for name in NODES: + nodes.append({ + "name": name, "ip": NODES[name], "online": name in sections, + "cpu": 0.0, "mem": 0, "maxmem": 0, "uptime": 0, "guests": 0, + }) + + # --- quorum --- + quorum = {"quorate": False, "votes": 0, "expected": 0} + for host in ("pvemini", "pve1", "pveelite"): + txt = sections.get(host, {}).get("quorum", "") + if not txt: + continue + m_q = re.search(r"^Quorate:\s*(\w+)", txt, re.MULTILINE) + m_v = re.search(r"^Total votes:\s*(\d+)", txt, re.MULTILINE) + m_e = re.search(r"^Expected votes:\s*(\d+)", txt, re.MULTILINE) + if m_q or m_v or m_e: + quorum = { + "quorate": bool(m_q and m_q.group(1).lower() == "yes"), + "votes": int(m_v.group(1)) if m_v else 0, + "expected": int(m_e.group(1)) if m_e else 0, + } + break + + # --- HA --- + ha: list[dict] = [] + for host in ("pvemini", "pve1", "pveelite"): + txt = sections.get(host, {}).get("ha", "") + if not txt: + continue + for line in txt.splitlines(): + m = re.match(r"^service\s+(\S+)\s+\((\S+),\s*(\w+)\)", line.strip()) + if m: + ha.append({"sid": m.group(1), "node": m.group(2), "state": m.group(3)}) + if ha: + break + + # --- replicare (merge peste toate nodurile, dedup pe id) --- + repl_jobs: dict[str, dict] = {} + for host, secs in sections.items(): + txt = secs.get("repl", "").strip() + if not txt: + continue + try: + arr = json.loads(txt) + except (ValueError, TypeError): + continue + if isinstance(arr, list): + for job in arr: + jid = job.get("id") + if jid: + repl_jobs[jid] = job + replication = { + "ok": sum(1 for j in repl_jobs.values() if not j.get("fail_count")), + "total": len(repl_jobs), + "last": max((j.get("last_sync", 0) for j in repl_jobs.values()), default=0), + "failed": [jid for jid, j in repl_jobs.items() if j.get("fail_count")], + } + + # --- jobs (systemd-run infra-*) --- + jobs: dict[tuple[str, str], dict] = {} + for host, secs in sections.items(): + txt = secs.get("jobs", "") + if not txt: + continue + for line in txt.splitlines(): + m = re.match(r"^infra-(\S+)\.service\s+\S+\s+(\S+)\s+(\S+)", line.strip()) + if m: + jid, load_state, active_state = m.group(1), m.group(2), m.group(3) + key = (host, jid) + jobs.setdefault(key, {"id": jid, "host": host, "active": False, "log": None}) + jobs[key]["active"] = active_state == "running" + continue + m2 = re.match(r"^([a-z0-9-]+)-\d{8}-\d{6}\.log$", line.strip()) + if m2: + jid = m2.group(1) + key = (host, jid) + entry = jobs.setdefault(key, {"id": jid, "host": host, "active": False, "log": None}) + if entry["log"] is None: + entry["log"] = f"/var/log/infra-actions/{line.strip()}" + job_list = list(jobs.values()) + + # --- UPS --- + ups = None + for host in ("pvemini", "pve1", "pveelite"): + txt = sections.get(host, {}).get("ups") + if txt is None: + continue + kv = {} + for line in txt.splitlines(): + if ":" in line: + k, _, v = line.partition(":") + kv[k.strip()] = v.strip() + if kv: + try: + charge = float(kv.get("battery.charge", "0")) + except ValueError: + charge = 0.0 + runtime_raw = kv.get("battery.runtime") + runtime_s = None + if runtime_raw: + try: + runtime_s = int(float(runtime_raw)) + except ValueError: + runtime_s = None + ups = {"status": kv.get("ups.status", ""), "charge": charge, "runtime_s": runtime_s} + break + + # --- ZFS readonly -> dr_failover_active --- + dr_failover_active = None + for host in ("pvemini", "pve1", "pveelite"): + txt = sections.get(host, {}).get("zfs") + if txt is None: + continue + val = txt.strip() + if val: + dr_failover_active = val == "off" + break + + # --- maintenance flag --- + maint = 0 + for host in ("pvemini", "pve1", "pveelite"): + txt = sections.get(host, {}).get("maint") + if txt is None: + continue + try: + maint = int(txt.strip() or "0") + except ValueError: + maint = 0 + break + + # --- SSL --- + ssl = None + best = None + for host in ("pvemini", "pve1", "pveelite"): + txt = sections.get(host, {}).get("ssl") + if not txt: + continue + for line in txt.splitlines(): + parts = line.split() + if len(parts) == 2: + domain, days_raw = parts + try: + days = int(days_raw) + except ValueError: + continue + if best is None or days < best[1]: + best = (domain, days) + break + if best: + ssl = {"min_days": best[1], "domain": best[0]} + + # --- backup age --- + backup = None + for host in ("pveelite", "pvemini", "pve1"): + txt = sections.get(host, {}).get("backup") + if txt is None: + continue + lines = [l for l in txt.splitlines() if l.strip()] + now = time.time() + + def _age_h(line: str) -> float | None: + parts = line.split(None, 1) + if not parts: + return None + try: + return (now - float(parts[0])) / 3600.0 + except ValueError: + return None + + full_age = _age_h(lines[0]) if len(lines) >= 1 else None + cum_age = _age_h(lines[1]) if len(lines) >= 2 else None + backup = {"full_age_h": full_age, "cum_age_h": cum_age} + break + + return { + "at": time.time(), + "errors": errors, + "nodes": nodes, + "quorum": quorum, + "guests": {"running": guests_running, "total": guests_total}, + "ha": ha, + "replication": replication, + "ups": ups, + "backup": backup, + "ssl": ssl, + "vm109": vm109, + "dr_failover_active": dr_failover_active, + "maint": maint, + "jobs": job_list, + } + + +_cache: dict = {"at": 0.0, "data": None} +_cache_lock = threading.Lock() + + +def _empty_status(error: str) -> dict: + return { + "at": time.time(), "errors": {"_": error}, "nodes": [], "quorum": {"quorate": False, "votes": 0, "expected": 0}, + "guests": {"running": 0, "total": 0}, "ha": [], "replication": {"ok": 0, "total": 0, "last": 0, "failed": []}, + "ups": None, "backup": None, "ssl": None, "vm109": None, "dr_failover_active": None, "maint": 0, "jobs": [], + } + + +def status(fresh: bool = False) -> dict: + """Stare cluster, cache 30s. Nu arunca niciodata.""" + now = time.time() + cached = _cache["data"] + if not fresh and cached is not None and now - _cache["at"] < _CACHE_TTL_S: + return cached + with _cache_lock: + now2 = time.time() + cached = _cache["data"] + # alt thread a reimprospatat deja cat am asteptat pe lock + if cached is not None and now2 - _cache["at"] < _CACHE_TTL_S: + return cached + try: + data = parse_status(_collect()) + except Exception as exc: # pragma: no cover - plasa de siguranta + data = _empty_status(str(exc)) + try: + data["actions"] = { + aid: {"ok": (r := check(aid, data, False))[0], "reason": r[1]} + for aid in ACTIONS + } + except Exception: # pragma: no cover + data["actions"] = {} + data["startup_text"] = STARTUP_TEXT + _cache["data"] = data + _cache["at"] = time.time() + return data + + +# ------------------------------------------------------------------- check + +def _node(st: dict, name: str) -> dict | None: + return next((n for n in st.get("nodes", []) if n["name"] == name), None) + + +def _online(st: dict, name: str) -> bool: + n = _node(st, name) + return bool(n and n["online"]) + + +def _active_jobs(st: dict) -> list[dict]: + return [j for j in st.get("jobs", []) if j.get("active")] + + +def _job_active(st: dict, *prefixes: str) -> bool: + return any(j["id"].startswith(prefixes) for j in _active_jobs(st)) + + +def check(action_id: str, st: dict, dry_run: bool) -> tuple[bool, str]: + """Precondi\u021bii pentru o actiune. Functie pura, nu atinge reteaua.""" + if action_id not in ACTIONS: + return False, "actiune necunoscuta" + + if action_id in ("stare", "sarcini"): + return True, "" + + if action_id == "oprire": + if dry_run: + return (True, "") if _online(st, "pvemini") else (False, "pvemini nu raspunde") + nodes = st.get("nodes", []) + if len(nodes) != 3 or not all(n["online"] for n in nodes): + return False, "nu toate cele 3 noduri sunt online" + if not st.get("quorum", {}).get("quorate"): + return False, "clusterul nu are cvorum" + if _active_jobs(st): + return False, "exista un job activ" + return True, "" + + if action_id == "pornire": + if not _online(st, "pvemini"): + return False, "pvemini nu raspunde" + if dry_run: + return True, "" + if not st.get("quorum", {}).get("quorate"): + return False, "clusterul nu are cvorum" + if _job_active(st, "oprire", "pornire"): + return False, "exista deja un job de oprire/pornire" + return True, "" + + if action_id == "wol": + nodes = st.get("nodes", []) + if not nodes: + return False, "stare necunoscuta" + if len(nodes) == 3 and all(n["online"] for n in nodes): + return False, "toate nodurile sunt deja online" + return True, "" + + if action_id == "ups-test": + ups = st.get("ups") + if not ups: + return False, "UPS necunoscut" + if ups.get("status") != "OL": + return False, f"UPS nu e OL (status={ups.get('status')})" + if ups.get("charge", 0) < 90: + return False, f"incarcare sub 90% ({ups.get('charge')}%)" + if _job_active(st, "ups-test"): + return False, "testul UPS ruleaza deja" + return True, "" + + if action_id in ("ups-simulare", "ups-istoric"): + return (True, "") if _online(st, "pvemini") else (False, "pvemini nu raspunde") + + if action_id in ("dr-test", "dr-patch"): + if not st.get("vm109"): + return False, "stare VM 109 necunoscuta" + if st["vm109"].get("status") != "stopped": + return False, "VM 109 e pornit" + if _job_active(st, "dr-test", "dr-patch", "vm109"): + return False, "exista deja un job DR activ" + return True, "" + + if action_id == "vm109-pornire": + if not st.get("vm109"): + return False, "stare VM 109 necunoscuta" + if st["vm109"].get("status") != "stopped": + return False, "VM 109 e deja pornit" + if _job_active(st, "dr-test", "dr-patch", "vm109"): + return False, "exista un job DR activ" + return True, "" + + if action_id == "vm109-oprire": + if not st.get("vm109"): + return False, "stare VM 109 necunoscuta" + if st["vm109"].get("status") != "running": + return False, "VM 109 e deja oprit" + if _job_active(st, "dr-test", "dr-patch"): + return False, "exista un job DR activ" + return True, "" + + if action_id == "backup-pe-pvemini": + if _online(st, "pveelite"): + return False, "pveelite e online" + if not _online(st, "pvemini"): + return False, "pvemini nu raspunde" + if st.get("dr_failover_active") is True: + return False, "failover-ul e deja activ" + return True, "" + + if action_id == "backup-pe-pveelite": + if not _online(st, "pveelite"): + return False, "pveelite nu e online" + if st.get("dr_failover_active") is not True: + return False, "failover-ul nu e activ" + return True, "" + + return False, "actiune fara verificare" # pragma: no cover - completitudine + + +# --------------------------------------------------------------------- run + +def _resolve_host(spec_host: str, st: dict) -> str: + if spec_host == "vm109": + vm109 = st.get("vm109") or {} + return vm109.get("node") or "pveelite" + return spec_host + + +def run(action_id: str, dry_run: bool) -> dict: + if action_id not in ACTIONS: + return {"ok": False, "code": 400, "error": f"actiune necunoscuta: {action_id}"} + + spec = ACTIONS[action_id] + st = status(fresh=True) + ok, reason = check(action_id, st, dry_run) + if not ok: + return {"ok": False, "code": 412, "host": spec["host"], "error": reason} + + cmd = spec["dry"] if dry_run else spec["cmd"] + if dry_run and cmd is None: + return {"ok": False, "code": 400, "error": "aceasta actiune nu are simulare"} + if cmd is None: + return {"ok": True, "code": 200, "host": spec["host"]} + + host = _resolve_host(spec["host"], st) + + warning = None + if action_id == "oprire" and not dry_run: + warning = post_startup_instructions() + + if spec["job"]: + result = _run_job(action_id, host, cmd) + else: + result = _run_sync(host, cmd) + if warning: + result["warning"] = warning + return result + + +def _run_job(action_id: str, host: str, cmd: str) -> dict: + remote_cmd = cmd + '; echo "[infra] cod iesire: $?"' + launcher = ( + f"mkdir -p /var/log/infra-actions && " + f"L=/var/log/infra-actions/{action_id}-$(date +%Y%m%d-%H%M%S).log && " + f"systemd-run --unit=infra-{action_id} --collect --quiet " + f"--setenv=INFRA_LOG=$L -p StandardOutput=append:$L -p StandardError=append:$L " + f"/bin/bash -c {shlex.quote(remote_cmd)} && echo $L" + ) + try: + proc = subprocess.run( + [sys.executable, INFRA_BIN, host, launcher], + capture_output=True, text=True, timeout=30, + ) + except subprocess.TimeoutExpired: + return {"ok": False, "code": 500, "host": host, "error": "timeout la lansarea job-ului"} + combined = proc.stdout + proc.stderr + if proc.returncode != 0: + if "already exists" in combined or "already loaded" in combined: + return {"ok": False, "code": 409, "host": host, "error": "job deja activ"} + return {"ok": False, "code": 500, "host": host, "error": combined.strip()[:2000]} + lines = [l for l in proc.stdout.strip().splitlines() if l.strip()] + log_path = lines[-1] if lines else None + return {"ok": True, "code": 200, "host": host, "log": log_path} + + +def _run_sync(host: str, cmd: str) -> dict: + try: + proc = subprocess.run( + [sys.executable, INFRA_BIN, host, cmd], + capture_output=True, text=True, timeout=120, + ) + except subprocess.TimeoutExpired: + return {"ok": False, "code": 500, "host": host, "error": "timeout"} + ok = proc.returncode == 0 + out = {"ok": ok, "code": 200 if ok else 500, "host": host, "output": proc.stdout} + if not ok: + out["error"] = proc.stderr.strip()[:2000] + return out + + +def stop(action_id: str) -> dict: + if action_id not in ACTIONS: + return {"ok": False, "code": 400, "error": "actiune necunoscuta"} + st = status() + host = _resolve_host(ACTIONS[action_id]["host"], st) + try: + proc = subprocess.run( + [sys.executable, INFRA_BIN, host, f"systemctl stop infra-{action_id}"], + capture_output=True, text=True, timeout=30, + ) + except subprocess.TimeoutExpired: + return {"ok": False, "code": 500, "host": host, "error": "timeout"} + ok = proc.returncode == 0 + out = {"ok": ok, "code": 200 if ok else 500, "host": host} + if not ok: + out["error"] = proc.stderr.strip()[:2000] + return out + + +def log_tail(action_id: str, n: int = 60) -> dict: + if action_id not in ACTIONS: + return {"file": None, "lines": [], "error": "actiune necunoscuta"} + st = status() + host = _resolve_host(ACTIONS[action_id]["host"], st) + remote = ( + f'F=$(ls -t /var/log/infra-actions/{action_id}-*.log 2>/dev/null | head -1); ' + f'[ -n "$F" ] && echo "@@file $F" && tail -n {int(n)} "$F"' + ) + try: + proc = subprocess.run( + [sys.executable, INFRA_BIN, host, remote], + capture_output=True, text=True, timeout=30, + ) + except subprocess.TimeoutExpired: + return {"file": None, "lines": [], "error": "timeout"} + out = proc.stdout + if not out.startswith("@@file "): + return {"file": None, "lines": []} + first, _, rest = out.partition("\n") + return {"file": first[len("@@file "):].strip(), "lines": rest.splitlines()} + + +def maintenance_pending(st: dict) -> bool: + """Cluster sus 3/3, cvorum complet, dar inca in starea de dupa oprire + (crontab-ul are inca marcajul #MENTENANTA) si niciun job de oprire/pornire + in desfasurare -- adica e nevoie sa se dea 'pornire' ca sa se termine.""" + q = st.get("quorum", {}) + if not q.get("quorate"): + return False + if q.get("votes", 0) < 3: + return False + if not st.get("maint", 0): + return False + if _job_active(st, "oprire", "pornire"): + return False + return True + + +def post_startup_instructions() -> str | None: + """Posteaza (si incearca sa fixeze) instructiunile de pornire in Discord. + + Nu blocheaza actiunea de oprire: orice eroare devine mesaj de avertisment. + """ + token = config.get("DISCORD_TOKEN", "") + chans = config.get_list("DISCORD_CHANNEL_IDS") or config.get_list("DISCORD_CHANNEL_ID") + if not token or not chans: + return "lipseste DISCORD_TOKEN sau canalul de Discord" + channel_id = chans[0] + try: + req = urllib.request.Request( + f"https://discord.com/api/v10/channels/{channel_id}/messages", + data=json.dumps({"content": STARTUP_TEXT}).encode("utf-8"), + method="POST", + headers={"Authorization": f"Bot {token}", "Content-Type": "application/json"}, + ) + with urllib.request.urlopen(req, timeout=10) as resp: + msg = json.loads(resp.read().decode("utf-8")) + except (urllib.error.URLError, urllib.error.HTTPError, ValueError, OSError) as exc: + return f"nu s-a putut posta mesajul de pornire: {exc}" + msg_id = msg.get("id") + if not msg_id: + return "raspuns Discord fara id de mesaj" + try: + pin_req = urllib.request.Request( + f"https://discord.com/api/v10/channels/{channel_id}/pins/{msg_id}", + method="PUT", + headers={"Authorization": f"Bot {token}"}, + ) + urllib.request.urlopen(pin_req, timeout=10) + except (urllib.error.URLError, urllib.error.HTTPError, OSError) as exc: + return f"mesaj postat, dar fixarea (pin) a esuat: {exc}" + return None diff --git a/proxmox/lxc171-claude-agent/discord-bridge/security/infra b/proxmox/lxc171-claude-agent/discord-bridge/security/infra index 7367a73..af63239 100755 --- a/proxmox/lxc171-claude-agent/discord-bridge/security/infra +++ b/proxmox/lxc171-claude-agent/discord-bridge/security/infra @@ -53,6 +53,9 @@ DEFAULT_HOSTS: dict[str, dict] = { "oracle-dr": {"addr": "10.0.20.37", "user": "romfast", "prod": True, "desc": "server Oracle DR"}, "roacentral": {"addr": "10.0.20.122", "user": "romfast", "prod": True, "desc": "VM 201 Windows, IIS reverse proxy"}, "oracle-test": {"addr": "10.0.20.130", "user": "romfast", "prod": False, "desc": "VM 302 mediu de test"}, + # statia de admin cu WoL pentru cluster (portul/cheia vin din ~/.ssh/config) + "oracle-prod-admin": {"addr": "oracle-prod-admin", "user": "Administrator", "prod": True, + "desc": "10.0.20.36 ca Administrator (WoL cluster)"}, } SSH_OPTS = [ diff --git a/proxmox/lxc171-claude-agent/discord-bridge/tests/test_infra.py b/proxmox/lxc171-claude-agent/discord-bridge/tests/test_infra.py index 08175e3..e30a508 100644 --- a/proxmox/lxc171-claude-agent/discord-bridge/tests/test_infra.py +++ b/proxmox/lxc171-claude-agent/discord-bridge/tests/test_infra.py @@ -163,8 +163,13 @@ def test_forma_scurta_addr_ca_string(home): def test_toate_hosturile_implicite_au_adresa(): mod = load_infra() + # oracle-prod-admin e un alias din ~/.ssh/config (portul/cheia vin de acolo), + # nu o adresa IP -- restul hosturilor raman IP-uri clasice. for name, spec in mod.DEFAULT_HOSTS.items(): - assert spec["addr"].count(".") == 3, name + if name == "oracle-prod-admin": + assert spec["addr"], name + else: + assert spec["addr"].count(".") == 3, name assert spec["user"], name # hosturile de productie sunt marcate ca atare assert mod.DEFAULT_HOSTS["oracle-prod"]["prod"] is True diff --git a/proxmox/lxc171-claude-agent/discord-bridge/tests/test_infra_actions.py b/proxmox/lxc171-claude-agent/discord-bridge/tests/test_infra_actions.py new file mode 100644 index 0000000..fff084b --- /dev/null +++ b/proxmox/lxc171-claude-agent/discord-bridge/tests/test_infra_actions.py @@ -0,0 +1,342 @@ +"""Teste pentru infra_actions.py (WP4). + +Fara retea: subprocess.run e inlocuit peste tot cu un dublu. Singurele lucruri +verificate impotriva realitatii sunt forma registrului ACTIONS si logica pura +(check, parse_status). +""" + +from __future__ import annotations + +import pathlib +import subprocess +import sys + +import pytest + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent)) + +import infra_actions as ia # noqa: E402 + + +def _cp(stdout: str = "", rc: int = 0, stderr: str = "") -> subprocess.CompletedProcess: + return subprocess.CompletedProcess(args=[], returncode=rc, stdout=stdout, stderr=stderr) + + +@pytest.fixture(autouse=True) +def _sin_cache(): + """Fiecare test porneste fara cache-ul de status intre ele.""" + ia._cache["data"] = None + ia._cache["at"] = 0.0 + yield + ia._cache["data"] = None + ia._cache["at"] = 0.0 + + +# ------------------------------------------------------------------ registru + +def test_id_uri_valide(): + for aid in ia.ACTIONS: + assert ia._VALID_ID.match(aid), aid + + +def test_chei_obligatorii(): + required = {"label", "desc", "host", "cmd", "dry", "job", "confirm", "pre"} + for aid, spec in ia.ACTIONS.items(): + assert required <= spec.keys(), aid + + +def test_nimic_fara_dry_run_pe_ups_shutdown(): + for aid, spec in ia.ACTIONS.items(): + cmds = [c for c in (spec["cmd"], spec["dry"]) if c] + for c in cmds: + if "ups-shutdown-cluster.sh" in c: + assert "--dry-run" in c, aid + + +def test_nimic_cu_vm201(): + for aid, spec in ia.ACTIONS.items(): + for c in (spec["cmd"], spec["dry"]): + if c: + assert "vm201" not in c.lower(), aid + + +def test_confirm_pentru_tot_ce_modifica_real(): + # singurele actiuni fara confirmare sunt cele read-only / fara efect real + fara_confirm = {aid for aid, s in ia.ACTIONS.items() if not s["confirm"]} + assert fara_confirm == {"stare", "sarcini", "wol", "ups-simulare", "ups-istoric"} + + +def test_oprire_pornire_au_yes(): + assert "--yes" in ia.ACTIONS["oprire"]["cmd"] + assert "--yes" in ia.ACTIONS["pornire"]["cmd"] + assert "--dry-run" in ia.ACTIONS["oprire"]["dry"] + assert "--dry-run" in ia.ACTIONS["pornire"]["dry"] + + +# ----------------------------------------------------------------------- run + +def test_run_id_necunoscut_fara_subprocess(monkeypatch): + apelat = [] + monkeypatch.setattr(subprocess, "run", lambda *a, **k: apelat.append(a) or _cp()) + res = ia.run("rm -rf /", False) + assert res == {"ok": False, "code": 400, "error": "actiune necunoscuta: rm -rf /"} + assert apelat == [] + + +def _st_full(**overrides) -> dict: + st = { + "nodes": [ + {"name": "pve1", "online": True}, {"name": "pvemini", "online": True}, + {"name": "pveelite", "online": True}, + ], + "quorum": {"quorate": True, "votes": 3, "expected": 3}, + "jobs": [], "ups": {"status": "OL", "charge": 100.0, "runtime_s": None}, + "vm109": {"node": "pveelite", "status": "stopped"}, + "dr_failover_active": False, + } + st.update(overrides) + return st + + +def test_run_oprire_dry_run_argv(monkeypatch): + monkeypatch.setattr(ia, "status", lambda fresh=False: _st_full()) + vazut = {} + + def fake_run(argv, **kw): + vazut["argv"] = argv + return _cp(stdout="") + + monkeypatch.setattr(subprocess, "run", fake_run) + res = ia.run("oprire", True) + assert res["ok"] is True + argv = vazut["argv"] + assert "--dry-run" in argv[-1] + assert ia.ACTIONS["oprire"]["dry"].split()[0] in argv[-1] + + +def test_run_precondictie_esuata_412(monkeypatch): + monkeypatch.setattr(ia, "status", lambda fresh=False: _st_full( + nodes=[{"name": "pve1", "online": False}, {"name": "pvemini", "online": True}, + {"name": "pveelite", "online": True}])) + apelat = [] + monkeypatch.setattr(subprocess, "run", lambda *a, **k: apelat.append(a) or _cp()) + res = ia.run("oprire", False) + assert res["code"] == 412 + assert res["ok"] is False + assert apelat == [] # precondictia pica inainte de orice subprocess + + +def test_run_backup_pe_pvemini_blocat_cu_pveelite_online(monkeypatch): + monkeypatch.setattr(ia, "status", lambda fresh=False: _st_full()) + res = ia.run("backup-pe-pvemini", False) + assert res["code"] == 412 + + +def test_run_backup_pe_pveelite_blocat_fara_failover_activ(monkeypatch): + monkeypatch.setattr(ia, "status", lambda fresh=False: _st_full(dr_failover_active=False)) + res = ia.run("backup-pe-pveelite", False) + assert res["code"] == 412 + + +def test_run_vm109_pornire_blocat_cu_job_dr_activ(monkeypatch): + monkeypatch.setattr(ia, "status", lambda fresh=False: _st_full( + jobs=[{"id": "dr-test", "host": "pveelite", "active": True, "log": None}])) + res = ia.run("vm109-pornire", False) + assert res["code"] == 412 + + +def test_run_wol_blocat_cu_toate_online(monkeypatch): + monkeypatch.setattr(ia, "status", lambda fresh=False: _st_full()) + res = ia.run("wol", False) + assert res["code"] == 412 + + +def test_run_job_lansat_prin_wrapper(monkeypatch): + monkeypatch.setattr(ia, "status", lambda fresh=False: _st_full()) + monkeypatch.setattr(ia, "post_startup_instructions", lambda: None) + vazut = {} + + def fake_run(argv, **kw): + vazut["argv"] = argv + return _cp(stdout="/var/log/infra-actions/ups-test-20260913-120000.log\n") + + monkeypatch.setattr(subprocess, "run", fake_run) + res = ia.run("ups-test", False) + assert res["ok"] is True + assert res["code"] == 200 + assert res["log"].endswith(".log") + argv = vazut["argv"] + assert argv[0] == sys.executable + assert argv[1] == ia.INFRA_BIN + assert argv[2] == "pvemini" + assert "systemd-run" in argv[3] + assert "infra-ups-test" in argv[3] + + +def test_run_job_deja_activ_409(monkeypatch): + monkeypatch.setattr(ia, "status", lambda fresh=False: _st_full()) + monkeypatch.setattr(ia, "post_startup_instructions", lambda: None) + monkeypatch.setattr(subprocess, "run", lambda *a, **k: _cp(rc=1, stderr="Unit infra-ups-test.service already exists")) + res = ia.run("ups-test", False) + assert res["code"] == 409 + + +def test_run_dry_fara_simulare_400(monkeypatch): + monkeypatch.setattr(ia, "status", lambda fresh=False: _st_full()) + res = ia.run("ups-test", True) + assert res["code"] == 400 + + +def test_run_vm109_host_rezolvat_din_status(monkeypatch): + monkeypatch.setattr(ia, "status", lambda fresh=False: _st_full(vm109={"node": "pveelite", "status": "stopped"})) + monkeypatch.setattr(ia, "post_startup_instructions", lambda: None) + vazut = {} + + def fake_run(argv, **kw): + vazut["argv"] = argv + return _cp(stdout="x.log") + + monkeypatch.setattr(subprocess, "run", fake_run) + ia.run("dr-test", False) + assert vazut["argv"][2] == "pveelite" + + +def test_stop_id_necunoscut(): + res = ia.stop("nu-exista") + assert res == {"ok": False, "code": 400, "error": "actiune necunoscuta"} + + +# --------------------------------------------------------------------- check + +def test_check_oprire_ok(): + ok, reason = ia.check("oprire", _st_full(), False) + assert ok is True and reason == "" + + +def test_check_pornire_blocat_de_job(): + ok, reason = ia.check("pornire", _st_full(jobs=[{"id": "oprire", "active": True}]), False) + assert ok is False + + +def test_check_ups_test_charge_mic(): + ok, _ = ia.check("ups-test", _st_full(ups={"status": "OL", "charge": 40.0, "runtime_s": None}), False) + assert ok is False + + +def test_check_id_necunoscut(): + ok, reason = ia.check("nu-exista", _st_full(), False) + assert ok is False + assert "necunoscuta" in reason + + +# ---------------------------------------------------------------- parse_status + +def test_parse_status_host_cu_timeout_nu_arunca(): + outputs = { + "pve1": subprocess.TimeoutExpired(cmd="ssh", timeout=20), + "pvemini": "", + "pveelite": "", + } + st = ia.parse_status(outputs) + assert "pve1" in st["errors"] + assert st["nodes"] == [ + {"name": "pve1", "ip": "10.0.20.200", "online": False, "cpu": 0.0, "mem": 0, "maxmem": 0, "uptime": 0, "guests": 0}, + {"name": "pvemini", "ip": "10.0.20.201", "online": True, "cpu": 0.0, "mem": 0, "maxmem": 0, "uptime": 0, "guests": 0}, + {"name": "pveelite", "ip": "10.0.20.202", "online": True, "cpu": 0.0, "mem": 0, "maxmem": 0, "uptime": 0, "guests": 0}, + ] + + +def test_parse_status_cluster_json(): + cluster = ( + '[{"type":"node","node":"pvemini","status":"online","cpu":0.1,"mem":100,"maxmem":200,"uptime":10},' + '{"type":"node","node":"pve1","status":"online","cpu":0.0,"mem":0,"maxmem":0,"uptime":0},' + '{"type":"node","node":"pveelite","status":"online","cpu":0.0,"mem":0,"maxmem":0,"uptime":0},' + '{"type":"qemu","vmid":109,"node":"pveelite","status":"stopped"},' + '{"type":"lxc","vmid":171,"node":"pvemini","status":"running"}]' + ) + outputs = { + "pve1": "", "pveelite": "", + "pvemini": f"@@cluster\n{cluster}\n@@quorum\nQuorate: Yes\nTotal votes: 3\nExpected votes: 3\n", + } + st = ia.parse_status(outputs) + pvemini = next(n for n in st["nodes"] if n["name"] == "pvemini") + assert pvemini["guests"] == 1 + assert pvemini["cpu"] == 0.1 + assert st["vm109"] == {"node": "pveelite", "status": "stopped"} + assert st["guests"] == {"running": 1, "total": 2} + assert st["quorum"] == {"quorate": True, "votes": 3, "expected": 3} + + +def test_parse_status_ups_si_zfs(): + outputs = { + "pve1": "", "pveelite": "", + "pvemini": "@@ups\nups.status: OL\nbattery.charge: 97\n@@zfs\noff\n@@maint\n1\n", + } + st = ia.parse_status(outputs) + assert st["ups"] == {"status": "OL", "charge": 97.0, "runtime_s": None} + assert st["dr_failover_active"] is True + assert st["maint"] == 1 + + +def test_parse_status_jobs(): + outputs = { + "pve1": "", "pveelite": "", + "pvemini": ( + "@@jobs\n" + "infra-ups-test.service loaded active running /bin/bash -c ...\n" + "ups-test-20260913-120000.log\n" + "oprire-20260901-010000.log\n" + ), + } + st = ia.parse_status(outputs) + by_id = {j["id"]: j for j in st["jobs"]} + assert by_id["ups-test"]["active"] is True + assert by_id["ups-test"]["log"].endswith("ups-test-20260913-120000.log") + assert by_id["oprire"]["active"] is False + + +def test_parse_status_fara_date_e_sigur(): + st = ia.parse_status({"pve1": Exception("boom"), "pvemini": Exception("x"), "pveelite": Exception("y")}) + assert st["ups"] is None + assert st["ssl"] is None + assert st["backup"] is None + assert st["vm109"] is None + assert st["dr_failover_active"] is None + assert set(st["errors"]) == {"pve1", "pvemini", "pveelite"} + + +# ------------------------------------------------------------- maintenance + +def test_maintenance_pending_true(): + st = _st_full() + st["maint"] = 1 + assert ia.maintenance_pending(st) is True + + +def test_maintenance_pending_false_fara_marcaj(): + st = _st_full() + st["maint"] = 0 + assert ia.maintenance_pending(st) is False + + +def test_maintenance_pending_false_cu_job_activ(): + st = _st_full(jobs=[{"id": "pornire", "active": True}]) + st["maint"] = 1 + assert ia.maintenance_pending(st) is False + + +# --------------------------------------------------------- post_startup_instructions + +def test_post_startup_instructions_fara_config(): + import config + config.reload({"__nu_exista__": True} and "/tmp/nu-exista-claude-discord-test") + # fara DISCORD_TOKEN in acest mediu izolat + import os + old = os.environ.pop("DISCORD_TOKEN", None) + try: + res = ia.post_startup_instructions() + assert res is not None + finally: + if old is not None: + os.environ["DISCORD_TOKEN"] = old + config.reload()