fix(worker): keepalive RAR ca dashboard-ul sa nu afiseze fals "RAR inaccesibil"
Dashboard-ul deduce starea RAR din vechimea ultimului login reusit (>30h -> "indisponibil?"). Cand coada e goala, worker-ul nu are de ce sa se logheze, deci timestamp-ul devine stale si banner-ul "Blocat: RAR inaccesibil — declaratiile NU pleaca" apare fals, desi RAR raspunde. Worker-ul face acum un login de proba o data pe zi (interval configurabil, 24h < pragul de 30h) cand coada e goala: pe succes reimprospateaza last_rar_login_ok; pe esec real last_rar_login_ok ramane vechi -> dashboard degradeaza corect. Forteaza login real (invalideaza sesiunea) ca proba sa fie autentica. Gating: cel mult o sondare pe interval, sa nu hartuiasca RAR jos. _keepalive_target sare conturile ale caror creds NU se decripteaza sub cheia curenta (start.sh both genereaza cheie efemera noua la fiecare pornire -> creds durabile vechi dau decrypt None) si cade pe creds <test> in dev. Teste: tests/test_worker_keepalive_rar.py (6). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -99,6 +99,11 @@ class Settings(BaseSettings):
|
|||||||
# Dev: foloseste creds <test> din settings.xml pt login worker. In productie
|
# Dev: foloseste creds <test> din settings.xml pt login worker. In productie
|
||||||
# creds vin per-cerere de la ROAAUTO — lasa False.
|
# creds vin per-cerere de la ROAAUTO — lasa False.
|
||||||
worker_use_test_creds: bool = False
|
worker_use_test_creds: bool = False
|
||||||
|
# Keepalive RAR: cand coada e goala, worker-ul face un login de proba la fiecare
|
||||||
|
# atata timp ca sa pastreze last_rar_login_ok proaspat (sub pragul de 30h al
|
||||||
|
# dashboard-ului) — altfel banner-ul "RAR inaccesibil" apare fals doar din lipsa
|
||||||
|
# de trafic. 0 = dezactivat. Implicit o data pe zi (24h < 30h, margine de 6h).
|
||||||
|
worker_rar_keepalive_interval_s: int = 86400
|
||||||
worker_sending_lease_s: int = 120 # rand 'sending' mai vechi de atat = orfan (worker mort mid-POST)
|
worker_sending_lease_s: int = 120 # rand 'sending' mai vechi de atat = orfan (worker mort mid-POST)
|
||||||
worker_retry_base_s: int = 5 # backoff = base * 2^retry (plafonat la max)
|
worker_retry_base_s: int = 5 # backoff = base * 2^retry (plafonat la max)
|
||||||
worker_retry_max_s: int = 300
|
worker_retry_max_s: int = 300
|
||||||
|
|||||||
@@ -34,7 +34,7 @@ import httpx
|
|||||||
from .. import errors
|
from .. import errors
|
||||||
from ..config import Settings, get_settings, load_test_credentials
|
from ..config import Settings, get_settings, load_test_credentials
|
||||||
from ..crypto import decrypt_creds
|
from ..crypto import decrypt_creds
|
||||||
from ..db import get_connection, init_db, write_heartbeat
|
from ..db import get_connection, init_db, read_heartbeat, write_heartbeat
|
||||||
from ..observ import log_event, set_source
|
from ..observ import log_event, set_source
|
||||||
from ..mapping import DEFAULT_ACCOUNT_ID, upsert_nomenclator
|
from ..mapping import DEFAULT_ACCOUNT_ID, upsert_nomenclator
|
||||||
from ..payload import build_rar_payload
|
from ..payload import build_rar_payload
|
||||||
@@ -428,6 +428,68 @@ def _creds_from_account(conn, account_id: int) -> dict | None:
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _keepalive_target(conn, settings: Settings) -> tuple[int | None, dict | None]:
|
||||||
|
"""Un cont cu creds durabile pentru login-ul de proba (sau creds <test> in dev).
|
||||||
|
|
||||||
|
Sare conturile ale caror creds NU se decripteaza sub cheia curenta — in dev
|
||||||
|
`start.sh both` genereaza o cheie efemera noua la fiecare pornire, deci creds-urile
|
||||||
|
durabile criptate sub cheia veche dau decrypt -> None. Fallback la creds <test>.
|
||||||
|
"""
|
||||||
|
rows = conn.execute(
|
||||||
|
"SELECT id, rar_creds_enc FROM accounts "
|
||||||
|
"WHERE rar_creds_enc IS NOT NULL ORDER BY id"
|
||||||
|
).fetchall()
|
||||||
|
for row in rows:
|
||||||
|
creds = decrypt_creds(row["rar_creds_enc"])
|
||||||
|
if creds and creds.get("email") and creds.get("password"):
|
||||||
|
return row["id"], creds
|
||||||
|
if settings.worker_use_test_creds:
|
||||||
|
return DEFAULT_ACCOUNT_ID, load_test_credentials()
|
||||||
|
return None, None
|
||||||
|
|
||||||
|
|
||||||
|
def _maybe_keepalive(conn, settings: Settings, sessions: "AccountSessions", state: dict) -> None:
|
||||||
|
"""Login de proba periodic cand coada e goala — verifica reachability RAR si
|
||||||
|
pastreaza last_rar_login_ok proaspat ca dashboard-ul sa nu afiseze fals
|
||||||
|
'RAR inaccesibil' doar din lipsa de trafic.
|
||||||
|
|
||||||
|
Sondeaza la cel mult o data pe interval (si pe succes, si pe esec): pe succes
|
||||||
|
heartbeat-ul se reimprospateaza singur; pe esec real (RAR jos) last_rar_login_ok
|
||||||
|
ramane vechi -> dashboard-ul degradeaza corect. Forteaza login real (invalideaza
|
||||||
|
sesiunea cache-uita) ca proba sa fie autentica, nu un token vechi din cache.
|
||||||
|
"""
|
||||||
|
interval = settings.worker_rar_keepalive_interval_s
|
||||||
|
if interval <= 0:
|
||||||
|
return
|
||||||
|
hb = read_heartbeat(conn)
|
||||||
|
last = hb["last_rar_login_ok"] if hb else None
|
||||||
|
if last:
|
||||||
|
try:
|
||||||
|
age = (datetime.now(timezone.utc) - datetime.fromisoformat(last)).total_seconds()
|
||||||
|
if age < interval:
|
||||||
|
return # login inca proaspat — nimic de facut
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
pass
|
||||||
|
now_ts = time.time()
|
||||||
|
if now_ts - state["last_attempt"] < interval:
|
||||||
|
return # deja am incercat recent (nu hartui RAR daca e jos)
|
||||||
|
state["last_attempt"] = now_ts
|
||||||
|
|
||||||
|
account_id, creds = _keepalive_target(conn, settings)
|
||||||
|
if account_id is None or not creds:
|
||||||
|
return # niciun cont cu creds durabile — nimic de sondat
|
||||||
|
sessions.invalidate(account_id) # forteaza login real, nu token din cache
|
||||||
|
try:
|
||||||
|
sessions.get_token(conn, account_id, creds) # reimprospateaza last_rar_login_ok la succes
|
||||||
|
except RarAuthError:
|
||||||
|
pass # creds invalide — deja logat in get_token (WARNING)
|
||||||
|
except Exception as exc:
|
||||||
|
# RAR indisponibil: last_rar_login_ok ramane vechi (corect). Nu propaga.
|
||||||
|
log_event("rar_keepalive", nivel="WARNING", account_id=account_id,
|
||||||
|
mesaj=f"keepalive RAR esuat (cont {account_id}): {type(exc).__name__}",
|
||||||
|
context={"rezultat": "esuat"}, conn=conn, sursa="worker")
|
||||||
|
|
||||||
|
|
||||||
def run() -> int:
|
def run() -> int:
|
||||||
signal.signal(signal.SIGTERM, _stop)
|
signal.signal(signal.SIGTERM, _stop)
|
||||||
signal.signal(signal.SIGINT, _stop)
|
signal.signal(signal.SIGINT, _stop)
|
||||||
@@ -440,6 +502,7 @@ def run() -> int:
|
|||||||
|
|
||||||
sessions = AccountSessions(settings)
|
sessions = AccountSessions(settings)
|
||||||
_last_purge_time: float = 0.0
|
_last_purge_time: float = 0.0
|
||||||
|
_keepalive_state = {"last_attempt": 0.0}
|
||||||
|
|
||||||
while _running:
|
while _running:
|
||||||
try:
|
try:
|
||||||
@@ -466,6 +529,9 @@ def run() -> int:
|
|||||||
# Nimic de trimis: recupereaza orfanii conturilor deja logate.
|
# Nimic de trimis: recupereaza orfanii conturilor deja logate.
|
||||||
for acct, rar, tok in sessions.active():
|
for acct, rar, tok in sessions.active():
|
||||||
recover_orphans(conn, settings, rar, tok, account_id=acct)
|
recover_orphans(conn, settings, rar, tok, account_id=acct)
|
||||||
|
# Login de proba periodic ca dashboard-ul sa nu afiseze fals
|
||||||
|
# "RAR inaccesibil" din lipsa de trafic (vezi _maybe_keepalive).
|
||||||
|
_maybe_keepalive(conn, settings, sessions, _keepalive_state)
|
||||||
time.sleep(settings.worker_poll_interval_s)
|
time.sleep(settings.worker_poll_interval_s)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
|||||||
185
tests/test_worker_keepalive_rar.py
Normal file
185
tests/test_worker_keepalive_rar.py
Normal file
@@ -0,0 +1,185 @@
|
|||||||
|
"""Teste keepalive RAR — login de proba periodic ca dashboard-ul sa nu afiseze
|
||||||
|
fals "RAR inaccesibil" doar din lipsa de trafic.
|
||||||
|
|
||||||
|
Comportament asteptat (_maybe_keepalive):
|
||||||
|
- login vechi/lipsa + creds durabile -> sondeaza (get_token apelat) si forteaza
|
||||||
|
login real (invalidate inainte);
|
||||||
|
- login proaspat (sub interval) -> NU sondeaza;
|
||||||
|
- interval=0 -> dezactivat;
|
||||||
|
- fara cont cu creds durabile -> nu sondeaza;
|
||||||
|
- gating: dupa o incercare, nu re-sondeaza in cadrul intervalului (nu hartui RAR).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import tempfile
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture()
|
||||||
|
def env(monkeypatch):
|
||||||
|
tmp = tempfile.mkdtemp()
|
||||||
|
monkeypatch.setenv("AUTOPASS_DB_PATH", os.path.join(tmp, "t.db"))
|
||||||
|
from app.config import get_settings
|
||||||
|
get_settings.cache_clear()
|
||||||
|
from app.db import get_connection, init_db
|
||||||
|
init_db()
|
||||||
|
conn = get_connection()
|
||||||
|
yield conn, get_settings()
|
||||||
|
conn.close()
|
||||||
|
get_settings.cache_clear()
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeSessions:
|
||||||
|
"""Imita AccountSessions: get_token reusit reimprospateaza heartbeat-ul (ca realul)."""
|
||||||
|
|
||||||
|
def __init__(self, conn, *, fail: bool = False):
|
||||||
|
self._conn = conn
|
||||||
|
self._fail = fail
|
||||||
|
self.invalidated: list[int] = []
|
||||||
|
self.tokens: list[int] = []
|
||||||
|
|
||||||
|
def invalidate(self, account_id: int) -> None:
|
||||||
|
self.invalidated.append(account_id)
|
||||||
|
|
||||||
|
def get_token(self, conn, account_id: int, creds) -> str | None:
|
||||||
|
self.tokens.append(account_id)
|
||||||
|
if self._fail:
|
||||||
|
raise RuntimeError("RAR jos")
|
||||||
|
from app.db import write_heartbeat
|
||||||
|
write_heartbeat(conn, rar_login_ok=True, detail=f"login proba (cont {account_id})")
|
||||||
|
return "tok"
|
||||||
|
|
||||||
|
|
||||||
|
def _set_last_login(conn, *, ago_s: float | None):
|
||||||
|
"""Seteaza last_rar_login_ok la now-ago_s (None = niciun login)."""
|
||||||
|
from app.db import write_heartbeat
|
||||||
|
write_heartbeat(conn, detail="poll") # asigura randul heartbeat
|
||||||
|
if ago_s is None:
|
||||||
|
conn.execute("UPDATE worker_heartbeat SET last_rar_login_ok=NULL WHERE id=1")
|
||||||
|
else:
|
||||||
|
ts = (datetime.now(timezone.utc) - timedelta(seconds=ago_s)).isoformat()
|
||||||
|
conn.execute("UPDATE worker_heartbeat SET last_rar_login_ok=? WHERE id=1", (ts,))
|
||||||
|
conn.commit()
|
||||||
|
|
||||||
|
|
||||||
|
def _account_cu_creds(conn) -> int:
|
||||||
|
from app.accounts import create_account
|
||||||
|
from app.crypto import encrypt_creds
|
||||||
|
acct = create_account(conn, "Service Cu Creds", email="svc@example.com")
|
||||||
|
enc = encrypt_creds({"email": "svc@example.com", "password": "secret"})
|
||||||
|
conn.execute("UPDATE accounts SET rar_creds_enc=? WHERE id=?", (enc, acct))
|
||||||
|
conn.commit()
|
||||||
|
return acct
|
||||||
|
|
||||||
|
|
||||||
|
def test_login_vechi_sondeaza_si_reimprospateaza(env):
|
||||||
|
"""Login mai vechi decat intervalul + creds durabile -> proba reala, heartbeat reimprospatat."""
|
||||||
|
from app.worker.__main__ import _maybe_keepalive
|
||||||
|
from app.db import read_heartbeat
|
||||||
|
|
||||||
|
conn, settings = env
|
||||||
|
settings.worker_rar_keepalive_interval_s = 86400
|
||||||
|
acct = _account_cu_creds(conn)
|
||||||
|
_set_last_login(conn, ago_s=100000) # > 24h
|
||||||
|
|
||||||
|
sessions = _FakeSessions(conn)
|
||||||
|
_maybe_keepalive(conn, settings, sessions, {"last_attempt": 0.0})
|
||||||
|
|
||||||
|
assert sessions.tokens == [acct] # a sondat contul cu creds
|
||||||
|
assert sessions.invalidated == [acct] # a fortat login real (nu token din cache)
|
||||||
|
last = read_heartbeat(conn)["last_rar_login_ok"]
|
||||||
|
age = (datetime.now(timezone.utc) - datetime.fromisoformat(last)).total_seconds()
|
||||||
|
assert age < 60 # heartbeat reimprospatat de proba
|
||||||
|
|
||||||
|
|
||||||
|
def test_login_proaspat_nu_sondeaza(env):
|
||||||
|
"""Login sub interval -> niciun login de proba."""
|
||||||
|
from app.worker.__main__ import _maybe_keepalive
|
||||||
|
|
||||||
|
conn, settings = env
|
||||||
|
settings.worker_rar_keepalive_interval_s = 86400
|
||||||
|
_account_cu_creds(conn)
|
||||||
|
_set_last_login(conn, ago_s=3600) # 1h < 24h
|
||||||
|
|
||||||
|
sessions = _FakeSessions(conn)
|
||||||
|
_maybe_keepalive(conn, settings, sessions, {"last_attempt": 0.0})
|
||||||
|
|
||||||
|
assert sessions.tokens == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_interval_zero_dezactivat(env):
|
||||||
|
"""interval=0 -> keepalive dezactivat, nicio proba chiar cu login vechi."""
|
||||||
|
from app.worker.__main__ import _maybe_keepalive
|
||||||
|
|
||||||
|
conn, settings = env
|
||||||
|
settings.worker_rar_keepalive_interval_s = 0
|
||||||
|
_account_cu_creds(conn)
|
||||||
|
_set_last_login(conn, ago_s=100000)
|
||||||
|
|
||||||
|
sessions = _FakeSessions(conn)
|
||||||
|
_maybe_keepalive(conn, settings, sessions, {"last_attempt": 0.0})
|
||||||
|
|
||||||
|
assert sessions.tokens == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_fara_creds_durabile_nu_sondeaza(env):
|
||||||
|
"""Niciun cont cu creds durabile + fara test-creds -> nimic de sondat."""
|
||||||
|
from app.worker.__main__ import _maybe_keepalive
|
||||||
|
|
||||||
|
conn, settings = env
|
||||||
|
settings.worker_rar_keepalive_interval_s = 86400
|
||||||
|
settings.worker_use_test_creds = False
|
||||||
|
_set_last_login(conn, ago_s=100000)
|
||||||
|
|
||||||
|
sessions = _FakeSessions(conn)
|
||||||
|
_maybe_keepalive(conn, settings, sessions, {"last_attempt": 0.0})
|
||||||
|
|
||||||
|
assert sessions.tokens == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_target_sare_creds_nedecriptabile(env):
|
||||||
|
"""Cont cu creds criptate sub alta cheie (decrypt -> None) e sarit; alege contul valid.
|
||||||
|
|
||||||
|
Reproduce bug-ul real: start.sh both genereaza o cheie efemera noua la fiecare
|
||||||
|
pornire, deci creds-urile durabile vechi nu se mai decripteaza.
|
||||||
|
"""
|
||||||
|
from app.worker.__main__ import _keepalive_target
|
||||||
|
from app.accounts import create_account
|
||||||
|
from app.crypto import encrypt_creds
|
||||||
|
|
||||||
|
conn, settings = env
|
||||||
|
settings.worker_use_test_creds = False
|
||||||
|
# Cont cu creds GUNOI (nedecriptabile sub cheia curenta), id mai mic.
|
||||||
|
bad = create_account(conn, "Cont Cheie Veche", email="old@example.com")
|
||||||
|
conn.execute("UPDATE accounts SET rar_creds_enc=? WHERE id=?", ("gAAAAA-token-invalid", bad))
|
||||||
|
# Cont cu creds valide, id mai mare.
|
||||||
|
good = create_account(conn, "Cont Valid", email="good@example.com")
|
||||||
|
enc = encrypt_creds({"email": "good@example.com", "password": "pw"})
|
||||||
|
conn.execute("UPDATE accounts SET rar_creds_enc=? WHERE id=?", (enc, good))
|
||||||
|
conn.commit()
|
||||||
|
|
||||||
|
acct_id, creds = _keepalive_target(conn, settings)
|
||||||
|
assert acct_id == good # a sarit contul nedecriptabil
|
||||||
|
assert creds and creds["email"] == "good@example.com"
|
||||||
|
|
||||||
|
|
||||||
|
def test_gating_nu_hartuieste_pe_esec(env):
|
||||||
|
"""Pe esec (RAR jos) login-ul ramane vechi; a doua trecere imediata NU re-sondeaza."""
|
||||||
|
from app.worker.__main__ import _maybe_keepalive
|
||||||
|
|
||||||
|
conn, settings = env
|
||||||
|
settings.worker_rar_keepalive_interval_s = 86400
|
||||||
|
_account_cu_creds(conn)
|
||||||
|
_set_last_login(conn, ago_s=100000)
|
||||||
|
|
||||||
|
state = {"last_attempt": 0.0}
|
||||||
|
sessions = _FakeSessions(conn, fail=True)
|
||||||
|
_maybe_keepalive(conn, settings, sessions, state) # incearca, esueaza
|
||||||
|
_maybe_keepalive(conn, settings, sessions, state) # gating: nu re-incearca
|
||||||
|
|
||||||
|
assert sessions.tokens == [sessions.invalidated[0]] # o singura proba
|
||||||
|
assert len(sessions.tokens) == 1
|
||||||
Reference in New Issue
Block a user