dcc5042586
Root cause of the 2GB prod import.db: the sync_run_orders audit junction recorded every order on every run; under the 1-minute scheduler ~98% of 21.7M rows were no-op ALREADY_IMPORTED re-observations. NSSM stdout/stderr also grew unbounded (rotation never applied to the live service). Changes: - sqlite_service: skip ALREADY_IMPORTED rows in sync_run_orders (write-side guard, _SKIP_JUNCTION_STATUSES); add prune_sync_history(retention_days) with incremental_vacuum. - maintenance_service (new): cleanup_old_logs + run_daily_maintenance. - scheduler_service: start_maintenance_job (daily CronTrigger). - main.py: RotatingFileHandler (sync_comenzi_current.log, 10MB x5) instead of a new timestamped file per start; schedule daily maintenance + one-shot catch-up at startup. - scripts/db_maintenance.py (new): one-shot prune + VACUUM + log cleanup, plain sqlite3, invoked by deploy.ps1 while the service is stopped. - deploy.ps1: stop -> run db_maintenance.py -> (re)apply NSSM AppRotate* idempotently -> start, so rotation reaches pre-existing services. Retention defaults: 7 days history, 7 days logs. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
104 lines
3.6 KiB
Python
104 lines
3.6 KiB
Python
import asyncio
|
|
from contextlib import asynccontextmanager
|
|
from fastapi import FastAPI
|
|
from fastapi.staticfiles import StaticFiles
|
|
from pathlib import Path
|
|
import logging
|
|
import logging.handlers
|
|
import os
|
|
|
|
from .config import settings
|
|
from .database import ensure_oracle_pool, close_oracle, init_sqlite
|
|
|
|
# Configure logging with both stream and file handlers
|
|
_log_level = getattr(logging, settings.LOG_LEVEL.upper(), logging.INFO)
|
|
_log_format = '%(asctime)s | %(levelname)s | %(name)s | %(message)s'
|
|
_formatter = logging.Formatter(_log_format)
|
|
|
|
_stream_handler = logging.StreamHandler()
|
|
_stream_handler.setFormatter(_formatter)
|
|
|
|
_log_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'logs')
|
|
os.makedirs(_log_dir, exist_ok=True)
|
|
# Rotating handler (10MB x 5 backups) instead of a new timestamped file per
|
|
# start — caps log growth and stops file proliferation across restarts. Fixed
|
|
# name still matches the QA glob `sync_comenzi_*.log`.
|
|
_file_handler = logging.handlers.RotatingFileHandler(
|
|
os.path.join(_log_dir, "sync_comenzi_current.log"),
|
|
maxBytes=10 * 1024 * 1024, backupCount=5, encoding='utf-8')
|
|
_file_handler.setFormatter(_formatter)
|
|
|
|
_root_logger = logging.getLogger()
|
|
_root_logger.setLevel(_log_level)
|
|
_root_logger.addHandler(_stream_handler)
|
|
_root_logger.addHandler(_file_handler)
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
@asynccontextmanager
|
|
async def lifespan(app: FastAPI):
|
|
"""Startup and shutdown events."""
|
|
logger.info("Starting GoMag Import Manager...")
|
|
|
|
# Initialize Oracle pool (non-fatal: app still starts if Oracle is down;
|
|
# each sync cycle calls ensure_oracle_pool() and self-heals when it returns)
|
|
if not ensure_oracle_pool():
|
|
logger.error("Oracle pool not ready at startup — will retry on each sync cycle")
|
|
|
|
# Initialize SQLite
|
|
init_sqlite()
|
|
|
|
# Initialize scheduler (restore saved config)
|
|
from .services import scheduler_service, sqlite_service
|
|
scheduler_service.init_scheduler()
|
|
try:
|
|
config = await sqlite_service.get_scheduler_config()
|
|
if config.get("enabled") == "True":
|
|
interval = int(config.get("interval_minutes", "10"))
|
|
scheduler_service.start_scheduler(interval)
|
|
except Exception:
|
|
pass
|
|
|
|
# Daily DB/log maintenance (prune audit history + cleanup old logs) + a
|
|
# one-shot catch-up so a long-down service reclaims immediately on start.
|
|
try:
|
|
from .services import maintenance_service
|
|
scheduler_service.start_maintenance_job()
|
|
asyncio.create_task(maintenance_service.run_daily_maintenance())
|
|
except Exception as e:
|
|
logger.warning(f"Maintenance scheduling failed: {e}")
|
|
|
|
logger.info("GoMag Import Manager started")
|
|
yield
|
|
|
|
# Shutdown
|
|
scheduler_service.shutdown_scheduler()
|
|
close_oracle()
|
|
logger.info("GoMag Import Manager stopped")
|
|
|
|
app = FastAPI(
|
|
title="GoMag Import Manager",
|
|
description="Import comenzi web GoMag → ROA Oracle",
|
|
version="1.0.0",
|
|
lifespan=lifespan
|
|
)
|
|
|
|
# Static files and templates
|
|
static_dir = Path(__file__).parent / "static"
|
|
templates_dir = Path(__file__).parent / "templates"
|
|
static_dir.mkdir(parents=True, exist_ok=True)
|
|
(static_dir / "css").mkdir(exist_ok=True)
|
|
(static_dir / "js").mkdir(exist_ok=True)
|
|
templates_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
app.mount("/static", StaticFiles(directory=str(static_dir)), name="static")
|
|
|
|
# Include routers
|
|
from .routers import health, dashboard, mappings, articles, validation, sync
|
|
app.include_router(health.router)
|
|
app.include_router(dashboard.router)
|
|
app.include_router(mappings.router)
|
|
app.include_router(articles.router)
|
|
app.include_router(validation.router)
|
|
app.include_router(sync.router)
|