feat: Migrate to ultrathin monolith architecture
Consolidate 3 separate applications (reports-app, data-entry-app, telegram-bot) into a unified
architecture with single backend and frontend:
Backend Changes:
- Unified FastAPI backend at backend/ with modular structure
- Modules: reports, data_entry, telegram in backend/modules/
- Centralized config.py and main.py with all routers registered
- Single worker mode (--workers 1) for Telegram bot compatibility
- Shared Oracle connection pool and JWT authentication
- Unified requirements.txt and environment configuration
Frontend Changes:
- Single Vue.js SPA with module-based routing
- Unified frontend at src/ with modules in src/modules/{reports,data-entry}/
- Shared components and stores in src/shared/
- Error boundaries for module isolation
- Dual API proxy in Vite for module communication
Infrastructure:
- New unified startup scripts: start-prod.sh, start-test.sh, start-backend.sh
- Environment templates: .env.dev.example, .env.test.example, .env.prod.example
- Updated deployment scripts for Windows IIS
- Simplified SSH tunnel management
Documentation:
- Comprehensive CLAUDE.md with architecture overview
- Module-specific docs in docs/{data-entry,telegram}/
- Architecture decision records in docs/ARCHITECTURE-DECISIONS.md
- Deployment guides consolidated in deployment/windows/docs/
This migration reduces complexity, improves maintainability, and enables easier
deployment while maintaining all existing functionality.
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
404
backend/modules/reports/cache/sqlite_cache.py
vendored
Normal file
404
backend/modules/reports/cache/sqlite_cache.py
vendored
Normal file
@@ -0,0 +1,404 @@
|
||||
"""
|
||||
SQLite persistent cache (L2 cache)
|
||||
Persistent, survives restarts, unlimited size
|
||||
"""
|
||||
import time
|
||||
import json
|
||||
import logging
|
||||
import aiosqlite
|
||||
from typing import Any, Optional, List, Dict
|
||||
from pathlib import Path
|
||||
from decimal import Decimal
|
||||
from datetime import datetime, date
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CustomJSONEncoder(json.JSONEncoder):
|
||||
"""Custom JSON encoder that handles Pydantic models, Decimal, datetime, etc."""
|
||||
def default(self, obj):
|
||||
# Handle Pydantic models
|
||||
if hasattr(obj, 'dict'):
|
||||
return obj.dict()
|
||||
if hasattr(obj, 'model_dump'): # Pydantic v2
|
||||
return obj.model_dump()
|
||||
# Handle Decimal
|
||||
if isinstance(obj, Decimal):
|
||||
return float(obj)
|
||||
# Handle datetime/date
|
||||
if isinstance(obj, (datetime, date)):
|
||||
return obj.isoformat()
|
||||
return super().default(obj)
|
||||
|
||||
|
||||
class SQLiteCache:
|
||||
"""
|
||||
SQLite-based persistent cache
|
||||
|
||||
Features:
|
||||
- Persistent storage (survives restarts)
|
||||
- JSON serialization for complex objects
|
||||
- Schema mappings (permanent cache for company->schema)
|
||||
- Watermarks for event-based invalidation
|
||||
- Performance tracking and benchmarks
|
||||
"""
|
||||
|
||||
def __init__(self, db_path: str):
|
||||
"""
|
||||
Initialize SQLite cache
|
||||
|
||||
Args:
|
||||
db_path: Path to SQLite database file
|
||||
"""
|
||||
self.db_path = db_path
|
||||
self._ensure_db_dir()
|
||||
|
||||
def _ensure_db_dir(self):
|
||||
"""Ensure database directory exists"""
|
||||
db_dir = Path(self.db_path).parent
|
||||
db_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
async def init_db(self):
|
||||
"""Initialize database schema with WAL mode enabled"""
|
||||
async with aiosqlite.connect(self.db_path) as db:
|
||||
# Enable Write-Ahead Logging (WAL) mode for better concurrency
|
||||
await db.execute("PRAGMA journal_mode=WAL")
|
||||
await db.commit()
|
||||
|
||||
# Table: cache_entries
|
||||
await db.execute("""
|
||||
CREATE TABLE IF NOT EXISTS cache_entries (
|
||||
cache_key TEXT PRIMARY KEY,
|
||||
cache_type TEXT NOT NULL,
|
||||
company_id INTEGER,
|
||||
data_json TEXT NOT NULL,
|
||||
created_at REAL NOT NULL,
|
||||
expires_at REAL NOT NULL,
|
||||
hit_count INTEGER DEFAULT 0,
|
||||
last_accessed REAL
|
||||
)
|
||||
""")
|
||||
await db.execute("CREATE INDEX IF NOT EXISTS idx_cache_type ON cache_entries(cache_type)")
|
||||
await db.execute("CREATE INDEX IF NOT EXISTS idx_company_id ON cache_entries(company_id)")
|
||||
await db.execute("CREATE INDEX IF NOT EXISTS idx_expires_at ON cache_entries(expires_at)")
|
||||
|
||||
# Table: schema_mappings (PERMANENT)
|
||||
await db.execute("""
|
||||
CREATE TABLE IF NOT EXISTS schema_mappings (
|
||||
id_firma INTEGER PRIMARY KEY,
|
||||
schema TEXT NOT NULL,
|
||||
created_at REAL NOT NULL,
|
||||
last_verified REAL
|
||||
)
|
||||
""")
|
||||
|
||||
# Table: query_benchmarks
|
||||
await db.execute("""
|
||||
CREATE TABLE IF NOT EXISTS query_benchmarks (
|
||||
cache_type TEXT PRIMARY KEY,
|
||||
avg_time_ms REAL NOT NULL,
|
||||
sample_count INTEGER DEFAULT 0,
|
||||
last_updated REAL
|
||||
)
|
||||
""")
|
||||
|
||||
# Table: performance_log
|
||||
await db.execute("""
|
||||
CREATE TABLE IF NOT EXISTS performance_log (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
cache_type TEXT NOT NULL,
|
||||
company_id INTEGER,
|
||||
cache_hit BOOLEAN NOT NULL,
|
||||
response_time_ms REAL NOT NULL,
|
||||
estimated_oracle_time_ms REAL,
|
||||
time_saved_ms REAL,
|
||||
username TEXT,
|
||||
timestamp REAL NOT NULL
|
||||
)
|
||||
""")
|
||||
await db.execute("CREATE INDEX IF NOT EXISTS idx_perf_timestamp ON performance_log(timestamp)")
|
||||
await db.execute("CREATE INDEX IF NOT EXISTS idx_perf_cache_type ON performance_log(cache_type)")
|
||||
|
||||
# Table: user_cache_settings
|
||||
await db.execute("""
|
||||
CREATE TABLE IF NOT EXISTS user_cache_settings (
|
||||
username TEXT PRIMARY KEY,
|
||||
cache_enabled BOOLEAN DEFAULT TRUE,
|
||||
created_at REAL,
|
||||
updated_at REAL
|
||||
)
|
||||
""")
|
||||
|
||||
# Table: cache_config
|
||||
await db.execute("""
|
||||
CREATE TABLE IF NOT EXISTS cache_config (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL,
|
||||
updated_at REAL
|
||||
)
|
||||
""")
|
||||
|
||||
# Table: cache_watermarks
|
||||
await db.execute("""
|
||||
CREATE TABLE IF NOT EXISTS cache_watermarks (
|
||||
company_id INTEGER PRIMARY KEY,
|
||||
schema TEXT NOT NULL,
|
||||
max_id_act INTEGER NOT NULL,
|
||||
checked_at REAL NOT NULL
|
||||
)
|
||||
""")
|
||||
|
||||
await db.commit()
|
||||
logger.info("SQLite cache database initialized")
|
||||
|
||||
async def get(self, key: str) -> Optional[Any]:
|
||||
"""
|
||||
Get value from cache
|
||||
|
||||
Args:
|
||||
key: Cache key
|
||||
|
||||
Returns:
|
||||
Cached value or None if not found/expired
|
||||
"""
|
||||
async with aiosqlite.connect(self.db_path) as db:
|
||||
async with db.execute("""
|
||||
SELECT data_json, expires_at
|
||||
FROM cache_entries
|
||||
WHERE cache_key = ?
|
||||
""", (key,)) as cursor:
|
||||
result = await cursor.fetchone()
|
||||
|
||||
if not result:
|
||||
return None
|
||||
|
||||
data_json, expires_at = result
|
||||
|
||||
# Check TTL expiration
|
||||
if expires_at < time.time():
|
||||
# Expired - delete and return None
|
||||
await db.execute("DELETE FROM cache_entries WHERE cache_key = ?", (key,))
|
||||
await db.commit()
|
||||
logger.debug(f"SQLite cache expired: {key}")
|
||||
return None
|
||||
|
||||
# Update hit_count and last_accessed
|
||||
await db.execute("""
|
||||
UPDATE cache_entries
|
||||
SET hit_count = hit_count + 1, last_accessed = ?
|
||||
WHERE cache_key = ?
|
||||
""", (time.time(), key))
|
||||
await db.commit()
|
||||
|
||||
logger.debug(f"SQLite cache HIT: {key}")
|
||||
return json.loads(data_json)
|
||||
|
||||
async def set(self, key: str, value: Any, cache_type: str, company_id: Optional[int], ttl: int):
|
||||
"""
|
||||
Set value in cache
|
||||
|
||||
Args:
|
||||
key: Cache key
|
||||
value: Value to cache
|
||||
cache_type: Type of cache entry
|
||||
company_id: Company ID (None for global caches)
|
||||
ttl: Time to live in seconds
|
||||
"""
|
||||
# Use custom encoder to handle Pydantic models, Decimal, datetime, etc.
|
||||
data_json = json.dumps(value, cls=CustomJSONEncoder)
|
||||
now = time.time()
|
||||
expires_at = now + ttl
|
||||
|
||||
async with aiosqlite.connect(self.db_path) as db:
|
||||
await db.execute("""
|
||||
INSERT OR REPLACE INTO cache_entries
|
||||
(cache_key, cache_type, company_id, data_json, created_at, expires_at, hit_count, last_accessed)
|
||||
VALUES (?, ?, ?, ?, ?, ?, 0, ?)
|
||||
""", (key, cache_type, company_id, data_json, now, expires_at, now))
|
||||
await db.commit()
|
||||
|
||||
logger.debug(f"SQLite cache SET: {key} (TTL: {ttl}s)")
|
||||
|
||||
async def delete(self, key: str) -> bool:
|
||||
"""Delete entry from cache"""
|
||||
async with aiosqlite.connect(self.db_path) as db:
|
||||
cursor = await db.execute("DELETE FROM cache_entries WHERE cache_key = ?", (key,))
|
||||
await db.commit()
|
||||
deleted = cursor.rowcount > 0
|
||||
if deleted:
|
||||
logger.debug(f"SQLite cache deleted: {key}")
|
||||
return deleted
|
||||
|
||||
async def clear(self):
|
||||
"""Clear all cache entries"""
|
||||
async with aiosqlite.connect(self.db_path) as db:
|
||||
cursor = await db.execute("DELETE FROM cache_entries")
|
||||
await db.commit()
|
||||
count = cursor.rowcount
|
||||
logger.info(f"SQLite cache cleared: {count} entries removed")
|
||||
|
||||
async def clear_by_company(self, company_id: int):
|
||||
"""Clear all entries for specific company"""
|
||||
async with aiosqlite.connect(self.db_path) as db:
|
||||
cursor = await db.execute("DELETE FROM cache_entries WHERE company_id = ?", (company_id,))
|
||||
await db.commit()
|
||||
count = cursor.rowcount
|
||||
logger.info(f"SQLite cache cleared for company {company_id}: {count} entries")
|
||||
|
||||
async def clear_by_type(self, cache_type: str):
|
||||
"""Clear all entries of specific type"""
|
||||
async with aiosqlite.connect(self.db_path) as db:
|
||||
cursor = await db.execute("DELETE FROM cache_entries WHERE cache_type = ?", (cache_type,))
|
||||
await db.commit()
|
||||
count = cursor.rowcount
|
||||
logger.info(f"SQLite cache cleared for type '{cache_type}': {count} entries")
|
||||
|
||||
async def cleanup_expired(self):
|
||||
"""Remove all expired entries"""
|
||||
async with aiosqlite.connect(self.db_path) as db:
|
||||
cursor = await db.execute("DELETE FROM cache_entries WHERE expires_at < ?", (time.time(),))
|
||||
await db.commit()
|
||||
count = cursor.rowcount
|
||||
if count > 0:
|
||||
logger.info(f"SQLite cache cleanup: {count} expired entries removed")
|
||||
|
||||
# Schema Mappings (PERMANENT)
|
||||
|
||||
async def get_schema_mapping(self, company_id: int) -> Optional[str]:
|
||||
"""Get permanent cached schema for company"""
|
||||
async with aiosqlite.connect(self.db_path) as db:
|
||||
async with db.execute("""
|
||||
SELECT schema
|
||||
FROM schema_mappings
|
||||
WHERE id_firma = ?
|
||||
""", (company_id,)) as cursor:
|
||||
result = await cursor.fetchone()
|
||||
return result[0] if result else None
|
||||
|
||||
async def set_schema_mapping(self, company_id: int, schema: str):
|
||||
"""Set permanent schema mapping (never expires)"""
|
||||
async with aiosqlite.connect(self.db_path) as db:
|
||||
await db.execute("""
|
||||
INSERT OR REPLACE INTO schema_mappings
|
||||
(id_firma, schema, created_at, last_verified)
|
||||
VALUES (?, ?, ?, ?)
|
||||
""", (company_id, schema, time.time(), time.time()))
|
||||
await db.commit()
|
||||
|
||||
# Benchmarks
|
||||
|
||||
async def get_benchmark(self, cache_type: str) -> Optional[float]:
|
||||
"""Get average benchmark time for cache type"""
|
||||
async with aiosqlite.connect(self.db_path) as db:
|
||||
async with db.execute("""
|
||||
SELECT avg_time_ms
|
||||
FROM query_benchmarks
|
||||
WHERE cache_type = ?
|
||||
""", (cache_type,)) as cursor:
|
||||
result = await cursor.fetchone()
|
||||
return result[0] if result else None
|
||||
|
||||
async def set_benchmark(self, cache_type: str, avg_time_ms: float, sample_count: int):
|
||||
"""Set/update benchmark"""
|
||||
async with aiosqlite.connect(self.db_path) as db:
|
||||
await db.execute("""
|
||||
INSERT OR REPLACE INTO query_benchmarks
|
||||
(cache_type, avg_time_ms, sample_count, last_updated)
|
||||
VALUES (?, ?, ?, ?)
|
||||
""", (cache_type, avg_time_ms, sample_count, time.time()))
|
||||
await db.commit()
|
||||
|
||||
# Performance Tracking
|
||||
|
||||
async def log_performance(self, cache_type: str, company_id: Optional[int], cache_hit: bool,
|
||||
response_time_ms: float, estimated_oracle_time_ms: Optional[float],
|
||||
time_saved_ms: Optional[float], username: Optional[str]):
|
||||
"""Log performance metric"""
|
||||
async with aiosqlite.connect(self.db_path) as db:
|
||||
await db.execute("""
|
||||
INSERT INTO performance_log
|
||||
(cache_type, company_id, cache_hit, response_time_ms, estimated_oracle_time_ms,
|
||||
time_saved_ms, username, timestamp)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""", (cache_type, company_id, cache_hit, response_time_ms, estimated_oracle_time_ms,
|
||||
time_saved_ms, username, time.time()))
|
||||
await db.commit()
|
||||
|
||||
# User Settings
|
||||
|
||||
async def get_user_cache_enabled(self, username: str) -> bool:
|
||||
"""Get user cache setting (default True)"""
|
||||
async with aiosqlite.connect(self.db_path) as db:
|
||||
async with db.execute("""
|
||||
SELECT cache_enabled
|
||||
FROM user_cache_settings
|
||||
WHERE username = ?
|
||||
""", (username,)) as cursor:
|
||||
result = await cursor.fetchone()
|
||||
return bool(result[0]) if result else True # Default enabled, explicit bool conversion
|
||||
|
||||
async def set_user_cache_enabled(self, username: str, enabled: bool):
|
||||
"""Set user cache setting"""
|
||||
async with aiosqlite.connect(self.db_path) as db:
|
||||
await db.execute("""
|
||||
INSERT OR REPLACE INTO user_cache_settings
|
||||
(username, cache_enabled, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?)
|
||||
""", (username, enabled, time.time(), time.time()))
|
||||
await db.commit()
|
||||
|
||||
# Watermarks
|
||||
|
||||
async def get_watermark(self, company_id: int) -> Optional[int]:
|
||||
"""Get cached watermark (max_id_act) for company"""
|
||||
async with aiosqlite.connect(self.db_path) as db:
|
||||
async with db.execute("""
|
||||
SELECT max_id_act
|
||||
FROM cache_watermarks
|
||||
WHERE company_id = ?
|
||||
""", (company_id,)) as cursor:
|
||||
result = await cursor.fetchone()
|
||||
return result[0] if result else None
|
||||
|
||||
async def set_watermark(self, company_id: int, schema: str, max_id_act: int):
|
||||
"""Set/update watermark for company"""
|
||||
async with aiosqlite.connect(self.db_path) as db:
|
||||
await db.execute("""
|
||||
INSERT OR REPLACE INTO cache_watermarks
|
||||
(company_id, schema, max_id_act, checked_at)
|
||||
VALUES (?, ?, ?, ?)
|
||||
""", (company_id, schema, max_id_act, time.time()))
|
||||
await db.commit()
|
||||
|
||||
async def get_cached_company_ids(self) -> List[int]:
|
||||
"""Get list of company_ids with active cache entries"""
|
||||
async with aiosqlite.connect(self.db_path) as db:
|
||||
async with db.execute("""
|
||||
SELECT DISTINCT company_id
|
||||
FROM cache_entries
|
||||
WHERE company_id IS NOT NULL
|
||||
AND expires_at > ?
|
||||
""", (time.time(),)) as cursor:
|
||||
results = await cursor.fetchall()
|
||||
return [row[0] for row in results]
|
||||
|
||||
# Statistics
|
||||
|
||||
async def get_stats(self) -> Dict[str, Any]:
|
||||
"""Get cache statistics"""
|
||||
async with aiosqlite.connect(self.db_path) as db:
|
||||
# Total entries
|
||||
async with db.execute("SELECT COUNT(*) FROM cache_entries") as cursor:
|
||||
total_entries = (await cursor.fetchone())[0]
|
||||
|
||||
# Active entries (not expired)
|
||||
async with db.execute("""
|
||||
SELECT COUNT(*) FROM cache_entries WHERE expires_at > ?
|
||||
""", (time.time(),)) as cursor:
|
||||
active_entries = (await cursor.fetchone())[0]
|
||||
|
||||
return {
|
||||
'total_entries': total_entries,
|
||||
'active_entries': active_entries,
|
||||
'expired_entries': total_entries - active_entries
|
||||
}
|
||||
Reference in New Issue
Block a user