feat(ocr): Add memory management and auto-restart for OCR stability
OCR Service improvements: - Single worker ThreadPoolExecutor to prevent parallel OCR memory accumulation - Semaphore to ensure only one OCR operation at a time - Explicit numpy array cleanup after each OCR step - Forced garbage collection after every OCR request - Memory threshold check (2500MB) with pre-processing GC - Memory usage logging before/after processing Backend auto-restart: - Add run-with-restart.sh wrapper script for uvicorn - Auto-restart on crash (OOM, etc.) with max 10 restarts - Reset restart counter if process runs >60s (stable) - Graceful exit on SIGINT/SIGTERM - Update start-prod.sh and start-test.sh to use wrapper 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
51
backend/run-with-restart.sh
Normal file
51
backend/run-with-restart.sh
Normal file
@@ -0,0 +1,51 @@
|
||||
#!/bin/bash
|
||||
# Wrapper script that auto-restarts uvicorn on crash
|
||||
# Usage: ./run-with-restart.sh [port] [log_file]
|
||||
|
||||
PORT=${1:-8000}
|
||||
LOG_FILE=${2:-/tmp/unified_backend.log}
|
||||
MAX_RESTARTS=10
|
||||
RESTART_COUNT=0
|
||||
RESTART_DELAY=3
|
||||
|
||||
echo "[Backend Runner] Starting uvicorn with auto-restart (max $MAX_RESTARTS restarts)" | tee -a "$LOG_FILE"
|
||||
echo "[Backend Runner] Port: $PORT, Log: $LOG_FILE" | tee -a "$LOG_FILE"
|
||||
|
||||
while [ $RESTART_COUNT -lt $MAX_RESTARTS ]; do
|
||||
START_TIME=$(date +%s)
|
||||
echo "" | tee -a "$LOG_FILE"
|
||||
echo "[Backend Runner] $(date '+%Y-%m-%d %H:%M:%S') - Starting uvicorn (attempt $((RESTART_COUNT + 1))/$MAX_RESTARTS)..." | tee -a "$LOG_FILE"
|
||||
|
||||
# Run uvicorn - it will exit on crash (OOM, etc.)
|
||||
uvicorn main:app --host 0.0.0.0 --port "$PORT" 2>&1 | tee -a "$LOG_FILE"
|
||||
EXIT_CODE=$?
|
||||
|
||||
END_TIME=$(date +%s)
|
||||
RUNTIME=$((END_TIME - START_TIME))
|
||||
|
||||
echo "[Backend Runner] $(date '+%Y-%m-%d %H:%M:%S') - uvicorn exited with code $EXIT_CODE after ${RUNTIME}s" | tee -a "$LOG_FILE"
|
||||
|
||||
# If it ran for more than 60 seconds, reset restart counter (was stable)
|
||||
if [ $RUNTIME -gt 60 ]; then
|
||||
RESTART_COUNT=0
|
||||
echo "[Backend Runner] Process was stable (>${RUNTIME}s), resetting restart counter" | tee -a "$LOG_FILE"
|
||||
else
|
||||
RESTART_COUNT=$((RESTART_COUNT + 1))
|
||||
echo "[Backend Runner] Quick crash detected, restart count: $RESTART_COUNT/$MAX_RESTARTS" | tee -a "$LOG_FILE"
|
||||
fi
|
||||
|
||||
# Exit if user sent SIGINT (Ctrl+C) or SIGTERM
|
||||
if [ $EXIT_CODE -eq 130 ] || [ $EXIT_CODE -eq 143 ]; then
|
||||
echo "[Backend Runner] Received termination signal, exiting..." | tee -a "$LOG_FILE"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Wait before restart
|
||||
if [ $RESTART_COUNT -lt $MAX_RESTARTS ]; then
|
||||
echo "[Backend Runner] Restarting in ${RESTART_DELAY}s..." | tee -a "$LOG_FILE"
|
||||
sleep $RESTART_DELAY
|
||||
fi
|
||||
done
|
||||
|
||||
echo "[Backend Runner] Max restarts ($MAX_RESTARTS) reached. Giving up." | tee -a "$LOG_FILE"
|
||||
exit 1
|
||||
Reference in New Issue
Block a user