Marius Mutu
51d736addf
OCR Service improvements: - Single worker ThreadPoolExecutor to prevent parallel OCR memory accumulation - Semaphore to ensure only one OCR operation at a time - Explicit numpy array cleanup after each OCR step - Forced garbage collection after every OCR request - Memory threshold check (2500MB) with pre-processing GC - Memory usage logging before/after processing Backend auto-restart: - Add run-with-restart.sh wrapper script for uvicorn - Auto-restart on crash (OOM, etc.) with max 10 restarts - Reset restart counter if process runs >60s (stable) - Graceful exit on SIGINT/SIGTERM - Update start-prod.sh and start-test.sh to use wrapper 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
52 lines
1.9 KiB
Bash
52 lines
1.9 KiB
Bash
#!/bin/bash
|
|
# Wrapper script that auto-restarts uvicorn on crash
|
|
# Usage: ./run-with-restart.sh [port] [log_file]
|
|
|
|
PORT=${1:-8000}
|
|
LOG_FILE=${2:-/tmp/unified_backend.log}
|
|
MAX_RESTARTS=10
|
|
RESTART_COUNT=0
|
|
RESTART_DELAY=3
|
|
|
|
echo "[Backend Runner] Starting uvicorn with auto-restart (max $MAX_RESTARTS restarts)" | tee -a "$LOG_FILE"
|
|
echo "[Backend Runner] Port: $PORT, Log: $LOG_FILE" | tee -a "$LOG_FILE"
|
|
|
|
while [ $RESTART_COUNT -lt $MAX_RESTARTS ]; do
|
|
START_TIME=$(date +%s)
|
|
echo "" | tee -a "$LOG_FILE"
|
|
echo "[Backend Runner] $(date '+%Y-%m-%d %H:%M:%S') - Starting uvicorn (attempt $((RESTART_COUNT + 1))/$MAX_RESTARTS)..." | tee -a "$LOG_FILE"
|
|
|
|
# Run uvicorn - it will exit on crash (OOM, etc.)
|
|
uvicorn main:app --host 0.0.0.0 --port "$PORT" 2>&1 | tee -a "$LOG_FILE"
|
|
EXIT_CODE=$?
|
|
|
|
END_TIME=$(date +%s)
|
|
RUNTIME=$((END_TIME - START_TIME))
|
|
|
|
echo "[Backend Runner] $(date '+%Y-%m-%d %H:%M:%S') - uvicorn exited with code $EXIT_CODE after ${RUNTIME}s" | tee -a "$LOG_FILE"
|
|
|
|
# If it ran for more than 60 seconds, reset restart counter (was stable)
|
|
if [ $RUNTIME -gt 60 ]; then
|
|
RESTART_COUNT=0
|
|
echo "[Backend Runner] Process was stable (>${RUNTIME}s), resetting restart counter" | tee -a "$LOG_FILE"
|
|
else
|
|
RESTART_COUNT=$((RESTART_COUNT + 1))
|
|
echo "[Backend Runner] Quick crash detected, restart count: $RESTART_COUNT/$MAX_RESTARTS" | tee -a "$LOG_FILE"
|
|
fi
|
|
|
|
# Exit if user sent SIGINT (Ctrl+C) or SIGTERM
|
|
if [ $EXIT_CODE -eq 130 ] || [ $EXIT_CODE -eq 143 ]; then
|
|
echo "[Backend Runner] Received termination signal, exiting..." | tee -a "$LOG_FILE"
|
|
exit 0
|
|
fi
|
|
|
|
# Wait before restart
|
|
if [ $RESTART_COUNT -lt $MAX_RESTARTS ]; then
|
|
echo "[Backend Runner] Restarting in ${RESTART_DELAY}s..." | tee -a "$LOG_FILE"
|
|
sleep $RESTART_DELAY
|
|
fi
|
|
done
|
|
|
|
echo "[Backend Runner] Max restarts ($MAX_RESTARTS) reached. Giving up." | tee -a "$LOG_FILE"
|
|
exit 1
|