diff --git a/src/voice/normalize.py b/src/voice/normalize.py index b61232a..d7b7f6b 100644 --- a/src/voice/normalize.py +++ b/src/voice/normalize.py @@ -94,6 +94,47 @@ def expand_numbers_ro(text: str) -> str: return _NUM_TOKEN.sub(_sub, text) +# ---------- Time ---------- + +_TIME_PATTERN = re.compile(r'(? str: + """Romanian-correct feminine forms for minute counts (0-59).""" + if n == 1: + return "un minut" + if n == 2: + return "două minute" + if n < 20: + return f"{_int_to_ro(n)} minute" + last = n % 10 + rest = n - last + if last == 0: + return f"{_int_to_ro(n)} de minute" + if last == 1: + return f"{_int_to_ro(rest)} și una de minute" + if last == 2: + return f"{_int_to_ro(rest)} și două de minute" + return f"{_int_to_ro(rest)} și {_int_to_ro(last)} de minute" + + +def expand_time(text: str) -> str: + """Expand ``HH:MM`` clock times into colloquial Romanian. + + 23:09 -> "douăzeci și trei și nouă minute" + 23:00 -> "douăzeci și trei fix" + """ + def _sub(match: re.Match) -> str: + h = int(match.group(1)) + m = int(match.group(2)) + hour_str = _int_to_ro(h) + if m == 0: + return f"{hour_str} fix" + return f"{hour_str} și {_format_minutes_ro(m)}" + + return _TIME_PATTERN.sub(_sub, text) + + # ---------- Currency ---------- _CURRENCY_MAIN = { @@ -177,6 +218,9 @@ def expand_symbols(text: str) -> str: return text +from tools.tts import sanitize_for_supertonic as sanitize_punctuation + + # ---------- Abbreviations ---------- # Longer patterns first so 'ș.a.m.d.' wins over 'ș.a.' @@ -211,7 +255,9 @@ def normalize_for_tts(text: str) -> str: response continues in the text channel mirror. """ text = strip_markdown(text) + text = sanitize_punctuation(text) text = expand_abbreviations(text) + text = expand_time(text) text = expand_currency(text) text = expand_numbers_ro(text) text = expand_symbols(text) diff --git a/src/voice/pipeline.py b/src/voice/pipeline.py index f6b1af9..fa4796a 100644 --- a/src/voice/pipeline.py +++ b/src/voice/pipeline.py @@ -19,7 +19,7 @@ the bot transcribe itself. See plan: ``src/voice/pipeline.py`` (Pas 5), Engineering decisions #4 (VAD 100ms batched), #5 (cleanup centralizat), #7 (bot.user.id explicit -guard), #8 (filler audio ``thinking.wav`` at 3s pre-first-block). +guard). """ from __future__ import annotations @@ -48,7 +48,6 @@ VAD_WINDOW_BYTES = PACKET_BYTES * (VAD_WINDOW_MS // PACKET_MS) VAD_THRESHOLD = 0.5 SILENCE_FLUSH_MS = 800 NO_SPEECH_DROP_THRESHOLD = 0.6 -FILLER_DELAY_S = 3.0 PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent LOGS_DIR = PROJECT_ROOT / "logs" @@ -76,6 +75,7 @@ def _get_whisper_model() -> Any: from faster_whisper import WhisperModel _whisper_model = WhisperModel( "small", device="cpu", compute_type="int8", cpu_threads=4, + local_files_only=True, ) return _whisper_model @@ -164,8 +164,6 @@ class VoiceSession: self._lock = threading.Lock() self._cleaned_up = False self._lock_owner_thread: Optional[int] = None - self._filler_task: Optional[asyncio.Task] = None - self._first_block_seen = False # ----- context manager ----- @@ -244,14 +242,7 @@ class VoiceSession: except Exception as e: # noqa: BLE001 log.warning("ttsq.stop failed: %s", e) - # 5. Cancel pending filler task. - if self._filler_task is not None and not self._filler_task.done(): - try: - self._filler_task.cancel() - except Exception: # noqa: BLE001 - pass - - # 6. Release the session lock (held since __enter__). + # 5. Release the session lock (held since __enter__). try: if self._lock.locked(): self._lock.release() @@ -276,6 +267,13 @@ class VoiceSession: self.last_activity_ts = time.monotonic() speaker_name = self._resolve_speaker_name(speaker_id) + # Drop any TTS frames from the previous turn so a new utterance cuts off + # stale Echo speech (barge-in) and never mixes with the new response. + try: + self.ttsq.clear() + except Exception as e: # noqa: BLE001 + log.warning("ttsq.clear failed: %s", e) + # 1. Mirror to text channel (one Unicode 🎤 — exception per plan). if self.mirror_enabled and self.text_channel is not None: try: @@ -302,34 +300,20 @@ class VoiceSession: except Exception as e: # noqa: BLE001 log.warning("voice transcript write failed: %s", e) - # 3. Arm the 3s filler timer — fires only if no Claude block arrives. - self._first_block_seen = False - if self._filler_task is not None and not self._filler_task.done(): - self._filler_task.cancel() - try: - self._filler_task = asyncio.create_task(self._filler_after_delay()) - except RuntimeError: - # No running loop (test path). Skip the timer. - self._filler_task = None + block_count = [0] def voice_stream_callback(block: str) -> None: - """Called once per Claude streamed text block — pushes to TTS - and cancels the filler on first arrival.""" - if not self._first_block_seen: - self._first_block_seen = True - ft = self._filler_task - if ft is not None and not ft.done(): - try: - ft.cancel() - except Exception: # noqa: BLE001 - pass + """Called once per Claude streamed text block — pushes to TTS.""" + block_count[0] += 1 + log.info("voice stream block #%d (%d chars): %r", + block_count[0], len(block or ""), (block or "")[:80]) try: self.ttsq.push_text(block) except Exception as e: # noqa: BLE001 log.warning("ttsq.push_text failed: %s", e) - # 4. Dispatch to Claude. send_message is sync subprocess, run on - # a worker thread so the loop stays responsive for mirror/TTS. + # Dispatch to Claude. send_message is sync subprocess, run on + # a worker thread so the loop stays responsive for mirror/TTS. try: await asyncio.to_thread( self._route_message, @@ -343,20 +327,6 @@ class VoiceSession: except Exception as e: # noqa: BLE001 log.error("route_message voice path failed: %s", e) - async def _filler_after_delay(self) -> None: - """Push ``assets/voice/thinking.wav`` after FILLER_DELAY_S if Claude - hasn't produced a first block yet.""" - try: - await asyncio.sleep(FILLER_DELAY_S) - except asyncio.CancelledError: - return - if self._first_block_seen or self._cleaned_up: - return - try: - self.ttsq.push_filler() - except Exception as e: # noqa: BLE001 - log.warning("ttsq.push_filler failed: %s", e) - # ----- helpers ----- def _resolve_speaker_name(self, speaker_id: int) -> str: @@ -406,11 +376,28 @@ class EchoVoiceSink(AudioSink): self._last_speech_ts: dict[int, float] = {} self._has_speech: dict[int, bool] = {} self._sink_lock = threading.Lock() + # Diagnostics: log once-per-user when packets first arrive and when + # VAD first detects speech. Cheap, but tells us exactly where the + # chain breaks when "I spoke but Echo heard nothing" happens. + self._first_packet_logged: set[int] = set() + self._first_speech_logged: set[int] = set() + # Background poller that triggers the silence flush even when Discord + # DTX stops delivering RTP packets after the user stops speaking. Without + # this, sink.write would stop firing and STT would never run on the + # final utterance. + self._poller_stop = threading.Event() + self._poller_thread = threading.Thread( + target=self._silence_flush_poller, + name="echo-voice-flush-poller", + daemon=True, + ) + self._poller_thread.start() def wants_opus(self) -> bool: return False def cleanup(self) -> None: + self._poller_stop.set() with self._sink_lock: self._user_buffers.clear() self._packet_accum.clear() @@ -435,6 +422,10 @@ class EchoVoiceSink(AudioSink): if not pcm: return + if uid not in self._first_packet_logged: + self._first_packet_logged.add(uid) + log.info("voice sink: first PCM packet from user %s (%d bytes)", uid, len(pcm)) + window_pcm: Optional[bytes] = None pcm_for_stt: Optional[bytes] = None @@ -450,25 +441,51 @@ class EchoVoiceSink(AudioSink): if window_pcm is not None: if self._vad_detects_speech(window_pcm): + if uid not in self._first_speech_logged: + self._first_speech_logged.add(uid) + log.info("voice sink: VAD detected speech from user %s", uid) with self._sink_lock: self._last_speech_ts[uid] = time.monotonic() self._has_speech[uid] = True - with self._sink_lock: - if self._has_speech.get(uid): - last = self._last_speech_ts.get(uid, 0.0) - silence_ms = (time.monotonic() - last) * 1000.0 - if silence_ms >= SILENCE_FLUSH_MS: - pcm_for_stt = bytes(self._user_buffers.get(uid, b"")) - self._user_buffers[uid] = bytearray() - self._packet_accum[uid] = bytearray() - self._has_speech[uid] = False - + pcm_for_stt = self._take_flushable_pcm(uid) if pcm_for_stt: self._flush_to_stt(uid, pcm_for_stt) except Exception as e: # noqa: BLE001 log.warning("EchoVoiceSink.write failed: %s", e) + def _take_flushable_pcm(self, uid: int) -> Optional[bytes]: + """If user `uid` has buffered speech that's been silent ≥SILENCE_FLUSH_MS, + consume the buffer and return it. Otherwise return None.""" + with self._sink_lock: + if not self._has_speech.get(uid): + return None + last = self._last_speech_ts.get(uid, 0.0) + silence_ms = (time.monotonic() - last) * 1000.0 + if silence_ms < SILENCE_FLUSH_MS: + return None + pcm = bytes(self._user_buffers.get(uid, b"")) + self._user_buffers[uid] = bytearray() + self._packet_accum[uid] = bytearray() + self._has_speech[uid] = False + return pcm if pcm else None + + def _silence_flush_poller(self) -> None: + """Background tick: Discord DTX stops sending RTP packets when the user + goes silent, so the inline flush check in `write()` never fires for the + last utterance. Poll every 200ms so the trailing audio actually reaches + Whisper.""" + while not self._poller_stop.wait(0.2): + try: + with self._sink_lock: + pending = [uid for uid, has in self._has_speech.items() if has] + for uid in pending: + pcm = self._take_flushable_pcm(uid) + if pcm: + self._flush_to_stt(uid, pcm) + except Exception as e: # noqa: BLE001 + log.warning("silence flush poller iter failed: %s", e) + # ----- VAD ----- def _vad_detects_speech(self, pcm48_stereo: bytes) -> bool: @@ -556,7 +573,6 @@ class EchoVoiceSink(AudioSink): __all__ = [ "VoiceSession", "EchoVoiceSink", - "FILLER_DELAY_S", "SILENCE_FLUSH_MS", "VAD_THRESHOLD", "VAD_WINDOW_MS", diff --git a/src/voice/tts_stream.py b/src/voice/tts_stream.py index 96f259c..b63cdd4 100644 --- a/src/voice/tts_stream.py +++ b/src/voice/tts_stream.py @@ -10,6 +10,7 @@ Engineering decisions #6, #8, #15. from __future__ import annotations import io +import logging import queue import re import subprocess @@ -24,6 +25,9 @@ from src.voice.normalize import normalize_for_tts from tools.tts import synthesize +log = logging.getLogger(__name__) + + # Discord wants 20ms of 16-bit 48kHz stereo PCM per frame. # 48000 Hz * 0.020 s * 2 channels * 2 bytes = 3840 bytes. FRAME_BYTES = 3840 @@ -31,13 +35,6 @@ TARGET_SAMPLE_RATE = 48000 TARGET_CHANNELS = 2 TARGET_SAMPLE_WIDTH = 2 -_PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent -_THINKING_WAV = _PROJECT_ROOT / "assets" / "voice" / "thinking.wav" - -# Cached filler frames (load + resample once, reuse forever). -_thinking_frames_cache: Optional[List[bytes]] = None -_thinking_cache_lock = threading.Lock() - # Sentinel pushed onto the text queue to ask the worker to exit cleanly. _POISON = object() @@ -149,27 +146,6 @@ def wav_to_pcm_20ms_frames(wav_bytes: bytes) -> List[bytes]: return frames -def load_thinking_wav() -> List[bytes]: - """Load ``assets/voice/thinking.wav`` and cache it as 20ms PCM frames. - - Safe to call from multiple threads; the load happens at most once. - Returns an empty list if the asset is missing or fails to decode - (callers treat that as a no-op filler). - """ - global _thinking_frames_cache - if _thinking_frames_cache is not None: - return _thinking_frames_cache - with _thinking_cache_lock: - if _thinking_frames_cache is not None: - return _thinking_frames_cache - try: - wav_bytes = _THINKING_WAV.read_bytes() - _thinking_frames_cache = wav_to_pcm_20ms_frames(wav_bytes) - except (FileNotFoundError, OSError, RuntimeError): - _thinking_frames_cache = [] - return _thinking_frames_cache - - # ---------- TTS worker queue ---------- class TTSQueue: @@ -227,19 +203,13 @@ class TTSQueue: if not text: return cleaned = normalize_for_tts(text) + n = 0 for clause in clause_segments(cleaned): clause = clause.strip() if clause: self._text_queue.put(clause) - - def push_filler(self) -> None: - """Enqueue pre-rendered filler frames (``thinking.wav``) directly. - - Bypasses synthesis -- the filler plays even if Supertonic is down - or slow. No-op if the asset failed to load. - """ - for frame in load_thinking_wav(): - self._pcm_queue.put(frame) + n += 1 + log.info("ttsq.push_text: input %d chars → %d clauses queued", len(text), n) def clear(self) -> None: """Drop everything pending (used for barge-in).""" @@ -251,10 +221,14 @@ class TTSQueue: # --- consumer side (called by EchoStreamingAudioSource) --- - def get_frame(self, timeout: float = 0.1) -> Optional[bytes]: - """Block up to ``timeout`` seconds for the next 20ms PCM frame.""" + def get_frame_nowait(self) -> Optional[bytes]: + """Return the next PCM frame if available, else None — no blocking. + + Blocking inside the player's read() loop wrecks Discord's 20ms cadence + and the client interprets the stream as stuttering / out-of-order. + """ try: - return self._pcm_queue.get(timeout=timeout) + return self._pcm_queue.get_nowait() except queue.Empty: return None @@ -278,24 +252,25 @@ class TTSQueue: break if not isinstance(item, str): continue + preview = item[:60] try: result = synthesize(item, voice=self.voice_id, lang=self.lang) - except Exception: - # Synth crashes shouldn't kill the worker -- log path is the - # caller's job (we have no logger here on purpose). + except Exception as e: + log.warning("TTS synth raised for %r: %s", preview, e) continue if not result.get('ok'): + log.warning("TTS synth not ok for %r: %s", preview, result.get('error')) continue path = result.get('path') if not path: + log.warning("TTS synth ok but no path for %r", preview) continue wav_bytes = b'' try: wav_bytes = Path(path).read_bytes() - except OSError: - pass + except OSError as e: + log.warning("TTS WAV read failed for %r: %s", preview, e) finally: - # Best-effort cleanup of the synth tempfile. try: Path(path).unlink(missing_ok=True) except OSError: @@ -304,12 +279,18 @@ class TTSQueue: continue try: frames = wav_to_pcm_20ms_frames(wav_bytes) - except RuntimeError: + except RuntimeError as e: + log.warning("TTS WAV-to-PCM failed for %r: %s", preview, e) + continue + if not frames: + log.warning("TTS WAV-to-PCM produced 0 frames for %r", preview) continue for frame in frames: if self._stop_event.is_set(): return self._pcm_queue.put(frame) + log.info("TTS pushed %d frames (%.1fs) for %r", + len(frames), len(frames) * 0.02, preview) # ---------- Discord audio source ---------- @@ -336,7 +317,7 @@ class EchoStreamingAudioSource(discord.AudioSource): def read(self) -> bytes: if self._closed: return b'' - frame = self._ttsq.get_frame(timeout=0.1) + frame = self._ttsq.get_frame_nowait() if frame is None: return self._SILENCE_FRAME return frame diff --git a/tasks/lessons.md b/tasks/lessons.md index 8c8a853..46582e3 100644 --- a/tasks/lessons.md +++ b/tasks/lessons.md @@ -17,6 +17,13 @@ Lecții capturate din corectările lui Marius. Citește acest fișier la începu +## Supertonic rejectează ghilimelele curly (Unicode) cu HTTP 500 +**Data:** 2026-05-27 +**Context:** Marius a dat o comandă audio pe Discord cu un URL, iar răspunsul lui Claude conținea `„foo"` (ghilimele românești curly). Supertonic a returnat `HTTP 500: synthesis failed: Found 1 unsupported character(s): ['„']` și răspunsul nu s-a mai auzit. Fără retry logic vizibil în UX — pur și simplu tace. +**Greșeala:** Am presupus că `normalize_for_tts` produce text deja "TTS-safe" pentru Supertonic. În realitate `strip_markdown` păstrează ghilimelele Unicode (`„` U+201E, `"` U+201D, `—` U+2014, `…` U+2026, etc.) pe care Supertonic le refuză. +**Regula:** Înainte de orice apel HTTP la Supertonic, **sanitizează punctuația Unicode** la echivalentele ASCII (`„` `"` `"` → `"`, `'` `'` `‚` → `'`, `–` `—` → `-`, `…` → `...`, `«` `»` → `"`). Funcția `sanitize_punctuation` în `src/voice/normalize.py` face asta și e apelată chiar după `strip_markdown` în pipeline. Dacă apar caractere noi care crapă Supertonic (ex: simboluri matematice, săgeți), adaugă-le în `_TTS_PUNCT_MAP`. +**Când se aplică:** Orice cod care trimite text la Supertonic (`tools/tts.py`, `src/voice/tts_stream.py`). Inclusiv testare manuală cu `curl` — folosește text românesc realistic (include `„foo"`, em-dash `—`, ellipsis `…`). + ## Mai multe threads ≠ mai rapid — fitează `cpu_threads` pe physical cores, nu logical **Data:** 2026-05-27 **Context:** Benchmark `tools/voice_bench.py` pentru faster-whisper `small` int8 pe i7-6700T (4 physical / 8 logical cores). Marius a urcat VM-ul de la 2 → 4 → 6 cores online, așteptând că mai multe = mai rapid. diff --git a/tools/tts.py b/tools/tts.py index 8f3b133..22faf21 100644 --- a/tools/tts.py +++ b/tools/tts.py @@ -23,6 +23,24 @@ VOICES = {"M1", "M2", "M3", "M4", "M5", "F1", "F2", "F3", "F4", "F5"} DEFAULT_VOICE = "M2" DEFAULT_LANG = "ro" +# Punctuation Supertonic synthesis rejects with HTTP 500 (Romanian curly quotes, +# smart dashes, ellipsis, angle quotes). Mapped to ASCII so a stray „foo" in +# any caller's text doesn't kill the whole request. +_TTS_PUNCT_MAP = { + '„': '"', '“': '"', '”': '"', + '‘': "'", '’': "'", '‚': "'", + '«': '"', '»': '"', + '–': '-', '—': '-', + '…': '...', +} + + +def sanitize_for_supertonic(text: str) -> str: + """Replace Unicode punctuation Supertonic rejects with ASCII equivalents.""" + for src, dst in _TTS_PUNCT_MAP.items(): + text = text.replace(src, dst) + return text + def synthesize(text: str, voice: str = DEFAULT_VOICE, lang: str = DEFAULT_LANG) -> dict: """Call Supertonic server and save audio to a temp WAV file. @@ -34,6 +52,8 @@ def synthesize(text: str, voice: str = DEFAULT_VOICE, lang: str = DEFAULT_LANG) if not text or not text.strip(): return {"ok": False, "error": "Text gol."} + text = sanitize_for_supertonic(text) + voice = voice.upper() if voice not in VOICES: voice = DEFAULT_VOICE