"""Speech recognition via WhisperX or GigaAM (onnx-asr). Also exposes `transcribe_aligned()` for word-level alignment (used by LRC). Models are cached; aggressive VRAM cleanup happens after each file. """ from __future__ import annotations import gc import threading import time from pathlib import Path from typing import Optional # The model caches below are process-wide. They are intentionally never # cleared between files or batches: reloading WhisperX takes minutes and is # what made runs look like they reloaded the model every batch. The lock makes # the "load once, reuse forever" guarantee safe even if ASR is ever called # from multiple threads. _MODEL_LOCK = threading.RLock() _REUSE_LOGGED: set = set() # CUDA context failures that cannot be recovered by retrying or empty_cache(). # They require restarting the app (or switching to CPU). _DEAD_CUDA_MARKERS = ( "invalid device ordinal", "cudaerrorinvaliddevice", "device-side assert", "unspecified launch failure", ) def _is_dead_cuda(exc: BaseException) -> bool: err = str(exc).lower() return any(m in err for m in _DEAD_CUDA_MARKERS) def _clear_model_caches() -> None: _WhisperXModels._cache.clear() _AlignModels._cache.clear() _GigaAMModels._cache.clear() _REUSE_LOGGED.clear() class CudaOOMError(RuntimeError): """CUDA OOM, переживший все стратегии понижения нагрузки.""" def _is_cuda_error(exc: BaseException) -> bool: err = str(exc).lower() return ( "out of memory" in err or "cuda" in err or "device ordinal" in err or "cublas" in err or "cudnn" in err ) def _resolve_device(device: str) -> str: if device in ("cuda", "cpu"): return device try: import torch return "cuda" if torch.cuda.is_available() else "cpu" except Exception: return "cpu" def _retry(fn, attempts: int = 3, base_delay: float = 2.0): last = None for i in range(attempts): try: return fn() except Exception as e: last = e if i < attempts - 1: time.sleep(base_delay * (i + 1)) raise last def _free_vram() -> None: gc.collect() try: import torch if torch.cuda.is_available(): torch.cuda.empty_cache() except Exception: # noqa: BLE001 — cleanup must never mask the original error pass def _enable_tf32() -> None: """Enable TF32 matmul on Ampere+ GPUs (RTX 30/40). Called lazily right before a WhisperX model is loaded, so runs that do not use ASR never import Torch or create a CUDA context at startup. """ try: import torch if torch.cuda.is_available(): torch.backends.cuda.matmul.allow_tf32 = True torch.backends.cudnn.allow_tf32 = True except Exception: # noqa: BLE001 — purely an optimization pass def _model_transcribe_with_retry( model_obj, audio, language, batch_size, name="