From 841fc8902d9a81888517c0b3d3cd2d3988196efe Mon Sep 17 00:00:00 2001 From: jadssss Date: Wed, 28 Oct 2026 23:53:27 +0300 Subject: [PATCH] handle dead CUDA contexts and empty ASR results - detect unrecoverable CUDA errors before OOM retries and clear model caches\n- keep normal OOM retries to two attempts (configured batch, then 1)\n- do not create text.txt when ASR returns only whitespace\n- warn per file and report empty_result / lrc+empty in the summary\n- keep at least 12 log lines visible in the GUI --- core/lrc_pipeline.py | 7 +++++++ core/pipeline.py | 36 +++++++++++++++++++++++++----------- core/transcribe.py | 43 +++++++++++++++++++++++++++++++++++++------ gui/main_window.py | 4 +++- 4 files changed, 72 insertions(+), 18 deletions(-) diff --git a/core/lrc_pipeline.py b/core/lrc_pipeline.py index 0e75d4c..d940218 100644 --- a/core/lrc_pipeline.py +++ b/core/lrc_pipeline.py @@ -146,6 +146,13 @@ def run_lrc_pipeline( ) lrc_content = segments_to_lrc(segments) + if not lrc_content.strip(): + log(f" ⚠ {f.name}: текст не извлечён (пустой результат ASR)") + results.append(FileResult( + f, None, None, + "EmptyResult: ASR returned no text", "lrc+empty", + )) + continue lrc_path.write_text(lrc_content, encoding="utf-8") log(f" ✓ text.lrc ({len(segments)} строк, язык: {detected})") diff --git a/core/pipeline.py b/core/pipeline.py index 13de14e..75f3d9d 100644 --- a/core/pipeline.py +++ b/core/pipeline.py @@ -836,14 +836,21 @@ def _run_sequential( gigaam_provider=config.transcribe.gigaam_provider, local_files_only=config.transcribe.local_files_only, ) + text = (text or "").strip() source = config.transcribe.engine - log(f" ✓ распознано {len(text)} символов") - if (config.lyrics.enabled and config.lyrics.try_after_asr): - matched = _try_post_asr_match(text, config, log) - if matched: - text = matched - source = f"{source}+lrclib" - (tmp_root / "text.txt").write_text(text, encoding="utf-8") + if not text: + text = None + source = "empty_result" + error = "EmptyResult: ASR returned no text" + log(f" ⚠ {f.name}: текст не извлечён (пустой результат ASR)") + else: + log(f" ✓ распознано {len(text)} символов") + if (config.lyrics.enabled and config.lyrics.try_after_asr): + matched = _try_post_asr_match(text, config, log) + if matched: + text = matched + source = f"{source}+lrclib" + (tmp_root / "text.txt").write_text(text, encoding="utf-8") metadata_dict = _build_metadata_dict( ident.metadata, ident.shazam, source, input_path=f, @@ -873,7 +880,7 @@ def _run_sequential( except CudaOOMError as e: error = f"CudaOOMError: {e}" source = "oom_skipped" - log(f" ⊘ {f.name}: OOM после 3 попыток — пропущен, идём дальше") + log(f" ⊘ {f.name}: OOM после ретраев — пропущен, идём дальше") except Exception as e: # noqa: BLE001 error = f"{type(e).__name__}: {e}" log(f" ✗ Ошибка: {error}") @@ -1293,9 +1300,16 @@ def _run_staged_batch( gigaam_provider=config.transcribe.gigaam_provider, local_files_only=config.transcribe.local_files_only, ) - state[f]["text"] = text - state[f]["source"] = config.transcribe.engine - log(f" ✓ {f.name} ({len(text)} символов)") + text = (text or "").strip() + if not text: + state[f]["text"] = None + state[f]["source"] = "empty_result" + state[f]["error"] = "EmptyResult: ASR returned no text" + log(f" ⚠ {f.name}: текст не извлечён (пустой результат ASR)") + else: + state[f]["text"] = text + state[f]["source"] = config.transcribe.engine + log(f" ✓ {f.name} ({len(text)} символов)") except CudaOOMError as e: state[f]["error"] = f"CudaOOMError: {e}" state[f]["source"] = "oom_skipped" diff --git a/core/transcribe.py b/core/transcribe.py index ab0572f..17381bc 100644 --- a/core/transcribe.py +++ b/core/transcribe.py @@ -20,6 +20,27 @@ from typing import Optional _MODEL_LOCK = threading.RLock() _REUSE_LOGGED: set = set() +# CUDA context failures that cannot be recovered by retrying or empty_cache(). +# They require restarting the app (or switching to CPU). +_DEAD_CUDA_MARKERS = ( + "invalid device ordinal", + "cudaerrorinvaliddevice", + "device-side assert", + "unspecified launch failure", +) + + +def _is_dead_cuda(exc: BaseException) -> bool: + err = str(exc).lower() + return any(m in err for m in _DEAD_CUDA_MARKERS) + + +def _clear_model_caches() -> None: + _WhisperXModels._cache.clear() + _AlignModels._cache.clear() + _GigaAMModels._cache.clear() + _REUSE_LOGGED.clear() + class CudaOOMError(RuntimeError): """CUDA OOM, переживший все стратегии понижения нагрузки.""" @@ -91,11 +112,13 @@ def _model_transcribe_with_retry( The loaded model is intentionally kept in the cache between attempts: reloading it costs minutes and does not help with an OOM caused by a long audio file. VAD is not touched here — WhisperX expects it at model-load - time (``vad_model``/``vad_options``), not per transcription call. If even - batch=1 fails, raise CudaOOMError so the caller can skip this file and - continue with the rest of the batch. + time (``vad_model``/``vad_options``), not per transcription call. A dead + CUDA context is never retried: caches are dropped and CudaOOMError is + raised immediately. If even batch=1 fails with a normal OOM, raise + CudaOOMError so the caller can skip this file and continue with the rest + of the batch. """ - attempts = [batch_size, 1, 1] + attempts = [batch_size, 1] last: Optional[Exception] = None for i, bs in enumerate(attempts, 1): try: @@ -103,12 +126,20 @@ def _model_transcribe_with_retry( audio, batch_size=bs, language=language, ) except Exception as e: # noqa: BLE001 + if _is_dead_cuda(e): + _clear_model_caches() + raise CudaOOMError( + f"CUDA context died on {name}: {e}. " + "GPU state is unrecoverable — restart the app or switch to CPU." + ) from e if not _is_cuda_error(e): raise last = e - print(f" [ASR] OOM on {name} (attempt {i}/3, bs={bs}): {e}") + print(f" [ASR] OOM on {name} (attempt {i}/{len(attempts)}, bs={bs}): {e}") _free_vram() - raise CudaOOMError(f"OOM after 3 attempts on {name}: {last}") from last + raise CudaOOMError( + f"OOM after {len(attempts)} attempts on {name}: {last}" + ) from last class _WhisperXModels: diff --git a/gui/main_window.py b/gui/main_window.py index 02d0e7d..f2a577f 100644 --- a/gui/main_window.py +++ b/gui/main_window.py @@ -219,7 +219,9 @@ class MainWindow(QMainWindow): self.log_view = QPlainTextEdit() self.log_view.setObjectName("log") self.log_view.setReadOnly(True) - self.log_view.setMinimumHeight(240) + # Keep at least ~12 log lines visible regardless of the UI font size. + line_height = self.log_view.fontMetrics().lineSpacing() + self.log_view.setMinimumHeight(max(280, line_height * 12 + 24)) self.log_view.setPlaceholderText("Здесь появится лог выполнения…") root.addWidget(self.log_view, 1)