handle dead CUDA contexts and empty ASR results

- detect unrecoverable CUDA errors before OOM retries and clear model caches\n- keep normal OOM retries to two attempts (configured batch, then 1)\n- do not create text.txt when ASR returns only whitespace\n- warn per file and report empty_result / lrc+empty in the summary\n- keep at least 12 log lines visible in the GUI
This commit is contained in:
jadssss
2026-10-28 23:53:27 +03:00
parent 60846e3c6c
commit 841fc8902d
4 changed files with 72 additions and 18 deletions

View File

@@ -146,6 +146,13 @@ def run_lrc_pipeline(
)
lrc_content = segments_to_lrc(segments)
if not lrc_content.strip():
log(f" ⚠ {f.name}: текст не извлечён (пустой результат ASR)")
results.append(FileResult(
f, None, None,
"EmptyResult: ASR returned no text", "lrc+empty",
))
continue
lrc_path.write_text(lrc_content, encoding="utf-8")
log(f" ✓ text.lrc ({len(segments)} строк, язык: {detected})")

View File

@@ -836,14 +836,21 @@ def _run_sequential(
gigaam_provider=config.transcribe.gigaam_provider,
local_files_only=config.transcribe.local_files_only,
)
text = (text or "").strip()
source = config.transcribe.engine
log(f" ✓ распознано {len(text)} символов")
if (config.lyrics.enabled and config.lyrics.try_after_asr):
matched = _try_post_asr_match(text, config, log)
if matched:
text = matched
source = f"{source}+lrclib"
(tmp_root / "text.txt").write_text(text, encoding="utf-8")
if not text:
text = None
source = "empty_result"
error = "EmptyResult: ASR returned no text"
log(f" ⚠ {f.name}: текст не извлечён (пустой результат ASR)")
else:
log(f" ✓ распознано {len(text)} символов")
if (config.lyrics.enabled and config.lyrics.try_after_asr):
matched = _try_post_asr_match(text, config, log)
if matched:
text = matched
source = f"{source}+lrclib"
(tmp_root / "text.txt").write_text(text, encoding="utf-8")
metadata_dict = _build_metadata_dict(
ident.metadata, ident.shazam, source, input_path=f,
@@ -873,7 +880,7 @@ def _run_sequential(
except CudaOOMError as e:
error = f"CudaOOMError: {e}"
source = "oom_skipped"
log(f" ⊘ {f.name}: OOM после 3 попыток — пропущен, идём дальше")
log(f" ⊘ {f.name}: OOM после ретраев — пропущен, идём дальше")
except Exception as e: # noqa: BLE001
error = f"{type(e).__name__}: {e}"
log(f" ✗ Ошибка: {error}")
@@ -1293,9 +1300,16 @@ def _run_staged_batch(
gigaam_provider=config.transcribe.gigaam_provider,
local_files_only=config.transcribe.local_files_only,
)
state[f]["text"] = text
state[f]["source"] = config.transcribe.engine
log(f" ✓ {f.name} ({len(text)} символов)")
text = (text or "").strip()
if not text:
state[f]["text"] = None
state[f]["source"] = "empty_result"
state[f]["error"] = "EmptyResult: ASR returned no text"
log(f" ⚠ {f.name}: текст не извлечён (пустой результат ASR)")
else:
state[f]["text"] = text
state[f]["source"] = config.transcribe.engine
log(f" ✓ {f.name} ({len(text)} символов)")
except CudaOOMError as e:
state[f]["error"] = f"CudaOOMError: {e}"
state[f]["source"] = "oom_skipped"

View File

@@ -20,6 +20,27 @@ from typing import Optional
_MODEL_LOCK = threading.RLock()
_REUSE_LOGGED: set = set()
# CUDA context failures that cannot be recovered by retrying or empty_cache().
# They require restarting the app (or switching to CPU).
_DEAD_CUDA_MARKERS = (
"invalid device ordinal",
"cudaerrorinvaliddevice",
"device-side assert",
"unspecified launch failure",
)
def _is_dead_cuda(exc: BaseException) -> bool:
err = str(exc).lower()
return any(m in err for m in _DEAD_CUDA_MARKERS)
def _clear_model_caches() -> None:
_WhisperXModels._cache.clear()
_AlignModels._cache.clear()
_GigaAMModels._cache.clear()
_REUSE_LOGGED.clear()
class CudaOOMError(RuntimeError):
"""CUDA OOM, переживший все стратегии понижения нагрузки."""
@@ -91,11 +112,13 @@ def _model_transcribe_with_retry(
The loaded model is intentionally kept in the cache between attempts:
reloading it costs minutes and does not help with an OOM caused by a long
audio file. VAD is not touched here — WhisperX expects it at model-load
time (``vad_model``/``vad_options``), not per transcription call. If even
batch=1 fails, raise CudaOOMError so the caller can skip this file and
continue with the rest of the batch.
time (``vad_model``/``vad_options``), not per transcription call. A dead
CUDA context is never retried: caches are dropped and CudaOOMError is
raised immediately. If even batch=1 fails with a normal OOM, raise
CudaOOMError so the caller can skip this file and continue with the rest
of the batch.
"""
attempts = [batch_size, 1, 1]
attempts = [batch_size, 1]
last: Optional[Exception] = None
for i, bs in enumerate(attempts, 1):
try:
@@ -103,12 +126,20 @@ def _model_transcribe_with_retry(
audio, batch_size=bs, language=language,
)
except Exception as e: # noqa: BLE001
if _is_dead_cuda(e):
_clear_model_caches()
raise CudaOOMError(
f"CUDA context died on {name}: {e}. "
"GPU state is unrecoverable — restart the app or switch to CPU."
) from e
if not _is_cuda_error(e):
raise
last = e
print(f" [ASR] OOM on {name} (attempt {i}/3, bs={bs}): {e}")
print(f" [ASR] OOM on {name} (attempt {i}/{len(attempts)}, bs={bs}): {e}")
_free_vram()
raise CudaOOMError(f"OOM after 3 attempts on {name}: {last}") from last
raise CudaOOMError(
f"OOM after {len(attempts)} attempts on {name}: {last}"
) from last
class _WhisperXModels:

View File

@@ -219,7 +219,9 @@ class MainWindow(QMainWindow):
self.log_view = QPlainTextEdit()
self.log_view.setObjectName("log")
self.log_view.setReadOnly(True)
self.log_view.setMinimumHeight(240)
# Keep at least ~12 log lines visible regardless of the UI font size.
line_height = self.log_view.fontMetrics().lineSpacing()
self.log_view.setMinimumHeight(max(280, line_height * 12 + 24))
self.log_view.setPlaceholderText("Здесь появится лог выполнения…")
root.addWidget(self.log_view, 1)