handle dead CUDA contexts and empty ASR results
- detect unrecoverable CUDA errors before OOM retries and clear model caches\n- keep normal OOM retries to two attempts (configured batch, then 1)\n- do not create text.txt when ASR returns only whitespace\n- warn per file and report empty_result / lrc+empty in the summary\n- keep at least 12 log lines visible in the GUI
This commit is contained in:
@@ -146,6 +146,13 @@ def run_lrc_pipeline(
|
||||
)
|
||||
|
||||
lrc_content = segments_to_lrc(segments)
|
||||
if not lrc_content.strip():
|
||||
log(f" ⚠ {f.name}: текст не извлечён (пустой результат ASR)")
|
||||
results.append(FileResult(
|
||||
f, None, None,
|
||||
"EmptyResult: ASR returned no text", "lrc+empty",
|
||||
))
|
||||
continue
|
||||
lrc_path.write_text(lrc_content, encoding="utf-8")
|
||||
log(f" ✓ text.lrc ({len(segments)} строк, язык: {detected})")
|
||||
|
||||
|
||||
@@ -836,14 +836,21 @@ def _run_sequential(
|
||||
gigaam_provider=config.transcribe.gigaam_provider,
|
||||
local_files_only=config.transcribe.local_files_only,
|
||||
)
|
||||
text = (text or "").strip()
|
||||
source = config.transcribe.engine
|
||||
log(f" ✓ распознано {len(text)} символов")
|
||||
if (config.lyrics.enabled and config.lyrics.try_after_asr):
|
||||
matched = _try_post_asr_match(text, config, log)
|
||||
if matched:
|
||||
text = matched
|
||||
source = f"{source}+lrclib"
|
||||
(tmp_root / "text.txt").write_text(text, encoding="utf-8")
|
||||
if not text:
|
||||
text = None
|
||||
source = "empty_result"
|
||||
error = "EmptyResult: ASR returned no text"
|
||||
log(f" ⚠ {f.name}: текст не извлечён (пустой результат ASR)")
|
||||
else:
|
||||
log(f" ✓ распознано {len(text)} символов")
|
||||
if (config.lyrics.enabled and config.lyrics.try_after_asr):
|
||||
matched = _try_post_asr_match(text, config, log)
|
||||
if matched:
|
||||
text = matched
|
||||
source = f"{source}+lrclib"
|
||||
(tmp_root / "text.txt").write_text(text, encoding="utf-8")
|
||||
|
||||
metadata_dict = _build_metadata_dict(
|
||||
ident.metadata, ident.shazam, source, input_path=f,
|
||||
@@ -873,7 +880,7 @@ def _run_sequential(
|
||||
except CudaOOMError as e:
|
||||
error = f"CudaOOMError: {e}"
|
||||
source = "oom_skipped"
|
||||
log(f" ⊘ {f.name}: OOM после 3 попыток — пропущен, идём дальше")
|
||||
log(f" ⊘ {f.name}: OOM после ретраев — пропущен, идём дальше")
|
||||
except Exception as e: # noqa: BLE001
|
||||
error = f"{type(e).__name__}: {e}"
|
||||
log(f" ✗ Ошибка: {error}")
|
||||
@@ -1293,9 +1300,16 @@ def _run_staged_batch(
|
||||
gigaam_provider=config.transcribe.gigaam_provider,
|
||||
local_files_only=config.transcribe.local_files_only,
|
||||
)
|
||||
state[f]["text"] = text
|
||||
state[f]["source"] = config.transcribe.engine
|
||||
log(f" ✓ {f.name} ({len(text)} символов)")
|
||||
text = (text or "").strip()
|
||||
if not text:
|
||||
state[f]["text"] = None
|
||||
state[f]["source"] = "empty_result"
|
||||
state[f]["error"] = "EmptyResult: ASR returned no text"
|
||||
log(f" ⚠ {f.name}: текст не извлечён (пустой результат ASR)")
|
||||
else:
|
||||
state[f]["text"] = text
|
||||
state[f]["source"] = config.transcribe.engine
|
||||
log(f" ✓ {f.name} ({len(text)} символов)")
|
||||
except CudaOOMError as e:
|
||||
state[f]["error"] = f"CudaOOMError: {e}"
|
||||
state[f]["source"] = "oom_skipped"
|
||||
|
||||
@@ -20,6 +20,27 @@ from typing import Optional
|
||||
_MODEL_LOCK = threading.RLock()
|
||||
_REUSE_LOGGED: set = set()
|
||||
|
||||
# CUDA context failures that cannot be recovered by retrying or empty_cache().
|
||||
# They require restarting the app (or switching to CPU).
|
||||
_DEAD_CUDA_MARKERS = (
|
||||
"invalid device ordinal",
|
||||
"cudaerrorinvaliddevice",
|
||||
"device-side assert",
|
||||
"unspecified launch failure",
|
||||
)
|
||||
|
||||
|
||||
def _is_dead_cuda(exc: BaseException) -> bool:
|
||||
err = str(exc).lower()
|
||||
return any(m in err for m in _DEAD_CUDA_MARKERS)
|
||||
|
||||
|
||||
def _clear_model_caches() -> None:
|
||||
_WhisperXModels._cache.clear()
|
||||
_AlignModels._cache.clear()
|
||||
_GigaAMModels._cache.clear()
|
||||
_REUSE_LOGGED.clear()
|
||||
|
||||
|
||||
class CudaOOMError(RuntimeError):
|
||||
"""CUDA OOM, переживший все стратегии понижения нагрузки."""
|
||||
@@ -91,11 +112,13 @@ def _model_transcribe_with_retry(
|
||||
The loaded model is intentionally kept in the cache between attempts:
|
||||
reloading it costs minutes and does not help with an OOM caused by a long
|
||||
audio file. VAD is not touched here — WhisperX expects it at model-load
|
||||
time (``vad_model``/``vad_options``), not per transcription call. If even
|
||||
batch=1 fails, raise CudaOOMError so the caller can skip this file and
|
||||
continue with the rest of the batch.
|
||||
time (``vad_model``/``vad_options``), not per transcription call. A dead
|
||||
CUDA context is never retried: caches are dropped and CudaOOMError is
|
||||
raised immediately. If even batch=1 fails with a normal OOM, raise
|
||||
CudaOOMError so the caller can skip this file and continue with the rest
|
||||
of the batch.
|
||||
"""
|
||||
attempts = [batch_size, 1, 1]
|
||||
attempts = [batch_size, 1]
|
||||
last: Optional[Exception] = None
|
||||
for i, bs in enumerate(attempts, 1):
|
||||
try:
|
||||
@@ -103,12 +126,20 @@ def _model_transcribe_with_retry(
|
||||
audio, batch_size=bs, language=language,
|
||||
)
|
||||
except Exception as e: # noqa: BLE001
|
||||
if _is_dead_cuda(e):
|
||||
_clear_model_caches()
|
||||
raise CudaOOMError(
|
||||
f"CUDA context died on {name}: {e}. "
|
||||
"GPU state is unrecoverable — restart the app or switch to CPU."
|
||||
) from e
|
||||
if not _is_cuda_error(e):
|
||||
raise
|
||||
last = e
|
||||
print(f" [ASR] OOM on {name} (attempt {i}/3, bs={bs}): {e}")
|
||||
print(f" [ASR] OOM on {name} (attempt {i}/{len(attempts)}, bs={bs}): {e}")
|
||||
_free_vram()
|
||||
raise CudaOOMError(f"OOM after 3 attempts on {name}: {last}") from last
|
||||
raise CudaOOMError(
|
||||
f"OOM after {len(attempts)} attempts on {name}: {last}"
|
||||
) from last
|
||||
|
||||
|
||||
class _WhisperXModels:
|
||||
|
||||
Reference in New Issue
Block a user