handle dead CUDA contexts and empty ASR results
- detect unrecoverable CUDA errors before OOM retries and clear model caches\n- keep normal OOM retries to two attempts (configured batch, then 1)\n- do not create text.txt when ASR returns only whitespace\n- warn per file and report empty_result / lrc+empty in the summary\n- keep at least 12 log lines visible in the GUI
This commit is contained in:
@@ -146,6 +146,13 @@ def run_lrc_pipeline(
|
|||||||
)
|
)
|
||||||
|
|
||||||
lrc_content = segments_to_lrc(segments)
|
lrc_content = segments_to_lrc(segments)
|
||||||
|
if not lrc_content.strip():
|
||||||
|
log(f" ⚠ {f.name}: текст не извлечён (пустой результат ASR)")
|
||||||
|
results.append(FileResult(
|
||||||
|
f, None, None,
|
||||||
|
"EmptyResult: ASR returned no text", "lrc+empty",
|
||||||
|
))
|
||||||
|
continue
|
||||||
lrc_path.write_text(lrc_content, encoding="utf-8")
|
lrc_path.write_text(lrc_content, encoding="utf-8")
|
||||||
log(f" ✓ text.lrc ({len(segments)} строк, язык: {detected})")
|
log(f" ✓ text.lrc ({len(segments)} строк, язык: {detected})")
|
||||||
|
|
||||||
|
|||||||
@@ -836,7 +836,14 @@ def _run_sequential(
|
|||||||
gigaam_provider=config.transcribe.gigaam_provider,
|
gigaam_provider=config.transcribe.gigaam_provider,
|
||||||
local_files_only=config.transcribe.local_files_only,
|
local_files_only=config.transcribe.local_files_only,
|
||||||
)
|
)
|
||||||
|
text = (text or "").strip()
|
||||||
source = config.transcribe.engine
|
source = config.transcribe.engine
|
||||||
|
if not text:
|
||||||
|
text = None
|
||||||
|
source = "empty_result"
|
||||||
|
error = "EmptyResult: ASR returned no text"
|
||||||
|
log(f" ⚠ {f.name}: текст не извлечён (пустой результат ASR)")
|
||||||
|
else:
|
||||||
log(f" ✓ распознано {len(text)} символов")
|
log(f" ✓ распознано {len(text)} символов")
|
||||||
if (config.lyrics.enabled and config.lyrics.try_after_asr):
|
if (config.lyrics.enabled and config.lyrics.try_after_asr):
|
||||||
matched = _try_post_asr_match(text, config, log)
|
matched = _try_post_asr_match(text, config, log)
|
||||||
@@ -873,7 +880,7 @@ def _run_sequential(
|
|||||||
except CudaOOMError as e:
|
except CudaOOMError as e:
|
||||||
error = f"CudaOOMError: {e}"
|
error = f"CudaOOMError: {e}"
|
||||||
source = "oom_skipped"
|
source = "oom_skipped"
|
||||||
log(f" ⊘ {f.name}: OOM после 3 попыток — пропущен, идём дальше")
|
log(f" ⊘ {f.name}: OOM после ретраев — пропущен, идём дальше")
|
||||||
except Exception as e: # noqa: BLE001
|
except Exception as e: # noqa: BLE001
|
||||||
error = f"{type(e).__name__}: {e}"
|
error = f"{type(e).__name__}: {e}"
|
||||||
log(f" ✗ Ошибка: {error}")
|
log(f" ✗ Ошибка: {error}")
|
||||||
@@ -1293,6 +1300,13 @@ def _run_staged_batch(
|
|||||||
gigaam_provider=config.transcribe.gigaam_provider,
|
gigaam_provider=config.transcribe.gigaam_provider,
|
||||||
local_files_only=config.transcribe.local_files_only,
|
local_files_only=config.transcribe.local_files_only,
|
||||||
)
|
)
|
||||||
|
text = (text or "").strip()
|
||||||
|
if not text:
|
||||||
|
state[f]["text"] = None
|
||||||
|
state[f]["source"] = "empty_result"
|
||||||
|
state[f]["error"] = "EmptyResult: ASR returned no text"
|
||||||
|
log(f" ⚠ {f.name}: текст не извлечён (пустой результат ASR)")
|
||||||
|
else:
|
||||||
state[f]["text"] = text
|
state[f]["text"] = text
|
||||||
state[f]["source"] = config.transcribe.engine
|
state[f]["source"] = config.transcribe.engine
|
||||||
log(f" ✓ {f.name} ({len(text)} символов)")
|
log(f" ✓ {f.name} ({len(text)} символов)")
|
||||||
|
|||||||
@@ -20,6 +20,27 @@ from typing import Optional
|
|||||||
_MODEL_LOCK = threading.RLock()
|
_MODEL_LOCK = threading.RLock()
|
||||||
_REUSE_LOGGED: set = set()
|
_REUSE_LOGGED: set = set()
|
||||||
|
|
||||||
|
# CUDA context failures that cannot be recovered by retrying or empty_cache().
|
||||||
|
# They require restarting the app (or switching to CPU).
|
||||||
|
_DEAD_CUDA_MARKERS = (
|
||||||
|
"invalid device ordinal",
|
||||||
|
"cudaerrorinvaliddevice",
|
||||||
|
"device-side assert",
|
||||||
|
"unspecified launch failure",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _is_dead_cuda(exc: BaseException) -> bool:
|
||||||
|
err = str(exc).lower()
|
||||||
|
return any(m in err for m in _DEAD_CUDA_MARKERS)
|
||||||
|
|
||||||
|
|
||||||
|
def _clear_model_caches() -> None:
|
||||||
|
_WhisperXModels._cache.clear()
|
||||||
|
_AlignModels._cache.clear()
|
||||||
|
_GigaAMModels._cache.clear()
|
||||||
|
_REUSE_LOGGED.clear()
|
||||||
|
|
||||||
|
|
||||||
class CudaOOMError(RuntimeError):
|
class CudaOOMError(RuntimeError):
|
||||||
"""CUDA OOM, переживший все стратегии понижения нагрузки."""
|
"""CUDA OOM, переживший все стратегии понижения нагрузки."""
|
||||||
@@ -91,11 +112,13 @@ def _model_transcribe_with_retry(
|
|||||||
The loaded model is intentionally kept in the cache between attempts:
|
The loaded model is intentionally kept in the cache between attempts:
|
||||||
reloading it costs minutes and does not help with an OOM caused by a long
|
reloading it costs minutes and does not help with an OOM caused by a long
|
||||||
audio file. VAD is not touched here — WhisperX expects it at model-load
|
audio file. VAD is not touched here — WhisperX expects it at model-load
|
||||||
time (``vad_model``/``vad_options``), not per transcription call. If even
|
time (``vad_model``/``vad_options``), not per transcription call. A dead
|
||||||
batch=1 fails, raise CudaOOMError so the caller can skip this file and
|
CUDA context is never retried: caches are dropped and CudaOOMError is
|
||||||
continue with the rest of the batch.
|
raised immediately. If even batch=1 fails with a normal OOM, raise
|
||||||
|
CudaOOMError so the caller can skip this file and continue with the rest
|
||||||
|
of the batch.
|
||||||
"""
|
"""
|
||||||
attempts = [batch_size, 1, 1]
|
attempts = [batch_size, 1]
|
||||||
last: Optional[Exception] = None
|
last: Optional[Exception] = None
|
||||||
for i, bs in enumerate(attempts, 1):
|
for i, bs in enumerate(attempts, 1):
|
||||||
try:
|
try:
|
||||||
@@ -103,12 +126,20 @@ def _model_transcribe_with_retry(
|
|||||||
audio, batch_size=bs, language=language,
|
audio, batch_size=bs, language=language,
|
||||||
)
|
)
|
||||||
except Exception as e: # noqa: BLE001
|
except Exception as e: # noqa: BLE001
|
||||||
|
if _is_dead_cuda(e):
|
||||||
|
_clear_model_caches()
|
||||||
|
raise CudaOOMError(
|
||||||
|
f"CUDA context died on {name}: {e}. "
|
||||||
|
"GPU state is unrecoverable — restart the app or switch to CPU."
|
||||||
|
) from e
|
||||||
if not _is_cuda_error(e):
|
if not _is_cuda_error(e):
|
||||||
raise
|
raise
|
||||||
last = e
|
last = e
|
||||||
print(f" [ASR] OOM on {name} (attempt {i}/3, bs={bs}): {e}")
|
print(f" [ASR] OOM on {name} (attempt {i}/{len(attempts)}, bs={bs}): {e}")
|
||||||
_free_vram()
|
_free_vram()
|
||||||
raise CudaOOMError(f"OOM after 3 attempts on {name}: {last}") from last
|
raise CudaOOMError(
|
||||||
|
f"OOM after {len(attempts)} attempts on {name}: {last}"
|
||||||
|
) from last
|
||||||
|
|
||||||
|
|
||||||
class _WhisperXModels:
|
class _WhisperXModels:
|
||||||
|
|||||||
@@ -219,7 +219,9 @@ class MainWindow(QMainWindow):
|
|||||||
self.log_view = QPlainTextEdit()
|
self.log_view = QPlainTextEdit()
|
||||||
self.log_view.setObjectName("log")
|
self.log_view.setObjectName("log")
|
||||||
self.log_view.setReadOnly(True)
|
self.log_view.setReadOnly(True)
|
||||||
self.log_view.setMinimumHeight(240)
|
# Keep at least ~12 log lines visible regardless of the UI font size.
|
||||||
|
line_height = self.log_view.fontMetrics().lineSpacing()
|
||||||
|
self.log_view.setMinimumHeight(max(280, line_height * 12 + 24))
|
||||||
self.log_view.setPlaceholderText("Здесь появится лог выполнения…")
|
self.log_view.setPlaceholderText("Здесь появится лог выполнения…")
|
||||||
root.addWidget(self.log_view, 1)
|
root.addWidget(self.log_view, 1)
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user