Compare commits

...

8 Commits

Author SHA1 Message Date
jadssss
a2e0070ace update output layout hints in the GUI 2026-10-29 23:33:34 +03:00
jadssss
2ac7dfd7b7 store outputs under the input-relative path
- add _output_dir_for(): output/<relpath_without_ext>\n- use it in sequential, staged, LRC and embed pipelines\n- keep track folders unique for equal stems in different albums\n- log the layout and report folder collisions\n- add a safe one-shot flat-to-path migration script
2026-10-29 23:32:37 +03:00
jadssss
841fc8902d handle dead CUDA contexts and empty ASR results
- detect unrecoverable CUDA errors before OOM retries and clear model caches\n- keep normal OOM retries to two attempts (configured batch, then 1)\n- do not create text.txt when ASR returns only whitespace\n- warn per file and report empty_result / lrc+empty in the summary\n- keep at least 12 log lines visible in the GUI
2026-10-28 23:53:27 +03:00
jadssss
60846e3c6c load the ASR model once per process
- keep WhisperX, align and GigaAM caches process-wide and thread-safe\n- preload the ASR model at the start of the staged stage 3\n- log that the model is loaded once and then reused\n- never clear caches between files, batches or OOM retries
2026-10-28 09:54:06 +03:00
jadssss
e23acf1555 stop passing vad_filter to whisperx transcribe
- retry only lowers batch_size: configured, 1, 1\n- keep vad_filter parameters for future model-load-level control\n- do not log success in staged finalize when no text was produced
2026-10-28 09:47:27 +03:00
jadssss
5f16a6f87b base skip_existing on the last enabled stage
- add _final_artifact_present and use it for resume decisions\n- text.txt no longer marks a track done when separation is the last stage\n- apply the same rule to pre-scan, staged seeding and sequential skipping\n- keep identification-only runs resolving through text.txt\n- preserve existing metadata tags when a stage-only rerun has less info
2026-10-28 00:48:53 +03:00
jadssss
84d7a28c69 pre-scan finished tracks and load CUDA lazily
- pre-scan all inputs before chunking so ready files stay out of batches\n- start overall progress at the already-finished count\n- count skipped/resumed files correctly in sequential and staged runs\n- keep batches full and log ready/pending counts before starting\n- avoid importing Torch at startup or in cleanup for separation-only runs\n- move TF32 setup into lazy WhisperX model loading\n- detect the GPU through nvidia-smi for the GUI status line
2026-10-27 23:19:56 +03:00
jadssss
f88ca913ce make long-file CUDA OOM recoverable per file
- replace permanent _CUDA_BROKEN flag with CudaOOMError and retry helper
- retry WhisperX with configured settings, then batch=1 without VAD, then once more
- keep the loaded model cached between retries instead of reloading it
- actually pass vad_filter through transcribe() and use it in aligned ASR
- skip only the OOM file in sequential, staged and LRC pipelines
- mark OOM files with oom_skipped / lrc+oom for the summary
- fix staged vocal separation writing input_dir/vocals.wav when convert is off
- route original inputs and resumed files through safe per-file separation
- validate ffmpeg before starting the LRC task
2026-10-27 23:03:29 +03:00
6 changed files with 605 additions and 159 deletions

View File

@@ -12,9 +12,10 @@ from .convert import convert_to_wav
from .lrc import embed_lrc, is_lrc, read_embedded_lyrics, segments_to_lrc
from .metadata import read_metadata
from .pipeline import (
FileResult, _artifact_belongs_to, _save_unrecognized, collect_audio_files,
FileResult, _artifact_belongs_to, _output_dir_for, _save_unrecognized,
collect_audio_files,
)
from .transcribe import transcribe_aligned
from .transcribe import CudaOOMError, transcribe_aligned
LogFn = Callable[[str], None]
@@ -23,11 +24,13 @@ OverallFn = Callable[[int, int], None] # completed_files, total_files
def _pick_audio_source(
f: Path, output_dir: Path, config: PipelineConfig, work_dir: Path, log: LogFn,
f: Path, input_root: Path, output_dir: Path, config: PipelineConfig,
work_dir: Path, log: LogFn,
) -> Path:
mode = (config.lrc.audio_source or "auto").lower()
vocals = output_dir / f.stem / "vocals.wav"
convert = output_dir / f.stem / "convert.wav"
track_out = _output_dir_for(f, input_root, output_dir)
vocals = track_out / "vocals.wav"
convert = track_out / "convert.wav"
if mode == "vocals":
return vocals
if mode == "convert":
@@ -100,7 +103,7 @@ def run_lrc_pipeline(
break
log(f"\n▶ [{idx}/{total}] {f.name}")
track_out = output_dir / f.stem
track_out = _output_dir_for(f, input_dir, output_dir)
track_out.mkdir(parents=True, exist_ok=True)
lrc_path = track_out / "text.lrc"
error: Optional[str] = None
@@ -125,7 +128,9 @@ def run_lrc_pipeline(
results.append(FileResult(f, None, track_out, None, "embedded+lrc"))
continue
audio_path = _pick_audio_source(f, output_dir, config, tmp, log)
audio_path = _pick_audio_source(
f, input_dir, output_dir, config, tmp, log,
)
if not audio_path.exists():
raise FileNotFoundError(
f"Источник аудио не найден: {audio_path}. "
@@ -146,6 +151,13 @@ def run_lrc_pipeline(
)
lrc_content = segments_to_lrc(segments)
if not lrc_content.strip():
log(f" ⚠ {f.name}: текст не извлечён (пустой результат ASR)")
results.append(FileResult(
f, None, None,
"EmptyResult: ASR returned no text", "lrc+empty",
))
continue
lrc_path.write_text(lrc_content, encoding="utf-8")
log(f" ✓ text.lrc ({len(segments)} строк, язык: {detected})")
@@ -155,6 +167,10 @@ def run_lrc_pipeline(
metadata=read_metadata(f),
)
results.append(FileResult(f, None, track_out, None, "lrc"))
except CudaOOMError as e:
error = f"CudaOOMError: {e}"
log(f" ⊘ {f.name}: OOM — пропущен, идём дальше")
results.append(FileResult(f, None, None, error, "lrc+oom"))
except Exception as e: # noqa: BLE001
error = f"{type(e).__name__}: {e}"
log(f" ✗ Ошибка: {error}")
@@ -206,7 +222,8 @@ def run_embed_pipeline(
log("⏹ Отменено")
break
lrc_path = output_dir / f.stem / "text.lrc"
track_out = _output_dir_for(f, input_dir, output_dir)
lrc_path = track_out / "text.lrc"
error: Optional[str] = None
try:
@@ -220,7 +237,7 @@ def run_embed_pipeline(
if config.lrc.embed_in_place:
target = f
else:
target = output_dir / f.stem / f.name
target = track_out / f.name
target.parent.mkdir(parents=True, exist_ok=True)
if not target.exists() or config.overwrite:
shutil.copy2(f, target)

View File

@@ -4,6 +4,7 @@ from __future__ import annotations
import difflib
import json
import shutil
import sys
import tempfile
import threading
from concurrent.futures import ThreadPoolExecutor, as_completed
@@ -18,7 +19,9 @@ from .lyrics import LyricsMatch, search_by_identity, search_by_text
from .metadata import TrackMetadata, read_metadata
from .recognize import ShazamResult, recognize
from .separate import separate_vocals, separate_vocals_batch
from .transcribe import transcribe
from .transcribe import (
CudaOOMError, preload_transcription_model, transcribe,
)
from .lrc import is_lrc, read_embedded_lyrics
@@ -48,11 +51,18 @@ class Cancelled(Exception):
# ---------------------------------------------------------------- helpers
def _free_memory() -> None:
"""Free unreferenced RAM and VRAM. Keeps model caches alive."""
"""Free unreferenced RAM and VRAM. Keeps model caches alive.
Never import torch here: if the current run does not use ASR, importing it
would initialize a CUDA context and steal VRAM from separation. If torch is
already loaded by the ASR stage, we can safely empty its cache.
"""
import gc
gc.collect()
torch = sys.modules.get("torch")
if torch is None:
return
try:
import torch
if torch.cuda.is_available():
torch.cuda.empty_cache()
except Exception: # noqa: BLE001 — never fail cleanup on a broken CUDA context
@@ -150,6 +160,120 @@ def _write_metadata_json(path: Path, data: Optional[dict]) -> None:
path.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8")
def _merge_metadata_file(existing: Path, data: Optional[dict]) -> Optional[dict]:
"""Keep tags/shazam from an earlier run when the new data has less info.
Reprocessing a track for one stage only (e.g. separation) must not replace
a rich metadata.json with a path-only dict.
"""
if not data or not existing.is_file():
return data
try:
old = json.loads(existing.read_text(encoding="utf-8"))
except (OSError, ValueError):
return data
if not isinstance(old, dict):
return data
merged = dict(old)
merged.update(data)
return merged
def _output_dir_for(f: Path, input_root: Path, output_root: Path) -> Path:
"""Path-based track folder: output/<relpath_without_ext>/."""
try:
rel = f.relative_to(input_root)
except ValueError:
rel = Path(f.name)
return output_root / rel.with_suffix("")
def _existing_artifacts(
input_root: Path, output_root: Path, input_path: Path,
) -> tuple[bool, bool, bool]:
"""Return (has_text, has_vocals, has_convert) for the matching output folder.
Artifacts are ignored when ``metadata.json`` says the folder belongs to a
different input file with the same stem.
"""
track_out = _output_dir_for(input_path, input_root, output_root)
if not _artifact_belongs_to(track_out, input_path):
return False, False, False
return (
(track_out / "text.txt").is_file(),
(track_out / "vocals.wav").is_file(),
(track_out / "convert.wav").is_file(),
)
def _final_artifact_present(track_out: Path, config: PipelineConfig) -> bool:
"""True if the artifact of the last enabled stage already exists.
``skip_existing`` must not treat an old ``text.txt`` as "done" when the
current run only needs separation: the final stage here is ``vocals.wav``,
and without it the track still has work to do.
"""
if config.transcribe.enabled:
return (track_out / "text.txt").is_file()
if config.separate.enabled:
return (track_out / "vocals.wav").is_file()
if config.convert.enabled:
return (track_out / "convert.wav").is_file()
return False
def _is_track_finished(
track_out: Path, config: PipelineConfig, belongs: bool = True,
) -> bool:
"""True when ``skip_existing`` may skip this output folder entirely."""
if not belongs:
return False
if _final_artifact_present(track_out, config):
return True
# Identification-only runs still finish with text.txt (no other stage
# produces an artifact of its own).
if (not config.convert.enabled and not config.separate.enabled
and not config.transcribe.enabled
and (config.shazam.enabled or config.lyrics.enabled)):
return (track_out / "text.txt").is_file()
return False
def _needs_work(
config: PipelineConfig, input_root: Path, output_root: Path, f: Path,
) -> bool:
"""True if at least one enabled stage still has to do something for ``f``.
This mirrors the resume rules used by the staged/sequential pipelines:
* final artifact present + skip_existing -> resolved, nothing to do
* vocals.wav + resume_partial -> convert/separate are skipped
* convert.wav + resume_partial -> convert is skipped
Files with no pending work are filtered out before batching, so batches
stay full instead of being filled by already-finished tracks.
"""
track_out = _output_dir_for(f, input_root, output_root)
belongs = _artifact_belongs_to(track_out, f)
has_text, has_vocals, has_convert = _existing_artifacts(
input_root, output_root, f,
)
if config.skip_existing and _is_track_finished(track_out, config, belongs):
return False
vocals_ready = config.resume_partial and has_vocals
convert_ready = config.resume_partial and (has_vocals or has_convert)
if config.shazam.enabled or config.lyrics.enabled:
return True # identification still has to run
if config.convert.enabled and not convert_ready:
return True
if config.separate.enabled and not vocals_ready:
return True
if config.transcribe.enabled:
return True # text is missing or skip_existing is disabled
return False
def _save_unrecognized(
config: PipelineConfig,
f: Path,
@@ -532,7 +656,8 @@ def run_pipeline(
log("Аудиофайлы не найдены.")
return []
mode = (config.mode or "sequential").lower()
log(f"Найдено треков: {len(files)} | режим: {mode} | "
total_all = len(files)
log(f"Найдено треков: {total_all} | режим: {mode} | "
f"keep_intermediate={config.keep_intermediate}")
if config.separate.save_vocals_copy:
log(" ✓ Вокал будет сохраняться всегда (save_vocals_copy=True)")
@@ -541,17 +666,37 @@ def run_pipeline(
if config.lyrics.enabled:
log(f" LRCLIB: включён (порог fuzzy={config.lyrics.fuzzy_threshold:.2f})")
if config.skip_existing:
log(" Resume: готовые файлы (text.txt) будут пропущены")
log(" Resume: готовые файлы (результат последнего этапа) будут пропущены")
if config.resume_partial:
log(" Resume: существующие vocals.wav / convert.wav будут переиспользованы")
# Pre-scan before chunking: remove files that are already complete for the
# current set of enabled stages. Without this, skipped tracks consumed
# batch slots and a staged run produced many half-empty batches.
pending = [f for f in files if _needs_work(config, input_dir, output_dir, f)]
already_done = total_all - len(pending)
if already_done:
log(f" ✓ Уже готово: {already_done} — в батчи не попадут")
log(f" → К обработке: {len(pending)}")
files = pending
if not files:
log(" Все файлы уже готовы — работа не требуется.")
if on_overall_progress:
on_overall_progress(0, len(files))
on_overall_progress(total_all, total_all)
return []
if on_overall_progress:
on_overall_progress(already_done, total_all)
if mode == "staged":
return _run_staged(
config, files, log, on_progress, stop_event, on_overall_progress,
overall_base=already_done, overall_total=total_all,
input_root=input_dir,
)
return _run_sequential(
config, files, log, on_progress, stop_event, on_overall_progress,
overall_base=already_done, overall_total=total_all,
input_root=input_dir,
)
@@ -559,16 +704,24 @@ def _run_sequential(
config: PipelineConfig, files: list[Path], log: LogFn,
on_progress: Optional[ProgressFn], stop_event: Optional[threading.Event],
on_overall_progress: Optional[OverallFn] = None,
overall_base: int = 0,
overall_total: Optional[int] = None,
input_root: Optional[Path] = None,
) -> list[FileResult]:
total = len(files)
if overall_total is None:
overall_total = overall_base + total
if input_root is None:
input_root = Path(config.input_dir)
results: list[FileResult] = []
output_root = Path(config.output_dir)
log("Раскладка: output/<relpath_from_input>/<stem>/")
ident_tmp = Path(tempfile.mkdtemp(prefix="at_ident_"))
ident_files = files
if config.skip_existing:
ident_files = [
f for f in files
if not (output_root / f.stem / "text.txt").is_file()
if not (_output_dir_for(f, input_root, output_root) / "text.txt").is_file()
]
try:
ident_map = _run_identification_parallel(
@@ -585,22 +738,30 @@ def _run_sequential(
on_progress("file", idx - 1, total)
log(f"\n▶ [{idx}/{total}] {f.name}")
track_out = output_root / f.stem
track_out = _output_dir_for(f, input_root, output_root)
track_out.mkdir(parents=True, exist_ok=True)
belongs = _artifact_belongs_to(track_out, f)
if not belongs and any(
(track_out / name).exists()
for name in ("text.txt", "vocals.wav", "convert.wav")
):
log(f" ⚠ коллизия раскладки: {track_out} занят другим треком — обрабатываю заново")
if config.skip_existing:
if config.skip_existing and _is_track_finished(track_out, config, belongs):
existing_text = track_out / "text.txt"
if existing_text.is_file() and belongs:
existing: Optional[str] = None
if existing_text.is_file():
try:
existing = existing_text.read_text(encoding="utf-8")
except OSError:
existing = None
log(" ⊘ text.txt уже есть — пропуск")
log(" ⊘ результат последнего этапа уже есть — пропуск")
if on_progress:
on_progress("file", idx, total)
if on_overall_progress:
on_overall_progress(idx, total)
on_overall_progress(
min(overall_base + idx, overall_total), overall_total,
)
results.append(FileResult(f, existing, track_out, None, "existing"))
continue
@@ -699,7 +860,14 @@ def _run_sequential(
gigaam_provider=config.transcribe.gigaam_provider,
local_files_only=config.transcribe.local_files_only,
)
text = (text or "").strip()
source = config.transcribe.engine
if not text:
text = None
source = "empty_result"
error = "EmptyResult: ASR returned no text"
log(f" ⚠ {f.name}: текст не извлечён (пустой результат ASR)")
else:
log(f" ✓ распознано {len(text)} символов")
if (config.lyrics.enabled and config.lyrics.try_after_asr):
matched = _try_post_asr_match(text, config, log)
@@ -711,6 +879,10 @@ def _run_sequential(
metadata_dict = _build_metadata_dict(
ident.metadata, ident.shazam, source, input_path=f,
)
if belongs:
metadata_dict = _merge_metadata_file(
track_out / "metadata.json", metadata_dict,
)
if metadata_dict:
_write_metadata_json(tmp_root / "metadata.json", metadata_dict)
@@ -729,6 +901,10 @@ def _run_sequential(
log("⏹ Отменено")
was_cancelled = True
break
except CudaOOMError as e:
error = f"CudaOOMError: {e}"
source = "oom_skipped"
log(f" ⊘ {f.name}: OOM после ретраев — пропущен, идём дальше")
except Exception as e: # noqa: BLE001
error = f"{type(e).__name__}: {e}"
log(f" ✗ Ошибка: {error}")
@@ -758,7 +934,9 @@ def _run_sequential(
if on_progress:
on_progress("file", idx, total)
if on_overall_progress:
on_overall_progress(idx, total)
on_overall_progress(
min(overall_base + idx, overall_total), overall_total,
)
_log_summary(results, log)
return results
@@ -768,19 +946,28 @@ def _run_staged(
config: PipelineConfig, files: list[Path], log: LogFn,
on_progress: Optional[ProgressFn], stop_event: Optional[threading.Event],
on_overall_progress: Optional[OverallFn] = None,
overall_base: int = 0,
overall_total: Optional[int] = None,
input_root: Optional[Path] = None,
) -> list[FileResult]:
"""Chunked staged mode: process `chunk_size` files, free memory, repeat."""
output_root = Path(config.output_dir)
if input_root is None:
input_root = Path(config.input_dir)
total = len(files)
if overall_total is None:
overall_total = overall_base + total
chunk_size = max(1, int(config.chunk_size or total))
log("Раскладка: output/<relpath_from_input>/<stem>/")
if chunk_size >= total:
log(f"Один батч: {total} файлов (chunk_size={chunk_size})")
results = _run_staged_batch(
config, files, output_root, log, on_progress, stop_event,
overall_offset=0, overall_total=total,
overall_offset=overall_base, overall_total=overall_total,
on_overall_progress=on_overall_progress,
overall_counter=[0],
overall_counter=[overall_base],
input_root=input_root,
)
_free_memory()
_log_summary(results, log)
@@ -790,7 +977,7 @@ def _run_staged(
log(f"Батчей: {n_batches} × {chunk_size} файлов (всего {total})")
all_results: list[FileResult] = []
overall_counter = [0]
overall_counter = [overall_base]
for b in range(n_batches):
if stop_event is not None and stop_event.is_set():
log("⏹ Отменено")
@@ -805,9 +992,10 @@ def _run_staged(
try:
batch_results = _run_staged_batch(
config, chunk, output_root, log, on_progress, stop_event,
overall_offset=start, overall_total=total,
overall_offset=overall_base + start, overall_total=overall_total,
on_overall_progress=on_overall_progress,
overall_counter=overall_counter,
input_root=input_root,
)
all_results.extend(batch_results)
except Exception as e: # noqa: BLE001
@@ -831,9 +1019,12 @@ def _run_staged_batch(
overall_total: Optional[int] = None,
on_overall_progress: Optional[OverallFn] = None,
overall_counter: Optional[list[int]] = None,
input_root: Optional[Path] = None,
) -> list[FileResult]:
"""One chunk of the staged pipeline. All temps are tracked and removed
in the `finally` block at the end of this call."""
if input_root is None:
input_root = Path(config.input_dir)
if overall_total is None:
overall_total = len(batch_files)
@@ -890,7 +1081,7 @@ def _run_staged_batch(
resumed_partial = 0
mismatched_folders = 0
for f in batch_files:
track_out = output_root / f.stem
track_out = _output_dir_for(f, input_root, output_root)
audio_path: Path = f
skip_convert = False
skip_separate = False
@@ -906,13 +1097,14 @@ def _run_staged_batch(
)
if has_artifacts:
mismatched_folders += 1
log(f" ⚠ {f.name}: папка {track_out.name} относится к другому треку — обрабатываю заново")
log(f" ⚠ коллизия раскладки: {track_out} занят другим треком — обрабатываю заново")
text_file = track_out / "text.txt"
if config.skip_existing and belongs and text_file.is_file():
if config.skip_existing and _is_track_finished(track_out, config, belongs):
resolved = True
skipped_existing += 1
source = "existing"
if text_file.is_file():
try:
text = text_file.read_text(encoding="utf-8")
except OSError:
@@ -938,7 +1130,7 @@ def _run_staged_batch(
}
if skipped_existing:
log(f" ⊘ Пропущено готовых (text.txt): {skipped_existing}")
log(f" ⊘ Пропущено готовых (результат последнего этапа): {skipped_existing}")
if resumed_partial:
log(f" ↻ Подхвачено промежуточных (vocals/convert): {resumed_partial}")
if mismatched_folders:
@@ -979,7 +1171,7 @@ def _run_staged_batch(
ident.metadata, ident.shazam, ident.source,
input_path=f,
)
track_out = output_root / f.stem
track_out = _output_dir_for(f, input_root, output_root)
track_out.mkdir(parents=True, exist_ok=True)
if ident.text:
(track_out / "text.txt").write_text(ident.text, encoding="utf-8")
@@ -1010,6 +1202,21 @@ def _run_staged_batch(
def separate_single(f: Path) -> None:
try:
src = state[f]["audio_path"]
# Never write next to the original input: with
# convert.enabled=False src == f, so every track would
# overwrite input_dir/vocals.wav. Only reuse the track
# output folder when the source already lives there.
track_dir = _output_dir_for(f, input_root, output_root)
if src.parent == track_dir:
# Already in the persistent track folder (resume).
target = src.parent / "vocals.wav"
elif src.parent == f.parent:
# src is the original input file (convert disabled).
# Never write next to it: all tracks share input_dir
# and would overwrite input_dir/vocals.wav.
target = _mkdtemp("at_sep_") / "vocals.wav"
else:
# Freshly converted file in its own unique temp dir.
target = src.parent / "vocals.wav"
with tempfile.TemporaryDirectory() as tmp:
raw = separate_vocals(
@@ -1027,11 +1234,15 @@ def _run_staged_batch(
# Resumed convert.wav files already live in output_root/<stem>,
# while freshly converted files live in unique temp dirs.
# Original inputs (convert disabled) must also go through the
# one-by-one path: batching them shares input_dir and can
# collide on vocals.wav.
# unblend's `{parent}` batch layout would mix those roots, so
# separate resumed files one by one and batch the rest.
# separate resumed/original files one by one and batch the rest.
resumed = [
f for f in todo
if state[f]["audio_path"].parent == output_root / f.stem
if state[f]["audio_path"].parent == _output_dir_for(f, input_root, output_root)
or state[f]["audio_path"].parent == f.parent
]
resumed_set = set(resumed)
batchable = [f for f in todo if f not in resumed_set]
@@ -1092,6 +1303,20 @@ def _run_staged_batch(
if config.transcribe.enabled and not cancelled():
todo = active()
log(f"\n══ Этап 3 · Транскрибация ({len(todo)} файлов) ══")
if todo:
# Load once, after separation has finished. Every later file
# and every later batch reuses this instance.
try:
preload_transcription_model(
engine=config.transcribe.engine,
model=config.transcribe.model,
device=config.transcribe.device,
compute_type=config.transcribe.compute_type,
gigaam_provider=config.transcribe.gigaam_provider,
local_files_only=config.transcribe.local_files_only,
)
except Exception as e: # noqa: BLE001
log(f" ⚠ Не удалось загрузить модель заранее: {e}")
progress("transcribe", 0, len(todo))
for i, f in enumerate(todo, 1):
if cancelled():
@@ -1108,9 +1333,20 @@ def _run_staged_batch(
gigaam_provider=config.transcribe.gigaam_provider,
local_files_only=config.transcribe.local_files_only,
)
text = (text or "").strip()
if not text:
state[f]["text"] = None
state[f]["source"] = "empty_result"
state[f]["error"] = "EmptyResult: ASR returned no text"
log(f" ⚠ {f.name}: текст не извлечён (пустой результат ASR)")
else:
state[f]["text"] = text
state[f]["source"] = config.transcribe.engine
log(f" ✓ {f.name} ({len(text)} символов)")
except CudaOOMError as e:
state[f]["error"] = f"CudaOOMError: {e}"
state[f]["source"] = "oom_skipped"
log(f" ⊘ {f.name}: OOM — пропущен")
except Exception as e: # noqa: BLE001
state[f]["error"] = f"{type(e).__name__}: {e}"
log(f" ✗ {f.name}: {e}")
@@ -1133,7 +1369,7 @@ def _run_staged_batch(
if isinstance(src_audio, Path):
vocal = src_audio.parent / "vocals.wav"
if vocal.exists():
track_out = output_root / f.stem
track_out = _output_dir_for(f, input_root, output_root)
track_out.mkdir(parents=True, exist_ok=True)
dst = track_out / "vocals.wav"
if dst.exists():
@@ -1174,8 +1410,13 @@ def _run_staged_batch(
)
s["metadata_dict"] = metadata_dict
track_out = output_root / f.stem
track_out = _output_dir_for(f, input_root, output_root)
track_out.mkdir(parents=True, exist_ok=True)
if _artifact_belongs_to(track_out, f):
metadata_dict = _merge_metadata_file(
track_out / "metadata.json", metadata_dict,
)
s["metadata_dict"] = metadata_dict
audio_work = s["audio_path"]
if isinstance(audio_work, Path) and audio_work != f:
@@ -1200,7 +1441,10 @@ def _run_staged_batch(
config, f, text, source,
metadata=(ident.metadata if ident else None),
)
if text is not None or s["resolved"]:
log(f" ✓ {f.name}")
else:
log(f" ⊘ {f.name}: пропущен (нет текста)")
mark_overall(is_complete(s))
finally:
@@ -1216,7 +1460,7 @@ def _run_staged_batch(
return [
FileResult(
f, state[f]["text"],
output_root / f.stem if state[f]["error"] is None else None,
_output_dir_for(f, input_root, output_root) if state[f]["error"] is None else None,
state[f]["error"], state[f]["source"], state[f]["metadata_dict"],
)
for f in batch_files

View File

@@ -6,15 +6,44 @@ Models are cached; aggressive VRAM cleanup happens after each file.
from __future__ import annotations
import gc
import threading
import time
from pathlib import Path
from typing import Optional
# Set permanently once the CUDA context has died (e.g. after an unrecoverable
# OOM). Every later CUDA transcription fails instantly instead of re-loading
# the model forever and hanging the pipeline.
_CUDA_BROKEN = False
# The model caches below are process-wide. They are intentionally never
# cleared between files or batches: reloading WhisperX takes minutes and is
# what made runs look like they reloaded the model every batch. The lock makes
# the "load once, reuse forever" guarantee safe even if ASR is ever called
# from multiple threads.
_MODEL_LOCK = threading.RLock()
_REUSE_LOGGED: set = set()
# CUDA context failures that cannot be recovered by retrying or empty_cache().
# They require restarting the app (or switching to CPU).
_DEAD_CUDA_MARKERS = (
"invalid device ordinal",
"cudaerrorinvaliddevice",
"device-side assert",
"unspecified launch failure",
)
def _is_dead_cuda(exc: BaseException) -> bool:
err = str(exc).lower()
return any(m in err for m in _DEAD_CUDA_MARKERS)
def _clear_model_caches() -> None:
_WhisperXModels._cache.clear()
_AlignModels._cache.clear()
_GigaAMModels._cache.clear()
_REUSE_LOGGED.clear()
class CudaOOMError(RuntimeError):
"""CUDA OOM, переживший все стратегии понижения нагрузки."""
def _is_cuda_error(exc: BaseException) -> bool:
@@ -28,14 +57,6 @@ def _is_cuda_error(exc: BaseException) -> bool:
)
def _check_cuda_usable(device: str) -> None:
if device == "cuda" and _CUDA_BROKEN:
raise RuntimeError(
"CUDA context is broken (previous OOM was unrecoverable). "
"Restart the app or switch device to CPU."
)
def _resolve_device(device: str) -> str:
if device in ("cuda", "cpu"):
return device
@@ -68,6 +89,59 @@ def _free_vram() -> None:
pass
def _enable_tf32() -> None:
"""Enable TF32 matmul on Ampere+ GPUs (RTX 30/40).
Called lazily right before a WhisperX model is loaded, so runs that do not
use ASR never import Torch or create a CUDA context at startup.
"""
try:
import torch
if torch.cuda.is_available():
torch.backends.cuda.matmul.allow_tf32 = True
torch.backends.cudnn.allow_tf32 = True
except Exception: # noqa: BLE001 — purely an optimization
pass
def _model_transcribe_with_retry(
model_obj, audio, language, batch_size, name="<audio>",
):
"""Try progressively lighter settings so one long file can still finish.
The loaded model is intentionally kept in the cache between attempts:
reloading it costs minutes and does not help with an OOM caused by a long
audio file. VAD is not touched here — WhisperX expects it at model-load
time (``vad_model``/``vad_options``), not per transcription call. A dead
CUDA context is never retried: caches are dropped and CudaOOMError is
raised immediately. If even batch=1 fails with a normal OOM, raise
CudaOOMError so the caller can skip this file and continue with the rest
of the batch.
"""
attempts = [batch_size, 1]
last: Optional[Exception] = None
for i, bs in enumerate(attempts, 1):
try:
return model_obj.transcribe(
audio, batch_size=bs, language=language,
)
except Exception as e: # noqa: BLE001
if _is_dead_cuda(e):
_clear_model_caches()
raise CudaOOMError(
f"CUDA context died on {name}: {e}. "
"GPU state is unrecoverable — restart the app or switch to CPU."
) from e
if not _is_cuda_error(e):
raise
last = e
print(f" [ASR] OOM on {name} (attempt {i}/{len(attempts)}, bs={bs}): {e}")
_free_vram()
raise CudaOOMError(
f"OOM after {len(attempts)} attempts on {name}: {last}"
) from last
class _WhisperXModels:
_cache: dict = {}
@@ -77,7 +151,10 @@ class _WhisperXModels:
if compute_type is None:
compute_type = "int8_float16" if device == "cuda" else "int8"
key = (model_name, device, compute_type, local_files_only)
with _MODEL_LOCK:
if key not in cls._cache:
if device == "cuda":
_enable_tf32()
import whisperx
print(f" [ASR] loading WhisperX {model_name} on {device} "
f"({compute_type}, local_files_only={local_files_only})...")
@@ -90,7 +167,10 @@ class _WhisperXModels:
)
cls._cache[key] = _retry(load)
print(" [ASR] model loaded.")
print(" [ASR] model loaded once — subsequent files will reuse it.")
elif ("whisperx",) + key not in _REUSE_LOGGED:
_REUSE_LOGGED.add(("whisperx",) + key)
print(f" [ASR] reusing loaded WhisperX {model_name} ({device}).")
return cls._cache[key]
@@ -100,6 +180,7 @@ class _AlignModels:
@classmethod
def get(cls, language_code: str, device: str, model_name: Optional[str] = None):
key = (language_code, device, model_name or "")
with _MODEL_LOCK:
if key not in cls._cache:
import whisperx
print(f" [ALIGN] loading align model for {language_code} "
@@ -112,7 +193,10 @@ class _AlignModels:
return whisperx.load_align_model(**kwargs)
cls._cache[key] = _retry(load)
print(" [ALIGN] align model loaded.")
print(" [ALIGN] align model loaded once — reusing it.")
elif ("align",) + key not in _REUSE_LOGGED:
_REUSE_LOGGED.add(("align",) + key)
print(f" [ALIGN] reusing loaded align model for {language_code}.")
return cls._cache[key]
@@ -127,6 +211,7 @@ class _GigaAMModels:
"auto": ["CUDAExecutionProvider", "CPUExecutionProvider"],
}.get(provider, ["CPUExecutionProvider"])
key = tuple(providers)
with _MODEL_LOCK:
if key not in cls._cache:
import onnx_asr
@@ -137,6 +222,28 @@ class _GigaAMModels:
return cls._cache[key]
def preload_transcription_model(
engine: str = "whisperx",
model: str = "large-v3",
device: str = "auto",
compute_type: Optional[str] = None,
gigaam_provider: str = "auto",
local_files_only: bool = False,
) -> None:
"""Load the ASR model once so the per-file loop only reuses it.
All model caches are process-wide and are never cleared between files,
batches or pipeline runs. This helper just makes the first load explicit.
"""
device = _resolve_device(device)
if engine == "whisperx":
_WhisperXModels.get(model, device, compute_type, local_files_only)
elif engine == "gigaam":
_GigaAMModels.get(gigaam_provider)
else:
raise ValueError(f"Unknown ASR engine: {engine!r}")
def transcribe(
audio_path: Path,
engine: str = "whisperx",
@@ -149,37 +256,13 @@ def transcribe(
local_files_only: bool = False,
vad_filter: bool = True,
) -> str:
global _CUDA_BROKEN
device = _resolve_device(device)
_check_cuda_usable(device)
if engine == "whisperx":
try:
return _transcribe_whisperx(
audio_path, model, language, device, compute_type,
batch_size, local_files_only, vad_filter,
)
except Exception as e:
if not _is_cuda_error(e):
raise
print(f" [ASR] CUDA error: {e}. Clearing model cache and retrying...")
_WhisperXModels._cache.clear()
_free_vram()
try:
return _transcribe_whisperx(
audio_path, model, language, device, compute_type,
batch_size, local_files_only, vad_filter,
)
except Exception as retry_error:
if not _is_cuda_error(retry_error):
raise
_CUDA_BROKEN = True
print(" [ASR] CUDA context is unrecoverable — "
"GPU disabled for the rest of this run.")
raise RuntimeError(
"CUDA context is broken after OOM recovery failed. "
"Restart the app or switch device to CPU."
) from retry_error
if engine == "gigaam":
return _transcribe_gigaam(audio_path, gigaam_provider)
@@ -199,7 +282,9 @@ def _transcribe_whisperx(
import whisperx
model = _WhisperXModels.get(model_name, device, compute_type, local_files_only)
audio = whisperx.load_audio(str(audio_path))
result = model.transcribe(audio, batch_size=batch_size, language=language)
result = _model_transcribe_with_retry(
model, audio, language, batch_size, name=audio_path.name,
)
text = " ".join(seg["text"].strip() for seg in result["segments"]).strip()
del audio, result
_free_vram()
@@ -220,6 +305,7 @@ def transcribe_aligned(
batch_size: int = 8,
align_model: Optional[str] = None,
local_files_only: bool = False,
vad_filter: bool = True,
) -> tuple[list[dict], str]:
"""Transcribe + word-level align. Returns (aligned_segments, detected_language).
@@ -228,10 +314,11 @@ def transcribe_aligned(
"""
import whisperx
device = _resolve_device(device)
_check_cuda_usable(device)
model_obj = _WhisperXModels.get(model, device, compute_type, local_files_only)
audio = whisperx.load_audio(str(audio_path))
result = model_obj.transcribe(audio, batch_size=batch_size, language=language)
result = _model_transcribe_with_retry(
model_obj, audio, language, batch_size, name=audio_path.name,
)
detected = result.get("language") or language or "en"

View File

@@ -2,6 +2,7 @@
from __future__ import annotations
import shutil
import subprocess
from pathlib import Path
from typing import Optional
@@ -42,17 +43,30 @@ MODE_LABELS = {
def _detect_device_info() -> tuple[str, str]:
"""Report the GPU via nvidia-smi.
Torch must not be imported here: importing it initializes a CUDA context
and reserves VRAM even for runs that only need separation. The ASR stage
imports Torch lazily when it actually loads a model.
"""
try:
import torch
except ImportError:
return "⚠ PyTorch не установлен", "#f38ba8"
if torch.cuda.is_available():
out = subprocess.run(
["nvidia-smi", "--query-gpu=name,memory.total",
"--format=csv,noheader,nounits"],
capture_output=True, text=True, timeout=5, check=True,
).stdout.strip()
except (OSError, subprocess.SubprocessError):
out = ""
if out:
parts = [p.strip() for p in out.splitlines()[0].split(",")]
name = parts[0] or "CUDA GPU"
if len(parts) > 1:
try:
name = torch.cuda.get_device_name(0)
vram = torch.cuda.get_device_properties(0).total_memory / (1024 ** 3)
vram = float(parts[1]) / 1024 # MiB -> GiB
return f"✓ CUDA: {name} · {vram:.1f} GB VRAM", "#a6e3a1"
except Exception: # noqa: BLE001
return "✓ CUDA доступна", "#a6e3a1"
except ValueError:
pass
return f"✓ CUDA: {name}", "#a6e3a1"
return "⚠ CUDA не обнаружена — расчёт пойдёт на CPU (медленно)", "#f9e2af"
@@ -137,7 +151,7 @@ class MainWindow(QMainWindow):
resume_opts = QHBoxLayout()
self.skip_existing_cb = QCheckBox("Пропускать готовые (text.txt)")
self.skip_existing_cb.setToolTip(
"Не трогать файлы, у которых в <output>/<stem>/ уже есть text.txt.\n"
"Не трогать файлы, у которых в <output>/<relpath>/<stem>/ уже есть text.txt.\n"
"Позволяет продолжить большой прогон после сбоя без пересчёта.\n"
"Если в metadata.json записан path другого файла — папка не считается готовой."
)
@@ -205,7 +219,9 @@ class MainWindow(QMainWindow):
self.log_view = QPlainTextEdit()
self.log_view.setObjectName("log")
self.log_view.setReadOnly(True)
self.log_view.setMinimumHeight(240)
# Keep at least ~12 log lines visible regardless of the UI font size.
line_height = self.log_view.fontMetrics().lineSpacing()
self.log_view.setMinimumHeight(max(280, line_height * 12 + 24))
self.log_view.setPlaceholderText("Здесь появится лог выполнения…")
root.addWidget(self.log_view, 1)
@@ -352,7 +368,7 @@ class MainWindow(QMainWindow):
self.separate_save_vocals_cb = QCheckBox("Всегда сохранять vocals.wav в output")
self.separate_save_vocals_cb.setToolTip(
"Независимо от 'Сохранять промежуточные': вокал всегда будет лежать "
"в <output>/<stem>/vocals.wav — полезно для LRC-этапа."
"в <output>/<relpath>/<stem>/vocals.wav — полезно для LRC-этапа."
)
form.addRow("", self.separate_save_vocals_cb)
return w
@@ -396,7 +412,7 @@ class MainWindow(QMainWindow):
form.setSpacing(10)
info = QLabel(
"1) Сгенерировать LRC: WhisperX + align → <output>/<stem>/text.lrc.\n"
"1) Сгенерировать LRC: WhisperX + align → <output>/<relpath>/<stem>/text.lrc.\n"
"2) Встроить LRC: записывает LRC в тег lyrics аудио (USLT/LYRICS/©lyr).\n"
"Если 'in-place' выключено — оригинал не трогается, создаётся копия в output."
)
@@ -417,7 +433,7 @@ class MainWindow(QMainWindow):
form.addRow("", self.lrc_embed_enabled_cb)
self.lrc_skip_existing_cb = QCheckBox("Пропускать готовые text.lrc")
self.lrc_skip_existing_cb.setToolTip(
"Если <output>/<stem>/text.lrc уже существует и содержит корректный LRC,\n"
"Если <output>/<relpath>/<stem>/text.lrc уже существует и содержит корректный LRC,\n"
"файл не будет пересчитан. Папка с metadata.json от другого трека\n"
"готовой не считается."
)
@@ -447,7 +463,7 @@ class MainWindow(QMainWindow):
self.lrc_embed_in_place_cb = QCheckBox("Встраивать in-place (перезаписывать оригинал)")
self.lrc_embed_in_place_cb.setToolTip(
"Если выключено (по умолчанию) — создаётся копия в <output>/<stem>/<name>.\n"
"Если выключено (по умолчанию) — создаётся копия в <output>/<relpath>/<stem>/<name>.\n"
"Если включено — оригинальный файл модифицируется."
)
form.addRow("", self.lrc_embed_in_place_cb)
@@ -692,6 +708,14 @@ class MainWindow(QMainWindow):
f"Команда '{cfg.separate.unblend_path}' не найдена в PATH.")
return
if task == "lrc" and not shutil.which(cfg.convert.ffmpeg_path):
QMessageBox.warning(
self, "ffmpeg не найден",
f"'{cfg.convert.ffmpeg_path}' не найден в PATH.\n"
"LRC-этапу нужен ffmpeg для подготовки аудио."
)
return
if task == "lrc" and cfg.transcribe.engine != "whisperx":
QMessageBox.warning(
self, "LRC требует WhisperX",

12
main.py
View File

@@ -26,20 +26,8 @@ def preload_cudnn() -> None:
pass
def enable_tf32() -> None:
"""Enable TF32 for faster matmul on Ampere+ GPUs (RTX 30/40 series)."""
try:
import torch
if torch.cuda.is_available():
torch.backends.cuda.matmul.allow_tf32 = True
torch.backends.cudnn.allow_tf32 = True
except ImportError:
pass
def main() -> int:
preload_cudnn()
enable_tf32()
from PySide6.QtWidgets import QApplication
from gui.theme import DARK_STYLESHEET

View File

@@ -0,0 +1,86 @@
#!/usr/bin/env python3
"""Migrate flat output/<stem>/ to output/<relpath>/<stem>/ (one-shot)."""
from __future__ import annotations
import argparse, json, shutil, sys
from pathlib import Path
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--input-root", required=True, type=Path)
ap.add_argument("--output-root", required=True, type=Path)
ap.add_argument("--dry-run", action="store_true")
args = ap.parse_args()
input_root = args.input_root.resolve()
output_root = args.output_root.resolve()
if not input_root.is_dir():
print(f"input root not found: {input_root}", file=sys.stderr); return 1
if not output_root.is_dir():
print(f"output root not found: {output_root}", file=sys.stderr); return 1
moved = legacy = conflicts = errors = skipped = 0
artifact_names = (
"metadata.json", "text.txt", "text.lrc",
"vocals.wav", "convert.wav",
)
for track_dir in sorted(output_root.iterdir()):
if not track_dir.is_dir():
continue
if track_dir.name.startswith("_"):
continue
# Never treat an album/parent folder as a flat track folder.
if not any((track_dir / name).is_file() for name in artifact_names):
skipped += 1
continue
target = output_root / "_legacy" / track_dir.name
meta_file = track_dir / "metadata.json"
if meta_file.is_file():
try:
data = json.loads(meta_file.read_text(encoding="utf-8"))
stored = data.get("path") if isinstance(data, dict) else None
except (OSError, ValueError):
stored = None
if stored:
try:
rel = Path(stored).relative_to(input_root)
target = output_root / rel.with_suffix("")
except ValueError:
target = output_root / "_legacy" / track_dir.name
if target == track_dir:
continue
if track_dir in target.parents:
print(f"skip: {track_dir.name} -> target inside itself")
skipped += 1
continue
if target.exists():
print(f"conflict: {track_dir.name} -> "
f"{target.relative_to(output_root)}")
conflicts += 1
continue
if args.dry_run:
print(f"DRY: {track_dir.name} -> {target.relative_to(output_root)}")
continue
try:
target.parent.mkdir(parents=True, exist_ok=True)
shutil.move(str(track_dir), str(target))
if "_legacy" in target.parts:
legacy += 1
else:
moved += 1
except OSError as e:
print(f"error: {track_dir.name}: {e}")
errors += 1
print(f"\nmoved: {moved} quarantined: {legacy} "
f"conflicts: {conflicts} skipped: {skipped} errors: {errors} "
f"(dry_run={args.dry_run})")
return 0 if errors == 0 else 2
if __name__ == "__main__":
sys.exit(main())