base skip_existing on the last enabled stage

- add _final_artifact_present and use it for resume decisions\n- text.txt no longer marks a track done when separation is the last stage\n- apply the same rule to pre-scan, staged seeding and sequential skipping\n- keep identification-only runs resolving through text.txt\n- preserve existing metadata tags when a stage-only rerun has less info
This commit is contained in:
jadssss
2026-10-28 00:48:53 +03:00
parent 84d7a28c69
commit 5f16a6f87b

View File

@@ -158,6 +158,25 @@ def _write_metadata_json(path: Path, data: Optional[dict]) -> None:
path.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8") path.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8")
def _merge_metadata_file(existing: Path, data: Optional[dict]) -> Optional[dict]:
"""Keep tags/shazam from an earlier run when the new data has less info.
Reprocessing a track for one stage only (e.g. separation) must not replace
a rich metadata.json with a path-only dict.
"""
if not data or not existing.is_file():
return data
try:
old = json.loads(existing.read_text(encoding="utf-8"))
except (OSError, ValueError):
return data
if not isinstance(old, dict):
return data
merged = dict(old)
merged.update(data)
return merged
def _existing_artifacts( def _existing_artifacts(
output_root: Path, input_path: Path, output_root: Path, input_path: Path,
) -> tuple[bool, bool, bool]: ) -> tuple[bool, bool, bool]:
@@ -176,26 +195,61 @@ def _existing_artifacts(
) )
def _final_artifact_present(track_out: Path, config: PipelineConfig) -> bool:
"""True if the artifact of the last enabled stage already exists.
``skip_existing`` must not treat an old ``text.txt`` as "done" when the
current run only needs separation: the final stage here is ``vocals.wav``,
and without it the track still has work to do.
"""
if config.transcribe.enabled:
return (track_out / "text.txt").is_file()
if config.separate.enabled:
return (track_out / "vocals.wav").is_file()
if config.convert.enabled:
return (track_out / "convert.wav").is_file()
return False
def _is_track_finished(
track_out: Path, config: PipelineConfig, belongs: bool = True,
) -> bool:
"""True when ``skip_existing`` may skip this output folder entirely."""
if not belongs:
return False
if _final_artifact_present(track_out, config):
return True
# Identification-only runs still finish with text.txt (no other stage
# produces an artifact of its own).
if (not config.convert.enabled and not config.separate.enabled
and not config.transcribe.enabled
and (config.shazam.enabled or config.lyrics.enabled)):
return (track_out / "text.txt").is_file()
return False
def _needs_work(config: PipelineConfig, output_root: Path, f: Path) -> bool: def _needs_work(config: PipelineConfig, output_root: Path, f: Path) -> bool:
"""True if at least one enabled stage still has to do something for ``f``. """True if at least one enabled stage still has to do something for ``f``.
This mirrors the resume rules used by the staged/sequential pipelines: This mirrors the resume rules used by the staged/sequential pipelines:
* text.txt + skip_existing -> resolved, nothing to do * final artifact present + skip_existing -> resolved, nothing to do
* vocals.wav + resume_partial -> convert/separate are skipped * vocals.wav + resume_partial -> convert/separate are skipped
* convert.wav + resume_partial -> convert is skipped * convert.wav + resume_partial -> convert is skipped
Files with no pending work are filtered out before batching, so batches Files with no pending work are filtered out before batching, so batches
stay full instead of being filled by already-finished tracks. stay full instead of being filled by already-finished tracks.
""" """
track_out = output_root / f.stem
belongs = _artifact_belongs_to(track_out, f)
has_text, has_vocals, has_convert = _existing_artifacts(output_root, f) has_text, has_vocals, has_convert = _existing_artifacts(output_root, f)
if config.skip_existing and has_text: if config.skip_existing and _is_track_finished(track_out, config, belongs):
return False # resolved by the same rule the pipelines use return False
vocals_ready = config.resume_partial and has_vocals vocals_ready = config.resume_partial and has_vocals
convert_ready = config.resume_partial and (has_vocals or has_convert) convert_ready = config.resume_partial and (has_vocals or has_convert)
if config.shazam.enabled or config.lyrics.enabled: if config.shazam.enabled or config.lyrics.enabled:
return True # identification still has to run (no resolved text) return True # identification still has to run
if config.convert.enabled and not convert_ready: if config.convert.enabled and not convert_ready:
return True return True
if config.separate.enabled and not vocals_ready: if config.separate.enabled and not vocals_ready:
@@ -597,7 +651,7 @@ def run_pipeline(
if config.lyrics.enabled: if config.lyrics.enabled:
log(f" LRCLIB: включён (порог fuzzy={config.lyrics.fuzzy_threshold:.2f})") log(f" LRCLIB: включён (порог fuzzy={config.lyrics.fuzzy_threshold:.2f})")
if config.skip_existing: if config.skip_existing:
log(" Resume: готовые файлы (text.txt) будут пропущены") log(" Resume: готовые файлы (результат последнего этапа) будут пропущены")
if config.resume_partial: if config.resume_partial:
log(" Resume: существующие vocals.wav / convert.wav будут переиспользованы") log(" Resume: существующие vocals.wav / convert.wav будут переиспользованы")
@@ -667,22 +721,23 @@ def _run_sequential(
track_out.mkdir(parents=True, exist_ok=True) track_out.mkdir(parents=True, exist_ok=True)
belongs = _artifact_belongs_to(track_out, f) belongs = _artifact_belongs_to(track_out, f)
if config.skip_existing: if config.skip_existing and _is_track_finished(track_out, config, belongs):
existing_text = track_out / "text.txt" existing_text = track_out / "text.txt"
if existing_text.is_file() and belongs: existing: Optional[str] = None
if existing_text.is_file():
try: try:
existing = existing_text.read_text(encoding="utf-8") existing = existing_text.read_text(encoding="utf-8")
except OSError: except OSError:
existing = None existing = None
log(" ⊘ text.txt уже есть — пропуск") log(" ⊘ результат последнего этапа уже есть — пропуск")
if on_progress: if on_progress:
on_progress("file", idx, total) on_progress("file", idx, total)
if on_overall_progress: if on_overall_progress:
on_overall_progress( on_overall_progress(
min(overall_base + idx, overall_total), overall_total, min(overall_base + idx, overall_total), overall_total,
) )
results.append(FileResult(f, existing, track_out, None, "existing")) results.append(FileResult(f, existing, track_out, None, "existing"))
continue continue
tmp_root = Path(tempfile.mkdtemp(prefix=f"at_{f.stem}_")) tmp_root = Path(tempfile.mkdtemp(prefix=f"at_{f.stem}_"))
@@ -791,6 +846,10 @@ def _run_sequential(
metadata_dict = _build_metadata_dict( metadata_dict = _build_metadata_dict(
ident.metadata, ident.shazam, source, input_path=f, ident.metadata, ident.shazam, source, input_path=f,
) )
if belongs:
metadata_dict = _merge_metadata_file(
track_out / "metadata.json", metadata_dict,
)
if metadata_dict: if metadata_dict:
_write_metadata_json(tmp_root / "metadata.json", metadata_dict) _write_metadata_json(tmp_root / "metadata.json", metadata_dict)
@@ -999,14 +1058,15 @@ def _run_staged_batch(
log(f" ⚠ {f.name}: папка {track_out.name} относится к другому треку — обрабатываю заново") log(f" ⚠ {f.name}: папка {track_out.name} относится к другому треку — обрабатываю заново")
text_file = track_out / "text.txt" text_file = track_out / "text.txt"
if config.skip_existing and belongs and text_file.is_file(): if config.skip_existing and _is_track_finished(track_out, config, belongs):
resolved = True resolved = True
skipped_existing += 1 skipped_existing += 1
source = "existing" source = "existing"
try: if text_file.is_file():
text = text_file.read_text(encoding="utf-8") try:
except OSError: text = text_file.read_text(encoding="utf-8")
text = None except OSError:
text = None
elif config.resume_partial and belongs: elif config.resume_partial and belongs:
vocals = track_out / "vocals.wav" vocals = track_out / "vocals.wav"
convert = track_out / "convert.wav" convert = track_out / "convert.wav"
@@ -1028,7 +1088,7 @@ def _run_staged_batch(
} }
if skipped_existing: if skipped_existing:
log(f" ⊘ Пропущено готовых (text.txt): {skipped_existing}") log(f" ⊘ Пропущено готовых (результат последнего этапа): {skipped_existing}")
if resumed_partial: if resumed_partial:
log(f" ↻ Подхвачено промежуточных (vocals/convert): {resumed_partial}") log(f" ↻ Подхвачено промежуточных (vocals/convert): {resumed_partial}")
if mismatched_folders: if mismatched_folders:
@@ -1289,6 +1349,11 @@ def _run_staged_batch(
track_out = output_root / f.stem track_out = output_root / f.stem
track_out.mkdir(parents=True, exist_ok=True) track_out.mkdir(parents=True, exist_ok=True)
if _artifact_belongs_to(track_out, f):
metadata_dict = _merge_metadata_file(
track_out / "metadata.json", metadata_dict,
)
s["metadata_dict"] = metadata_dict
audio_work = s["audio_path"] audio_work = s["audio_path"]
if isinstance(audio_work, Path) and audio_work != f: if isinstance(audio_work, Path) and audio_work != f: