Spaces:
Running
Running
add granular logging + full stderr to diagnose preprocessing hang
Browse files
app.py
CHANGED
|
@@ -343,7 +343,12 @@ try:
|
|
| 343 |
torchaudio.load = _sf
|
| 344 |
|
| 345 |
log("[Step 1/2] Preprocessing audio...")
|
|
|
|
| 346 |
from acestep.training_v2.preprocess import preprocess_audio_files
|
|
|
|
|
|
|
|
|
|
|
|
|
| 347 |
result = preprocess_audio_files(
|
| 348 |
audio_dir="{audio_dir}",
|
| 349 |
output_dir="{output_dir}/preprocessed_tensors",
|
|
@@ -351,6 +356,8 @@ try:
|
|
| 351 |
variant="turbo", max_duration=60.0,
|
| 352 |
device="cpu", precision="float32",
|
| 353 |
)
|
|
|
|
|
|
|
| 354 |
processed = result.get("processed", 0)
|
| 355 |
failed = result.get("failed", 0)
|
| 356 |
log(f" Preprocessed: {{processed}}/{{result.get('total',0)}} (failed: {{failed}})")
|
|
@@ -429,8 +436,8 @@ finally:
|
|
| 429 |
parts.append(open(TRAIN_LOG).read())
|
| 430 |
stderr_log = os.path.join(ADAPTER_DIR, "test-lora", "train_stderr.log")
|
| 431 |
if os.path.exists(stderr_log) and os.path.getsize(stderr_log) > 0:
|
| 432 |
-
stderr = open(stderr_log).read()[-
|
| 433 |
-
parts.append(f"\n--- stderr (last
|
| 434 |
return "\n".join(parts) if parts else "No training log found."
|
| 435 |
|
| 436 |
# -- Build UI --
|
|
|
|
| 343 |
torchaudio.load = _sf
|
| 344 |
|
| 345 |
log("[Step 1/2] Preprocessing audio...")
|
| 346 |
+
log(" importing preprocess module...")
|
| 347 |
from acestep.training_v2.preprocess import preprocess_audio_files
|
| 348 |
+
log(" import done, calling preprocess_audio_files...")
|
| 349 |
+
import resource
|
| 350 |
+
mem_before = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss // 1024
|
| 351 |
+
log(f" RAM before preprocess: {{mem_before}} MB")
|
| 352 |
result = preprocess_audio_files(
|
| 353 |
audio_dir="{audio_dir}",
|
| 354 |
output_dir="{output_dir}/preprocessed_tensors",
|
|
|
|
| 356 |
variant="turbo", max_duration=60.0,
|
| 357 |
device="cpu", precision="float32",
|
| 358 |
)
|
| 359 |
+
mem_after = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss // 1024
|
| 360 |
+
log(f" RAM after preprocess: {{mem_after}} MB")
|
| 361 |
processed = result.get("processed", 0)
|
| 362 |
failed = result.get("failed", 0)
|
| 363 |
log(f" Preprocessed: {{processed}}/{{result.get('total',0)}} (failed: {{failed}})")
|
|
|
|
| 436 |
parts.append(open(TRAIN_LOG).read())
|
| 437 |
stderr_log = os.path.join(ADAPTER_DIR, "test-lora", "train_stderr.log")
|
| 438 |
if os.path.exists(stderr_log) and os.path.getsize(stderr_log) > 0:
|
| 439 |
+
stderr = open(stderr_log).read()[-8000:]
|
| 440 |
+
parts.append(f"\n--- stderr (last 8000 chars, {os.path.getsize(stderr_log)} bytes total) ---\n{stderr}")
|
| 441 |
return "\n".join(parts) if parts else "No training log found."
|
| 442 |
|
| 443 |
# -- Build UI --
|