Nekochu commited on
Commit
6cee8bd
·
1 Parent(s): 560b5e0

add granular logging + full stderr to diagnose preprocessing hang

Browse files
Files changed (1) hide show
  1. app.py +9 -2
app.py CHANGED
@@ -343,7 +343,12 @@ try:
343
  torchaudio.load = _sf
344
 
345
  log("[Step 1/2] Preprocessing audio...")
 
346
  from acestep.training_v2.preprocess import preprocess_audio_files
 
 
 
 
347
  result = preprocess_audio_files(
348
  audio_dir="{audio_dir}",
349
  output_dir="{output_dir}/preprocessed_tensors",
@@ -351,6 +356,8 @@ try:
351
  variant="turbo", max_duration=60.0,
352
  device="cpu", precision="float32",
353
  )
 
 
354
  processed = result.get("processed", 0)
355
  failed = result.get("failed", 0)
356
  log(f" Preprocessed: {{processed}}/{{result.get('total',0)}} (failed: {{failed}})")
@@ -429,8 +436,8 @@ finally:
429
  parts.append(open(TRAIN_LOG).read())
430
  stderr_log = os.path.join(ADAPTER_DIR, "test-lora", "train_stderr.log")
431
  if os.path.exists(stderr_log) and os.path.getsize(stderr_log) > 0:
432
- stderr = open(stderr_log).read()[-2000:]
433
- parts.append(f"\n--- stderr (last 2000 chars) ---\n{stderr}")
434
  return "\n".join(parts) if parts else "No training log found."
435
 
436
  # -- Build UI --
 
343
  torchaudio.load = _sf
344
 
345
  log("[Step 1/2] Preprocessing audio...")
346
+ log(" importing preprocess module...")
347
  from acestep.training_v2.preprocess import preprocess_audio_files
348
+ log(" import done, calling preprocess_audio_files...")
349
+ import resource
350
+ mem_before = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss // 1024
351
+ log(f" RAM before preprocess: {{mem_before}} MB")
352
  result = preprocess_audio_files(
353
  audio_dir="{audio_dir}",
354
  output_dir="{output_dir}/preprocessed_tensors",
 
356
  variant="turbo", max_duration=60.0,
357
  device="cpu", precision="float32",
358
  )
359
+ mem_after = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss // 1024
360
+ log(f" RAM after preprocess: {{mem_after}} MB")
361
  processed = result.get("processed", 0)
362
  failed = result.get("failed", 0)
363
  log(f" Preprocessed: {{processed}}/{{result.get('total',0)}} (failed: {{failed}})")
 
436
  parts.append(open(TRAIN_LOG).read())
437
  stderr_log = os.path.join(ADAPTER_DIR, "test-lora", "train_stderr.log")
438
  if os.path.exists(stderr_log) and os.path.getsize(stderr_log) > 0:
439
+ stderr = open(stderr_log).read()[-8000:]
440
+ parts.append(f"\n--- stderr (last 8000 chars, {os.path.getsize(stderr_log)} bytes total) ---\n{stderr}")
441
  return "\n".join(parts) if parts else "No training log found."
442
 
443
  # -- Build UI --