Spaces:
Running
Running
Fix coroutine .strip() bug — wrap await in parentheses
Browse filesawait self._ssh_exec(...).strip() calls .strip() on the coroutine,
not the result. Fixed all 11 occurrences to (await ...).strip().
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
src/content_engine/services/runpod_trainer.py
CHANGED
|
@@ -383,12 +383,12 @@ class RunPodTrainer:
|
|
| 383 |
install_cmds = []
|
| 384 |
|
| 385 |
# Check if already present in workspace
|
| 386 |
-
tuner_exist = await self._ssh_exec(ssh, f"test -f {tuner_dir}/pyproject.toml && echo EXISTS || echo MISSING").strip()
|
| 387 |
if tuner_exist == "EXISTS":
|
| 388 |
job._log("musubi-tuner found in workspace")
|
| 389 |
else:
|
| 390 |
# Check volume cache
|
| 391 |
-
vol_exist = await self._ssh_exec(ssh, "test -f /runpod-volume/musubi-tuner/pyproject.toml && echo EXISTS || echo MISSING").strip()
|
| 392 |
if vol_exist == "EXISTS":
|
| 393 |
job._log("Restoring musubi-tuner from volume cache...")
|
| 394 |
await self._ssh_exec(ssh, f"rm -rf {tuner_dir} 2>/dev/null; cp -r /runpod-volume/musubi-tuner {tuner_dir}")
|
|
@@ -408,7 +408,7 @@ class RunPodTrainer:
|
|
| 408 |
])
|
| 409 |
else:
|
| 410 |
# SD 1.5 / SDXL / FLUX.1 use sd-scripts
|
| 411 |
-
scripts_exist = await self._ssh_exec(ssh, "test -f /workspace/sd-scripts/setup.py && echo EXISTS || echo MISSING").strip()
|
| 412 |
if scripts_exist == "EXISTS":
|
| 413 |
job._log("Kohya sd-scripts already cached on volume, updating...")
|
| 414 |
install_cmds = [
|
|
@@ -443,9 +443,9 @@ class RunPodTrainer:
|
|
| 443 |
vae_path = f"{flux2_dir}/ae.safetensors" # Original BFL format (not diffusers)
|
| 444 |
te_path = f"{flux2_dir}/text_encoder/model-00001-of-00010.safetensors"
|
| 445 |
|
| 446 |
-
dit_exists = await self._ssh_exec(ssh, f"test -f {dit_path} && echo EXISTS || echo MISSING").strip()
|
| 447 |
-
vae_exists = await self._ssh_exec(ssh, f"test -f {vae_path} && echo EXISTS || echo MISSING").strip()
|
| 448 |
-
te_exists = await self._ssh_exec(ssh, f"test -f {te_path} && echo EXISTS || echo MISSING").strip()
|
| 449 |
|
| 450 |
if dit_exists != "EXISTS" or te_exists != "EXISTS":
|
| 451 |
missing = []
|
|
@@ -465,7 +465,7 @@ hf_hub_download('black-forest-labs/FLUX.2-dev', 'ae.safetensors', local_dir='{fl
|
|
| 465 |
print('Downloaded ae.safetensors')
|
| 466 |
" 2>&1 | tail -5""", timeout=600)
|
| 467 |
# Verify download
|
| 468 |
-
vae_check = await self._ssh_exec(ssh, f"test -f {vae_path} && echo EXISTS || echo MISSING").strip()
|
| 469 |
if vae_check != "EXISTS":
|
| 470 |
raise RuntimeError("Failed to download ae.safetensors")
|
| 471 |
job._log("VAE downloaded")
|
|
@@ -474,7 +474,7 @@ print('Downloaded ae.safetensors')
|
|
| 474 |
|
| 475 |
else:
|
| 476 |
# SD 1.5 / SDXL / FLUX.1 — download single model file
|
| 477 |
-
model_exists = await self._ssh_exec(ssh, f"test -f /workspace/models/{hf_filename} && echo EXISTS || echo MISSING").strip()
|
| 478 |
if model_exists == "EXISTS":
|
| 479 |
job._log(f"Base model already cached on volume: {model_name}")
|
| 480 |
else:
|
|
@@ -488,7 +488,7 @@ hf_hub_download('{hf_repo}', '{hf_filename}', local_dir='/workspace/models')
|
|
| 488 |
|
| 489 |
# For FLUX.1, download additional required models (CLIP, T5, VAE)
|
| 490 |
if model_type == "flux":
|
| 491 |
-
flux_files_check = await self._ssh_exec(ssh, "test -f /workspace/models/clip_l.safetensors && test -f /workspace/models/t5xxl_fp16.safetensors && test -f /workspace/models/ae.safetensors && echo EXISTS || echo MISSING").strip()
|
| 492 |
if flux_files_check == "EXISTS":
|
| 493 |
job._log("FLUX.1 auxiliary models already cached on volume")
|
| 494 |
else:
|
|
@@ -652,9 +652,9 @@ resolution = [{resolution}, {resolution}]
|
|
| 652 |
await self._ssh_exec(ssh, "mkdir -p /runpod-volume/loras")
|
| 653 |
remote_output = f"/workspace/output/{name}.safetensors"
|
| 654 |
# Find the output file
|
| 655 |
-
check = await self._ssh_exec(ssh, f"test -f {remote_output} && echo EXISTS || echo MISSING").strip()
|
| 656 |
if check == "MISSING":
|
| 657 |
-
remote_files = await self._ssh_exec(ssh, "ls /workspace/output/*.safetensors 2>/dev/null").strip()
|
| 658 |
if remote_files:
|
| 659 |
remote_output = remote_files.split("\n")[-1].strip()
|
| 660 |
else:
|
|
|
|
| 383 |
install_cmds = []
|
| 384 |
|
| 385 |
# Check if already present in workspace
|
| 386 |
+
tuner_exist = (await self._ssh_exec(ssh, f"test -f {tuner_dir}/pyproject.toml && echo EXISTS || echo MISSING")).strip()
|
| 387 |
if tuner_exist == "EXISTS":
|
| 388 |
job._log("musubi-tuner found in workspace")
|
| 389 |
else:
|
| 390 |
# Check volume cache
|
| 391 |
+
vol_exist = (await self._ssh_exec(ssh, "test -f /runpod-volume/musubi-tuner/pyproject.toml && echo EXISTS || echo MISSING")).strip()
|
| 392 |
if vol_exist == "EXISTS":
|
| 393 |
job._log("Restoring musubi-tuner from volume cache...")
|
| 394 |
await self._ssh_exec(ssh, f"rm -rf {tuner_dir} 2>/dev/null; cp -r /runpod-volume/musubi-tuner {tuner_dir}")
|
|
|
|
| 408 |
])
|
| 409 |
else:
|
| 410 |
# SD 1.5 / SDXL / FLUX.1 use sd-scripts
|
| 411 |
+
scripts_exist = (await self._ssh_exec(ssh, "test -f /workspace/sd-scripts/setup.py && echo EXISTS || echo MISSING")).strip()
|
| 412 |
if scripts_exist == "EXISTS":
|
| 413 |
job._log("Kohya sd-scripts already cached on volume, updating...")
|
| 414 |
install_cmds = [
|
|
|
|
| 443 |
vae_path = f"{flux2_dir}/ae.safetensors" # Original BFL format (not diffusers)
|
| 444 |
te_path = f"{flux2_dir}/text_encoder/model-00001-of-00010.safetensors"
|
| 445 |
|
| 446 |
+
dit_exists = (await self._ssh_exec(ssh, f"test -f {dit_path} && echo EXISTS || echo MISSING")).strip()
|
| 447 |
+
vae_exists = (await self._ssh_exec(ssh, f"test -f {vae_path} && echo EXISTS || echo MISSING")).strip()
|
| 448 |
+
te_exists = (await self._ssh_exec(ssh, f"test -f {te_path} && echo EXISTS || echo MISSING")).strip()
|
| 449 |
|
| 450 |
if dit_exists != "EXISTS" or te_exists != "EXISTS":
|
| 451 |
missing = []
|
|
|
|
| 465 |
print('Downloaded ae.safetensors')
|
| 466 |
" 2>&1 | tail -5""", timeout=600)
|
| 467 |
# Verify download
|
| 468 |
+
vae_check = (await self._ssh_exec(ssh, f"test -f {vae_path} && echo EXISTS || echo MISSING")).strip()
|
| 469 |
if vae_check != "EXISTS":
|
| 470 |
raise RuntimeError("Failed to download ae.safetensors")
|
| 471 |
job._log("VAE downloaded")
|
|
|
|
| 474 |
|
| 475 |
else:
|
| 476 |
# SD 1.5 / SDXL / FLUX.1 — download single model file
|
| 477 |
+
model_exists = (await self._ssh_exec(ssh, f"test -f /workspace/models/{hf_filename} && echo EXISTS || echo MISSING")).strip()
|
| 478 |
if model_exists == "EXISTS":
|
| 479 |
job._log(f"Base model already cached on volume: {model_name}")
|
| 480 |
else:
|
|
|
|
| 488 |
|
| 489 |
# For FLUX.1, download additional required models (CLIP, T5, VAE)
|
| 490 |
if model_type == "flux":
|
| 491 |
+
flux_files_check = (await self._ssh_exec(ssh, "test -f /workspace/models/clip_l.safetensors && test -f /workspace/models/t5xxl_fp16.safetensors && test -f /workspace/models/ae.safetensors && echo EXISTS || echo MISSING")).strip()
|
| 492 |
if flux_files_check == "EXISTS":
|
| 493 |
job._log("FLUX.1 auxiliary models already cached on volume")
|
| 494 |
else:
|
|
|
|
| 652 |
await self._ssh_exec(ssh, "mkdir -p /runpod-volume/loras")
|
| 653 |
remote_output = f"/workspace/output/{name}.safetensors"
|
| 654 |
# Find the output file
|
| 655 |
+
check = (await self._ssh_exec(ssh, f"test -f {remote_output} && echo EXISTS || echo MISSING")).strip()
|
| 656 |
if check == "MISSING":
|
| 657 |
+
remote_files = (await self._ssh_exec(ssh, "ls /workspace/output/*.safetensors 2>/dev/null")).strip()
|
| 658 |
if remote_files:
|
| 659 |
remote_output = remote_files.split("\n")[-1].strip()
|
| 660 |
else:
|