Switch training UI from epochs to max steps (default 1500)

- Replace Epochs field with Max Steps (1500 default, 1500-2000 recommended)
- Replace Save Every N Epochs with Save Every N Steps (500 default)
- Persist pod state to disk so it survives server restarts
- Increase generation polling timeout to 600s

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

src/content_engine/api/routes_training.py

@@ -76,14 +76,14 @@ async def start_training(
     captions_json: str = Form("{}"),
     base_model: str = Form("flux2_dev"),  # Model registry key (flux2_dev, sd15_realistic, sdxl_base)
     resolution: int | None = Form(None),  # None = use model default
-    num_epochs: int = Form(10),
-    max_train_steps: int | None = Form(None),  # If set, overrides epochs
+    num_epochs: int = Form(100),  # High default — max_steps controls actual limit
+    max_steps: int = Form(1500),  # Primary training length control
     learning_rate: float | None = Form(None),  # None = use model default
     network_rank: int | None = Form(None),  # None = use model default
     network_alpha: int | None = Form(None),  # None = use model default
     optimizer: str | None = Form(None),  # None = use model default
     train_batch_size: int = Form(1),
-    save_every_n_epochs: int = Form(2),
+    save_every_n_steps: int = Form(500),
     backend: str = Form("runpod"),  # Default to runpod for cloud training
     gpu_type: str = Form("NVIDIA GeForce RTX 4090"),
 ):
@@ -140,12 +140,12 @@ async def start_training(
             base_model=base_model,
             resolution=resolution,
             num_epochs=num_epochs,
-            max_train_steps=max_train_steps,
+            max_train_steps=max_steps,
             learning_rate=learning_rate,
             network_rank=network_rank,
             network_alpha=network_alpha,
             optimizer=optimizer,
-            save_every_n_epochs=save_every_n_epochs,
+            save_every_n_steps=save_every_n_steps,
             gpu_type=gpu_type,
         )
         job = _runpod_trainer.get_job(job_id)
@@ -184,7 +184,7 @@ async def start_training(
         network_alpha=network_alpha or model_cfg.get("network_alpha", 16),
         optimizer=optimizer or model_cfg.get("optimizer", "AdamW8bit"),
         train_batch_size=train_batch_size,
-        save_every_n_epochs=save_every_n_epochs,
+        save_every_n_epochs=save_every_n_steps,  # Local trainer uses epoch-based saving
     )
 
     job_id = await _trainer.start_training(config, image_paths)

src/content_engine/api/ui.html

@@ -1332,8 +1332,9 @@ select { cursor: pointer; }
           <div class="section-title">Training Settings</div>
           <div class="row" style="display:grid;grid-template-columns:1fr 1fr;gap:8px">
             <div>
-              <label>Epochs</label>
-              <input type="number" id="train-epochs" value="10" min="1" max="100">
+              <label>Max Steps</label>
+              <input type="number" id="train-max-steps" value="1500" min="50" max="10000" step="100">
+              <div style="font-size:10px;color:var(--text-secondary);margin-top:2px">1500-2000 recommended</div>
             </div>
             <div>
               <label>Network Rank (dim)</label>
@@ -1370,8 +1371,8 @@ select { cursor: pointer; }
               </select>
             </div>
             <div>
-              <label>Save Every N Epochs</label>
-              <input type="number" id="train-save-every" value="2" min="1">
+              <label>Save Every N Steps</label>
+              <input type="number" id="train-save-every" value="500" min="50" step="50">
             </div>
           </div>
 
@@ -2908,8 +2909,8 @@ async function startTraining() {
   formData.append('name', name);
   formData.append('trigger_word', document.getElementById('train-trigger').value);
   formData.append('base_model', document.getElementById('train-base-model').value);
-  formData.append('num_epochs', document.getElementById('train-epochs').value);
-  formData.append('save_every_n_epochs', document.getElementById('train-save-every').value);
+  formData.append('max_steps', document.getElementById('train-max-steps').value);
+  formData.append('save_every_n_steps', document.getElementById('train-save-every').value);
   formData.append('backend', selectedTrainBackend);
 
   // Optional params - only send if user explicitly set them (otherwise use model defaults)

src/content_engine/services/runpod_trainer.py

@@ -169,6 +169,7 @@ class RunPodTrainer:
         network_alpha: int | None = None,
         optimizer: str | None = None,
         save_every_n_epochs: int = 2,
+        save_every_n_steps: int = 500,
         gpu_type: str = DEFAULT_GPU,
     ) -> str:
         """Start a cloud training job. Returns job ID.
@@ -194,6 +195,7 @@ class RunPodTrainer:
             name=name,
             status="pending",
             total_epochs=num_epochs,
+            total_steps=final_steps,
             gpu_type=gpu_type,
             started_at=time.time(),
             base_model=base_model,
@@ -217,6 +219,7 @@ class RunPodTrainer:
             network_alpha=final_alpha,
             optimizer=final_optimizer,
             save_every_n_epochs=save_every_n_epochs,
+            save_every_n_steps=save_every_n_steps,
         ))
 
         return job_id
@@ -235,6 +238,7 @@ class RunPodTrainer:
         network_alpha: int,
         optimizer: str,
         save_every_n_epochs: int,
+        save_every_n_steps: int = 500,
     ):
         """Full cloud training pipeline: create pod -> upload -> train -> download -> cleanup."""
         ssh = None
@@ -578,6 +582,7 @@ resolution = [{resolution}, {resolution}]
                 network_alpha=network_alpha,
                 optimizer=optimizer,
                 save_every_n_epochs=save_every_n_epochs,
+                save_every_n_steps=save_every_n_steps,
                 model_cfg=model_cfg,
                 gpu_type=job.gpu_type,
             )
@@ -792,6 +797,7 @@ resolution = [{resolution}, {resolution}]
         network_alpha: int,
         optimizer: str,
         save_every_n_epochs: int,
+        save_every_n_steps: int = 500,
         model_cfg: dict,
         gpu_type: str = "",
     ) -> str:
@@ -875,7 +881,6 @@ resolution = [{resolution}, {resolution}]
                 ])
 
             args.extend([
-                f"--save_every_n_epochs={save_every_n_epochs}",
                 "--seed=42",
                 '--output_dir=/workspace/output',
                 f'--output_name={name}',
@@ -884,8 +889,13 @@ resolution = [{resolution}, {resolution}]
 
             if max_train_steps:
                 args.append(f"--max_train_steps={max_train_steps}")
+                if save_every_n_steps:
+                    args.append(f"--save_every_n_steps={save_every_n_steps}")
+                else:
+                    args.append(f"--save_every_n_epochs={save_every_n_epochs}")
             else:
                 args.append(f"--max_train_epochs={num_epochs}")
+                args.append(f"--save_every_n_epochs={save_every_n_epochs}")
 
             return " ".join(args) + " 2>&1"