updates

app.py

@@ -546,10 +546,25 @@ def start_training_advanced(
             result = subprocess.run(resnet_cmd, capture_output=True, text=True, check=False)
             
             if result.returncode == 0:
-                log_message += "✅ ResNet training completed successfully!\n\n"
+                log_message += "✅ ResNet training completed successfully!\n"
+                log_message += f"📊 ResNet Output:\n{result.stdout}\n\n"
             else:
                 log_message += f"❌ ResNet training failed: {result.stderr}\n\n"
-                return
+                return log_message
+            
+            # Wait a moment for file system sync and ensure ResNet is fully saved
+            import time
+            time.sleep(3)
+            log_message += "⏳ Waiting for ResNet checkpoint to be fully saved...\n"
+            
+            # Verify ResNet checkpoint exists before proceeding
+            resnet_checkpoint = os.path.join(export_dir, "resnet_item_embedder_custom.pth")
+            if not os.path.exists(resnet_checkpoint):
+                log_message += f"❌ ResNet checkpoint not found at {resnet_checkpoint}\n"
+                log_message += "Cannot proceed with ViT training without ResNet embeddings.\n"
+                return log_message
+            
+            log_message += f"✅ ResNet checkpoint verified: {resnet_checkpoint}\n"
             
             # Train ViT with custom parameters
             log_message += f"🚀 Starting ViT training with custom parameters...\n"
@@ -573,7 +588,8 @@ def start_training_advanced(
             result = subprocess.run(vit_cmd, capture_output=True, text=True, check=False)
             
             if result.returncode == 0:
-                log_message += "✅ ViT training completed successfully!\n\n"
+                log_message += "✅ ViT training completed successfully!\n"
+                log_message += f"📊 ViT Output:\n{result.stdout}\n\n"
                 log_message += "🎉 All training completed! Models saved to models/exports/\n"
                 log_message += "🔄 Reloading models for inference...\n"
                 service.reload_models()
@@ -629,15 +645,45 @@ def start_training_simple(dataset_size: str, res_epochs: int, vit_epochs: int):
             if dataset_size != "full":
                 dataset_args = ["--max_samples", dataset_size]
             
-            subprocess.run([
+            # Train ResNet first and wait for completion
+            log_message += f"\n🚀 Starting ResNet training on {dataset_size} samples...\n"
+            resnet_result = subprocess.run([
                 "python", "train_resnet.py", "--data_root", DATASET_ROOT, "--epochs", str(res_epochs),
                 "--out", os.path.join(export_dir, "resnet_item_embedder.pth")
-            ] + dataset_args, check=False)
-            log_message += f"\nTraining ViT (triplet) on {dataset_size} samples...\n"
-            subprocess.run([
+            ] + dataset_args, capture_output=True, text=True, check=False)
+            
+            if resnet_result.returncode == 0:
+                log_message += "✅ ResNet training completed successfully!\n"
+                log_message += f"📊 ResNet Output:\n{resnet_result.stdout}\n"
+            else:
+                log_message += f"❌ ResNet training failed: {resnet_result.stderr}\n"
+                return log_message
+            
+            # Wait a moment for file system sync
+            import time
+            time.sleep(2)
+            
+            # Verify ResNet checkpoint exists before proceeding
+            resnet_checkpoint = os.path.join(export_dir, "resnet_item_embedder.pth")
+            if not os.path.exists(resnet_checkpoint):
+                log_message += f"❌ ResNet checkpoint not found at {resnet_checkpoint}\n"
+                log_message += "Cannot proceed with ViT training without ResNet embeddings.\n"
+                return log_message
+            
+            log_message += f"✅ ResNet checkpoint verified: {resnet_checkpoint}\n"
+            
+            log_message += f"\n🚀 Starting ViT training on {dataset_size} samples...\n"
+            vit_result = subprocess.run([
                 "python", "train_vit_triplet.py", "--data_root", DATASET_ROOT, "--epochs", str(vit_epochs),
                 "--export", os.path.join(export_dir, "vit_outfit_model.pth")
-            ] + dataset_args, check=False)
+            ] + dataset_args, capture_output=True, text=True, check=False)
+            
+            if vit_result.returncode == 0:
+                log_message += "✅ ViT training completed successfully!\n"
+                log_message += f"📊 ViT Output:\n{vit_result.stdout}\n"
+            else:
+                log_message += f"❌ ViT training failed: {vit_result.stderr}\n"
+                return log_message
             service.reload_models()
             log_message += "\nDone. Artifacts in models/exports."
             

train_resnet.py

@@ -111,13 +111,17 @@ def main() -> None:
                 running_loss += loss.item()
                 steps += 1
                 
-                if batch_idx % 100 == 0:
+                if batch_idx % 10 == 0:  # More frequent logging
                     print(f"  Batch {batch_idx}/{len(loader)}: loss={loss.item():.4f}")
                     
             except Exception as e:
                 print(f"❌ Error in batch {batch_idx}: {e}")
                 continue
         
+        # Print final batch completion
+        print(f"  ✅ Batch {len(loader)-1}/{len(loader)}: loss={loss.item():.4f}")
+        print(f"  📊 Epoch {epoch+1} completed: {len(loader)} batches processed")
+        
         avg_loss = running_loss / max(1, steps)
         
         # Save checkpoint with better path handling

train_vit_triplet.py

@@ -148,13 +148,17 @@ def main() -> None:
                 running_loss += loss.item()
                 steps += 1
                 
-                if batch_idx % 50 == 0:
+                if batch_idx % 10 == 0:  # More frequent logging
                     print(f"  Batch {batch_idx}/{len(loader)}: loss={loss.item():.4f}")
                     
             except Exception as e:
                 print(f"❌ Error in batch {batch_idx}: {e}")
                 continue
 
+        # Print final batch completion
+        print(f"  ✅ Batch {len(loader)-1}/{len(loader)}: loss={loss.item():.4f}")
+        print(f"  📊 Epoch {epoch+1} completed: {len(loader)} batches processed")
+
         avg_loss = running_loss / max(1, steps)
         
         # Simple validation using a subset of training data as a proxy if no val split here