Upload app.py

app.py

@@ -630,31 +630,50 @@ Hinweis: Du musst eingeloggt sein und die Terms akzeptieren!"""
             # Training mit Progress-Updates
             import torch.optim as optim
             
-            def training_callback(epoch, total_epochs, loss=None):
-                progress_value = 0.7 + (epoch / total_epochs) * 0.25
-                desc = f"Epoch {epoch+1}/{total_epochs}"
-                if loss is not None:
-                    desc += f" - Loss: {loss:.4f}"
+            # Tracking-Variablen für Training
+            training_completed = False
+            actual_epochs_completed = 0
+            training_error = None
+            
+            def training_epoch_end_callback():
+                """Callback der nach jeder Epoch aufgerufen wird."""
+                nonlocal actual_epochs_completed, training_completed
+                # Hole aktuelle Epoch aus Model-Objekt
+                current_epoch = getattr(model, 'ep', actual_epochs_completed)
+                actual_epochs_completed = current_epoch
+                progress_value = 0.7 + (current_epoch / epochs) * 0.25
+                desc = f"🔥 Epoch {current_epoch}/{epochs}"
                 progress(progress_value, desc=desc)
+                
+                # Prüfe ob letzte Epoch erreicht wurde
+                if current_epoch >= epochs:
+                    training_completed = True
             
             # Starte Training
-            model.fit(
-                network,
-                criterion,
-                optim.Adam(network.parameters(), lr=main_config.LEARNING_RATE),
-                main_config.EPOCHS,
-                max_epochs=float("inf"),
-                pretrained_path=main_config.PRETRAINED_PATH,
-                keep=True,
-                backprop_freq=main_config.BATCH_STEP,
-                device_ids=main_config.DEVICE_IDS,
-                eval_metrics=eval_metrics,
-                keep_epoch=main_config.KEEP_EPOCH,
-                keep_optimizer=main_config.KEEP_OPTIMIZER,
-                config=None,
-                upload=False,
-                flush_cache_after_step=main_config.FLUSH_CACHE_AFTER_STEP,
-            )
+            try:
+                model.fit(
+                    network,
+                    criterion,
+                    optim.Adam(network.parameters(), lr=main_config.LEARNING_RATE),
+                    epochs,
+                    max_epochs=float("inf"),
+                    pretrained_path=main_config.PRETRAINED_PATH,
+                    keep=True,
+                    backprop_freq=main_config.BATCH_STEP,
+                    device_ids=main_config.DEVICE_IDS,
+                    eval_metrics=eval_metrics,
+                    keep_epoch=main_config.KEEP_EPOCH,
+                    keep_optimizer=main_config.KEEP_OPTIMIZER,
+                    config=None,
+                    upload=False,
+                    flush_cache_after_step=main_config.FLUSH_CACHE_AFTER_STEP,
+                    training_epoch_end=training_epoch_end_callback,
+                )
+                training_completed = True
+            except Exception as e:
+                training_error = str(e)
+                import traceback
+                training_error += f"\n\n{traceback.format_exc()}"
             
             progress(0.95, desc="💾 Speichere Modell...")
             
@@ -662,34 +681,77 @@ Hinweis: Du musst eingeloggt sein und die Terms akzeptieren!"""
             model_path = Path("/tmp/models")
             model_path.mkdir(exist_ok=True)
             
+            # Prüfe ob Training erfolgreich war
+            model_saved = False
+            best_model_path = None
+            
             # Finde bestes Modell
             runs_dir = netlistify_dir / "runs" / "FormalDatasetWindowedLinePair"
-            if runs_dir.exists():
-                latest_run = max(runs_dir.iterdir(), key=lambda x: x.stat().st_mtime)
-                best_model = latest_run / "best_train.pth"
-                if best_model.exists():
-                    shutil.copy2(best_model, model_path / "best_model.pth")
+            if runs_dir.exists() and runs_dir.is_dir():
+                try:
+                    run_dirs = [d for d in runs_dir.iterdir() if d.is_dir()]
+                    if run_dirs:
+                        latest_run = max(run_dirs, key=lambda x: x.stat().st_mtime)
+                        best_model = latest_run / "best_train.pth"
+                        if best_model.exists():
+                            best_model_path = model_path / "best_model.pth"
+                            shutil.copy2(best_model, best_model_path)
+                            model_saved = True
+                            
+                            # Prüfe auch latest.pth
+                            latest_model = latest_run / "latest.pth"
+                            if latest_model.exists():
+                                shutil.copy2(latest_model, model_path / "latest_model.pth")
+                except Exception as e:
+                    pass
             
             progress(1.0, desc="✅ Training abgeschlossen!")
             
-            return f"""
-✅ Training erfolgreich abgeschlossen!
-
-📊 **Training-Details:**
-- GPU: {gpu_name} ({gpu_memory:.1f} GB)
-- Epochs: {epochs}
-- Batch Size: {batch_size}
-- Learning Rate: {learning_rate}
-- Dataset-Größe: {len(img_files)} Bilder
-
-💾 **Modell gespeichert:**
-- Pfad: {model_path}
-- Bestes Modell: best_model.pth
-
-📁 **Nächste Schritte:**
-1. Lade das trainierte Modell herunter
-2. Verwende es für Inference in deiner Anwendung
-"""
+            # Erstelle Status-Report
+            status_lines = []
+            
+            if training_error:
+                status_lines.append("❌ **Training mit Fehler beendet:**")
+                status_lines.append(f"```\n{training_error}\n```")
+            elif training_completed:
+                status_lines.append("✅ **Training erfolgreich abgeschlossen!**")
+            else:
+                status_lines.append("⚠️ **Training-Status unklar**")
+            
+            status_lines.append("")
+            status_lines.append("📊 **Training-Details:**")
+            status_lines.append(f"- GPU: {gpu_name} ({gpu_memory:.1f} GB)")
+            status_lines.append(f"- Geplante Epochs: {epochs}")
+            status_lines.append(f"- Abgeschlossene Epochs: {actual_epochs_completed}")
+            status_lines.append(f"- Batch Size: {batch_size}")
+            status_lines.append(f"- Learning Rate: {learning_rate}")
+            status_lines.append(f"- Dataset-Größe: {len(img_files)} Bilder")
+            status_lines.append("")
+            
+            if model_saved:
+                status_lines.append("💾 **Modell gespeichert:**")
+                status_lines.append(f"- Pfad: {model_path}")
+                status_lines.append(f"- Bestes Modell: best_model.pth")
+                if best_model_path and best_model_path.exists():
+                    file_size = best_model_path.stat().st_size / (1024 * 1024)  # MB
+                    status_lines.append(f"- Dateigröße: {file_size:.2f} MB")
+            else:
+                status_lines.append("⚠️ **Modell nicht gefunden:**")
+                status_lines.append(f"- Erwarteter Pfad: {runs_dir}")
+                status_lines.append("- Prüfe Logs für Details")
+            
+            status_lines.append("")
+            
+            if training_completed and model_saved:
+                status_lines.append("📁 **Nächste Schritte:**")
+                status_lines.append("1. Lade das trainierte Modell herunter")
+                status_lines.append("2. Verwende es für Inference in deiner Anwendung")
+            elif not training_completed:
+                status_lines.append("⚠️ **Hinweis:** Training wurde möglicherweise nicht vollständig abgeschlossen.")
+                status_lines.append("- Prüfe die Logs für weitere Details")
+                status_lines.append("- Versuche Training erneut zu starten")
+            
+            return "\n".join(status_lines)
             
         except Exception as e:
             import traceback