Update 37 run training preparation in background worker

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## Update ke-37 - 2026-05-01
+- Perbaiki training yang berhenti pada tahap `preparing_dataset`.
+- Persiapan dataset, pseudo-label, training YOLO, penyimpanan `best.pt`, grafik PNG, confusion matrix, log, dan sync artefak kini berjalan dalam background worker sejak awal.
+- Tombol training sekarang hanya memulai worker dan langsung mengembalikan status, sehingga proses tidak bergantung pada koneksi browser/request Gradio.
+- Minor update dinaikkan ke-37.
+
 ## Update ke-36 - 2026-05-01
 - Reset alur training agar `best.pt` dan artefak training lama bisa dikosongkan tanpa menghapus opsi `Best.pt` dari UI.
 - `Best.pt` tidak lagi fallback diam-diam ke `YOLOv8n`; jika belum ada, aplikasi menampilkan pesan model belum tersedia.

README.md

@@ -51,6 +51,7 @@ Versi ini difokuskan untuk tahap awal:
   - file `training.log`, `training_curve_latest.png`, dan `confusion_matrix_latest.png` juga disimpan lokal
   - jika env Hugging Face diisi, `best.pt` dan artefak training otomatis di-sync ke repo HF
   - grafik training disimpan permanen sebagai gambar PNG di `training_outputs/training_curve_latest.png` dan `space_training_archive/<run>/training_curve.png`
+  - persiapan dataset dan pseudo-label berjalan di background worker, sehingga training tidak berhenti hanya karena koneksi browser terputus
 - **Tahap 3 - Input Gambar**
   - input gambar (upload/webcam)
   - tombol proses
@@ -128,6 +129,7 @@ Catatan auto-save hasil training:
 - File grafik terbaru juga disalin ke `training_outputs/training_curve_latest.png` agar tetap bisa ditampilkan ulang setelah Space rebuild.
 - Proses training juga langsung menyalin grafik YOLO `results.png` menjadi PNG permanen saat training selesai, sehingga tidak bergantung pada refresh browser.
 - Jika koneksi browser terputus saat training, artefak tetap aman setelah proses training selesai dan sync ke repo berhasil. Jika container berhenti sebelum training selesai, artefak final memang belum sempat dibuat.
+- Pseudo-label dari dataset tanpa label `.txt` dapat memakan waktu di CPU Space karena semua gambar harus diproses YOLOv8n sebelum epoch dimulai.
 
 Catatan auto-sync update file repo:
 - Default `HF_AUTOSYNC_ENABLED=false`, jadi tidak ada sync repo saat startup/runtime.

app.py

@@ -24,7 +24,7 @@ from ultralytics import YOLO
 CM_INIT = {"tp": 0, "fp": 0, "fn": 0, "tn": 0, "person": 0, "not_person": 0}
 EVAL_CLASSES = ["Person", "Not Person/Background"]
 APP_VERSION = os.getenv("APP_VERSION", "v0.9.0")
-APP_MINOR_UPDATE = os.getenv("APP_MINOR_UPDATE", "36")
+APP_MINOR_UPDATE = os.getenv("APP_MINOR_UPDATE", "37")
 
 
 # Gradio/Spaces can trigger noisy asyncio cleanup traces ("Invalid file descriptor: -1")
@@ -1662,6 +1662,144 @@ def poll_training_status() -> Tuple[str, Any, Any, Any, Any, np.ndarray, Any, np
         )
 
 
+def _append_training_log(log_path: Path, message: str) -> None:
+    with log_path.open("a", encoding="utf-8") as lf:
+        lf.write(f"[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] {message}\n")
+
+
+def training_worker_main() -> None:
+    base_dir = Path(os.environ["TRAIN_BASE_DIR"]).resolve()
+    paths = _train_paths(base_dir)
+    run_name = os.environ["TRAIN_RUN_NAME"]
+    project_dir = Path(os.environ["TRAIN_PROJECT_DIR"])
+    save_dir = Path(os.environ["TRAIN_SAVE_DIR"])
+    results_csv = save_dir / "results.csv"
+    train_model_choice = os.environ.get("TRAIN_MODEL_CHOICE", "YOLOv8n")
+
+    state: Dict[str, Any] = {
+        "status": "running",
+        "stage": "preparing_dataset",
+        "run_name": run_name,
+        "pid": os.getpid(),
+        "save_dir": str(save_dir),
+        "results_csv": str(results_csv),
+        "started_at": datetime.now().isoformat(),
+    }
+    _write_status(paths["status"], state)
+
+    try:
+        _append_training_log(paths["log"], f"Worker training aktif. PID={os.getpid()}")
+        _append_training_log(paths["log"], "Preparing dataset dan pseudo-label di background...")
+        data_yaml, prep_info = _prepare_training_dataset(
+            dataset_root=base_dir / "dataset",
+            use_pseudo_if_missing=_env_true("TRAIN_USE_PSEUDO", default=True),
+            pseudo_conf=float(os.environ.get("TRAIN_PSEUDO_CONF", "0.30")),
+            log_file=paths["log"],
+        )
+
+        model_path = resolve_model_path("Best.pt" if train_model_choice == "Best.pt" else "YOLOv8n")
+        state.update(
+            {
+                "stage": "training",
+                "data_yaml": str(data_yaml),
+                "model_path": str(model_path),
+            }
+        )
+        _write_status(paths["status"], state)
+        _append_training_log(paths["log"], f"Dataset siap: {prep_info}")
+        _append_training_log(paths["log"], f"Start training {run_name}")
+        _append_training_log(paths["log"], f"Model awal training: {train_model_choice} ({model_path})")
+
+        model = YOLO(str(model_path))
+        res = model.train(
+            data=str(data_yaml),
+            epochs=int(os.environ["TRAIN_EPOCHS"]),
+            imgsz=int(os.environ["TRAIN_IMGSZ"]),
+            batch=int(os.environ["TRAIN_BATCH"]),
+            device="cpu",
+            project=str(project_dir),
+            name=run_name,
+            exist_ok=False,
+            verbose=True,
+        )
+
+        final_save = Path(str(res.save_dir))
+        final_results_csv = final_save / "results.csv"
+        best_pt = final_save / "weights" / "best.pt"
+        best_out = base_dir / "best.pt"
+        hf_sync_msg = "HF sync dilewati: best.pt tidak ditemukan."
+        if best_pt.exists() and best_pt.is_file():
+            shutil.copy2(best_pt, best_out)
+            hf_sync_msg = sync_best_to_hf(best_out)
+
+        chart_file: str | None = None
+        chart_src = final_save / "results.png"
+        if chart_src.exists() and chart_src.is_file():
+            paths["chart"].parent.mkdir(parents=True, exist_ok=True)
+            shutil.copy2(chart_src, paths["chart"])
+            chart_file = str(paths["chart"])
+        elif final_results_csv.exists() and final_results_csv.is_file():
+            chart_file = save_training_curve_image(plot_training_curves(final_results_csv), paths["chart"])
+
+        cm_file = save_training_confusion_matrix({"save_dir": str(final_save)}, paths["train_cm"])
+        rows = _read_results_rows(final_results_csv if final_results_csv.exists() else None)
+        final_state = {
+            **state,
+            "status": "done",
+            "stage": "done",
+            "ended_at": datetime.now().isoformat(),
+            "save_dir": str(final_save),
+            "results_csv": str(final_results_csv),
+            "best_path": str(best_out if best_out.exists() else best_pt),
+            "hf_sync_msg": hf_sync_msg,
+        }
+        summary_file = save_training_summary(paths, final_state, len(rows))
+        artifact_sync_msg = sync_training_artifacts_to_hf(
+            run_name=run_name,
+            results_csv=final_results_csv if final_results_csv.exists() else None,
+            log_file=paths["log"] if paths["log"].exists() else None,
+            chart_file=Path(chart_file) if chart_file else None,
+            confusion_matrix_file=Path(cm_file) if cm_file else None,
+            summary_file=Path(summary_file),
+        )
+        final_state.update(
+            {
+                "artifacts_sync_attempted": True,
+                "artifacts_sync_msg": artifact_sync_msg,
+            }
+        )
+        save_training_summary(paths, final_state, len(rows))
+        _write_status(paths["status"], final_state)
+        _append_training_log(paths["log"], "TRAIN_DONE")
+    except Exception as err:
+        _append_training_log(paths["log"], f"TRAIN_FAILED: {err}")
+        failed_state = {
+            **state,
+            "status": "failed",
+            "stage": "failed",
+            "ended_at": datetime.now().isoformat(),
+            "error": str(err),
+        }
+        summary_file = save_training_summary(paths, failed_state, 0)
+        artifact_sync_msg = sync_training_artifacts_to_hf(
+            run_name=run_name,
+            results_csv=None,
+            log_file=paths["log"] if paths["log"].exists() else None,
+            chart_file=paths["chart"] if paths["chart"].exists() else None,
+            confusion_matrix_file=paths["train_cm"] if paths["train_cm"].exists() else None,
+            summary_file=Path(summary_file),
+        )
+        failed_state.update(
+            {
+                "artifacts_sync_attempted": True,
+                "artifacts_sync_msg": artifact_sync_msg,
+            }
+        )
+        save_training_summary(paths, failed_state, 0)
+        _write_status(paths["status"], failed_state)
+        raise
+
+
 def run_training(
     train_model_choice: str,
     epochs: float,
@@ -1681,198 +1819,27 @@ def run_training(
             return poll_training_status()
 
     try:
-        _write_status(
-            paths["status"],
-            {
-                "status": "running",
-                "stage": "preparing_dataset",
-                "run_name": current.get("run_name", "-"),
-                "started_at": datetime.now().isoformat(),
-            },
-        )
-        with paths["log"].open("a", encoding="utf-8") as lf:
-            lf.write(f"[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] Preparing dataset...\n")
-
-        dataset_root = base_dir / "dataset"
-        data_yaml, prep_info = _prepare_training_dataset(
-            dataset_root=dataset_root,
-            use_pseudo_if_missing=use_pseudo_if_missing,
-            pseudo_conf=float(pseudo_conf),
-            log_file=paths["log"],
-        )
-
-        if train_model_choice == "Best.pt":
-            model_path = resolve_model_path("Best.pt")
-        else:
-            model_path = str(base_dir / "yolov8n.pt") if (base_dir / "yolov8n.pt").exists() else resolve_model_path("YOLOv8n")
         run_name = f"classroom_train_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
         project_dir = base_dir / "runs_train"
         save_dir = project_dir / run_name
         results_csv = save_dir / "results.csv"
-
-        with paths["log"].open("a", encoding="utf-8") as lf:
-            lf.write(f"\n[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] Start training {run_name}\n")
-            lf.write(f"Model awal training: {train_model_choice} ({model_path})\n")
-            lf.write(f"{prep_info}\n")
-
-        train_code = r"""
-        import os, json, shutil
-        from datetime import datetime
-        from pathlib import Path
-        from ultralytics import YOLO
-        from huggingface_hub import HfApi
-        
-        def write_status(path, payload):
-            p = Path(path)
-            tmp = p.with_suffix(".tmp")
-            tmp.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
-            tmp.replace(p)
-        
-        status_path = os.environ["TRAIN_STATUS_PATH"]
-        base_dir = Path(os.environ["TRAIN_BASE_DIR"])
-        run_name = os.environ["TRAIN_RUN_NAME"]
-        save_dir = Path(os.environ["TRAIN_SAVE_DIR"])
-        results_csv = save_dir / "results.csv"
-        
-        state = {
-            "status": "running",
-            "run_name": run_name,
-            "pid": os.getpid(),
-            "save_dir": str(save_dir),
-            "results_csv": str(results_csv),
-            "started_at": datetime.now().isoformat(),
-        }
-        write_status(status_path, state)
-        
-        try:
-            model = YOLO(os.environ["TRAIN_MODEL_PATH"])
-            res = model.train(
-                data=os.environ["TRAIN_DATA_YAML"],
-                epochs=int(os.environ["TRAIN_EPOCHS"]),
-                imgsz=int(os.environ["TRAIN_IMGSZ"]),
-                batch=int(os.environ["TRAIN_BATCH"]),
-                device="cpu",
-                project=os.environ["TRAIN_PROJECT_DIR"],
-                name=run_name,
-                exist_ok=False,
-                verbose=True,
-            )
-            final_save = Path(str(res.save_dir))
-            best_pt = final_save / "weights" / "best.pt"
-            best_out = base_dir / "best.pt"
-            artifact_dir = base_dir / "training_outputs"
-            artifact_dir.mkdir(parents=True, exist_ok=True)
-            chart_out = artifact_dir / "training_curve_latest.png"
-            cm_out = artifact_dir / "confusion_matrix_latest.png"
-            summary_out = artifact_dir / "training_summary_latest.json"
-        
-            chart_src = final_save / "results.png"
-            if chart_src.exists():
-                shutil.copy2(chart_src, chart_out)
-        
-            for cm_src in (final_save / "confusion_matrix.png", final_save / "confusion_matrix_normalized.png"):
-                if cm_src.exists():
-                    shutil.copy2(cm_src, cm_out)
-                    break
-        
-            hf_sync_msg = "HF sync dilewati: best.pt tidak ditemukan."
-            if best_pt.exists():
-                shutil.copy2(best_pt, best_out)
-                hf_repo = os.getenv("HF_MODEL_REPO", "").strip()
-                hf_file = os.getenv("HF_MODEL_FILE", "best.pt").strip() or "best.pt"
-                hf_repo_type = os.getenv("HF_MODEL_REPO_TYPE", "space").strip() or "space"
-                hf_token = os.getenv("HF_TOKEN", "").strip() or None
-                if hf_repo:
-                    api = HfApi(token=hf_token)
-                    api.upload_file(
-                        path_or_fileobj=str(best_out),
-                        path_in_repo=hf_file,
-                        repo_id=hf_repo,
-                        repo_type=hf_repo_type,
-                        commit_message=f"Auto-update {hf_file} ({datetime.now().strftime('%Y-%m-%d %H:%M:%S')})",
-                    )
-                    hf_sync_msg = f"HF sync sukses: {hf_repo} ({hf_repo_type}) -> {hf_file}"
-        
-            artifact_sync_msg = "HF training artifact sync dilewati: set HF_MODEL_REPO."
-            try:
-                hf_repo = os.getenv("HF_MODEL_REPO", "").strip()
-                hf_repo_type = os.getenv("HF_MODEL_REPO_TYPE", "space").strip() or "space"
-                hf_token = os.getenv("HF_TOKEN", "").strip() or None
-                repo_dir = os.getenv("HF_TRAINING_DIR", "space_training_archive").strip().strip("/") or "space_training_archive"
-                if hf_repo:
-                    api = HfApi(token=hf_token)
-                    summary_payload = {
-                        "saved_at": datetime.now().isoformat(),
-                        "status": "done",
-                        "run_name": run_name,
-                        "save_dir": str(final_save),
-                        "results_csv": str(final_save / "results.csv"),
-                        "best_path": str(best_out if best_out.exists() else best_pt),
-                        "hf_sync_msg": hf_sync_msg,
-                    }
-                    summary_out.write_text(json.dumps(summary_payload, ensure_ascii=False, indent=2), encoding="utf-8")
-                    upload_items = [
-                        (final_save / "results.csv", f"{repo_dir}/{run_name}/results.csv"),
-                        (Path(os.environ["TRAIN_LOG_PATH"]), f"{repo_dir}/{run_name}/training.log"),
-                        (chart_out, f"{repo_dir}/{run_name}/training_curve.png"),
-                        (cm_out, f"{repo_dir}/{run_name}/confusion_matrix.png"),
-                        (summary_out, f"{repo_dir}/{run_name}/training_summary.json"),
-                        (chart_out, "training_outputs/training_curve_latest.png"),
-                        (cm_out, "training_outputs/confusion_matrix_latest.png"),
-                        (summary_out, "training_outputs/training_summary_latest.json"),
-                    ]
-                    uploaded = 0
-                    for local_path, repo_path in upload_items:
-                        if local_path.exists() and local_path.is_file():
-                            api.upload_file(
-                                path_or_fileobj=str(local_path),
-                                path_in_repo=repo_path,
-                                repo_id=hf_repo,
-                                repo_type=hf_repo_type,
-                                commit_message=f"Save permanent training artifact {run_name}",
-                            )
-                            uploaded += 1
-                    artifact_sync_msg = f"HF training artifact sync sukses: {uploaded} file -> {repo_dir}/{run_name}"
-            except Exception as sync_err:
-                artifact_sync_msg = f"HF training artifact sync gagal: {sync_err}"
-        
-            state.update({
-                "status": "done",
-                "ended_at": datetime.now().isoformat(),
-                "save_dir": str(final_save),
-                "results_csv": str(final_save / "results.csv"),
-                "best_path": str(best_out if best_out.exists() else best_pt),
-                "hf_sync_msg": hf_sync_msg,
-                "artifacts_sync_attempted": True,
-                "artifacts_sync_msg": artifact_sync_msg,
-            })
-            write_status(status_path, state)
-            print("TRAIN_DONE", flush=True)
-        except Exception as e:
-            state.update({
-                "status": "failed",
-                "ended_at": datetime.now().isoformat(),
-                "error": str(e),
-            })
-            write_status(status_path, state)
-            print("TRAIN_FAILED: " + str(e), flush=True)
-        """
         env = os.environ.copy()
         env["TRAIN_BASE_DIR"] = str(base_dir)
-        env["TRAIN_MODEL_PATH"] = str(model_path)
-        env["TRAIN_DATA_YAML"] = str(data_yaml)
+        env["TRAIN_MODEL_CHOICE"] = str(train_model_choice)
         env["TRAIN_EPOCHS"] = str(int(epochs))
         env["TRAIN_IMGSZ"] = str(int(imgsz))
         env["TRAIN_BATCH"] = str(int(batch))
+        env["TRAIN_USE_PSEUDO"] = "true" if use_pseudo_if_missing else "false"
+        env["TRAIN_PSEUDO_CONF"] = str(float(pseudo_conf))
         env["TRAIN_PROJECT_DIR"] = str(project_dir)
         env["TRAIN_RUN_NAME"] = run_name
         env["TRAIN_SAVE_DIR"] = str(save_dir)
         env["TRAIN_STATUS_PATH"] = str(paths["status"])
-        env["TRAIN_LOG_PATH"] = str(paths["log"])
 
         with paths["log"].open("a", encoding="utf-8") as lf:
+            lf.write(f"[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] Worker background dimulai: {run_name}\n")
             proc = subprocess.Popen(
-                [sys.executable, "-u", "-c", train_code],
+                [sys.executable, "-u", __file__, "--train-worker"],
                 stdout=lf,
                 stderr=subprocess.STDOUT,
                 cwd=str(base_dir),
@@ -1884,7 +1851,7 @@ def run_training(
             paths["status"],
             {
                 "status": "running",
-                "stage": "training",
+                "stage": "preparing_dataset",
                 "run_name": run_name,
                 "pid": proc.pid,
                 "save_dir": str(save_dir),
@@ -1895,7 +1862,7 @@ def run_training(
         return poll_training_status()
     except Exception as err:
         with paths["log"].open("a", encoding="utf-8") as lf:
-            lf.write(f"[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] Training gagal: {err}\n")
+            lf.write(f"[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] Training gagal sebelum worker aktif: {err}\n")
         _write_status(
             paths["status"],
             {
@@ -2251,10 +2218,14 @@ with gr.Blocks(title="Pengecekan dan Klasifikasi Mahasiswa - YOLOv8") as demo:
     )
 
 
-if __name__ == "__main__":
-    if _env_true("HF_AUTOSYNC_ENABLED", default=False):
-        print(start_repo_autosync(Path(__file__).resolve().parent))
-    else:
+if __name__ == "__main__":
+    if len(sys.argv) > 1 and sys.argv[1] == "--train-worker":
+        training_worker_main()
+        raise SystemExit(0)
+
+    if _env_true("HF_AUTOSYNC_ENABLED", default=False):
+        print(start_repo_autosync(Path(__file__).resolve().parent))
+    else:
         print("HF autosync nonaktif (HF_AUTOSYNC_ENABLED=false).")
     demo.queue()
     demo.launch(css=CUSTOM_CSS, ssr_mode=False)

space_training_archive/-/training.log

@@ -1,4 +0,0 @@
-
-[2026-05-01 16:09:37] Trigger training request
-[2026-05-01 16:09:37] Preparing dataset...
-[2026-05-01 16:09:37] Prepare split `train`: total gambar=80

space_training_archive/-/training_summary.json

@@ -1,11 +0,0 @@
-{
-  "saved_at": "2026-05-01T16:09:38.448561",
-  "status": "failed",
-  "stage": "preparing_dataset",
-  "run_name": "-",
-  "rows_count": 0,
-  "results_csv": null,
-  "best_path": null,
-  "hf_sync_msg": null,
-  "error": "Proses berhenti tanpa status akhir."
-}

training_outputs/training_summary_latest.json

@@ -1,11 +0,0 @@
-{
-  "saved_at": "2026-05-01T16:09:38.448561",
-  "status": "failed",
-  "stage": "preparing_dataset",
-  "run_name": "-",
-  "rows_count": 0,
-  "results_csv": null,
-  "best_path": null,
-  "hf_sync_msg": null,
-  "error": "Proses berhenti tanpa status akhir."
-}