Upload folder using huggingface_hub

app.py

@@ -88,11 +88,15 @@ def _mlflow_event(run_name: str, params: dict | None = None, metrics: dict | Non
 
 def _build_flow_markdown(model_loaded=False, inference_ready=False, task_states=None):
     lines = ["### Execution Flow"]
+    task_states = task_states or {}
+    any_task_activity = any(v != "pending" for v in task_states.values()) if task_states else False
     for i, step in enumerate(FLOW_STEPS, start=1):
         status = "⬜"
         if model_loaded and i <= 3:
             status = "✅"
-        if inference_ready and i <= 11:
+        if (inference_ready or model_loaded) and i <= 11:
+            status = "✅"
+        if i == 12 and any_task_activity:
             status = "✅"
         lines.append(f"{status} {i}. {step}")
     if task_states:
@@ -477,9 +481,17 @@ def _run_analysis_cmd(task, ckpt_path, output_dir, input_text="dharmo rakṣati
     env.setdefault("HF_DATASETS_CACHE", "/tmp/hf_datasets")
     env.setdefault("HF_HUB_CACHE", "/tmp/hf_hub")
 
-    proc = subprocess.run(cmd, capture_output=True, text=True, env=env)
-    log = f"$ {' '.join(cmd)}\n\n{proc.stdout}\n{proc.stderr}"
-    return proc.returncode, log, False
+    timeout_map = {"1": 120, "2": 180, "3": 240, "4": 300, "5": 240}
+    timeout_s = int(os.environ.get("TASK_TIMEOUT_S", timeout_map.get(str(task), 180)))
+    try:
+        proc = subprocess.run(cmd, capture_output=True, text=True, env=env, timeout=timeout_s)
+        log = f"$ {' '.join(cmd)}\n\n{proc.stdout}\n{proc.stderr}"
+        return proc.returncode, log, False
+    except subprocess.TimeoutExpired as e:
+        out = e.stdout or ""
+        err = e.stderr or ""
+        log = f"$ {' '.join(cmd)}\n\n[timeout after {timeout_s}s]\n{out}\n{err}"
+        return 124, log, False
 
 
 def _bundle_task_outputs(model_bundle, output_dir):
@@ -648,24 +660,34 @@ def _bg_worker(job_id: str, model_bundle, output_dir: str, input_text: str, task
                 "updated": datetime.now().isoformat(),
             }
         )
-        code, log, used_bundled = _run_analysis_cmd(
-            task,
-            model_bundle["ckpt_path"],
-            output_dir,
-            input_text,
-            task4_phase,
-            task5_cfg.get("samples", 50),
-        )
-        logs.append(f"\n\n{'='*22} TASK {task} {'='*22}\n{log}")
-        if code != 0:
+        try:
+            code, log, used_bundled = _run_analysis_cmd(
+                task,
+                model_bundle["ckpt_path"],
+                output_dir,
+                input_text,
+                task4_phase,
+                task5_cfg.get("samples", 50),
+            )
+            logs.append(f"\n\n{'='*22} TASK {task} {'='*22}\n{log}")
+            if code != 0:
+                failures += 1
+                try:
+                    logs.append(f"\n[Live fallback]\n{_live_task_analysis(model_bundle, task, input_text, task5_cfg)}\n")
+                    _BG_JOBS[job_id]["task_states"][task] = "done(live-fast)"
+                except Exception as live_e:
+                    _BG_JOBS[job_id]["task_states"][task] = "failed"
+                    logs.append(f"\n[Live fallback failed]\n{live_e}\n")
+            elif used_bundled:
+                _BG_JOBS[job_id]["task_states"][task] = "done(bundled)"
+                logs.append(f"\n[Live bundled summary]\n{_live_task_analysis(model_bundle, task, input_text, task5_cfg)}\n")
+            else:
+                _BG_JOBS[job_id]["task_states"][task] = "done"
+        except Exception as e:
             failures += 1
             _BG_JOBS[job_id]["task_states"][task] = "failed"
-            logs.append(f"\n[Live fallback]\n{_live_task_analysis(model_bundle, task, input_text, task5_cfg)}\n")
-        elif used_bundled:
-            _BG_JOBS[job_id]["task_states"][task] = "done(bundled)"
-            logs.append(f"\n[Live bundled summary]\n{_live_task_analysis(model_bundle, task, input_text, task5_cfg)}\n")
-        else:
-            _BG_JOBS[job_id]["task_states"][task] = "done"
+            logs.append(f"\n\n{'='*22} TASK {task} {'='*22}\n[worker exception]\n{e}\n")
+            code, used_bundled = 1, False
         _BG_JOBS[job_id].update(
             {
                 "log": "".join(logs),
@@ -740,7 +762,7 @@ def start_run_all_background(model_bundle, output_dir, input_text, task4_phase,
         daemon=True,
     )
     th.start()
-    flow = _build_flow_markdown(model_loaded=True, inference_ready=False, task_states=_BG_JOBS[job_id]["task_states"])
+    flow = _build_flow_markdown(model_loaded=True, inference_ready=True, task_states=_BG_JOBS[job_id]["task_states"])
     return f"Background run started. Job ID: {job_id}", f"Job {job_id} queued...", job_id, _BG_JOBS[job_id]["task_states"], flow
 
 
@@ -756,7 +778,7 @@ def poll_run_all_background(job_id, output_dir):
         f"failures={j['failures']} | updated={j['updated']}"
     )
     outputs = refresh_task_outputs(output_dir)
-    flow = _build_flow_markdown(model_loaded=True, inference_ready=False, task_states=j.get("task_states", {}))
+    flow = _build_flow_markdown(model_loaded=True, inference_ready=True, task_states=j.get("task_states", {}))
     return status, j.get("log", ""), j.get("task_states", {}), flow, *outputs
 
 
@@ -778,9 +800,14 @@ def run_single_task(model_bundle, task, output_dir, input_text, task4_phase, tas
     elapsed = (time.perf_counter() - t0) * 1000.0
     if code != 0:
         _bundle_task_outputs(model_bundle, output_dir)
-        log = f"{log}\n\n--- Live task analysis ---\n{_live_task_analysis(model_bundle, task, input_text, task5_cfg)}"
-        status = f"Task {task} fallback mode: bundled reports + live input analysis."
-        task_states[str(task)] = "failed"
+        try:
+            log = f"{log}\n\n--- Live task analysis ---\n{_live_task_analysis(model_bundle, task, input_text, task5_cfg)}"
+            status = f"Task {task} fallback mode: bundled reports + live input analysis."
+            task_states[str(task)] = "done(live-fast)"
+        except Exception as e:
+            log = f"{log}\n\n--- Live task analysis failed ---\n{e}"
+            status = f"Task {task} failed (and live fallback failed)."
+            task_states[str(task)] = "failed"
     else:
         if used_bundled:
             _bundle_task_outputs(model_bundle, output_dir)
@@ -805,7 +832,7 @@ def run_single_task(model_bundle, task, output_dir, input_text, task4_phase, tas
         },
         tags={"source": "hf_space", "mode": "single_task"},
     )
-    flow = _build_flow_markdown(model_loaded=True, inference_ready=False, task_states=task_states)
+    flow = _build_flow_markdown(model_loaded=True, inference_ready=True, task_states=task_states)
     return status, log, task_states, flow