Update Deployment_UI_BE.py

Deployment_UI_BE.py

@@ -755,83 +755,63 @@ async def api_middleware_infer(req: Request):
     route = _INST.get("predictRoute") or "/predict"
     _INST["predictRoute"] = route
 
+    # Build deterministic proxy URL instead of waiting on readiness
+    pid = (_INST.get("podId") or "").strip()
+    if not pid:
+        try:
+            _load_state()
+            pid = (_INST.get("podId") or "").strip()
+        except Exception:
+            pass
+    if not pid:
+        return JSONResponse({"error": "no podId yet (create/start first)"}, status_code=400)
+
+    cspec = _get_container_spec()
+    internal, _ = _get_port_and_proto(cspec)
+    if not internal:
+        return JSONResponse({"error": "cannot resolve internal port from blob"}, status_code=400)
+
+    base = f"https://{pid}-{internal}.proxy.runpod.net"
+    url = f"{base}{route}"
+    _log_status(f"PROMPT_ENDPOINT {url}")
+    _job_log("compute", f"[MW] Forwarding infer to {url}")
+
     payload = await req.json()
     prompt = payload.get("prompt")
     if not isinstance(prompt, str) or not prompt.strip():
-        return JSONResponse({"error": "Missing 'prompt' in request body."}, status_code=400)
+        return JSONResponse({"error": "Missing 'prompt' in request body."}, 400)
 
-    # HF text-classification shim: wrap into Vertex-style instances
+    # HF text-classification shim
     img = (_get_container_spec().get("imageUri","")).lower()
     if "huggingface-pytorch-inference" in img and isinstance(payload.get("prompt"), str):
         payload = {"instances": [payload["prompt"]]}
 
-    try:
-        # Prefer proxy base; fall back to direct IP if proxy not cached
-        pid = (_INST.get("podId") or "").strip()
-        proxy_base = None
+    # Send immediately; no readiness checks or polling gates
+    bodies = [payload, {"prompt": prompt}, {"text": prompt}, {"inputs": prompt}, {"input": prompt}]
+    for body in bodies:
         try:
-            cspec = _get_container_spec()
-            internal, _ = _get_port_and_proto(cspec)
-            if pid and internal:
-                proxy_base = f"https://{pid}-{internal}.proxy.runpod.net"
-                _log_status(f"PROMPT_BASE proxy={proxy_base}")
-        except Exception:
-            pass
-        if not proxy_base:
-            ip, port = _INST.get("ip"), _INST.get("port")
-            if ip and port:
-                proxy_base = f"http://{ip}:{port}"
-                _log_status(f"PROMPT_BASE direct={proxy_base}")
-
-        # If neither path is available, do not gate—return clear error
-        if not proxy_base:
-            return JSONResponse({"error": "no reachable base yet (no proxy or ip/port)"}, status_code=503)
-
-        url = f"{proxy_base}{route}"
-        _log_status(f"PROMPT_ENDPOINT {url}")
-        _job_log("compute", f"[MW] Forwarding infer to {url}")
-
-        # Try multiple prompt body formats until success
-        bodies = [
-            payload,
-            {"prompt": prompt},
-            {"text": prompt},
-            {"inputs": prompt},
-            {"input": prompt},
-        ]
-        rp, data = None, None
-        for body in bodies:
-            try:
-                rp = requests.post(url, json=body, timeout=120)
-                _log_status(f"PREDICT_RESP code={rp.status_code} len={len(rp.text)}")
-                if rp.ok:
-                    break
-            except Exception as e:
-                _log_status(f"PREDICT_ERR {e}")
-
-        if not rp:
-            return JSONResponse({"error": "no response from model"}, status_code=504)
-
-        ct = (rp.headers.get("content-type") or "").lower()
-        data = _as_json(rp) if "application/json" in ct else {"_raw": rp.text}
-
-        if isinstance(data, dict):
-            if "image_b64" in data:
-                return JSONResponse({"image_b64": data["image_b64"], "timings": data.get("timings")}, status_code=rp.status_code)
-            if isinstance(data.get("output"), str):
-                return JSONResponse({"output": data["output"]}, status_code=rp.status_code)
-            if "_raw" in data:
-                return JSONResponse({"output": data["_raw"]}, status_code=rp.status_code)
-            return JSONResponse({"output": json.dumps(data, ensure_ascii=False)}, status_code=rp.status_code)
-
-        return JSONResponse({"output": str(data)}, status_code=rp.status_code)
-
-    except HTTPException as he:
-        _job_log("compute", f"[MW] ERROR {he.status_code}: {he.detail}")
-        return JSONResponse({"error": he.detail}, status_code=he.status_code)
-    except Exception as e:
-        _job_log("compute", f"[MW] ERROR infer: {e}")
-        return JSONResponse({"error": f"middleware infer failed: {e}"}, status_code=502)
+            rp = requests.post(url, json=body, timeout=120)
+            _log_status(f"PREDICT_RESP code={rp.status_code} len={len(rp.text)}")
+            if rp.ok:
+                ct = (rp.headers.get("content-type") or "").lower()
+                data = _as_json(rp) if "application/json" in ct else {"_raw": rp.text}
+                if isinstance(data, dict):
+                    if "image_b64" in data:
+                        return JSONResponse({"image_b64": data["image_b64"], "timings": data.get("timings")}, rp.status_code)
+                    if isinstance(data.get("output"), str):
+                        return JSONResponse({"output": data["output"]}, rp.status_code)
+                    if "_raw" in data:
+                        return JSONResponse({"output": data["_raw"]}, rp.status_code)
+                    return JSONResponse({"output": json.dumps(data, ensure_ascii=False)}, rp.status_code)
+                return JSONResponse({"output": str(data)}, rp.status_code)
+        except Exception as e:
+            _log_status(f"PREDICT_ERR {e}")
+
+    # Fallthrough: show last response or generic error
+    try:
+        return JSONResponse({"error": rp.text[:400]}, status_code=rp.status_code)
+    except Exception:
+        return JSONResponse({"error": "no response from model"}, status_code=504)
 # ---------------------------------------------------------------------
 # Job progress + callback routes
 # ---------------------------------------------------------------------