Update bot/integrations/http.py

bot/integrations/http.py

@@ -1,25 +1,116 @@
 # PATH: bot/integrations/http.py
+import asyncio
+from typing import Any, Dict, Optional
+
 import httpx
-from bot.core.settings import HTTP_TIMEOUT_SEC, MAX_RETRIES
 
-_async_client: httpx.AsyncClient | None = None
+# One global client (HF no persistence, but process lives)
+_client: Optional[httpx.AsyncClient] = None
+
 
 def get_client() -> httpx.AsyncClient:
-    global _async_client
-    if _async_client is None:
-        _async_client = httpx.AsyncClient(timeout=HTTP_TIMEOUT_SEC, follow_redirects=True)
-    return _async_client
+    """
+    Shared HTTP client for Worker calls.
+    - Retries handled in request_json()
+    - trust_env=False avoids weird proxy/env issues on HF
+    """
+    global _client
+    if _client is not None:
+        return _client
+
+    timeout = httpx.Timeout(
+        connect=10.0,
+        read=60.0,
+        write=60.0,
+        pool=10.0,
+    )
+
+    limits = httpx.Limits(
+        max_connections=50,
+        max_keepalive_connections=20,
+        keepalive_expiry=30.0,
+    )
+
+    _client = httpx.AsyncClient(
+        timeout=timeout,
+        limits=limits,
+        follow_redirects=True,
+        http2=True,
+        trust_env=False,
+        headers={
+            "user-agent": "YouTubeLoaderHF/1.0",
+            "accept": "application/json,text/plain,*/*",
+        },
+    )
+    return _client
 
-async def post_json(url: str, headers: dict, payload: dict) -> dict:
+
+async def request_json(
+    method: str,
+    url: str,
+    *,
+    headers: Optional[Dict[str, str]] = None,
+    json_body: Optional[Dict[str, Any]] = None,
+    data: Optional[Any] = None,
+    retries: int = 4,
+    backoff_base: float = 0.6,
+) -> Dict[str, Any]:
+    """
+    Standard wrapper used by cf_worker1/cf_worker2.
+    Returns a dict always:
+      - on success: parsed JSON dict (or {"ok": True, "raw": "..."} if non-json)
+      - on error: {"ok": False, "status": <int>, "err": "<msg>"}
+
+    Handles DNS glitches like:
+      [Errno -5] No address associated with hostname
+    """
     c = get_client()
     last_err = None
-    for _ in range(MAX_RETRIES + 1):
+
+    for attempt in range(retries):
         try:
-            r = await c.post(url, headers=headers, json=payload)
-            data = r.json() if r.headers.get("content-type","").startswith("application/json") else {}
-            if r.status_code >= 400:
-                return {"ok": False, "status": r.status_code, "data": data, "text": r.text[:300]}
-            return data
+            r = await c.request(
+                method.upper(),
+                url,
+                headers=headers,
+                json=json_body,
+                data=data,
+            )
+
+            # Try parse JSON always
+            try:
+                j = r.json()
+            except Exception:
+                txt = (r.text or "").strip()
+                j = {"ok": r.is_success, "raw": txt}
+
+            if r.is_success:
+                # if API didn't include ok, add it
+                if isinstance(j, dict) and "ok" not in j:
+                    j["ok"] = True
+                return j
+
+            # non-2xx -> return error payload
+            if isinstance(j, dict):
+                # keep server error if present
+                j.setdefault("ok", False)
+                j.setdefault("status", r.status_code)
+                if "err" not in j:
+                    j["err"] = f"http_{r.status_code}"
+                return j
+
+            return {"ok": False, "status": r.status_code, "err": f"http_{r.status_code}"}
+
+        except (httpx.ConnectError, httpx.ConnectTimeout, httpx.ReadTimeout, httpx.RemoteProtocolError) as e:
+            last_err = e
+        except OSError as e:
+            # DNS / low-level network issues (includes Errno -5)
+            last_err = e
         except Exception as e:
             last_err = e
-    return {"ok": False, "status": 0, "err": str(last_err)}
+
+        # retry with exponential backoff
+        if attempt < retries - 1:
+            await asyncio.sleep(backoff_base * (2 ** attempt))
+
+    return {"ok": False, "status": 0, "err": str(last_err) if last_err else "unknown_error"}