Update bot/integrations/http.py

bot/integrations/http.py

@@ -1,118 +1,182 @@
 # PATH: bot/integrations/http.py
-import asyncio
-from typing import Any, Dict, Optional
+import ssl
+import json as _json
+from typing import Any, Dict, Optional, Tuple, List
+from urllib.parse import urlparse, urlunparse
 
 import httpx
 
-# One global client
+
+_DEFAULT_TIMEOUT = httpx.Timeout(20.0, connect=10.0)
+
 _client: Optional[httpx.AsyncClient] = None
+_client_insecure: Optional[httpx.AsyncClient] = None
 
 
 def get_client() -> httpx.AsyncClient:
-    """
-    Shared HTTP client for Worker calls.
-    - trust_env=False avoids proxy/env issues on HF
-    """
     global _client
-    if _client is not None:
-        return _client
-
-    timeout = httpx.Timeout(
-        connect=10.0,
-        read=60.0,
-        write=60.0,
-        pool=10.0,
-    )
-
-    limits = httpx.Limits(
-        max_connections=50,
-        max_keepalive_connections=20,
-        keepalive_expiry=30.0,
-    )
-
-    _client = httpx.AsyncClient(
-        timeout=timeout,
-        limits=limits,
-        follow_redirects=True,
-        http2=True,
-        trust_env=False,
-        headers={
-            "user-agent": "YouTubeLoaderHF/1.0",
-            "accept": "application/json,text/plain,*/*",
-        },
-    )
+    if _client is None:
+        _client = httpx.AsyncClient(
+            timeout=_DEFAULT_TIMEOUT,
+            follow_redirects=True,
+            http2=True,
+        )
     return _client
 
 
-async def request_json(
-    method: str,
-    url: str,
-    *,
-    headers: Optional[Dict[str, str]] = None,
-    json_body: Optional[Dict[str, Any]] = None,
-    data: Optional[Any] = None,
-    retries: int = 4,
-    backoff_base: float = 0.6,
-) -> Dict[str, Any]:
+def get_client_insecure() -> httpx.AsyncClient:
     """
-    Returns dict always:
-      - success: JSON dict (adds ok=True if missing)
-      - error:   {"ok": False, "status": <int>, "err": "<msg>"}
+    Insecure client used only for IP fallback (TLS verify off),
+    because we connect to https://<IP> with Host header.
     """
-    c = get_client()
-    last_err: Optional[BaseException] = None
-
-    for attempt in range(retries):
-        try:
-            r = await c.request(
-                method.upper(),
-                url,
-                headers=headers,
-                json=json_body,
-                data=data,
-            )
-
-            # parse JSON if possible
-            try:
-                j = r.json()
-            except Exception:
-                txt = (r.text or "").strip()
-                j = {"raw": txt}
+    global _client_insecure
+    if _client_insecure is None:
+        _client_insecure = httpx.AsyncClient(
+            timeout=_DEFAULT_TIMEOUT,
+            follow_redirects=True,
+            http2=True,
+            verify=False,
+        )
+    return _client_insecure
 
-            if r.is_success:
-                if isinstance(j, dict):
-                    j.setdefault("ok", True)
-                    return j
-                return {"ok": True, "raw": j}
 
-            # non-2xx
-            if isinstance(j, dict):
-                j.setdefault("ok", False)
-                j.setdefault("status", r.status_code)
-                j.setdefault("err", f"http_{r.status_code}")
-                return j
+def _is_dns_error(e: Exception) -> bool:
+    s = str(e).lower()
+    return ("no address associated with hostname" in s) or ("gaierror" in s) or ("name or service not known" in s)
 
-            return {"ok": False, "status": r.status_code, "err": f"http_{r.status_code}"}
 
-        except (httpx.ConnectError, httpx.ConnectTimeout, httpx.ReadTimeout, httpx.RemoteProtocolError) as e:
-            last_err = e
-        except OSError as e:
-            # DNS / low-level network issues (includes Errno -5)
-            last_err = e
-        except Exception as e:
-            last_err = e
+async def doh_resolve_a(host: str) -> List[str]:
+    """
+    Resolve A records via DNS-over-HTTPS.
+    Uses dns.google (since google.com resolves fine in your env).
+    """
+    c = get_client()
+    try:
+        r = await c.get("https://dns.google/resolve", params={"name": host, "type": "A"})
+        if r.status_code != 200:
+            return []
+        j = r.json()
+        ans = j.get("Answer") or []
+        ips = []
+        for a in ans:
+            if a.get("type") == 1 and a.get("data"):
+                ips.append(str(a["data"]))
+        # unique keep order
+        out = []
+        for ip in ips:
+            if ip not in out:
+                out.append(ip)
+        return out
+    except Exception:
+        return []
+
+
+def _split_url(url: str) -> Tuple[str, str, str]:
+    """
+    Returns (scheme, host, rest(path+query+fragment))
+    """
+    u = urlparse(url)
+    scheme = u.scheme or "https"
+    host = u.netloc
+    rest = urlunparse(("", "", u.path or "/", u.params, u.query, u.fragment))
+    return scheme, host, rest
 
-        if attempt < retries - 1:
-            await asyncio.sleep(backoff_base * (2 ** attempt))
 
-    return {"ok": False, "status": 0, "err": str(last_err) if last_err else "unknown_error"}
+async def fetch_status(url: str) -> str:
+    """
+    Returns status code as string, or ERR:...
+    Uses same DNS fallback logic.
+    """
+    try:
+        r = await request_text("GET", url)
+        return str(r[0])
+    except Exception as e:
+        return f"ERR:{type(e).__name__}:{e}"
 
 
-# ----------------- Backward-compatible helpers -----------------
-# (your old files import these names)
+async def request_text(method: str, url: str, headers: Optional[Dict[str, str]] = None) -> Tuple[int, str]:
+    """
+    Low-level text request with DNS fallback.
+    Returns (status_code, text)
+    """
+    headers = dict(headers or {})
+    c = get_client()
+    try:
+        r = await c.request(method, url, headers=headers)
+        return r.status_code, r.text
+    except Exception as e:
+        # DNS fail -> DoH fallback
+        if not _is_dns_error(e):
+            raise
+
+        scheme, host, rest = _split_url(url)
+        if not host:
+            raise
+
+        ips = await doh_resolve_a(host)
+        if not ips:
+            raise
+
+        ic = get_client_insecure()
+        last_err = e
+        for ip in ips:
+            try:
+                # connect to IP but keep routing via Host header
+                h2 = dict(headers)
+                h2["Host"] = host
+                r = await ic.request(method, f"{scheme}://{ip}{rest}", headers=h2)
+                return r.status_code, r.text
+            except Exception as e2:
+                last_err = e2
+                continue
+        raise last_err
+
+
+async def post_json(url: str, headers: Optional[Dict[str, str]] = None, payload: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
+    """
+    POST JSON helper with DNS fallback.
+    Returns dict: {ok, status, data?, err?}
+    """
+    headers = dict(headers or {})
+    headers.setdefault("Content-Type", "application/json")
 
-async def post_json(url: str, payload: Dict[str, Any], headers: Optional[Dict[str, str]] = None) -> Dict[str, Any]:
-    return await request_json("POST", url, headers=headers, json_body=payload)
+    c = get_client()
+    payload = payload or {}
 
-async def get_json(url: str, headers: Optional[Dict[str, str]] = None) -> Dict[str, Any]:
-    return await request_json("GET", url, headers=headers)
+    try:
+        r = await c.post(url, headers=headers, json=payload)
+        try:
+            data = r.json()
+        except Exception:
+            data = {"raw": r.text}
+        return {"ok": r.status_code < 400, "status": r.status_code, "data": data}
+    except Exception as e:
+        if not _is_dns_error(e):
+            return {"ok": False, "status": 0, "err": str(e)}
+
+        # DNS fail -> DoH -> IP fallback (verify off + Host header)
+        scheme, host, rest = _split_url(url)
+        if not host:
+            return {"ok": False, "status": 0, "err": str(e)}
+
+        ips = await doh_resolve_a(host)
+        if not ips:
+            return {"ok": False, "status": 0, "err": str(e)}
+
+        ic = get_client_insecure()
+        last_err: Exception = e
+        for ip in ips:
+            try:
+                h2 = dict(headers)
+                h2["Host"] = host
+                r = await ic.post(f"{scheme}://{ip}{rest}", headers=h2, json=payload)
+                try:
+                    data = r.json()
+                except Exception:
+                    data = {"raw": r.text}
+                return {"ok": r.status_code < 400, "status": r.status_code, "data": data}
+            except Exception as e2:
+                last_err = e2
+                continue
+
+        return {"ok": False, "status": 0, "err": str(last_err)}