Update app.py

app.py

@@ -11,8 +11,10 @@ Gradio App – robuste Async-Ausführung via Status-Endpoints
 - FIX: exclude_hash richtig laden/berücksichtigen (paginierte Wholix-Suche)
 - FIX: pro Lead harte Fehlerisolierung -> kein Gradio-Absturz
 - NEU: Speichert IMMER in Wholix. Falls keine echte E-Mail vorhanden ist,
-       wird eine stabile Platzhalter-Adresse generiert (no-email.invalid)
-       und ein Tag "no-email" gesetzt.
+       wird eine stabile Platzhalter-Adresse generiert (example.com) und
+       ein Tag "no-email" gesetzt.
+- NEU: 422-Robustheit -> progressive Degradation & Minimal-Record-Fallback
+- NEU: URL-Normalisierung (Umlaute/IDNA) für Pfad/Query/Fragment
 """
 
 import json
@@ -40,7 +42,7 @@ WHOLIX_BASE_URL = "https://api.wholix.ai"
 
 MAX_LEADS       = 100          # Sicherheitskappe
 MAX_WORKERS     = 16           # gleichzeitige Pipelines am Server
-JOB_TTL_SEC     = 60 * 60     # wie lange fertige Jobs im Speicher bleiben (1h)
+JOB_TTL_SEC     = 60 * 60      # wie lange fertige Jobs im Speicher bleiben (1h)
 
 # ============================== LOGGING ====================================
 
@@ -148,6 +150,46 @@ def req(
             return _safe_json(text)
     return text
 
+# ========================= URL NORMALISIERUNG ==============================
+
+from urllib.parse import urlsplit, urlunsplit, quote
+
+def _normalize_url(u: Optional[str]) -> Optional[str]:
+    """
+    Normalisiert URLs für strenge Validatoren:
+    - Scheme ergänzen (https)
+    - Host -> IDNA (punycode)
+    - Path/Query/Fragment UTF-8 percent-encoden
+    """
+    if not u:
+        return None
+    u = u.strip()
+    if not re.match(r"^[a-zA-Z][a-zA-Z0-9+\-.]*://", u):
+        u = "https://" + u
+    try:
+        parts = urlsplit(u)
+        if not parts.scheme or not parts.netloc:
+            return None
+        # Host zu IDNA
+        try:
+            host = parts.hostname.encode("idna").decode("ascii") if parts.hostname else ""
+        except Exception:
+            return None
+        netloc = host
+        if parts.port:
+            netloc += f":{parts.port}"
+        if parts.username:
+            auth = parts.username
+            if parts.password:
+                auth += f":{parts.password}"
+            netloc = f"{auth}@{netloc}"
+        path = quote(parts.path or "", safe="/-._~")
+        query = quote(parts.query or "", safe="=&?/-._~")
+        fragment = quote(parts.fragment or "", safe="/-._~")
+        return urlunsplit((parts.scheme.lower(), netloc.lower(), path, query, fragment))
+    except Exception:
+        return None
+
 # ========================= Async Start & Poll ==============================
 
 def _looks_like_ngrok_html(body: Any) -> bool:
@@ -410,6 +452,8 @@ def wholix_login(email: str, password: str) -> str:
 
 # ===================== Helfer für Platzhalter-E-Mail =======================
 
+EMAIL_RE = re.compile(r"^[^@\s]+@[^@\s]+\.[^@\s]+$")
+
 def _slug(val: Any, maxlen: int = 24) -> str:
     s = "" if val is None else str(val)
     s = s.lower().strip()
@@ -419,25 +463,70 @@ def _slug(val: Any, maxlen: int = 24) -> str:
 
 
 def _make_placeholder_email(record: dict) -> str:
+    # Stable, valide, kurz
     fn = _slug(record.get("firstname"))
     ln = _slug(record.get("lastname"))
-    co = _slug(record.get("company_name"))
+    base = (fn + "." + ln).strip(".") or "lead"
     cid = _slug(record.get("exclude_hash") or uuid.uuid4().hex[:8], maxlen=16)
-    local = "-".join([p for p in [fn, ln, co] if p]) or "lead"
-    local = re.sub(r"\.+", ".", local.replace("-", "."))
-    local = local.strip(".")[:48]
-    return f"{local}.{cid}@no-email.invalid"
+    local = f"{base}.{cid}"[:48]
+    return f"{local}@example.com"  # RFC-Reservedomäne, von Validatoren akzeptiert
+
+# ===================== Wholix Store + Fallbacks ============================
+
+def _store_with_fallbacks(token: str, payload: dict, module: str) -> dict:
+    """
+    Versucht 3 Stufen, um 422 zu vermeiden:
+    1) Original-Payload
+    2) Mit reparierten/entfernten URLs
+    3) Minimal-Record (email, firstname, lastname, company_name, exclude_hash, tags)
+    """
+    url = f"{WHOLIX_BASE_URL}/api/v1/table-object-data/store-objects"
+    headers = {"Authorization": f"Bearer {token}"}
+
+    def _post(body):
+        return req(url, method="POST", headers=headers, json_body=body, timeout=(5.0, 30.0))
+
+    # 1: Original
+    body1 = {"module": module, "action": "store", "data": [payload]}
+    try:
+        return _post(body1)
+    except HTTPError as e:
+        if e.status != 422:
+            raise
+
+    # 2: URLs reparieren/entfernen
+    p2 = dict(payload)
+    for k in ("linkedin_url", "company_url"):
+        if p2.get(k):
+            fixed = _normalize_url(p2.get(k))
+            if fixed:
+                p2[k] = fixed
+            else:
+                p2.pop(k, None)
+    try:
+        return _post({"module": module, "action": "store", "data": [p2]})
+    except HTTPError as e:
+        if e.status != 422:
+            raise
+
+    # 3: Minimal-Record
+    minimal = {
+        "email": p2.get("email") or _make_placeholder_email(p2),
+        "firstname": p2.get("firstname") or "",
+        "lastname": p2.get("lastname") or "",
+        "company_name": p2.get("company_name") or "",
+        "exclude_hash": p2.get("exclude_hash") or _slug(uuid.uuid4().hex[:8]),
+        "tags": p2.get("tags") or {"keys": ["no-email"], "values": ["no-email"]},
+    }
+    return _post({"module": module, "action": "store", "data": [minimal]})
 
 
 def wholix_store_contact(token: str, record: dict, module: str = "Contacts", allow_placeholder: bool = True) -> dict:
     """
     Sendet NUR erlaubte Felder an Wholix und saniert problematische Werte.
-    NEU: Wenn keine E-Mail vorhanden ist und allow_placeholder=True,
-         wird automatisch eine Platzhalter-Adresse erzeugt (no-email.invalid),
-         damit der Datensatz *immer* gespeichert werden kann.
+    IMMER speicherbar: erzeugt bei Bedarf Placeholder-Mail (example.com) + Tag.
+    Nutzt _store_with_fallbacks gegen 422.
     """
-    import urllib.parse as _urlparse
-
     if not isinstance(record, dict):
         raise ValueError("Wholix: record muss ein dict sein.")
 
@@ -449,14 +538,15 @@ def wholix_store_contact(token: str, record: dict, module: str = "Contacts", all
         return s if s else None
 
     email = _clean_str(record.get("email"))
-    if not email and allow_placeholder:
-        email = _make_placeholder_email(record)
-        record["email"] = email
-        # Kennzeichnungs-Tag ergänzen (wird später sauber normiert)
-        if isinstance(record.get("tags"), dict):
-            keys = list({*record["tags"].get("keys", []), "no-email"})
-            vals = list({*record["tags"].get("values", []), "no-email"})
-            record["tags"] = {"keys": keys, "values": vals}
+    if not email or not EMAIL_RE.match(email):
+        if allow_placeholder:
+            email = _make_placeholder_email(record)
+            record["email"] = email
+            if isinstance(record.get("tags"), dict):
+                for k in ("keys", "values"):
+                    record["tags"].setdefault(k, [])
+                    if "no-email" not in record["tags"][k]:
+                        record["tags"][k].append("no-email")
 
     ALLOWED = {
         "firstname",
@@ -491,20 +581,6 @@ def wholix_store_contact(token: str, record: dict, module: str = "Contacts", all
                 v = v[1:-1].strip().strip("'\"")
         return _clean_str(v)
 
-    def _normalize_url(u):
-        u = _clean_str(u)
-        if not u:
-            return None
-        if not re.match(r"^[a-zA-Z][a-zA-Z0-9+\-.]*://", u):
-            u = "https://" + u
-        try:
-            pr = _urlparse.urlparse(u)
-            if not pr.scheme or not pr.netloc:
-                return None
-            return u
-        except Exception:
-            return None
-
     out = {}
     for k in ALLOWED:
         if k not in record:
@@ -527,10 +603,8 @@ def wholix_store_contact(token: str, record: dict, module: str = "Contacts", all
     # Pflichtfeld sicher (jetzt inkl. Platzhalter möglich)
     out["email"] = _clean_str(email)
 
-    url = f"{WHOLIX_BASE_URL}/api/v1/table-object-data/store-objects"
-    headers = {"Authorization": f"Bearer {token}"}
-    body = {"module": module, "action": "store", "data": [out]}
-    return req(url, method="POST", headers=headers, json_body=body, timeout=(5.0, 30.0))
+    # POST mit Fallback-Logik
+    return _store_with_fallbacks(token, out, module)
 
 
 # ======= NEW: Wholix-Excludes paginiert laden (wie in deinem JS) ===========
@@ -847,19 +921,7 @@ def run_pipeline_bg(job_id: str, curl_text: str, n_leads_ui: int):
         return s or None
 
     def norm_url(u: Any) -> Optional[str]:
-        u = "" if u is None else str(u).strip()
-        if not u:
-            return None
-        if not re.match(r"^[a-zA-Z][a-zA-Z0-9+\-.]*://", u):
-            u = "https://" + u
-        try:
-            from urllib.parse import urlparse
-            pr = urlparse(u)
-            if pr.scheme and pr.netloc:
-                return u
-        except Exception:
-            pass
-        return None
+        return _normalize_url(u)
 
     for i in range(1, n_leads + 1):
         try:
@@ -899,7 +961,7 @@ def run_pipeline_bg(job_id: str, curl_text: str, n_leads_ui: int):
                 "Checkliste_Landingpage": checklist,
                 "homepage_url": homepage_url,   # optional
                 "tags": raw_tag,                # kommt als "[AI]" → später normiert
-                "Touch_Point": "LinkedIn DM",   # optional
+                "Touch_Point": "LinkedIn DM",  # optional
             }
             try:
                 draft = email_generate_async(token_id, variables, items)