Update app.py

send always to wholix, even if a mail is missing to avoid endless retries, because the exclude_hash is not stored.

app.py

@@ -10,6 +10,9 @@ Gradio App – robuste Async-Ausführung via Status-Endpoints
 - Logging in Konsole + logs/<job_id>.log
 - FIX: exclude_hash richtig laden/berücksichtigen (paginierte Wholix-Suche)
 - FIX: pro Lead harte Fehlerisolierung -> kein Gradio-Absturz
+- NEU: Speichert IMMER in Wholix. Falls keine echte E-Mail vorhanden ist,
+       wird eine stabile Platzhalter-Adresse generiert (no-email.invalid)
+       und ein Tag "no-email" gesetzt.
 """
 
 import json
@@ -342,8 +345,6 @@ def email_generate_async(token_id: str, variables: dict, items: List[dict]) -> d
     if not items:
         raise ValueError("items fehlt (mindestens 1 Lead erforderlich).")
 
-    # Nur die Keys weiterreichen, die dein Backend tatsächlich kennt/erwartet.
-    # Wir filtern NICHT um – wir vertrauen der übergebenen payload (keine Fantasie-Felder).
     root_allowed = {
         "Produkt_und_Dienstleistungsbeschreibung",
         "CTA",
@@ -356,14 +357,11 @@ def email_generate_async(token_id: str, variables: dict, items: List[dict]) -> d
 
     safe_vars = {}
     for k, v in (variables or {}).items():
-        # 1:1 durchlassen für die bekannten Felder
         if k in root_allowed:
             safe_vars[k] = v
 
-    # Payload für den echten Endpoint: Root-Variablen + Items
     start_body = {**safe_vars, "items": items, "item_index": 0}
 
-    # Debug-Log: zeigen, welche Keys wir wirklich senden (inkl. Signatur)
     try:
         sent_keys = sorted(list(start_body.keys()))
         sig_preview = str(start_body.get("Signatur") or "")[:120]
@@ -372,7 +370,6 @@ def email_generate_async(token_id: str, variables: dict, items: List[dict]) -> d
     except Exception:
         pass
 
-    # Async-Start & Poll
     res_job_id = start_async_job(
         url=f"{LEAD_BASE_URL}/email/generate?async=1",
         body=start_body,
@@ -385,7 +382,6 @@ def email_generate_async(token_id: str, variables: dict, items: List[dict]) -> d
         session=GLOBAL_SES,
     )
 
-    # Ergebnis 1:1 auswerten – nur echte Felder
     raw = unwrap_result(st) or {}
     results = raw.get("results") if isinstance(raw, dict) else None
     first = (results[0] if isinstance(results, list) and results else {}) or {}
@@ -412,27 +408,60 @@ def wholix_login(email: str, password: str) -> str:
         raise RuntimeError("Wholix-Login fehlgeschlagen.")
     return token
 
-def wholix_store_contact(token: str, record: dict, module: str = "Contacts") -> dict:
+# ===================== Helfer für Platzhalter-E-Mail =======================
+
+def _slug(val: Any, maxlen: int = 24) -> str:
+    s = "" if val is None else str(val)
+    s = s.lower().strip()
+    s = re.sub(r"[^a-z0-9]+", "-", s)
+    s = re.sub(r"-+", "-", s).strip("-")
+    return s[:maxlen] or "x"
+
+
+def _make_placeholder_email(record: dict) -> str:
+    fn = _slug(record.get("firstname"))
+    ln = _slug(record.get("lastname"))
+    co = _slug(record.get("company_name"))
+    cid = _slug(record.get("exclude_hash") or uuid.uuid4().hex[:8], maxlen=16)
+    local = "-".join([p for p in [fn, ln, co] if p]) or "lead"
+    local = re.sub(r"\.+", ".", local.replace("-", "."))
+    local = local.strip(".")[:48]
+    return f"{local}.{cid}@no-email.invalid"
+
+
+def wholix_store_contact(token: str, record: dict, module: str = "Contacts", allow_placeholder: bool = True) -> dict:
     """
-    Sendet NUR erlaubte Felder an Wholix und saniert problematische Werte:
-      - URLs (linkedin_url/company_url): Schema ergänzen, invalid droppen
-      - departments: immer Plain-Text
-      - Multi-Select (status_field/tags): nur korrektes {keys,values}
-      - Leere Strings -> None und weglassen
+    Sendet NUR erlaubte Felder an Wholix und saniert problematische Werte.
+    NEU: Wenn keine E-Mail vorhanden ist und allow_placeholder=True,
+         wird automatisch eine Platzhalter-Adresse erzeugt (no-email.invalid),
+         damit der Datensatz *immer* gespeichert werden kann.
     """
     import urllib.parse as _urlparse
 
     if not isinstance(record, dict):
         raise ValueError("Wholix: record muss ein dict sein.")
 
-    email = str((record.get("email") or "")).strip()
-    if not email:
-        raise ValueError("Wholix: 'email' ist Pflichtfeld.")
+    # E-Mail prüfen / ggf. generieren
+    def _clean_str(v):
+        if v is None:
+            return None
+        s = str(v).strip()
+        return s if s else None
+
+    email = _clean_str(record.get("email"))
+    if not email and allow_placeholder:
+        email = _make_placeholder_email(record)
+        record["email"] = email
+        # Kennzeichnungs-Tag ergänzen (wird später sauber normiert)
+        if isinstance(record.get("tags"), dict):
+            keys = list({*record["tags"].get("keys", []), "no-email"})
+            vals = list({*record["tags"].get("values", []), "no-email"})
+            record["tags"] = {"keys": keys, "values": vals}
 
     ALLOWED = {
         "firstname",
         "lastname",
-        "email",                 # Pflichtfeld
+        "email",                 # Pflichtfeld (ggf. Platzhalter)
         "adress",                # (sic) genau so
         "city",
         "postcode",
@@ -451,12 +480,6 @@ def wholix_store_contact(token: str, record: dict, module: str = "Contacts") ->
         "tags",                  # Multi-Select: {keys:[], values:[]}
     }
 
-    def _clean_str(v):
-        if v is None:
-            return None
-        s = str(v).strip()
-        return s if s else None
-
     def _coerce_departments(v):
         if v is None:
             return None
@@ -464,7 +487,6 @@ def wholix_store_contact(token: str, record: dict, module: str = "Contacts") ->
             v = ", ".join(str(x).strip() for x in v if str(x).strip())
         else:
             v = str(v).strip()
-            # String wie "[Marketing]" -> "Marketing"
             if v.startswith("[") and v.endswith("]"):
                 v = v[1:-1].strip().strip("'\"")
         return _clean_str(v)
@@ -494,7 +516,6 @@ def wholix_store_contact(token: str, record: dict, module: str = "Contacts") ->
         elif k in ("linkedin_url", "company_url"):
             val = _normalize_url(val)
         elif k in ("status_field", "tags"):
-            # nur korrektes Schema durchlassen
             if not (isinstance(val, dict) and "keys" in val and "values" in val):
                 val = None
         else:
@@ -503,7 +524,7 @@ def wholix_store_contact(token: str, record: dict, module: str = "Contacts") ->
         if val is not None:
             out[k] = val
 
-    # Pflichtfeld sicher
+    # Pflichtfeld sicher (jetzt inkl. Platzhalter möglich)
     out["email"] = _clean_str(email)
 
     url = f"{WHOLIX_BASE_URL}/api/v1/table-object-data/store-objects"
@@ -615,8 +636,8 @@ def _gc_jobs():
 
 LEAD_COUNTS = [1, 2, 3, 4, 5, 10, 15, 20, 40, 80, 100, 200, 300, 400, 500, 1000]
 
-CURL_DATA_RE = re.compile(r"""--data(?:-raw)?\s+(?P<q>['"])(?P<body>.*?)(?P=q)""", re.DOTALL)
-HDR_XTOKEN_RE = re.compile(r"""-H\s+(?P<q>['"])X-Token-Id:\s*(?P<val>[^'"]+)(?P=q)""", re.IGNORECASE)
+CURL_DATA_RE = re.compile(r"""--data(?:-raw)?\s+(?P<q>['\"])(?P<body>.*?)(?P=q)""", re.DOTALL)
+HDR_XTOKEN_RE = re.compile(r"""-H\s+(?P<q>['\"])X-Token-Id:\s*(?P<val>[^'\"]+)(?P=q)""", re.IGNORECASE)
 
 def _find_data_quote_start(s: str) -> Tuple[int, Optional[str]]:
     m = re.search(r"--data(?:-raw)?\s+(['\"])", s)
@@ -653,14 +674,11 @@ def parse_curl(curl_text: str) -> Tuple[str, Dict[str, Any]]:
     if not isinstance(curl_text, str) or not curl_text.strip():
         raise ValueError("Leerer curl-Text.")
 
-    # 1) Token aus Header holen (robust für einfache/doppelte Quotes)
     token_id = ""
-    # -H 'X-Token-Id: abc'   oder   -H "X-Token-Id: abc"
     m = re.search(r"""-H\s+(["'])X-Token-Id:\s*([^"']+)\1""", curl_text, re.IGNORECASE)
     if m:
         token_id = m.group(2).strip()
 
-    # fallback: unquoted header
     if not token_id:
         m2 = re.search(r"""X-Token-Id:\s*([A-Za-z0-9\-\._]+)""", curl_text, re.IGNORECASE)
         if m2:
@@ -673,19 +691,16 @@ def parse_curl(curl_text: str) -> Tuple[str, Dict[str, Any]]:
     if not token_id:
         raise ValueError("Konnte keinen X-Token-Id Header im curl (oder env X_TOKEN_ID) finden.")
 
-    # 2) JSON-Body nach --data / --data-raw extrahieren (einfach/doppelt-quoted)
-    qmatch = re.search(r"""--data(?:-raw)?\s+(['"])""", curl_text)
+    qmatch = re.search(r"""--data(?:-raw)?\s+(['\"])""", curl_text)
     if not qmatch:
         raise ValueError("Konnte den JSON Body nicht finden (erwarte --data-raw '...').")
     quote = qmatch.group(1)
     start = qmatch.end(1)
 
-    # Payload bis zum passenden schließenden Quote lesen (beachtet Escapes)
     out = []
     i = start
     n = len(curl_text)
     if quote == "'":
-        # Bash-Rule: in single quotes sind nur Sequenzen '\"'\"' als eingebettetes einzelnes '
         while i < n:
             if i + 4 < n and curl_text[i:i+5] == "'\"'\"'":
                 out.append("'")
@@ -697,7 +712,6 @@ def parse_curl(curl_text: str) -> Tuple[str, Dict[str, Any]]:
             out.append(ch)
             i += 1
     else:
-        # Doppel-Quotes: Backslashes beachten
         while i < n:
             ch = curl_text[i]
             if ch == "\\" and i + 1 < n:
@@ -710,11 +724,9 @@ def parse_curl(curl_text: str) -> Tuple[str, Dict[str, Any]]:
             i += 1
 
     body_str = "".join(out).strip()
-    # 3) JSON laden (ohne Felder zu „erdenken“)
     try:
         payload = json.loads(body_str)
     except json.JSONDecodeError:
-        # CRLF -> LF und nochmal versuchen
         payload = json.loads(body_str.replace("\r\n", "\n").replace("\r", "\n"))
 
     return token_id, payload
@@ -801,18 +813,11 @@ def run_pipeline_bg(job_id: str, curl_text: str, n_leads_ui: int):
         return None
 
     def norm_tags(raw) -> dict:
-        """
-        Multi-Select für Wholix: {keys:[…],values:[…]}
-        - String "[AI]" -> "AI"
-        - String "AI"   -> "AI"
-        - Liste ["AI","X"] -> entsprechend erweitern
-        """
         if raw is None:
             return {"keys": [], "values": []}
         vals: List[str] = []
         if isinstance(raw, str):
             s = raw.strip()
-            # wenn JSON-Array als String übergeben wurde
             if (s.startswith("[") and s.endswith("]")):
                 try:
                     arr = json.loads(s)
@@ -828,8 +833,6 @@ def run_pipeline_bg(job_id: str, curl_text: str, n_leads_ui: int):
             vals = [str(x).strip() for x in raw if str(x).strip()]
         else:
             vals = [str(raw).strip()]
-
-        # leere filtern
         vals = [v for v in vals if v]
         return {"keys": vals, "values": vals}
 
@@ -913,75 +916,69 @@ def run_pipeline_bg(job_id: str, curl_text: str, n_leads_ui: int):
             _job_emit(job_id, f"   ✉️ Subject: {subj_flat}")
             _job_emit(job_id, f"   ✉️ Message: {body_flat}")
 
-            # 4) Speichern in Wholix
+            # 4) Speichern in Wholix (IMMER – auch ohne echte E-Mail)
             step += 1
             _job_emit(job_id, "   → Speichere in Wholix …", progress=int(step / total_steps * 100))
 
             email_to = str(person.get("email") or "").strip()
-            if not email_to:
-                _job_emit(job_id, "   ⚠️ Keine E-Mail vorhanden – Speichern übersprungen.")
-                stored_ok = False
-            else:
-                try:
-                    # --- Felder befüllen NUR wenn real vorhanden -----------------
-                    firstname   = pick(person, "first_name")
-                    lastname    = pick(person, "last_name")
-                    job_title   = pick(person, "job_title", "title", "position")
-                    departments = norm_departments(pick(person, "departments"))
-                    linkedin    = pick(person, "linkedin_url")
-
-                    # Kontakt-Adressdaten: erst person, dann company (falls vorhanden)
-                    phonenumber = pick(person, "phone", "phonenumber") or pick(company, "phone", "company_phone", "phonenumber")
-                    adress      = pick(person, "adress", "address", "street") or pick(company, "adress", "address", "street")
-                    city        = pick(person, "city") or pick(company, "city")
-                    postcode    = pick(person, "postcode", "postal_code", "zip") or pick(company, "postcode", "postal_code", "zip")
-
-                    company_name = pick(company, "name", "company_name")
-                    company_url  = norm_url(pick(company, "url", "website_url", "website", "domain"))
-
-                    # Tags korrekt als Multi-Select (Dropdown-Wert ohne [])
-                    tags_ms = norm_tags(raw_tag)
-
-                    record = {
-                        "firstname":  firstname,
-                        "lastname":   lastname,
-                        "email":      email_to,
-                        "phonenumber": phonenumber,
-                        "adress":     adress,
-                        "city":       city,
-                        "postcode":   postcode,
-
-                        "job_title":  job_title,
-                        "departments": departments,
-                        "linkedin_url": linkedin,
-
-                        "company_name": company_name,
-                        "company_url":  company_url,
-
-                        "message_mail_subject": subj_raw or None,
-                        "message_mail":         body_raw or None,
-                        "message_followup1":    draft.get("followup1") or None,
-                        "message_followup2":    draft.get("followup2") or None,
-
-                        "exclude_hash": combined_id or None,
-
-                        "status_field": {"keys": ["Kontakt aufgenommen"], "values": ["Kontakt aufgenommen"]},
-                        "tags":         tags_ms,
-                    }
-
-                    # Log: kurze Vorschau der wichtigsten Felder
-                    try:
-                        APP_LOG.info(f"Store-> firstname={firstname} lastname={lastname} email={email_to} job_title={job_title} dept={departments} phone={phonenumber} city={city} postcode={postcode} company_url={company_url}")
-                        APP_LOG.info(f"Store-> tags={tags_ms}")
-                    except Exception:
-                        pass
 
-                    store_res = wholix_store_contact(wh_token, record)
-                    stored_ok = bool(store_res)
-                except Exception:
-                    logging.exception(f"[{job_id}] Wholix-Store-Fehler")
-                    _job_emit(job_id, "❌ Wholix-Store-Fehler: siehe Logs")
-                    stored_ok = False
+            try:
+                # --- Felder befüllen NUR wenn real vorhanden -----------------
+                firstname   = pick(person, "first_name")
+                lastname    = pick(person, "last_name")
+                job_title   = pick(person, "job_title", "title", "position")
+                departments = norm_departments(pick(person, "departments"))
+                linkedin    = pick(person, "linkedin_url")
+
+                phonenumber = pick(person, "phone", "phonenumber") or pick(company, "phone", "company_phone", "phonenumber")
+                adress      = pick(person, "adress", "address", "street") or pick(company, "adress", "address", "street")
+                city        = pick(person, "city") or pick(company, "city")
+                postcode    = pick(person, "postcode", "postal_code", "zip") or pick(company, "postcode", "postal_code", "zip")
+
+                company_name = pick(company, "name", "company_name")
+                company_url  = norm_url(pick(company, "url", "website_url", "website", "domain"))
+
+                tags_ms = norm_tags(raw_tag)
+                # Wenn keine echte E-Mail: Tag ergänzen
+                if not email_to:
+                    tags_ms = {"keys": list({*tags_ms.get("keys", []), "no-email"}),
+                               "values": list({*tags_ms.get("values", []), "no-email"})}
+
+                record = {
+                    "firstname":  firstname,
+                    "lastname":   lastname,
+                    "email":      email_to or None,  # kann None sein → Platzhalter wird im Store erzeugt
+                    "phonenumber": phonenumber,
+                    "adress":     adress,
+                    "city":       city,
+                    "postcode":   postcode,
+
+                    "job_title":  job_title,
+                    "departments": departments,
+                    "linkedin_url": linkedin,
+
+                    "company_name": company_name,
+                    "company_url":  company_url,
+
+                    "message_mail_subject": subj_raw or None,
+                    "message_mail":         body_raw or None,
+                    "message_followup1":    draft.get("followup1") or None,
+                    "message_followup2":    draft.get("followup2") or None,
+
+                    "exclude_hash": combined_id or None,
+
+                    "status_field": {"keys": ["Kontakt aufgenommen"], "values": ["Kontakt aufgenommen"]},
+                    "tags":         tags_ms,
+                }
+
+                store_res = wholix_store_contact(wh_token, record, allow_placeholder=True)
+                stored_ok = bool(store_res)
+                if not email_to:
+                    _job_emit(job_id, "   ℹ️ Keine echte E-Mail – Platzhalter gespeichert und Tag 'no-email' gesetzt.")
+            except Exception:
+                logging.exception(f"[{job_id}] Wholix-Store-Fehler")
+                _job_emit(job_id, "❌ Wholix-Store-Fehler: siehe Logs")
+                stored_ok = False
 
             if combined_id:
                 excludes_set.add(combined_id)
@@ -989,10 +986,10 @@ def run_pipeline_bg(job_id: str, curl_text: str, n_leads_ui: int):
 
             _job_emit(job_id, rows_append={
                 "person":  f"{person.get('first_name','')} {person.get('last_name','')}".strip(),
-                "email":   email_to,
+                "email":   email_to or "(placeholder)",
                 "company": company.get("name") or company.get("company_name") or "",
                 "subject": subj_flat,
-                "message": body_flat,   # Body bleibt Body
+                "message": body_flat,
                 "stored_ok": stored_ok,
             })