Spaces:

metricspace
/

leadgen

Sleeping

App Files Files Community

MichaelWelsch commited on Oct 7, 2025

Commit

f4915e7

verified ·

1 Parent(s): 016c9c4

Update app.py

Browse files

Files changed (1) hide show

app.py +133 -73

app.py CHANGED Viewed

@@ -1,3 +1,6 @@
 """
 Gradio UI (robust gegen kurzzeitigen Browser-Verbindungsverlust):
   - Startet einen Hintergrund-Job und speichert Status/Progress/Ergebnisse per job_id.
@@ -97,7 +100,6 @@ CURL_DATA_RE = re.compile(
     re.DOTALL
 )
 HDR_XTOKEN_RE = re.compile(r"""-H\s+(?P<q>['"])X-Token-Id:\s*(?P<val>[^'"]+)(?P=q)""", re.IGNORECASE)
-HDR_AUTH_RE  = re.compile(r"""-H\s+(?P<q>['"])Authorization:\s*Bearer\s+(?P<val>[^'"]+)(?P=q)""", re.IGNORECASE)
 def parse_curl(curl_text: str) -> Tuple[str, Dict[str, Any]]:
     """
@@ -110,7 +112,6 @@ def parse_curl(curl_text: str) -> Tuple[str, Dict[str, Any]]:
     if m:
         token_id = m.group("val").strip()
-    # Some users might pass token in env; if absent, try to look for it inline as 'X-Token-Id: ...'
     if not token_id:
         hdr_inline = re.search(r"X-Token-Id:\s*([a-zA-Z0-9\-\._]+)", curl_text, re.IGNORECASE)
         if hdr_inline:
@@ -120,7 +121,6 @@ def parse_curl(curl_text: str) -> Tuple[str, Dict[str, Any]]:
     if md:
         body_str = md.group("body").strip()
     else:
-        # fallback for -d '...'
         md2 = re.search(r"-d\s+(?P<q>['\"])(?P<body>.*?)(?P=q)", curl_text, re.DOTALL)
         if md2:
             body_str = md2.group("body").strip()
@@ -128,16 +128,13 @@ def parse_curl(curl_text: str) -> Tuple[str, Dict[str, Any]]:
     if not body_str:
         raise ValueError("Konnte den JSON Body aus dem curl nicht finden (erwarte --data-raw '...').")
-    # The curl example uses single quotes around valid JSON → parse directly
     try:
         payload = json.loads(body_str)
-    except json.JSONDecodeError as e:
-        # attempt to unescape smart single quotes or stray CRs
         candidate = body_str.replace("\r\n", "\n").replace("\r", "\n")
         payload = json.loads(candidate)
     if not token_id:
-        # allow token in the payload via "X-Token-Id" or env var
         env_token = os.getenv("X_TOKEN_ID", "").strip()
         if env_token:
             token_id = env_token
@@ -228,9 +225,11 @@ def lead_suggest(token_id: str, filters: Dict[str, Any], icp_text: str, exclude_
         time.sleep(min(2.4, 0.6 * attempt))
 def _ci_get(d: Dict[str, Any], key: str) -> Any:
     if key in d and str(d[key]).strip() != "":
         return d[key]
-    k = next((k for k in d.keys() if k.lower() == key.lower() and str(d[k]).strip() != ""), None)
     return d.get(k) if k else None
 def _normalize_draft_result(raw: Any) -> Dict[str, Any]:
@@ -261,7 +260,7 @@ def _normalize_draft_result(raw: Any) -> Dict[str, Any]:
             if isinstance(src, dict):
                 for k in keys:
                     v = _ci_get(src, k)
-                    if v is not None:
                         return v
         return ""
@@ -318,7 +317,7 @@ def email_generate(token_id: str, variables: Dict[str, Any], items: List[Dict[st
         attempt += 1
         time.sleep(min(2.4, 0.6 * attempt))
-# ======== DROP-IN: JS-kompatible Wholix-Mapping & Store (1:1) ========
 ALLOWED_FIELDS = {
     "firstname",
@@ -344,7 +343,6 @@ ALLOWED_FIELDS = {
 def filter_wholix_contact_fields(obj: dict) -> dict:
     """
-    1:1 wie JS filterWholixContactFields:
     - nur erlaubte Felder
     - email immer getrimmt
     - Strings getrimmt; leere Werte raus
@@ -366,24 +364,25 @@ def filter_wholix_contact_fields(obj: dict) -> dict:
 def normalize_wholix_dropdown(val):
     """
-    1:1 wie JS normalizeWholixDropdown:
     akzeptiert {keys,values}, Array oder String
     → normalisiert zu {keys:[...], values:[...]} oder None
     """
     if isinstance(val, dict) and ("keys" in val or "values" in val):
-        ks = [str(x) for x in (val.get("keys") or []) if str(x).strip()]
-        vs = [str(x) for x in (val.get("values") or []) if str(x).strip()]
         if not vs and ks:
             vs = ks[:]
         return {"keys": ks, "values": vs} if (ks or vs) else None
     if isinstance(val, list):
-        ks = [str(x) for x in val if str(x).strip()]
         return {"keys": ks, "values": ks} if ks else None
     if isinstance(val, str) and val.strip():
         s = val.strip()
         return {"keys": [s], "values": [s]}
     return None
 def _first_non_empty(*vals):
     for v in vals:
         if isinstance(v, str) and v.strip():
@@ -399,14 +398,73 @@ def _from_ci(d: dict, *keys, default=None):
         if k in d and str(d[k]).strip() != "":
             return d[k]
         for dk in d.keys():
-            if dk.lower() == k.lower() and str(d[dk]).strip() != "":
                 return d[dk]
     return default
 def _join_nonempty(parts, sep=" "):
     return sep.join([str(x).strip() for x in parts if str(x or "").strip()])
 def map_to_wholix_record(lead: dict, draft: dict, tag_text: str = "AI") -> dict:
     p   = (lead or {}).get("person")   or {}
     c   = (lead or {}).get("company")  or {}
     m   = (lead or {}).get("messages") or {}
@@ -419,40 +477,29 @@ def map_to_wholix_record(lead: dict, draft: dict, tag_text: str = "AI") -> dict:
         e.name = "ValidationError"
         raise e
-    # --- Departments as TEXT + we’ll mirror into tags dropdown later ---
-    depts_raw = p.get("departments")
-    if isinstance(depts_raw, list):
-        departments_txt = ", ".join([str(x).strip() for x in depts_raw if str(x).strip()]) or None
-        depts_list_for_tags = [str(x).strip() for x in depts_raw if str(x).strip()]
-    else:
-        departments_txt = str(depts_raw).strip() if depts_raw not in (None, "", []) else None
-        depts_list_for_tags = [departments_txt] if departments_txt else []
     # --- Company URL with wide fallbacks ---
     company_url = _first_non_empty(
-        c.get("url"),
-        c.get("website"),
-        c.get("domain"),
-        c.get("homepage_url"),
-        c.get("website_url"),
-        c.get("url_normalized"),
-        ctx.get("url"),
-        (lead or {}).get("homepage_url"),
     )
-    # --- Message fields: PULL FROM 'draft' (fix) ---
     draft = draft or {}
     draft_email  = draft.get("email") if isinstance(draft, dict) else {}
-    # tolerate shapes like {'email': {'subject','body','to'}} + followups at root
     msg_subject  = _first_non_empty(
         _from_ci(draft_email, "subject", "email_subject"),
         _from_ci(draft, "subject", "email_subject", "Betreff"),
-        _from_ci(m, "message_mail_subject"),  # legacy
     )
     msg_body     = _first_non_empty(
         _from_ci(draft_email, "body", "text", "content"),
         _from_ci(draft, "body", "Text", "content", "email_body"),
-        _from_ci(m, "message_mail"),  # legacy
     )
     followup1    = _first_non_empty(
         _from_ci(draft, "followup1", "FollowUp1", "LinkedIn", "linkedin", "li"),
@@ -463,30 +510,35 @@ def map_to_wholix_record(lead: dict, draft: dict, tag_text: str = "AI") -> dict:
         _from_ci(m, "followup2", "message_followup2"),
     )
-    # --- Address with wide fallbacks ---
     street = _first_non_empty(
         _join_nonempty([c.get("street_name"), c.get("street_number")]),
-        c.get("address"),
-        c.get("address1"),
-        c.get("address_line1"),
-        c.get("street"),
-        c.get("street_address"),
     )
-    city = _first_non_empty(
-        c.get("city"),
-        c.get("town"),
-        c.get("locality"),
     )
-    postcode = _first_non_empty(
-        c.get("zip_code"),
-        c.get("postal_code"),
-        c.get("postcode"),
-        c.get("zip"),
     )
-    # --- exclude_hash with fallbacks ---
     exclude_hash = _first_non_empty(
         lead.get("exclude_hash"),
         c.get("exclude_hash"),
@@ -494,11 +546,8 @@ def map_to_wholix_record(lead: dict, draft: dict, tag_text: str = "AI") -> dict:
         lead.get("combined_id"),
     )
-    # --- Build tags dropdown: keep existing tag_text and add departments (if any) ---
-    tag_items = [str(tag_text).strip("[]")] if str(tag_text).strip("[]") else []
-    for d in depts_list_for_tags:
-        if d and d not in tag_items:
-            tag_items.append(d)
     tags_dropdown = {"keys": tag_items, "values": tag_items} if tag_items else None
     payload = {
@@ -506,19 +555,19 @@ def map_to_wholix_record(lead: dict, draft: dict, tag_text: str = "AI") -> dict:
         "firstname":    p.get("first_name") or None,
         "lastname":     p.get("last_name")  or None,
         "email":        email,
-        "adress":       street or None,       # (sic) exact key
         "city":         city or None,
         "postcode":     postcode or None,
         "phonenumber":  p.get("phone") or None,
-        "job_title":    p.get("job_title") or None,
-        "departments":  departments_txt,      # text field, as before
-        "linkedin_url": p.get("linkedin_url") or None,
         # Company
-        "company_name": c.get("name") or c.get("company_name") or None,
         "company_url":  company_url or None,
-        # Message (now from 'draft')
         "message_mail":          msg_body or None,
         "message_mail_subject":  msg_subject or None,
         "message_followup1":     followup1 or None,
@@ -534,7 +583,7 @@ def map_to_wholix_record(lead: dict, draft: dict, tag_text: str = "AI") -> dict:
     normalized = filter_wholix_contact_fields(payload)
-    # Normalize dropdowns (status/tags). Departments stays text.
     if "status_field" in normalized:
         fixed = normalize_wholix_dropdown(normalized["status_field"])
         if fixed: normalized["status_field"] = fixed
@@ -547,10 +596,12 @@ def map_to_wholix_record(lead: dict, draft: dict, tag_text: str = "AI") -> dict:
     return normalized
 def wholix_store_contact(token: str, record: dict, module: str = "Contacts") -> dict:
     """
-    JS wholixStoreContact exakt nachgebildet (mit Fallbacks)
     """
     email = str((record or {}).get("email") or "").strip()
     if not email:
@@ -623,8 +674,6 @@ def wholix_store_contact(token: str, record: dict, module: str = "Contacts") ->
     stripped.pop("tags", None)
     body3 = {"module": module, "action": "store", "data": [stripped]}
     return req(url, method="POST", headers=headers, json_body=body3, timeout=60)
-# ======== END DROP-IN =====================================================
 # ====================== Background-Jobs (robust UI) =======================
@@ -671,12 +720,14 @@ def _job_finish(job_id: str, error: Optional[str] = None):
 def run_pipeline_bg(job_id: str, curl_text: str, n_leads: int):
     """
-    1:1 wie vorherige run_pipeline(), aber ohne yield – stattdessen Status ins JOBS-Store schreiben.
     """
     results: List[Dict[str, Any]] = []
     def log(msg: str):
-        # kleine Bequemlichkeit, auto-emit mit aktuellem Fortschritt
         st = JOBS.get(job_id, {})
         prog = st.get("progress", 0.0)
         _job_emit(job_id, msg=msg, progress=prog, rows=results)
@@ -700,7 +751,16 @@ def run_pipeline_bg(job_id: str, curl_text: str, n_leads: int):
     signature    = payload.get("Signatur") or ""
     cta          = payload.get("CTA") or ""
     homepage_url = payload.get("icp_homepage_url") or ""
-    tag_text     = payload.get("Wholic_tag") or payload.get("Wholix_tag") or "AI"
     total_steps = max(1, n_leads) * 4 + 2  # login + excludes + (lead + email + store)*N
     step = 0
@@ -756,7 +816,7 @@ def run_pipeline_bg(job_id: str, curl_text: str, n_leads: int):
             _job_emit(job_id, msg=f"❌ Email-Generate-Fehler: {e}", progress=step/total_steps, rows=results)
             continue
-        # 5) Store
         step += 1
         _job_emit(job_id, msg="   → Speichere Kontakt + Nachricht in Wholix …", progress=step/total_steps, rows=results)
         try:

+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
 """
 Gradio UI (robust gegen kurzzeitigen Browser-Verbindungsverlust):
   - Startet einen Hintergrund-Job und speichert Status/Progress/Ergebnisse per job_id.
     re.DOTALL
 )
 HDR_XTOKEN_RE = re.compile(r"""-H\s+(?P<q>['"])X-Token-Id:\s*(?P<val>[^'"]+)(?P=q)""", re.IGNORECASE)
 def parse_curl(curl_text: str) -> Tuple[str, Dict[str, Any]]:
     """
     if m:
         token_id = m.group("val").strip()
     if not token_id:
         hdr_inline = re.search(r"X-Token-Id:\s*([a-zA-Z0-9\-\._]+)", curl_text, re.IGNORECASE)
         if hdr_inline:
     if md:
         body_str = md.group("body").strip()
     else:
         md2 = re.search(r"-d\s+(?P<q>['\"])(?P<body>.*?)(?P=q)", curl_text, re.DOTALL)
         if md2:
             body_str = md2.group("body").strip()
     if not body_str:
         raise ValueError("Konnte den JSON Body aus dem curl nicht finden (erwarte --data-raw '...').")
     try:
         payload = json.loads(body_str)
+    except json.JSONDecodeError:
         candidate = body_str.replace("\r\n", "\n").replace("\r", "\n")
         payload = json.loads(candidate)
     if not token_id:
         env_token = os.getenv("X_TOKEN_ID", "").strip()
         if env_token:
             token_id = env_token
         time.sleep(min(2.4, 0.6 * attempt))
 def _ci_get(d: Dict[str, Any], key: str) -> Any:
+    if not isinstance(d, dict):
+        return None
     if key in d and str(d[key]).strip() != "":
         return d[key]
+    k = next((k for k in d.keys() if isinstance(k, str) and k.lower() == key.lower() and str(d[k]).strip() != ""), None)
     return d.get(k) if k else None
 def _normalize_draft_result(raw: Any) -> Dict[str, Any]:
             if isinstance(src, dict):
                 for k in keys:
                     v = _ci_get(src, k)
+                    if v is not None and str(v).strip() != "":
                         return v
         return ""
         attempt += 1
         time.sleep(min(2.4, 0.6 * attempt))
+# ======== DROP-IN: Wholix-Mapping & Store (FIXED) ========================
 ALLOWED_FIELDS = {
     "firstname",
 def filter_wholix_contact_fields(obj: dict) -> dict:
     """
     - nur erlaubte Felder
     - email immer getrimmt
     - Strings getrimmt; leere Werte raus
 def normalize_wholix_dropdown(val):
     """
     akzeptiert {keys,values}, Array oder String
     → normalisiert zu {keys:[...], values:[...]} oder None
     """
     if isinstance(val, dict) and ("keys" in val or "values" in val):
+        ks = [str(x).strip() for x in (val.get("keys") or []) if str(x).strip()]
+        vs = [str(x).strip() for x in (val.get("values") or []) if str(x).strip()]
         if not vs and ks:
             vs = ks[:]
         return {"keys": ks, "values": vs} if (ks or vs) else None
     if isinstance(val, list):
+        ks = [str(x).strip() for x in val if str(x).strip()]
         return {"keys": ks, "values": ks} if ks else None
     if isinstance(val, str) and val.strip():
         s = val.strip()
         return {"keys": [s], "values": [s]}
     return None
+# ---------- helpers for robust mapping ----------
 def _first_non_empty(*vals):
     for v in vals:
         if isinstance(v, str) and v.strip():
         if k in d and str(d[k]).strip() != "":
             return d[k]
         for dk in d.keys():
+            if isinstance(dk, str) and dk.lower() == k.lower() and str(d[dk]).strip() != "":
                 return d[dk]
     return default
 def _join_nonempty(parts, sep=" "):
     return sep.join([str(x).strip() for x in parts if str(x or "").strip()])
+def _parse_maybe_json_list(value):
+    """
+    Accept list, JSON-string list, or bracketed string → return list[str]
+    """
+    if value is None:
+        return []
+    if isinstance(value, list):
+        return [str(x).strip() for x in value if str(x).strip()]
+    s = str(value).strip()
+    if not s:
+        return []
+    # try JSON
+    if (s.startswith("[") and s.endswith("]")) or (s.startswith("(") and s.endswith(")")):
+        try:
+            arr = json.loads(s.replace("(", "[").replace(")", "]"))
+            if isinstance(arr, list):
+                return [str(x).strip() for x in arr if str(x).strip()]
+        except Exception:
+            # crude fallback
+            s2 = s.strip("[]()")
+            parts = [p.strip().strip("'").strip('"') for p in s2.split(",")]
+            return [p for p in parts if p]
+    # plain string, maybe delimited
+    if "," in s:
+        return [p.strip() for p in s.split(",") if p.strip()]
+    return [s]
+def _normalize_tag_items(tag_text):
+    """
+    Accepts: "AI", "[AI]", "AI, Sales", '["AI","Sales"]' → returns list[str]
+    """
+    if isinstance(tag_text, list):
+        return [str(x).strip() for x in tag_text if str(x).strip()]
+    if tag_text is None:
+        return []
+    s = str(tag_text).strip()
+    if not s:
+        return []
+    try:
+        if s.startswith("[") and s.endswith("]"):
+            arr = json.loads(s)
+            if isinstance(arr, list):
+                return [str(x).strip() for x in arr if str(x).strip()]
+    except Exception:
+        pass
+    for sep in [",", "|", ";"]:
+        if sep in s:
+            return [p.strip() for p in s.split(sep) if p.strip()]
+    return [s.strip("[]")]
+# ---------- the fixed mapper ----------
 def map_to_wholix_record(lead: dict, draft: dict, tag_text: str = "AI") -> dict:
+    """
+    FIXED:
+      - nutzt jetzt 'draft' für message_mail/subject/followups
+      - bereinigt departments (keine ["..."] Reste)
+      - breite Fallbacks für job_title / linkedin_url / adress / city / postcode / company_url / exclude_hash
+      - Tags: nur Benutzer-Tags, KEINE Departments mehr
+    """
     p   = (lead or {}).get("person")   or {}
     c   = (lead or {}).get("company")  or {}
     m   = (lead or {}).get("messages") or {}
         e.name = "ValidationError"
         raise e
+    # --- Departments (TEXT) — clean up list-like strings ---
+    depts_list = _parse_maybe_json_list(p.get("departments"))
+    departments_txt = ", ".join(depts_list) if depts_list else None
     # --- Company URL with wide fallbacks ---
     company_url = _first_non_empty(
+        c.get("url"), c.get("website"), c.get("domain"),
+        c.get("homepage_url"), c.get("website_url"), c.get("url_normalized"),
+        ctx.get("url"), (lead or {}).get("homepage_url"),
     )
+    # --- Message from generated draft ---
     draft = draft or {}
     draft_email  = draft.get("email") if isinstance(draft, dict) else {}
     msg_subject  = _first_non_empty(
         _from_ci(draft_email, "subject", "email_subject"),
         _from_ci(draft, "subject", "email_subject", "Betreff"),
+        _from_ci(m, "message_mail_subject"),
     )
     msg_body     = _first_non_empty(
         _from_ci(draft_email, "body", "text", "content"),
         _from_ci(draft, "body", "Text", "content", "email_body"),
+        _from_ci(m, "message_mail"),
     )
     followup1    = _first_non_empty(
         _from_ci(draft, "followup1", "FollowUp1", "LinkedIn", "linkedin", "li"),
         _from_ci(m, "followup2", "message_followup2"),
     )
+    # --- Address / City / Postcode fallbacks ---
     street = _first_non_empty(
         _join_nonempty([c.get("street_name"), c.get("street_number")]),
+        c.get("address"), c.get("address1"), c.get("address_line1"),
+        c.get("street"), c.get("street_address"),
     )
+    city = _first_non_empty(c.get("city"), c.get("town"), c.get("locality"))
+    postcode = _first_non_empty(c.get("zip_code"), c.get("postal_code"), c.get("postcode"), c.get("zip"))
+    # --- Job title with fallbacks ---
+    job_title = _first_non_empty(
+        p.get("job_title"),
+        p.get("job_title_de_DE"),
+        p.get("title"),
+        p.get("position"),
+        _from_ci(p, "role"),
     )
+    # --- LinkedIn URL with fallbacks ---
+    linkedin_url = _first_non_empty(
+        p.get("linkedin_url"),
+        p.get("linkedin"),
+        p.get("linkedin_profile"),
+        p.get("linkedinUrl"),
+        p.get("li"),
+        p.get("li_url"),
     )
+    # --- exclude_hash fallbacks ---
     exclude_hash = _first_non_empty(
         lead.get("exclude_hash"),
         c.get("exclude_hash"),
         lead.get("combined_id"),
     )
+    # --- Tags: ONLY what user provided (no departments mirroring) ---
+    tag_items = _normalize_tag_items(tag_text)
     tags_dropdown = {"keys": tag_items, "values": tag_items} if tag_items else None
     payload = {
         "firstname":    p.get("first_name") or None,
         "lastname":     p.get("last_name")  or None,
         "email":        email,
+        "adress":       street or None,           # (sic)
         "city":         city or None,
         "postcode":     postcode or None,
         "phonenumber":  p.get("phone") or None,
+        "job_title":    job_title or None,
+        "departments":  departments_txt,
+        "linkedin_url": linkedin_url or None,
         # Company
+        "company_name": _first_non_empty(c.get("name"), c.get("company_name")),
         "company_url":  company_url or None,
+        # Message
         "message_mail":          msg_body or None,
         "message_mail_subject":  msg_subject or None,
         "message_followup1":     followup1 or None,
     normalized = filter_wholix_contact_fields(payload)
+    # Normalize dropdowns
     if "status_field" in normalized:
         fixed = normalize_wholix_dropdown(normalized["status_field"])
         if fixed: normalized["status_field"] = fixed
     return normalized
 def wholix_store_contact(token: str, record: dict, module: str = "Contacts") -> dict:
     """
+    Wholix-Store mit Dropdown-Fallbacks:
+      1) Normales {keys,values}
+      2) Legacy {value}
+      3) Ohne problematische Felder
     """
     email = str((record or {}).get("email") or "").strip()
     if not email:
     stripped.pop("tags", None)
     body3 = {"module": module, "action": "store", "data": [stripped]}
     return req(url, method="POST", headers=headers, json_body=body3, timeout=60)
 # ====================== Background-Jobs (robust UI) =======================
 def run_pipeline_bg(job_id: str, curl_text: str, n_leads: int):
     """
+    Background-Pipeline:
+      1) Wholix-Login
+      2) Excludes laden
+      3..N) Lead holen → Nachricht generieren → in Wholix speichern
     """
     results: List[Dict[str, Any]] = []
     def log(msg: str):
         st = JOBS.get(job_id, {})
         prog = st.get("progress", 0.0)
         _job_emit(job_id, msg=msg, progress=prog, rows=results)
     signature    = payload.get("Signatur") or ""
     cta          = payload.get("CTA") or ""
     homepage_url = payload.get("icp_homepage_url") or ""
+    # Tags: in beliebigen Formen erlauben (AI | [AI] | "AI, Sales" | ["AI","Sales"])
+    raw_tag      = payload.get("Wholic_tag") or payload.get("Wholix_tag") or "AI"
+    tag_text     = raw_tag  # Mapper parst das robust
+    # optional limit aus Payload
+    try:
+        n_leads = int(payload.get("limit", n_leads))
+    except Exception:
+        pass
+    n_leads = max(1, n_leads)
     total_steps = max(1, n_leads) * 4 + 2  # login + excludes + (lead + email + store)*N
     step = 0
             _job_emit(job_id, msg=f"❌ Email-Generate-Fehler: {e}", progress=step/total_steps, rows=results)
             continue
+        # 5) Store (MAPPER FIXED)
         step += 1
         _job_emit(job_id, msg="   → Speichere Kontakt + Nachricht in Wholix …", progress=step/total_steps, rows=results)
         try: