Update app.py

app.py

@@ -1,4 +1,4 @@
-
+# (paste this into your existing file, replacing the old content)
 import os
 import time
 import re
@@ -26,7 +26,6 @@ from email.mime.text import MIMEText
 # ============================
 # CONFIG (ENV VARS recommended)
 # ============================
-# IMPORTANT: set these as Space "Secrets" (see README below)
 API_KEY = os.environ.get("GOOGLE_API_KEY", "YOUR_GOOGLE_API_KEY")
 CX      = os.environ.get("GOOGLE_CSE_ID", "YOUR_CSE_ID")
 DEFAULT_COUNTRY = "Ghana"
@@ -38,11 +37,9 @@ ALLY_AI_NAME = os.environ.get("ALLY_AI_NAME", "Ally AI Assistant")
 ALLY_AI_LOGO_URL_DEFAULT = os.environ.get("ALLY_AI_LOGO_URL",
                                          "https://i.ibb.co/7nZqz0H/ai-logo.png")
 
-# Optional country maps for search bias & phone parsing
 COUNTRY_TLD_MAP   = {"Ghana":"gh","Nigeria":"ng","Kenya":"ke","South Africa":"za","USA":"us","United Kingdom":"uk"}
 COUNTRY_REGION_MAP= {"Ghana":"GH","Nigeria":"NG","Kenya":"KE","South Africa":"ZA","USA":"US","United Kingdom":"GB"}
 
-# HTTP + Regex
 HEADERS = {"User-Agent":"Mozilla/5.0 (X11; Linux x86_64)"}
 EMAIL_REGEX = re.compile(r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}")
 
@@ -124,19 +121,67 @@ def extract_phones(text, region="GH"):
             pass
     return list(set(phones))
 
+def _domain_from_url(url):
+    try:
+        return urllib.parse.urlparse(url).netloc
+    except Exception:
+        return url
+
 def scrape_contacts(url, region="GH"):
+    """
+    Extended scraping: returns emails, phones, and a guessed 'org' name
+    extracted from meta tags, headings, or via NER on page text.
+    """
     try:
         res = requests.get(url, headers=HEADERS, timeout=12)
         if not res.ok or not res.text:
-            return {"emails": [], "phones": []}
-        text = BeautifulSoup(res.text, "html.parser").get_text(separator=" ")
+            return {"emails": [], "phones": [], "org": None}
+        soup = BeautifulSoup(res.text, "html.parser")
+
+        # raw text for phone/email/NER
+        text = soup.get_text(separator=" ")
         text = " ".join(text.split())[:300000]
+
+        # emails & phones
         emails = list(set(EMAIL_REGEX.findall(text)))
         phones = extract_phones(text, region)
-        return {"emails": emails, "phones": phones}
+
+        # try meta og:site_name or twitter site meta
+        org_name = None
+        meta_og = soup.find("meta", property="og:site_name") or soup.find("meta", attrs={"name":"og:site_name"})
+        if meta_og and meta_og.get("content"):
+            org_name = meta_og.get("content").strip()
+
+        # fallback to <title> or first <h1>
+        if not org_name:
+            title_tag = soup.find("title")
+            if title_tag and title_tag.get_text(strip=True):
+                org_name = title_tag.get_text(strip=True)
+        if not org_name:
+            h1 = soup.find("h1")
+            if h1 and h1.get_text(strip=True):
+                org_name = h1.get_text(strip=True)
+
+        # run NER to find ORG mentions in the page text and prefer that if short and clean
+        try:
+            people, orgs, locs = extract_entities(text)
+            if orgs:
+                # choose first short/clean org
+                for o in orgs:
+                    if len(o) > 1 and len(o) < 80:
+                        org_name = o
+                        break
+        except Exception:
+            pass
+
+        # final fallback: domain
+        if not org_name:
+            org_name = _domain_from_url(url)
+
+        return {"emails": emails, "phones": phones, "org": org_name}
     except Exception as e:
         print(f"[scrape error] {url} -> {e}")
-        return {"emails": [], "phones": []}
+        return {"emails": [], "phones": [], "org": _domain_from_url(url)}
 
 # ============================
 # NER + STORY → PROFESSIONS
@@ -172,7 +217,7 @@ def build_queries(story: str, country: str):
         if p == "gbv":
             cores += ["GBV support organizations", "gender based violence help"]
         else:
-            cores += [f"{p} for GBV", f"{p} for sexual assault"]
+            cores += [f"{p} for GBV", f"{p} for sexual assault", f"{p} near me {p} {country}"]
     unique_cores, seen = [], set()
     for c in cores:
         if c not in seen:
@@ -230,29 +275,44 @@ def find_professionals_from_story(story, country=DEFAULT_COUNTRY, results_per_qu
         return {"summary":"No results found. Try a different country or wording.",
                 "professionals":[], "queries_used":queries}
 
-    # NER on titles/snippets
+    # NER on titles/snippets for context
     all_people, all_orgs, all_locs = [], [], []
     for r in search_results:
         ctx = f"{r.get('title','')}. {r.get('snippet','')}"
         p,o,l = extract_entities(ctx)
         all_people += p; all_orgs += o; all_locs += l
 
-    # Scrape contacts concurrently
+    # Scrape contacts concurrently, extracting org names from page content
     professionals = []
     with ThreadPoolExecutor(max_workers=MAX_SCRAPE_WORKERS) as ex:
         futures = {ex.submit(scrape_contacts, r["link"], region): r for r in search_results}
         for fut in as_completed(futures):
             r = futures[fut]
-            contacts = {"emails": [], "phones": []}
+            contacts = {"emails": [], "phones": [], "org": None}
             try:
                 contacts = fut.result()
             except Exception as e:
                 print("[scrape future error]", r["link"], e)
+
+            # choose a single email/phone if available
+            email = contacts["emails"][0] if contacts.get("emails") else None
+            phone = contacts["phones"][0] if contacts.get("phones") else None
+            org_name = contacts.get("org") or ""
+            # attempt to choose profession tag from the query used
+            prof_tag = None
+            qlower = (r.get("query") or "").lower()
+            for p in professions_from_story(story):
+                if p in qlower:
+                    prof_tag = p
+                    break
+            prof_tag = prof_tag or (professions_from_story(story)[0] if professions_from_story(story) else "gbv")
+
             professionals.append({
-                "title": r.get("title",""),
+                "org": org_name,
                 "url": r.get("link",""),
-                "email": contacts["emails"][0] if contacts["emails"] else "Not found",
-                "phone": contacts["phones"][0] if contacts["phones"] else "Not found",
+                "email": email if email else "Not found",
+                "phone": phone if phone else "Not found",
+                "profession": prof_tag,
                 "source_query": r.get("query","")
             })
 
@@ -268,26 +328,57 @@ def find_professionals_from_story(story, country=DEFAULT_COUNTRY, results_per_qu
 # DRAFT (mailto + .eml)
 # ============================
 def build_mailto_and_eml(to_addr, subject, body, default_from="noreply@ally.ai"):
-    from email.message import EmailMessage
-    import time
+    """
+    Creates a proper .eml file and returns (mailto_link, absolute_eml_path).
+    Ensures the file is actually written and non-empty. If .eml fails, writes a .txt fallback.
+    """
+    # sanitize inputs
+    to_addr = (to_addr or "").strip()
+    subject = subject or ""
+    body = body or ""
 
+    # Create EmailMessage object
     msg = EmailMessage()
     msg["From"] = default_from
     msg["To"] = to_addr
     msg["Subject"] = subject
     msg.set_content(body)
 
-    # ✅ Save to a writable directory (current working dir or "tmp")
-    os.makedirs("tmp", exist_ok=True)
-    fname = os.path.join("tmp", f"email_draft_{int(time.time())}.eml")
+    # ensure output dir exists and use absolute path (more robust for HF Spaces / Colab)
+    out_dir = os.path.abspath("tmp")
+    os.makedirs(out_dir, exist_ok=True)
 
-    with open(fname, "wb") as f:
-        f.write(msg.as_bytes())
+    fname = os.path.join(out_dir, f"email_draft_{int(time.time())}.eml")
 
-    # Create mailto link (this part is fine)
-    mailto = f"mailto:{to_addr}?subject={subject}&body={body}"
+    try:
+        # write bytes
+        with open(fname, "wb") as f:
+            f.write(msg.as_bytes())
+
+        # verify file exists and is non-empty
+        if os.path.exists(fname) and os.path.getsize(fname) > 0:
+            mailto = f"mailto:{urllib.parse.quote(to_addr)}?subject={urllib.parse.quote(subject)}&body={urllib.parse.quote(body)}"
+            return mailto, fname
+
+        # fallback: create a plain text copy (useful for debugging)
+        fallback = fname + ".txt"
+        with open(fallback, "w", encoding="utf-8") as f:
+            f.write(f"To: {to_addr}\nSubject: {subject}\n\n{body}")
+        mailto = f"mailto:{urllib.parse.quote(to_addr)}?subject={urllib.parse.quote(subject)}&body={urllib.parse.quote(body)}"
+        return mailto, fallback
 
-    return mailto, fname
+    except Exception as e:
+        # If writing .eml fails entirely, create a .txt fallback and return that path
+        fallback = os.path.join(out_dir, f"email_draft_{int(time.time())}.txt")
+        try:
+            with open(fallback, "w", encoding="utf-8") as f:
+                f.write(f"Error writing .eml: {e}\n\nTo: {to_addr}\nSubject: {subject}\n\n{body}")
+            mailto = f"mailto:{urllib.parse.quote(to_addr)}?subject={urllib.parse.quote(subject)}&body={urllib.parse.quote(body)}"
+            return mailto, fallback
+        except Exception as e2:
+            # ultimate fallback: return no file and an informative mailto
+            mailto = f"mailto:{urllib.parse.quote(to_addr)}?subject={urllib.parse.quote(subject)}&body={urllib.parse.quote(body)}"
+            return mailto, None
 
 # ============================
 # SENDER (SMTP) — Ally AI branding
@@ -296,16 +387,8 @@ def send_ally_ai_email(to_email, subject, body, user_email,
                        sender_email, sender_password,
                        ai_name=ALLY_AI_NAME,
                        logo_url=ALLY_AI_LOGO_URL_DEFAULT):
-    """
-    Sends an HTML email branded as Ally AI.
-    to_email: recipient (organization)
-    subject: subject line
-    body: main message (already anonymized or full text)
-    user_email: survivor's email (included for reply inside body)
-    sender_email/sender_password: SMTP credentials (use Gmail App Password with Gmail)
-    """
     if not to_email or to_email == "Not found":
-        return "❌ No recipient email found — choose a contact with an email."
+        return "❌ No recipient email found — choose a contact or provide a manual email."
 
     msg = MIMEMultipart("alternative")
     msg["Subject"] = subject or "Request for support"
@@ -336,7 +419,7 @@ def send_ally_ai_email(to_email, subject, body, user_email,
     try:
         server = smtplib.SMTP("smtp.gmail.com", 587)
         server.starttls()
-        server.login(sender_email, sender_password)  # Gmail App Password recommended
+        server.login(sender_email, sender_password)
         server.sendmail(sender_email, [to_email], msg.as_string())
         server.quit()
         return f"✅ Email sent successfully to {to_email}"
@@ -346,13 +429,7 @@ def send_ally_ai_email(to_email, subject, body, user_email,
 # ============================
 # GRADIO UI
 # ============================
-# ------- Replace existing run_search and _on_search with these -------
-
 def run_search(story, country):
-    """
-    Robust search wrapper: returns (summary, table_records, dropdown_options, anonymized_text).
-    Avoids returning gr.update(...) to prevent KeyError during serialization.
-    """
     try:
         out = find_professionals_from_story(story, country=country, results_per_query=RESULTS_PER_QUERY)
     except Exception as e:
@@ -362,18 +439,18 @@ def run_search(story, country):
 
     pros = out.get("professionals", []) or []
 
-    # build table records
+    # build table records with org instead of article title
     try:
         records = pd.DataFrame(pros).to_dict(orient="records") if pros else []
     except Exception:
         records = []
 
-    # build dropdown options as list of strings (guarantee at least one)
+    # build dropdown options as list of strings
     options = []
     for i, r in enumerate(pros):
         label_contact = r.get("email") if r.get("email") and r.get("email") != "Not found" else (r.get("phone", "No contact"))
-        title = r.get("title") or r.get("url") or "(no title)"
-        label = f"{i} — {title} ({label_contact})"
+        org_label = r.get("org") or r.get("url") or "(no org)"
+        label = f"{i} — {org_label} ({label_contact})"
         options.append(label)
 
     if not options:
@@ -389,30 +466,8 @@ def run_search(story, country):
     summary = out.get("summary", "No results found.")
     return summary, records, options, anon
 
-
-def _on_search(story, country):
-    """
-    Function wired to the search button.
-    Returns exactly 5 outputs to match:
-      [summary_out, results_table, dropdown_sel, anon_out, message_in]
-    """
-    summary, records, options, anon = run_search(story, country)
-
-    # pre-fill message body with anonymized text (user email left empty for now)
-    prefill = make_body(anon, story, True, "")
-
-    # Return plain serializable values (not gr.update)
-    # summary -> str
-    # records -> list[dict] (or [])
-    # options -> list[str] for dropdown (Gradio will accept it)
-    # anon -> str
-    # prefill -> str (message body)
-    return summary, records, options, anon, prefill
-
-
 def make_body(anon_text, full_story, use_anon, user_email):
     core = (anon_text or "").strip() if use_anon else (full_story or "").strip()
-    # polite template with user email included in body
     lines = [
         core,
         "",
@@ -422,31 +477,41 @@ def make_body(anon_text, full_story, use_anon, user_email):
     ]
     return "\n".join([l for l in lines if l is not None])
 
-def preview_contact(dropdown_value, df_json, subject, message_text):
+def preview_contact(dropdown_value, df_json, subject, message_text, manual_email):
     if not dropdown_value:
         return "No contact selected.", ""
     try:
         idx = int(str(dropdown_value).split(" — ")[0])
         rows = pd.DataFrame(df_json)
         contact = rows.iloc[idx].to_dict()
-        recipient = contact.get("email") if contact.get("email") and contact.get("email")!="Not found" else "[no email]"
+
+        # choose recipient from manual_email if provided & valid, else scraped email
+        recipient = None
+        if manual_email and EMAIL_REGEX.search(manual_email):
+            recipient = manual_email.strip()
+        else:
+            recipient = contact.get("email") if contact.get("email") and contact.get("email")!="Not found" else "[no email]"
+
+        org_display = contact.get('org') or contact.get('url') or "(no org)"
         html = f"""
         <h3>Preview</h3>
         <b>To:</b> {recipient}<br/>
-        <b>Organization:</b> <a href="{contact.get('url')}" target="_blank" rel="noopener">{contact.get('title')}</a><br/>
+        <b>Organization:</b> <a href="{contact.get('url')}" target="_blank" rel="noopener">{org_display}</a><br/>
+        <b>Profession tag:</b> {contact.get('profession')}<br/>
         <b>Subject:</b> {subject}<br/>
         <hr/>
         <pre style="white-space:pre-wrap;">{message_text}</pre>
         """
-        text = f"To: {recipient}\nSubject: {subject}\n\n{message_text[:600]}{'...' if len(message_text)>600 else ''}"
+        text = f"To: {recipient}\nOrganization: {org_display}\nSubject: {subject}\n\n{message_text[:600]}{'...' if len(message_text)>600 else ''}"
         return text, html
     except Exception as e:
         return f"Preview error: {e}", ""
 
 def confirm_action(mode, dropdown_value, df_json, subject, message_text,
-                   user_email, sender_email, sender_password, logo_url):
+                   user_email, sender_email, sender_password, logo_url, manual_email):
     """
     mode: "Draft only" or "Send via SMTP (Gmail)"
+    manual_email: optional override to use when scraped email not found
     """
     if not dropdown_value:
         return "❌ No contact selected.", "", None
@@ -459,11 +524,18 @@ def confirm_action(mode, dropdown_value, df_json, subject, message_text,
     except Exception as e:
         return f"❌ Selection error: {e}", "", None
 
-    recipient = contact.get("email")
+    scraped_recipient = contact.get("email")
+    # use manual if valid
+    recipient = None
+    if manual_email and EMAIL_REGEX.search(manual_email):
+        recipient = manual_email.strip()
+    elif scraped_recipient and scraped_recipient != "Not found":
+        recipient = scraped_recipient
+
     if mode.startswith("Send"):
         # Validate required fields
-        if not recipient or recipient == "Not found":
-            return "❌ This contact has no email address. Choose another contact.", "", None
+        if not recipient:
+            return "❌ No recipient email found — either pick a contact with an email or provide a manual email.", "", None
         if not user_email or "@" not in user_email:
             return "❌ Please enter your email (so the organisation can contact you).", "", None
         if not sender_email or not sender_password:
@@ -479,21 +551,27 @@ def confirm_action(mode, dropdown_value, df_json, subject, message_text,
             ai_name=ALLY_AI_NAME,
             logo_url=logo_url or ALLY_AI_LOGO_URL_DEFAULT
         )
-        # also provide an .eml draft copy (optional)
         _, eml_path = build_mailto_and_eml(recipient, subject, message_text, default_from=sender_email)
         file_out = eml_path if eml_path and os.path.exists(eml_path) else None
         return status, "", file_out
     else:
-        # Draft-only path
-        recip_for_draft = recipient if (recipient and recipient!="Not found") else ""
+        # Draft-only path (mailto + .eml)
+        recip_for_draft = recipient or ""
         mailto, eml_path = build_mailto_and_eml(recip_for_draft, subject, message_text, default_from="noreply@ally.ai")
-        html_link = f'<a href="{mailto}" target="_blank" rel="noopener">Open draft in email client</a>'
-        file_out = eml_path if eml_path and os.path.exists(eml_path) else None
-        return "✅ Draft created (no email sent).", html_link, file_out
+        if eml_path and os.path.exists(eml_path) and os.path.getsize(eml_path) > 0:
+            html_link = f'<a href="{mailto}" target="_blank" rel="noopener">Open draft in email client</a>'
+            file_out = eml_path
+            return "✅ Draft created (no email sent).", html_link, file_out
+        elif eml_path and os.path.exists(eml_path):
+            # file exists but is empty
+            return "⚠️ Draft file created but it's empty. Check the message body or try manual email.", "", eml_path
+        else:
+            return "❌ Failed to create draft file.", "", None
+
 
 with gr.Blocks() as demo:
     gr.Markdown("## Ally AI — GBV Help Finder & Email Assistant\n"
-                "This tool searches local organizations, lets you select a contact, and creates an email draft or sends a branded email via SMTP.\n"
+                "This tool searches local professionals/organizations (not article pages), lets you select a contact or enter an email manually, and creates an email draft or sends a branded email via SMTP.\n"
                 "**Privacy tip:** Prefer anonymized summaries unless you’re comfortable sharing details.")
 
     with gr.Row():
@@ -502,7 +580,8 @@ with gr.Blocks() as demo:
 
     search_btn = gr.Button("Search for professionals")
     summary_out = gr.Textbox(label="Search summary (AI)", interactive=False)
-    results_table = gr.Dataframe(headers=["title","url","email","phone","source_query"], label="Search results")
+    # updated headers: use org (organization name) instead of article title
+    results_table = gr.Dataframe(headers=["org","url","email","phone","profession","source_query"], label="Search results")
 
     dropdown_sel = gr.Dropdown(label="Select organization (from results)", choices=[])
 
@@ -515,6 +594,9 @@ with gr.Blocks() as demo:
     subject_in = gr.Textbox(value="Request for GBV support", label="Email subject")
     message_in = gr.Textbox(label="Message body", lines=10)
 
+    # Manual override for organization email (new)
+    manual_email_in = gr.Textbox(label="Manual org email (optional) - used when a scraped email isn't available")
+
     with gr.Accordion("Sending options (for automatic sending via Ally AI SMTP)", open=False):
         mode = gr.Radio(choices=["Draft only (mailto + .eml)", "Send via SMTP (Gmail)"], value="Draft only (mailto + .eml)", label="Delivery mode")
         sender_email_in = gr.Textbox(label="Ally AI sender email (SMTP account)")
@@ -534,8 +616,8 @@ with gr.Blocks() as demo:
     # Wire: Search
     def _on_search(story, country):
         s, records, options, anon = run_search(story, country)
-        # set dropdown + anonymized text and prefill message
         prefill = make_body(anon, story, True, "")  # user email unknown yet
+        # return updated dropdown choices (value is first option)
         return s, records, gr.update(choices=options, value=(options[0] if options else None)), anon, prefill
 
     search_btn.click(_on_search,
@@ -553,13 +635,13 @@ with gr.Blocks() as demo:
 
     # Preview
     preview_btn.click(preview_contact,
-                      inputs=[dropdown_sel, results_table, subject_in, message_in],
+                      inputs=[dropdown_sel, results_table, subject_in, message_in, manual_email_in],
                       outputs=[preview_text_out, preview_html_out])
 
-    # Confirm (create draft or send)
+    # Confirm (create draft or send) - manual_email_in passed as last arg
     confirm_btn.click(confirm_action,
                       inputs=[mode, dropdown_sel, results_table, subject_in, message_in,
-                              user_email_in, sender_email_in, sender_pass_in, logo_url_in],
+                              user_email_in, sender_email_in, sender_pass_in, logo_url_in, manual_email_in],
                       outputs=[status_out, mailto_html_out, eml_file_out])
 
-demo.launch(share=False)
+demo.launch(share=False)