Project Uploaded

Dockerfile

@@ -0,0 +1,39 @@
+# Use a base image with Python
+FROM python:3.10-slim
+
+# Set the working directory
+WORKDIR /app
+
+# Set environment variables to prevent interactive prompts during installation
+ENV DEBIAN_FRONTEND=noninteractive
+ENV PYTHONUNBUFFERED=1
+ENV PYTHONIOENCODING=utf-8
+ENV FLASK_APP=api_server.py
+ENV FLASK_RUN_HOST=0.0.0.0
+ENV FLASK_RUN_PORT=7860
+
+# Install system dependencies for Selenium and Chrome
+RUN apt-get update && apt-get install -y \
+    wget \
+    gnupg \
+    unzip \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install Google Chrome
+RUN wget -q -O - https://dl.google.com/linux/linux_signing_key.pub | apt-key add - \
+    && echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list \
+    && apt-get update && apt-get install -y google-chrome-stable \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy the requirements file and install Python dependencies
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy the rest of the application files
+COPY . .
+
+# Expose the port the app runs on
+EXPOSE 7860
+
+# Command to run the Flask application
+CMD ["flask", "run"]

api_server.py

@@ -0,0 +1,388 @@
+import os, re, json, time, base64, pickle, subprocess, threading, traceback, html
+from datetime import datetime
+from dataclasses import dataclass, field
+from typing import List, Dict, Any, Optional
+from flask import Flask, request, jsonify, send_from_directory
+from flask_cors import CORS
+from google_auth_oauthlib.flow import InstalledAppFlow
+from google.auth.transport.requests import Request
+from googleapiclient.discovery import build
+from googleapiclient.errors import HttpError
+from dotenv import load_dotenv
+
+load_dotenv()
+
+# Decode secrets at startup
+if 'CREDENTIALS_B64' in os.environ:
+    with open('credentials.json', 'w') as f:
+        f.write(base64.b64decode(os.environ['CREDENTIALS_B64']).decode('utf-8'))
+
+if 'FB_COOKIES_B64' in os.environ:
+    with open('facebook_cookies.pkl', 'wb') as f:
+        f.write(base64.b64decode(os.environ['FB_COOKIES_B64']))
+
+GROUPS_TXT   = os.environ.get("GROUPS_TXT", "groups.txt")
+SCRAPE_OUTDIR = os.environ.get("SCRAPE_OUTDIR", "scraped")
+ANALYSIS_OUTDIR = os.environ.get("ANALYSIS_OUTDIR", "analysis")
+FINAL5_PATH  = os.environ.get("FINAL5_PATH", "final5.py")
+PYTHON_BIN   = os.environ.get("PYTHON_BIN", "python")
+SENDER_EMAIL = os.environ.get("SENDER_EMAIL", "smahato@hillsidemedicalgroup.com")
+
+GEMINI_KEYS = []
+for i in range(1, 6):
+    key = os.environ.get(f"GEMINI_API_KEY_{i}")
+    if key:
+        GEMINI_KEYS.append(key)
+
+GMAIL_SCOPES = [
+    "https://www.googleapis.com/auth/gmail.send",
+    "https://www.googleapis.com/auth/gmail.metadata",
+]
+os.makedirs(SCRAPE_OUTDIR, exist_ok=True)
+os.makedirs(ANALYSIS_OUTDIR, exist_ok=True)
+
+@dataclass
+class GroupRun:
+    link: str
+    stage: str = "pending"
+    scraped_json: str = ""
+    analysis_json: str = ""
+    scraped_posts: int = 0
+    detected_posts: int = 0
+    emails_sent_by_final5: int = 0
+    error: str = ""
+
+@dataclass
+class PipelineState:
+    running: bool = False
+    message: str = "idle"
+    progress: int = 0
+    current: int = 0
+    total: int = 0
+    groups: List[GroupRun] = field(default_factory=list)
+    recipients: List[str] = field(default_factory=list)
+    summary_path: str = ""
+
+app = Flask(__name__, static_folder='.', static_url_path='')
+CORS(app)
+
+class LogBuffer:
+    def __init__(self, max_items: int = 10000):
+        self._buf: List[Dict[str, Any]] = []
+        self._lock = threading.Lock()
+        self._next_id = 1
+        self._max = max_items
+    def append(self, msg: str, level: str = "info", source: str = "server"):
+        ts = datetime.now().strftime("%H:%M:%S")
+        line = {"id": self._next_id, "ts": ts, "level": level, "source": source, "msg": msg}
+        with self._lock:
+            self._buf.append(line); self._next_id += 1
+            if len(self._buf) > self._max: self._buf = self._buf[-self._max:]
+    def clear(self):
+        with self._lock: self._buf.clear()
+    def get_after(self, after_id: int, limit: int = 500):
+        with self._lock:
+            if after_id <= 0: data = self._buf[-limit:]
+            else: data = [x for x in self._buf if x["id"] > after_id][:limit]
+            last_id = self._buf[-1]["id"] if self._buf else after_id
+        return data, last_id
+
+logs = LogBuffer()
+def log(msg: str, level: str = "info", source: str = "server"):
+    logs.append(msg, level, source)
+    print(f"[{level.upper()}][{source}] {msg}", flush=True)
+
+live_lock = threading.Lock()
+live_state: Dict[str, Any] = {
+    "group": None,
+    "counts": {"total_posts": 0, "kw_hits": 0, "ai_done": 0, "confirmed": 0, "emails": 0},
+    "posts": []
+}
+
+def reset_live_state(group_link: str):
+    with live_lock:
+        live_state["group"] = group_link
+        live_state["counts"] = {"total_posts": 0, "kw_hits": 0, "ai_done": 0, "confirmed": 0, "emails": 0}
+        live_state["posts"] = []
+
+def ensure_post_obj(pid: int) -> Dict[str, Any]:
+    with live_lock:
+        for p in live_state["posts"]:
+            if p.get("id") == pid:
+                return p
+        p = {"id": pid, "text": "", "group_link": live_state.get("group")}
+        live_state["posts"].append(p)
+        return p
+
+def load_scraped_into_live(path: str):
+    try:
+        with open(path, "r", encoding="utf-8") as f:
+            posts = json.load(f)
+    except Exception as e:
+        log(f"live load error: {e}", "error", "LIVE")
+        return
+    with live_lock:
+        live_state["posts"] = posts
+        live_state["counts"]["total_posts"] = len(posts)
+
+def handle_event_line(line: str):
+    if not line.startswith("::"): return
+    try:
+        if "::SCRAPE_SAVED::" in line:
+            path = line.split("::SCRAPE_SAVED::", 1)[1].strip()
+            if path: load_scraped_into_live(path)
+        elif "::KW_HIT::" in line:
+            d = json.loads(line.split("::KW_HIT::", 1)[1].strip())
+            p = ensure_post_obj(int(d["id"]))
+            p["found_keywords"] = d.get("found_keywords", [])
+            with live_lock: live_state["counts"]["kw_hits"] += 1
+        elif "::AI_RESULT::" in line:
+            d = json.loads(line.split("::AI_RESULT::", 1)[1].strip())
+            p = ensure_post_obj(int(d["id"]))
+            ai = d.get("ai", {})
+            p["ai"] = ai
+            with live_lock:
+                live_state["counts"]["ai_done"] += 1
+                if ai.get("is_medical_seeking"): live_state["counts"]["confirmed"] += 1
+        elif "::EMAIL_SENT::" in line:
+            d = json.loads(line.split("::EMAIL_SENT::", 1)[1].strip())
+            p = ensure_post_obj(int(d["id"]))
+            sent = int(d.get("sent", 0))
+            p["email_sent"] = sent > 0
+            if sent > 0:
+                with live_lock: live_state["counts"]["emails"] += sent
+    except Exception as e:
+        log(f"live parse error: {e}", "error", "LIVE")
+
+def read_groups(path: str) -> List[str]:
+    if not os.path.exists(path): return []
+    with open(path, "r", encoding="utf-8") as f:
+        return [ln.strip() for ln in f.read().splitlines() if ln.strip()]
+
+def slugify(url: str) -> str:
+    s = re.sub(r"[^a-zA-Z0-9]+", "-", url)
+    return s.strip("-").lower()
+
+def build_gmail_service():
+    creds = None
+    if os.path.exists("token.pickle"):
+        with open("token.pickle", "rb") as token: creds = pickle.load(token)
+    if not creds or not creds.valid:
+        if creds and creds.expired and creds.refresh_token:
+            creds.refresh(Request())
+        else:
+            if not os.path.exists("credentials.json"):
+                log("credentials.json missing; Gmail unavailable", "warn", "gmail")
+                return None
+            flow = InstalledAppFlow.from_client_secrets_file("credentials.json", GMAIL_SCOPES)
+            creds = flow.run_local_server(port=0)
+        with open("token.pickle", "wb") as token: pickle.dump(creds, token)
+    try:
+        return build("gmail", "v1", credentials=creds)
+    except Exception as e:
+        log(f"Gmail service build failed: {e}", "error", "gmail")
+        return None
+
+gmail_service = build_gmail_service()
+
+def send_html_email(to_emails: List[str], subject: str, html_content: str) -> int:
+    if not gmail_service:
+        log("Gmail not configured; skipping email", "warn", "gmail")
+        return 0
+    from email.message import EmailMessage
+    sent = 0
+    for to in to_emails:
+        try:
+            msg = EmailMessage()
+            msg["to"] = to
+            msg["from"] = SENDER_EMAIL
+            msg["subject"] = subject
+            msg.set_content(html_content, subtype="html")
+            raw = base64.urlsafe_b64encode(msg.as_bytes()).decode("utf-8")
+            gmail_service.users().messages().send(userId="me", body={"raw": raw}).execute()
+            sent += 1
+        except HttpError as e:
+            log(f"Gmail HTTP error to {to}: {e}", "error", "gmail")
+        except Exception as e:
+            log(f"Gmail send error to {to}: {e}", "error", "gmail")
+    return sent
+
+def build_confirmed_posts_email(groups_run: List[GroupRun], all_confirmed_posts: List[Dict[str, Any]]) -> str:
+    total_groups, total_scraped, total_confirmed = len(groups_run), sum(g.scraped_posts for g in groups_run), len(all_confirmed_posts)
+    table_rows = "".join(f"""
+      <tr>
+        <td style="padding: 8px; border-bottom: 1px solid #eee;"><a href="{g.link}" target="_blank">{g.link}</a></td>
+        <td style="padding: 8px; border-bottom: 1px solid #eee; text-align: center;">{g.scraped_posts}</td>
+        <td style="padding: 8px; border-bottom: 1px solid #eee; text-align: center;">{g.detected_posts}</td>
+        <td style="padding: 8px; border-bottom: 1px solid #eee;">{"OK" if g.stage == "done" else "ERROR"}</td>
+      </tr>""" for g in groups_run)
+    summary_table_html = f"""<h3>Group Summary</h3><table style="width: 100%; border-collapse: collapse; margin-top: 8px; border: 1px solid #ddd;"><thead><tr style="background: #0f172a; color: #fff;"><th style="text-align: left; padding: 8px;">Group Link</th><th style="text-align: center; padding: 8px;">Posts Scraped</th><th style="text-align: center; padding: 8px;">Confirmed Posts</th><th style="text-align: left; padding: 8px;">Status</th></tr></thead><tbody>{table_rows}</tbody></table>"""
+    if all_confirmed_posts:
+        posts_html = "".join(f"""
+            <div style="margin-bottom: 25px; padding: 12px; border: 1px solid #ddd; border-radius: 5px; background-color: #fafafa;">
+              <h4 style="margin-top: 0; margin-bottom: 8px;">Post ID: {p.get("id", "N/A")} | Urgency: {p.get("ai_analysis", {}).get("urgency_level", "N/A")} | Confidence: {p.get("ai_analysis", {}).get("confidence", "N/A")}</h4>
+              <p style="margin: 5px 0;"><strong>Summary:</strong> {html.escape(p.get("ai_analysis", {}).get("medical_summary", "N/A"))}</p>
+              <p style="margin: 5px 0;"><strong>Text:</strong></p>
+              <pre style="white-space: pre-wrap; background-color: #f0f0f0; padding: 8px; border: 1px solid #eee; border-radius: 3px; font-family: monospace; font-size: 0.9em;">{html.escape(p.get("text", "N/A"))}</pre>
+              <p style="margin: 5px 0;"><a href="{p.get("group_link", "#")}" target="_blank">View Group</a></p>
+            </div>""" for p in all_confirmed_posts)
+    else: posts_html = "<p>No confirmed medical posts were found during this run.</p>"
+    return f"""<!DOCTYPE html><html><head><title>Hillside Medical Group - Confirmed Medical Posts Summary</title></head><body style="font-family: Arial, sans-serif; margin: 0; padding: 0; background-color: #f5f5f5;"><div style="max-width: 900px; margin: 20px auto; padding: 20px; background-color: #ffffff; border: 1px solid #e0e0e0; border-radius: 8px;"><div style="background: #1e3c72; color: #fff; padding: 16px 20px; border-radius: 6px 6px 0 0;"><h2 style="margin: 0;">Hillside Medical Group - Confirmed Medical Posts</h2><div style="font-size: 0.9em;">Run completed: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}</div></div><div style="padding: 16px;"><p><strong>Overall Summary:</strong> Processed {total_groups} groups, scraped {total_scraped} posts, found {total_confirmed} confirmed medical posts.</p><hr style="margin: 20px 0; border: 0; border-top: 1px solid #eee;">{summary_table_html}<hr style="margin: 20px 0; border: 0; border-top: 1px solid #eee;"><h3>Confirmed Posts Details</h3>{posts_html}</div><div style="margin-top: 20px; padding: 10px; font-size: 0.8em; color: #666; border-top: 1px solid #eee;"><p>This email contains posts identified as potentially seeking personal medical help. Please review and take appropriate action.</p><p><em>Note: The link provided is to the group. Direct post links are not currently extracted.</em></p></div></div></body></html>"""
+
+state = PipelineState()
+
+def stream_process_lines(args: List[str], env: Optional[Dict[str, str]] = None, tag: str = "FINAL5") -> int:
+    log(f"Exec: {' '.join(args)}", "info", tag)
+    proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, bufsize=1, universal_newlines=True, env=env or os.environ.copy())
+    def pump(pipe, name):
+        for raw in pipe:
+            line = (raw or "").rstrip("\n")
+            if not line: continue
+            if line.startswith("::"):
+                try: handle_event_line(line)
+                except Exception as e: log(f"event parse error: {e}", "error", tag)
+            log(line, "info" if name == "stdout" else "warn", tag)
+    t1 = threading.Thread(target=pump, args=(proc.stdout, "stdout"), daemon=True)
+    t2 = threading.Thread(target=pump, args=(proc.stderr, "stderr"), daemon=True)
+    t1.start(); t2.start()
+    rc = proc.wait()
+    t1.join(timeout=0.2); t2.join(timeout=0.2)
+    log(f"Exit code: {rc}", "info", tag)
+    return rc
+
+def call_final5_for_group(group_url: str, out_json: str, analysis_json: str, recipients: List[str]) -> Dict[str, Any]:
+    args = [PYTHON_BIN, FINAL5_PATH, "--group", group_url, "--out", out_json, "--analysis-out", analysis_json, "--recipients", ",".join(recipients), "--sender", SENDER_EMAIL, "--headless"]
+    if GEMINI_KEYS: args.extend(["--gemini-keys", ",".join(GEMINI_KEYS)])
+    env = os.environ.copy()
+    env["PYTHONUNBUFFERED"] = "1"
+    env["PYTHONIOENCODING"] = "utf-8"
+    rc = stream_process_lines(args, env=env, tag="FINAL5")
+    return {"ok": rc == 0, "code": rc}
+
+def run_pipeline(recipients: List[str]):
+    try:
+        logs.clear()
+        log("Pipeline starting", "info", "ORCHESTRATOR")
+        state.running, state.message, state.progress, state.recipients = True, "initializing", 0, recipients
+        state.groups.clear()
+        links = read_groups(GROUPS_TXT)
+        state.total = len(links)
+        if not links:
+            log("No groups found in groups.txt", "warn", "ORCHESTRATOR")
+            state.message, state.running = "No groups", False
+            return
+        all_confirmed_posts = []
+        for i, link in enumerate(links, start=1):
+            reset_live_state(link)
+            g = GroupRun(link=link, stage="running")
+            state.groups.append(g)
+            state.current, state.message, state.progress = i, f"Processing {link}", int(((i - 1) / max(1, state.total)) * 100)
+            log(f"[{i}/{state.total}] Processing group: {link}", "info", "ORCHESTRATOR")
+            slug = slugify(link)
+            out_json, analysis_json = os.path.join(SCRAPE_OUTDIR, f"{slug}.json"), os.path.join(ANALYSIS_OUTDIR, f"analysis_{slug}.json")
+            g.scraped_json, g.analysis_json = out_json, analysis_json
+            result = call_final5_for_group(link, out_json, analysis_json, recipients)
+            if not result.get("ok"):
+                g.stage, g.error = "error", f"final5 exit code {result.get('code')}"
+                log(f"final5 failed for {link}: code {result.get('code')}", "error", "ORCHESTRATOR")
+            else:
+                try:
+                    if os.path.exists(out_json):
+                        with open(out_json, "r", encoding="utf-8") as f: g.scraped_posts = len(json.load(f))
+                    if os.path.exists(analysis_json):
+                        with open(analysis_json, "r", encoding="utf-8") as f: a = json.load(f)
+                        g.detected_posts = a.get("confirmed_medical", 0)
+                        g.emails_sent_by_final5 = a.get("emails_sent", 0)
+                        confirmed_posts = a.get("posts", [])
+                        for post in confirmed_posts:
+                            if "group_link" not in post: post["group_link"] = link
+                        all_confirmed_posts.extend(confirmed_posts)
+                    g.stage = "done"
+                    log(f"Group done: scraped={g.scraped_posts}, confirmed={g.detected_posts}", "info", "ORCHESTRATOR")
+                except Exception as e:
+                    g.stage, g.error = "error", f"parse_error: {e}"
+                    log(f"Parsing outputs failed for {link}: {e}", "error", "ORCHESTRATOR")
+            state.progress = int((i / max(1, state.total)) * 100)
+        try:
+            html_content = build_confirmed_posts_email(state.groups, all_confirmed_posts)
+            subject = f"🩺 Hillside - Confirmed Medical Posts Found ({len(all_confirmed_posts)} total)"
+            sent_count = send_html_email(recipients, subject, html_content)
+            log(f"Consolidated email sent to {len(recipients)} recipient(s), {sent_count} successful", "info", "GMAIL")
+        except Exception as e:
+            log(f"Error building or sending consolidated email: {e}", "error", "ORCHESTRATOR")
+        summary = {"run_date": datetime.now().isoformat(), "groups": [g.__dict__ for g in state.groups]}
+        summary_path = os.path.join(ANALYSIS_OUTDIR, "analysis_summary.json")
+        with open(summary_path, "w", encoding="utf-8") as f: json.dump(summary, f, ensure_ascii=False, indent=2)
+        state.summary_path, state.message, state.progress, state.running = summary_path, "All groups processed", 100, False
+        log("Pipeline finished", "info", "ORCHESTRATOR")
+    except Exception as e:
+        state.message, state.running = f"pipeline_error: {e}", False
+        log(f"Pipeline error: {e}\n{traceback.format_exc()}", "error", "ORCHESTRATOR")
+
+@app.route("/")
+def index():
+    return send_from_directory('.', 'index.html')
+
+@app.get("/api/system/status")
+def system_status():
+    return jsonify({
+        "gmail": gmail_service is not None, "groups_file_exists": os.path.exists(GROUPS_TXT),
+        "groups_count": len(read_groups(GROUPS_TXT)), "scrape_outdir": SCRAPE_OUTDIR,
+        "analysis_outdir": ANALYSIS_OUTDIR, "sender_email": SENDER_EMAIL,
+        "final5_exists": os.path.exists(FINAL5_PATH), "gemini_keys_count": len(GEMINI_KEYS)
+    })
+
+@app.get("/api/groups")
+def api_groups():
+    return jsonify({"groups": read_groups(GROUPS_TXT)})
+
+@app.post("/api/process/start")
+def api_process_start():
+    if state.running: return jsonify({"success": False, "message": "Already running"}), 409
+    data = request.json or {}
+    recips = data.get("recipients") or [SENDER_EMAIL]
+    if isinstance(recips, str): recips = [e.strip() for e in recips.split(",") if e.strip()]
+    threading.Thread(target=run_pipeline, args=(recips,), daemon=True).start()
+    log(f"Start requested by client; recipients={recips}", "info", "API")
+    return jsonify({"success": True, "message": "Pipeline started", "recipients": recips})
+
+@app.get("/api/process/status")
+def api_process_status():
+    return jsonify({"running": state.running, "message": state.message, "progress": state.progress,
+        "current": state.current, "total": state.total, "groups": [g.__dict__ for g in state.groups]})
+
+@app.get("/api/process/logs")
+def api_process_logs():
+    data, last_id = logs.get_after(int(request.args.get("after", "0")), limit=int(request.args.get("limit", "500")))
+    return jsonify({"entries": data, "last": last_id})
+
+@app.post("/api/process/clear-logs")
+def api_clear_logs():
+    logs.clear()
+    log("Logs cleared by client", "info", "API")
+    return jsonify({"success": True})
+
+@app.get("/api/live/state")
+def api_live_state():
+    with live_lock: return jsonify({"success": True, "data": live_state})
+
+@app.get("/api/results/summary")
+def api_results_summary():
+    p = state.summary_path or os.path.join(ANALYSIS_OUTDIR, "analysis_summary.json")
+    if not os.path.exists(p): return jsonify({"success": False, "message": "No summary yet"}), 404
+    with open(p, "r", encoding="utf-8") as f: return jsonify({"success": True, "data": json.load(f)})
+
+@app.get("/api/recipients")
+def api_get_recipients():
+    recipients_path = "recipients.json"
+    if not os.path.exists(recipients_path): return jsonify({"success": False, "message": "recipients.json not found"}), 404
+    try:
+        with open(recipients_path, "r", encoding="utf-8") as f: data = json.load(f)
+        if not isinstance(data, list): return jsonify({"success": False, "message": "Invalid format"}), 500
+        return jsonify({"success": True, "data": data})
+    except Exception as e:
+        return jsonify({"success": False, "message": f"Error reading file: {str(e)}"}), 500
+
+if __name__ == "__main__":
+    port = int(os.environ.get("PORT", 7860))
+    app.run(host="0.0.0.0", port=port)

final5.py

@@ -0,0 +1,462 @@
+import os, re, sys, time, json, base64, pickle, argparse, traceback
+from typing import List, Dict, Any, Tuple
+from datetime import datetime
+
+try:
+    sys.stdout.reconfigure(encoding="utf-8", errors="replace")
+    sys.stderr.reconfigure(encoding="utf-8", errors="replace")
+except Exception:
+    pass
+
+from selenium import webdriver
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.common.exceptions import (
+    StaleElementReferenceException, NoSuchElementException, TimeoutException
+)
+from google_auth_oauthlib.flow import InstalledAppFlow
+from google.auth.transport.requests import Request
+from googleapiclient.discovery import build
+from googleapiclient.errors import HttpError
+import google.generativeai as genai
+from google.api_core.exceptions import ResourceExhausted
+
+def get_args():
+    p = argparse.ArgumentParser(description="Scrape one FB group, analyze, and email alerts.")
+    p.add_argument("--group", required=True)
+    p.add_argument("--out", required=True)
+    p.add_argument("--analysis-out", required=True)
+    p.add_argument("--recipients", default="")
+    p.add_argument("--sender", default=os.environ.get("SENDER_EMAIL", ""))
+    p.add_argument("--cookies-file", default=os.environ.get("FB_COOKIES_FILE","facebook_cookies.pkl"))
+    p.add_argument("--max-scrolls", type=int, default=int(os.environ.get("MAX_SCROLLS","5")))
+    p.add_argument("--scroll-pause", type=float, default=float(os.environ.get("SCROLL_PAUSE","3")))
+    p.add_argument("--gemini-keys", default="")
+    p.add_argument("--headless", action="store_true", help="Prefer headless browser")
+    return p.parse_args()
+
+GMAIL_SCOPES = [
+    "https://www.googleapis.com/auth/gmail.send",
+    "https://www.googleapis.com/auth/gmail.metadata",
+]
+
+def build_gmail_service():
+    creds = None
+    if os.path.exists("token.pickle"):
+        try:
+            with open("token.pickle", "rb") as token:
+                creds = pickle.load(token)
+        except Exception:
+            creds = None
+    if not creds or not creds.valid:
+        if creds and creds.expired and creds.refresh_token:
+            try:
+                creds.refresh(Request())
+            except Exception:
+                creds = None
+        if not creds:
+            if not os.path.exists("credentials.json"):
+                print("[GMAIL] credentials.json missing; Gmail unavailable")
+                return None
+            flow = InstalledAppFlow.from_client_secrets_file("credentials.json", GMAIL_SCOPES)
+            creds = flow.run_local_server(port=0)
+        with open("token.pickle", "wb") as token:
+            pickle.dump(creds, token)
+    try:
+        svc = build("gmail", "v1", credentials=creds)
+        _ = svc.users().getProfile(userId="me").execute()
+        return svc
+    except Exception as e:
+        print(f"[GMAIL] service build failed: {e}")
+        return None
+
+def send_html_email(service, sender: str, to_list: List[str], subject: str, html: str) -> int:
+    if not service:
+        print("[GMAIL] service not available; skipping email")
+        return 0
+    from email.message import EmailMessage
+    sent = 0
+    for to in to_list:
+        try:
+            msg = EmailMessage()
+            msg["to"] = to
+            msg["from"] = sender
+            msg["subject"] = subject
+            msg.set_content(html, subtype="html")
+            raw = base64.urlsafe_b64encode(msg.as_bytes()).decode("utf-8")
+            service.users().messages().send(userId="me", body={"raw": raw}).execute()
+            sent += 1
+        except HttpError as e:
+            print(f"[GMAIL] http error to {to}: {e}")
+        except Exception as e:
+            print(f"[GMAIL] send error to {to}: {e}")
+    return sent
+
+GEMINI_MODEL = "gemini-1.5-flash"
+
+class GeminiManager:
+    def __init__(self, api_keys: List[str]):
+        self.api_keys = api_keys
+        self.current_key_index = 0
+        self.model = None
+        self._setup_model()
+
+    def _setup_model(self):
+        if not self.api_keys:
+            print("[GEMINI] No API keys provided")
+            self.model = None
+            return
+        while self.current_key_index < len(self.api_keys):
+            try:
+                api_key = self.api_keys[self.current_key_index]
+                genai.configure(api_key=api_key)
+                self.model = genai.GenerativeModel(GEMINI_MODEL)
+                print(f"[GEMINI] Using API key {self.current_key_index + 1}")
+                return
+            except Exception as e:
+                print(f"[GEMINI] Failed to setup with key {self.current_key_index + 1}: {e}")
+                self.current_key_index += 1
+        print("[GEMINI] All API keys failed")
+        self.model = None
+
+    def rotate_key(self):
+        self.current_key_index += 1
+        self._setup_model()
+
+    def is_available(self):
+        return self.model is not None
+
+    def generate_content(self, prompt: str):
+        if not self.is_available():
+            raise Exception("No available Gemini model")
+        try:
+            return self.model.generate_content(prompt)
+        except ResourceExhausted as e:
+            self.rotate_key()
+            if self.is_available():
+                return self.model.generate_content(prompt)
+            else:
+                raise e
+
+def parse_retry_seconds_from_error(err: Exception) -> int:
+    s = str(err)
+    m1 = re.search(r"retry[_ ]delay\s*\{\s*seconds:\s*(\d+)", s, re.IGNORECASE)
+    if m1: return int(m1.group(1))
+    m2 = re.search(r'"retryDelay"\s*:\s*"(\d+)s"', s)
+    if m2: return int(m2.group(1))
+    return 45
+
+def ai_medical_intent(gemini_manager: GeminiManager, post_text: str, found_keywords: List[str]) -> Dict[str,Any]:
+    fallback = {
+        "is_medical_seeking": False,
+        "confidence": "low",
+        "medical_summary": "Not a medical request (AI unavailable/throttled)",
+        "suggested_services": [],
+        "urgency_level": "low",
+        "analysis": "Keyword-based fallback",
+        "reasoning": "short explanation",
+        "matched_keywords": found_keywords
+    }
+    if not gemini_manager or not gemini_manager.is_available():
+        return fallback
+    keywords_str = ", ".join(found_keywords) if found_keywords else "none"
+    prompt = f"""
+Analyze this social post and decide if the author is genuinely seeking medical help, doctor/hospital recommendations, or healthcare services for PERSONAL HEALTH NEEDS (not business, donations, or casual mentions).
+KEYWORDS FOUND IN POST: {keywords_str}
+CRITICAL RULES:
+1. ONLY flag posts where someone is seeking medical care for themselves or a loved one
+2. IGNORE posts about:
+   - Business services (e.g., "Looking for a doctor for my clinic")
+   - Donations or fundraising (e.g., "Raising money for surgery")
+   - Selling medical products
+   - Job postings for medical professionals
+   - General health information sharing
+   - Research or academic inquiries
+3. ONLY flag if it's a PERSONAL HEALTH NEED
+Post: "{post_text}"
+Return ONLY JSON:
+{{
+  "is_medical_seeking": true/false,
+  "confidence": "high/medium/low",
+  "medical_summary": "short summary",
+  "suggested_services": ["service1","service2"],
+  "urgency_level": "high/medium/low",
+  "analysis": "why it's seeking help",
+  "reasoning": "short explanation",
+  "matched_keywords": ["keyword1", "keyword2"]
+}}
+"""
+    for attempt in range(1, 5):
+        try:
+            resp = gemini_manager.generate_content(prompt)
+            txt = (resp.text or "").strip()
+            s, e = txt.find("{"), txt.rfind("}") + 1
+            if s >= 0 and e > s:
+                result = json.loads(txt[s:e])
+                result["is_medical_seeking"] = bool(result.get("is_medical_seeking", False))
+                if "matched_keywords" not in result:
+                    result["matched_keywords"] = found_keywords
+                return result
+            return fallback
+        except ResourceExhausted as e:
+            wait_s = min(parse_retry_seconds_from_error(e) + 2, 120)
+            print(f"[GEMINI] 429 rate limit; backoff {wait_s}s (attempt {attempt}/4)")
+            time.sleep(wait_s)
+            if gemini_manager.is_available():
+                continue
+            else:
+                return fallback
+        except Exception as e:
+            print(f"[GEMINI] error: {e}")
+            gemini_manager.rotate_key()
+            if not gemini_manager.is_available():
+                return fallback
+    return fallback
+
+MEDICAL_KEYWORDS = [
+    "doctor","physician","primary care","healthcare","medical","clinic","hospital",
+    "urgent care","emergency","er","specialist","pediatrician","dentist",
+    "gynecologist","obgyn","women's health","health center","family doctor",
+    "maternity","prenatal","postnatal","labor","delivery",
+    "need doctor","looking for doctor","find doctor","recommend doctor",
+    "medical help","health help","appointment","checkup","treatment",
+    "prescription","medicine","surgery","best hospital","best clinic",
+    "where to go","doctor recommendation",
+    "pregnancy","birth control","contraception","fertility",
+    "hillside","medical group","wellness center"
+]
+
+def contains_keywords(text: str) -> Tuple[bool, List[str]]:
+    tl = (text or "").lower()
+    hits = [kw for kw in MEDICAL_KEYWORDS if kw in tl]
+    return (len(hits) > 0, hits)
+
+def new_driver(headless: bool):
+    options = webdriver.ChromeOptions()
+    options.add_argument("--disable-notifications")
+    options.add_argument("--disable-web-security")
+    options.add_argument("--disable-features=IsolateOrigins,site-per-process")
+    options.add_argument("--disable-blink-features=AutomationControlled")
+    options.add_experimental_option("useAutomationExtension", False)
+    options.add_experimental_option("excludeSwitches", ["enable-automation"])
+    options.add_argument("--window-size=1920,1080")
+    options.add_argument("--lang=en-US,en")
+    options.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36")
+    if headless:
+        options.add_argument("--headless=new")
+        options.add_argument("--disable-gpu")
+        options.add_argument("--disable-dev-shm-usage")
+        options.add_argument("--no-sandbox")
+        options.add_argument("--disable-extensions")
+        options.add_argument("--disable-plugins")
+        options.add_argument("--disable-images")
+    driver = webdriver.Chrome(options=options)
+    try:
+        driver.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
+            "source": "Object.defineProperty(navigator, 'webdriver', { get: () => undefined });"
+        })
+    except Exception:
+        pass
+    return driver
+
+def load_cookies(driver, cookies_file: str):
+    print("[FB] Loading Facebook homepage...")
+    driver.get("https://www.facebook.com")
+    time.sleep(3)
+    try:
+        with open(cookies_file, "rb") as f:
+            cookies = pickle.load(f)
+        for cookie in cookies:
+            if "sameSite" in cookie and cookie["sameSite"] not in ["Strict","Lax","None"]:
+                cookie["sameSite"] = "Lax"
+            try:
+                driver.add_cookie(cookie)
+            except Exception:
+                pass
+        print("[FB] Cookies loaded. Refreshing page...")
+        driver.refresh()
+        time.sleep(5)
+    except FileNotFoundError:
+        raise RuntimeError(f"[FB] Cookies file not found: {cookies_file}")
+    except Exception as e:
+        raise RuntimeError(f"[FB] Cookie load error: {e}")
+
+def wait_group_feed(driver, wait):
+    wait.until(EC.presence_of_element_located((By.TAG_NAME, "body")))
+    feed_loaded = False
+    start = time.time(); timeout = 30
+    while not feed_loaded and (time.time() - start) < timeout:
+        try:
+            driver.find_element(By.XPATH, "//div[@data-pagelet='GroupFeed' or @role='feed']")
+            feed_loaded = True; break
+        except NoSuchElementException:
+            try:
+                driver.find_element(By.XPATH, "//div[@role='article']")
+                feed_loaded = True; break
+            except NoSuchElementException:
+                pass
+        time.sleep(1)
+    if not feed_loaded:
+        raise TimeoutException("Timed out waiting for group feed")
+
+def find_message_nodes(driver):
+    nodes = driver.find_elements(By.XPATH, "//div[@data-ad-preview='message']")
+    if nodes: return nodes
+    nodes = driver.find_elements(By.XPATH, "//div[@data-ad-comet-preview='message']")
+    if nodes: return nodes
+    return driver.find_elements(By.XPATH, "//div[@role='article']//div[@dir='auto' and string-length(normalize-space())>0]")
+
+def scrape_group(driver, wait, group_url: str, max_scrolls: int, pause: float):
+    print(f"[SCRAPE] Navigating to group: {group_url}")
+    driver.get(group_url)
+    wait_group_feed(driver, wait)
+    posts, seen, rects = [], set(), set()
+    total = 0
+    for s in range(max_scrolls):
+        print(f"[SCRAPE] --- Scroll {s+1}/{max_scrolls} ---")
+        driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
+        try:
+            wait.until(lambda d: d.execute_script("return document.readyState") == "complete")
+        except Exception:
+            pass
+        time.sleep(pause)
+        try:
+            divs = find_message_nodes(driver)
+            print(f"[SCRAPE] Nodes found: {len(divs)}")
+        except Exception as e:
+            print(f"[SCRAPE] find error: {e}")
+            continue
+        added = 0
+        for i, d in enumerate(divs):
+            try:
+                rect = (d.rect.get('x'), d.rect.get('y'), d.rect.get('width'), d.rect.get('height'))
+                if rect in rects: continue
+                rects.add(rect)
+            except Exception:
+                pass
+            try:
+                txt = (d.text or "").strip()
+                if len(txt) < 20:
+                    try:
+                        art = d.find_element(By.XPATH, "ancestor::div[@role='article']")
+                        txt = (art.text or "").strip()
+                    except Exception:
+                        pass
+            except StaleElementReferenceException:
+                continue
+            if not txt or len(txt) < 20: continue
+            if txt in seen: continue
+            wc = len(re.findall(r"\b\w+\b", txt))
+            if wc > 7 and not any(j in txt for j in ["LikeCommentShare","Write a comment","View more comments"]):
+                seen.add(txt)
+                total += 1
+                posts.append({"id": total, "text": txt, "group_link": group_url})
+                added += 1
+        print(f"[SCRAPE] New posts this scroll: {added}")
+    print(f"[SCRAPE] Total unique posts: {total}")
+    return posts
+
+def try_scrape_with_fallback(group_url: str, cookies_file: str, max_scrolls: int, pause: float):
+    driver = new_driver(headless=True)
+    wait = WebDriverWait(driver, 15)
+    try:
+        load_cookies(driver, cookies_file)
+        posts = scrape_group(driver, wait, group_url, max_scrolls, pause)
+        return posts, driver
+    except Exception as e:
+        try:
+            driver.quit()
+        except Exception:
+            pass
+        print(f"[SCRAPE] Error in headless mode: {e}")
+        return [], None
+
+def main():
+    args = get_args()
+    os.makedirs(os.path.dirname(args.out) or ".", exist_ok=True)
+    os.makedirs(os.path.dirname(args.analysis_out) or ".", exist_ok=True)
+    
+    gemini_keys = []
+    if args.gemini_keys:
+        gemini_keys = [k.strip() for k in args.gemini_keys.split(",") if k.strip()]
+    else:
+        for i in range(1, 6):
+            key = os.environ.get(f"GEMINI_API_KEY_{i}")
+            if key:
+                gemini_keys.append(key)
+    gemini_manager = GeminiManager(gemini_keys) if gemini_keys else None
+    
+    gmail = build_gmail_service()
+    
+    posts, driver = try_scrape_with_fallback(args.group, args.cookies_file, args.max_scrolls, args.scroll_pause)
+    if driver:
+        try:
+            driver.quit()
+        except Exception:
+            pass
+            
+    try:
+        with open(args.out, "w", encoding="utf-8") as f:
+            json.dump(posts, f, ensure_ascii=False, indent=2)
+        print(f"[SCRAPE] Saved scraped posts to {args.out}")
+        print(f"::SCRAPE_SAVED::{args.out}")
+    except Exception as e:
+        print(f"[SCRAPE] Error saving posts: {e}")
+        
+    keyword_hits, confirmed = [], []
+    for p in posts:
+        has, hits = contains_keywords(p.get("text",""))
+        if has:
+            p["found_keywords"] = hits
+            keyword_hits.append(p)
+            print(f"::KW_HIT::{json.dumps({'id': p['id'], 'found_keywords': hits}, ensure_ascii=False)}")
+            
+    per_call_sleep = 7
+    analyzed_posts = []
+    for idx, p in enumerate(keyword_hits, start=1):
+        found_kws = p.get("found_keywords", [])
+        ai = ai_medical_intent(gemini_manager, p.get("text",""), found_kws)
+        p["ai_analysis"] = ai
+        print(f"::AI_RESULT::{json.dumps({'id': p['id'], 'ai': ai}, ensure_ascii=False)}")
+        if ai.get("is_medical_seeking"):
+            confirmed.append(p)
+            analyzed_posts.append(p)
+        if idx < len(keyword_hits):
+            time.sleep(per_call_sleep)
+            
+    report = {
+        "analysis_date": datetime.now().isoformat(),
+        "group_link": args.group,
+        "total_posts": len(posts),
+        "keyword_hits": len(keyword_hits),
+        "confirmed_medical": len(confirmed),
+        "emails_sent": 0,
+        "posts": confirmed
+    }
+    
+    try:
+        with open(args.analysis_out, "w", encoding="utf-8") as f:
+            json.dump(report, f, ensure_ascii=False, indent=2)
+        print(f"[ANALYSIS] Saved analysis to {args.analysis_out}")
+        print(f"::ANALYSIS_SAVED::{args.analysis_out}")
+    except Exception as e:
+        print(f"[ANALYSIS] Error saving analysis: {e}")
+
+if __name__ == "__main__":
+    try:
+        # Decode secrets from environment variables and write to files
+        if 'CREDENTIALS_B64' in os.environ:
+            with open('credentials.json', 'w') as f:
+                f.write(base64.b64decode(os.environ['CREDENTIALS_B64']).decode('utf-8'))
+        
+        if 'FB_COOKIES_B64' in os.environ:
+            with open('facebook_cookies.pkl', 'wb') as f:
+                f.write(base64.b64decode(os.environ['FB_COOKIES_B64']))
+
+        main()
+    except Exception as e:
+        print("Unhandled error:")
+        print(e)
+        print(traceback.format_exc())
+        raise

groups.txt

@@ -0,0 +1,2 @@
+https://www.facebook.com/groups/Lifepaths
+https://www.facebook.com/groups/1324510239249728

index.html

@@ -0,0 +1,586 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="utf-8"/>
+<meta name="viewport" content="width=device-width,initial-scale=1"/>
+<title>Hillside Medical Group - Social Media Monitor</title>
+<style>
+  :root{
+    --bg:#f6f7fb; --ink:#0f172a; --muted:#475569;
+    --card:#ffffff; --border:#e5e7eb; --shadow:0 6px 24px rgba(15,23,42,.06);
+    --grad1:#2c5aa0; --grad2:#1e3c72; /* Blue theme */
+    --ok:#22c55e; --warn:#f59e0b; --err:#ef4444;
+    --term-bg:#0b1221; --term-ink:#22c55e; --term-hdr:#07101d; --term-border:#1f2937; --term-meta:#86efac;
+    --badge:#e2e8f0; --badge-ink:#0f172a;
+    --kw:#f59e0b; --kw-ink:#1f2937;
+    --pill:#dcfce7; --pill-ink:#166534;
+  }
+  *{box-sizing:border-box}
+  body{margin:0;font-family:Inter,system-ui,Arial,sans-serif;background:var(--bg);color:var(--ink)}
+  .wrap{max-width:1400px;margin:24px auto;padding:0 16px}
+  .header{background:linear-gradient(135deg,var(--grad1),var(--grad2));color:#fff;border-radius:14px;padding:18px;box-shadow:0 8px 28px rgba(15,23,42,.18); text-align: center;} /* Centered header text */
+  h1{margin:0 0 6px 0;font-size:22px;font-weight:600}
+  .grid{display:grid;gap:16px;grid-template-columns:1fr 420px}
+  .card{background:var(--card);border:1px solid var(--border);border-radius:14px;box-shadow:var(--shadow);padding:18px}
+  .btn{background:var(--ok);color:#fff;border:none;border-radius:10px;padding:10px 14px;font-weight:600;cursor:pointer}
+  .btn.red{background:var(--err)} .btn.gray{background:#64748b}
+  .btn[disabled]{opacity:.5;cursor:not-allowed}
+  input, select{width:100%;padding:10px 12px;border:1px solid var(--border);border-radius:10px} /* Added select style */
+  label{font-weight:600; display: block; margin-bottom: 6px;} /* Improved label spacing */
+  .muted{color:var(--muted);font-size:13px}
+  .bar{height:12px;background:var(--border);border-radius:999px;overflow:hidden}
+  .fill{height:100%;background:linear-gradient(90deg,var(--grad1),#667eea);width:0%}
+  .mono{font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace;font-size:12px;color:#334155;word-break:break-all}
+
+  /* Terminal */
+  .terminal{background:var(--term-bg);border-radius:14px;border:1px solid var(--term-border);height:520px;display:flex;flex-direction:column}
+  .term-header{color:#a7f3d0;background:var(--term-hdr);border-radius:14px 14px 0 0;padding:10px 12px;font-weight:700;font-size:13px;border-bottom:1px solid var(--term-border)}
+  .term-body{flex:1;overflow:auto;padding:10px 12px}
+  .term-line{font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace;font-size:12px;color:var(--term-ink);white-space:pre-wrap;word-break:break-word;margin:0}
+  .term-meta{color:var(--term-meta)}
+  .term-warn{color:#facc15}
+  .term-err{color:#f87171}
+  .term-footer{padding:8px 12px;border-top:1px solid var(--term-border);display:flex;gap:8px}
+
+  /* Analysis */
+  .tabs{display:flex;gap:10px;margin-bottom:10px}
+  .tab{background:#e2e8f0;color:#0f172a;border:none;border-radius:10px;padding:8px 12px;font-weight:600;cursor:pointer}
+  .tab.active{background:#1e293b;color:#fff}
+  .flex{display:flex;gap:10px;flex-wrap:wrap;align-items:center}
+  .pill{display:inline-block;background:var(--pill);color:var(--pill-ink);padding:2px 10px;border-radius:999px;font-size:12px;font-weight:700}
+  .stat{display:grid;grid-template-columns:repeat(4,minmax(120px,1fr));gap:10px;margin-top:10px}
+  .stat .box{background:#f8fafc;border:1px solid var(--border);border-radius:12px;padding:10px;text-align:center}
+  .stat .num{font-weight:800;font-size:22px}
+  .filters{display:flex;gap:8px;flex-wrap:wrap;align-items:center;margin-top:10px}
+  .filter{background:#e5e7eb;border:none;border-radius:999px;padding:6px 10px;font-weight:600;cursor:pointer}
+  .filter.active{background:#1e293b;color:#fff}
+  .search{flex:1;min-width:240px}
+  .posts{margin-top:12px;display:grid;gap:10px}
+  .post{background:#ffffff;border:1px solid var(--border);border-radius:12px;padding:12px}
+  .post-hdr{display:flex;justify-content:space-between;gap:8px;flex-wrap:wrap}
+  .badge{display:inline-block;background:var(--badge);color:var(--badge-ink);padding:2px 8px;border-radius:999px;font-size:12px;font-weight:700}
+  .kw{display:inline-block;background:#fff3cd;color:var(--kw-ink);border:1px solid #fde68a;padding:2px 8px;border-radius:999px;font-size:12px;margin:2px 4px 0 0}
+  .ai{margin-top:8px;background:#f1f5f9;border:1px solid var(--border);border-radius:10px;padding:10px}
+  .ai-row{display:flex;gap:8px;flex-wrap:wrap;margin-bottom:6px}
+  .ai-pill{display:inline-block;border-radius:999px;padding:2px 10px;font-size:12px;font-weight:800}
+  .ai-ok{background:#dcfce7;color:#166534}
+  .ai-mid{background:#fef9c3;color:#92400e}
+  .ai-low{background:#fee2e2;color:#991b1b}
+  .email{font-size:12px;font-weight:700}
+  .email.ok{color:#166534} .email.no{color:#991b1b}
+  .reason{margin-top:6px;background:#fff;border:1px dashed var(--border);border-radius:10px;padding:8px;display:none}
+  .reason.show{display:block}
+
+  /* Recipient Dropdown */
+  .recipient-container { margin-top: 10px; position: relative; } /* Container for dropdown and custom input */
+  .recipient-select { margin-bottom: 10px; } /* Space below select if custom input appears */
+  #custom-recipient { margin-top: 6px; display: none; } /* Initially hidden custom input */
+</style>
+</head>
+<body>
+<div class="wrap">
+  <div class="header">
+    <h1>🩺 Hillside Medical Group - Social Media Monitor</h1>
+    <div class="muted">Automated monitoring and analysis of medical help requests in Facebook groups.</div>
+  </div>
+
+  <div class="grid" style="margin-top:16px">
+    <div class="left">
+      <div class="card">
+        <h3>System Status</h3>
+        <div id="sys"></div>
+        <button class="btn" style="margin-top:8px" onclick="refreshSystem()">Refresh Status</button>
+      </div>
+
+      <div class="card" style="margin-top:16px">
+        <h3>Start Monitoring Process</h3>
+        <label for="recipient-select">Report Recipients</label>
+        <div class="recipient-container">
+            <select id="recipient-select" class="recipient-select" onchange="handleRecipientChange()">
+                <option value="">-- Loading Recipients --</option>
+            </select>
+            <input type="email" id="custom-recipient" placeholder="Enter custom email address..." />
+        </div>
+        <div class="muted" style="margin:6px 0 10px">
+            Select a recipient from the list or choose 'Custom' to enter an email address.
+            The report summary will be sent to the selected address(es) after processing.
+        </div>
+        <div class="flex">
+          <button id="start" class="btn" onclick="startProcess()">Start Monitoring</button>
+          <button class="btn gray" onclick="refreshLive()">Refresh Live View</button>
+        </div>
+        <div style="margin-top:12px">
+          <div style="display:flex;justify-content:space-between"><b>Overall Progress</b><span id="pct" class="muted">0%</span></div>
+          <div class="bar"><div id="fill" class="fill"></div></div>
+          <div id="msg" class="muted" style="margin-top:6px">idle</div>
+        </div>
+      </div>
+
+      <div class="card" style="margin-top:16px">
+        <div class="tabs">
+          <button id="tab-groups" class="tab active" onclick="switchSection('groups')">Groups</button>
+          <button id="tab-analysis" class="tab" onclick="switchSection('analysis')">Analysis (Live)</button>
+          <button id="tab-summary" class="tab" onclick="switchSection('summary')">Summary</button>
+        </div>
+
+        <!-- Groups -->
+        <div id="section-groups">
+          <h3>Configured Groups (from groups.txt)</h3>
+          <div id="groups"></div>
+        </div>
+
+        <!-- Analysis Live -->
+        <div id="section-analysis" style="display:none">
+          <div class="flex">
+            <div class="pill" id="live-group">Group: –</div>
+          </div>
+
+          <div class="stat">
+            <div class="box"><div class="num" id="cnt-total">0</div><div class="muted">Total Posts</div></div>
+            <div class="box"><div class="num" id="cnt-kw">0</div><div class="muted">Keyword Hits</div></div>
+            <div class="box"><div class="num" id="cnt-ai">0</div><div class="muted">AI Analyzed</div></div>
+            <div class="box"><div class="num" id="cnt-confirmed">0</div><div class="muted">Confirmed</div></div>
+          </div>
+
+          <div class="filters">
+            <button class="filter active" id="flt-all" onclick="setFilter('all')">All</button>
+            <button class="filter" id="flt-kw" onclick="setFilter('kw')">Keyword</button>
+            <button class="filter" id="flt-confirmed" onclick="setFilter('confirmed')">Confirmed</button>
+            <input id="search" class="search" placeholder="Search post text..."/>
+          </div>
+
+          <div class="posts" id="posts"></div>
+        </div>
+
+        <!-- Summary -->
+        <div id="section-summary" style="display:none">
+          <h3>Last Run Summary</h3>
+          <div id="summary" class="muted">No summary available yet.</div>
+        </div>
+      </div>
+    </div>
+
+    <div class="right">
+      <div class="terminal">
+        <div class="term-header">Process Logs (Live)</div>
+        <div id="term" class="term-body"></div>
+        <div class="term-footer">
+          <button class="btn" onclick="scrollBottom()">Scroll to Bottom</button>
+          <button class="btn red" onclick="clearLogs()">Clear Logs</button>
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+
+<script>
+const API = "/api";  //Server re babu
+
+async function refreshSystem(){
+  try {
+    const r = await fetch(`${API}/system/status`);
+    const j = await r.json();
+    const chip = (b)=>`<span style="padding:2px 8px;border-radius:999px;font-size:12px;font-weight:700;background:${b?'#dcfce7':'#fee2e2'};color:${b?'#166534':'#991b1b'}">${b?'OK':'Missing'}</span>`;
+    document.getElementById('sys').innerHTML = `
+      <table>
+        <tr><td>Gmail Service</td><td>${chip(j.gmail)}</td></tr>
+        <tr><td>Groups File (groups.txt)</td><td>${chip(j.groups_file_exists)} • Count: ${j.groups_count}</td></tr>
+        <tr><td>Processing Script (final5.py)</td><td>${chip(j.final5_exists)}</td></tr>
+        <tr><td>Default Sender</td><td class="mono">${j.sender_email}</td></tr>
+        <tr><td>Data Folders</td><td class="mono">${j.scrape_outdir} • ${j.analysis_outdir}</td></tr>
+      </table>`;
+  } catch (error) {
+      console.error("Error refreshing system status:", error);
+      document.getElementById('sys').innerHTML = `<div class="muted">Error loading system status.</div>`;
+  }
+}
+
+async function loadGroups(){
+  try {
+    const r = await fetch(`${API}/groups`);
+    const j = await r.json();
+    const list = j.groups || [];
+    if(!list.length){
+        document.getElementById('groups').innerHTML = `<div class="muted">Please add Facebook group links to 'groups.txt' (one per line).</div>`;
+        return;
+    }
+    document.getElementById('groups').innerHTML = list.map((g,i)=>`
+      <div style="border:1px solid var(--border);border-radius:12px;padding:10px;margin:6px 0;background:#fff">
+        <div style="display:flex;gap:8px;align-items:center;flex-wrap:wrap"><b>#${i+1}</b><span class="mono">${g}</span></div>
+        <div id="g-${i}" class="muted">Status: Pending</div>
+      </div>
+    `).join("");
+  } catch (error) {
+      console.error("Error loading groups:", error);
+      document.getElementById('groups').innerHTML = `<div class="muted">Error loading groups list.</div>`;
+  }
+}
+
+// ---------------- Recipient Management ----------------
+let recipientList = []; // Store fetched recipients
+
+async function loadRecipients() {
+    const selectElement = document.getElementById('recipient-select');
+    selectElement.innerHTML = '<option value="">-- Loading Recipients --</option>';
+    try {
+        const response = await fetch(`${API}/recipients`);
+        const result = await response.json();
+
+        if (result.success && Array.isArray(result.data)) {
+            recipientList = result.data;
+            populateRecipientDropdown();
+        } else {
+            console.warn("API did not return a successful recipient list:", result);
+            selectElement.innerHTML = '<option value="">-- Error Loading --</option>';
+        }
+    } catch (error) {
+        console.error("Error fetching recipients:", error);
+        selectElement.innerHTML = '<option value="">-- Network Error --</option>';
+    }
+}
+
+function populateRecipientDropdown() {
+    const selectElement = document.getElementById('recipient-select');
+    selectElement.innerHTML = ''; // Clear loading option
+
+    // Add default recipient first
+    const defaultRecipient = "smahato@hillsidemedicalgroup.com";
+    let defaultOptionFound = false;
+    recipientList.forEach(recipient => {
+        const option = document.createElement('option');
+        option.value = recipient.email;
+        option.textContent = `${recipient.name} (${recipient.email})`;
+        if (recipient.email === defaultRecipient) {
+            option.selected = true;
+            defaultOptionFound = true;
+        }
+        selectElement.appendChild(option);
+    });
+
+    // Add Custom option
+    const customOption = document.createElement('option');
+    customOption.value = "custom";
+    customOption.textContent = "-- Custom Email --";
+    selectElement.appendChild(customOption);
+
+    // If default wasn't in the list, add it and select it
+    if (!defaultOptionFound) {
+        const defaultOption = document.createElement('option');
+        defaultOption.value = defaultRecipient;
+        defaultOption.textContent = `Subash Mahato (Default) (${defaultRecipient})`;
+        defaultOption.selected = true;
+        selectElement.insertBefore(defaultOption, selectElement.firstChild); // Add to top
+    }
+}
+
+function handleRecipientChange() {
+    const selectElement = document.getElementById('recipient-select');
+    const customInput = document.getElementById('custom-recipient');
+    if (selectElement.value === 'custom') {
+        customInput.style.display = 'block';
+        customInput.focus();
+    } else {
+        customInput.style.display = 'none';
+        customInput.value = ''; // Clear if deselected
+    }
+}
+
+function getSelectedRecipients() {
+    const selectElement = document.getElementById('recipient-select');
+    const customInput = document.getElementById('custom-recipient');
+    let emails = [];
+
+    if (selectElement.value === 'custom' && customInput.value.trim() !== '') {
+        // Validate custom email (basic check)
+        const emailRegex = /^[^\s@]+@[^\s@]+\.[^\s@]+$/;
+        if (emailRegex.test(customInput.value.trim())) {
+            emails.push(customInput.value.trim());
+        } else {
+             alert("Please enter a valid custom email address.");
+             return null; // Signal error
+        }
+    } else if (selectElement.value) {
+        emails.push(selectElement.value);
+    } else {
+        // No recipient selected (shouldn't happen with default)
+        alert("Please select a recipient.");
+        return null;
+    }
+    return emails;
+}
+
+
+// ---------------- Process control ----------------
+async function startProcess(){
+  document.getElementById('start').disabled = true;
+
+  const recipients = getSelectedRecipients();
+  if (!recipients) {
+      // Error message already shown in getSelectedRecipients
+      document.getElementById('start').disabled = false;
+      return;
+  }
+
+  try {
+      await fetch(`${API}/process/clear-logs`, {method:'POST'}); // Clear previous logs
+      const r = await fetch(`${API}/process/start`, {
+          method:'POST',
+          headers:{'Content-Type':'application/json'},
+          body: JSON.stringify({recipients: recipients}) // Send selected recipients
+      });
+      const j = await r.json();
+      if(!j.success){
+          alert(j.message||'Failed to start the process.');
+          document.getElementById('start').disabled = false;
+          return;
+      }
+      pollStatus(); // Start polling for status updates
+  } catch (error) {
+      console.error("Error starting process:", error);
+      alert("An error occurred while trying to start the process.");
+      document.getElementById('start').disabled = false;
+  }
+}
+
+async function pollStatus(){
+  try {
+    const r = await fetch(`${API}/process/status`);
+    const j = await r.json();
+    document.getElementById('msg').innerText = j.message || 'Idle';
+    document.getElementById('pct').innerText = `${j.progress||0}%`;
+    document.getElementById('fill').style.width = `${j.progress||0}%`;
+    (j.groups||[]).forEach((g,idx)=>{
+      const el = document.getElementById(`g-${idx}`);
+      if(!el) return;
+      el.innerHTML = `Status: ${g.stage} • Scraped: ${g.scraped_posts} • Confirmed: ${g.detected_posts}${g.error?(' • Error: '+g.error):''}`;
+    });
+    if(j.running){
+        setTimeout(pollStatus, 1200); // Poll every 1.2 seconds while running
+    } else {
+        document.getElementById('start').disabled = false;
+        loadSummary(); // Load the final summary when done
+    }
+  } catch (error) {
+      console.error("Error polling status:", error);
+      // Optionally, retry or show error state
+      setTimeout(pollStatus, 3000); // Retry after a longer delay on error
+  }
+}
+
+// ---------------- Logs (terminal) ----------------
+let lastLogId = 0;
+function appendLogs(entries){
+  const term = document.getElementById('term');
+  const nearBottom = term.scrollTop + term.clientHeight >= term.scrollHeight - 40;
+  entries.forEach(e=>{
+    const div = document.createElement('div');
+    div.className = 'term-line';
+    const color = e.level === 'error' ? 'term-err' : e.level === 'warn' ? 'term-warn' : '';
+    div.innerHTML = `<span class="term-meta">[${e.ts}] [${e.source}]</span> <span class="${color}">${escapeHtml(e.msg)}</span>`;
+    term.appendChild(div);
+  });
+  if(nearBottom || entries.length){ term.scrollTop = term.scrollHeight; }
+}
+async function pollLogs(){
+  try{
+    const r = await fetch(`${API}/process/logs?after=${lastLogId}&limit=500`);
+    const j = await r.json();
+    if(j.entries && j.entries.length){
+      appendLogs(j.entries);
+      lastLogId = j.last || lastLogId;
+    }
+  }catch(e){
+      console.error("Error polling logs:", e);
+      // Continue polling even on error
+  }
+  setTimeout(pollLogs, 900); // Poll logs every 0.9 seconds
+}
+function scrollBottom(){
+    const term = document.getElementById('term');
+    term.scrollTop = term.scrollHeight;
+}
+async function clearLogs(){
+    try {
+        await fetch(`${API}/process/clear-logs`, {method:'POST'});
+        document.getElementById('term').innerHTML = '';
+        lastLogId = 0;
+    } catch (error) {
+        console.error("Error clearing logs:", error);
+        // Optionally inform user
+    }
+}
+function escapeHtml(s){
+    return (s||'').replace(/[&<>"']/g, c => ({'&':'&amp;','<':'<','>':'>','"':'&quot;',"'":'&#039;'}[c]));
+}
+
+// ---------------- Live Analysis ----------------
+let liveFilter = 'all';  // 'all' | 'kw' | 'confirmed'
+let liveSearch = '';
+
+function setFilter(f){
+  liveFilter = f;
+  document.getElementById('flt-all').classList.toggle('active', f==='all');
+  document.getElementById('flt-kw').classList.toggle('active', f==='kw');
+  document.getElementById('flt-confirmed').classList.toggle('active', f==='confirmed');
+  refreshLive();
+}
+document.getElementById('search').addEventListener('input', e=>{
+  liveSearch = e.target.value.toLowerCase();
+  refreshLive();
+});
+
+async function refreshLive(){
+  try {
+    const r = await fetch(`${API}/live/state`);
+    const j = await r.json();
+    if(!j.success) {
+        console.warn("Live state API returned not success:", j);
+        return;
+    }
+    const data = j.data || {};
+    renderLive(data);
+  } catch (error) {
+      console.error("Error refreshing live state:", error);
+      // Optionally update UI to show error
+  }
+}
+
+function renderLive(data){
+  // header + counts
+  document.getElementById('live-group').innerText = `Current Group: ${data.group || '–'}`;
+  const c = data.counts || {};
+  document.getElementById('cnt-total').innerText = c.total_posts || 0;
+  document.getElementById('cnt-kw').innerText = c.kw_hits || 0;
+  document.getElementById('cnt-ai').innerText = c.ai_done || 0;
+  document.getElementById('cnt-confirmed').innerText = c.confirmed || 0;
+
+  const posts = Array.isArray(data.posts) ? data.posts : [];
+  // filter/search
+  const filtered = posts.filter(p=>{
+    const hasKW = Array.isArray(p.found_keywords) && p.found_keywords.length>0;
+    const isConfirmed = p.ai && p.ai.is_medical_seeking;
+    if(liveFilter==='kw' && !hasKW) return false;
+    if(liveFilter==='confirmed' && !isConfirmed) return false;
+    if(liveSearch && !(p.text||'').toLowerCase().includes(liveSearch)) return false;
+    return true;
+  });
+
+  const html = filtered.map(p=>{
+    const hasKW = Array.isArray(p.found_keywords) && p.found_keywords.length>0;
+    const ai = p.ai || null;
+    const confirm = !!(ai && ai.is_medical_seeking);
+    const conf = ai ? (ai.confidence||'').toLowerCase() : '';
+    const urg = ai ? (ai.urgency_level||'').toLowerCase() : '';
+    const confClass = conf==='high' ? 'ai-ok' : conf==='medium' ? 'ai-mid' : 'ai-low';
+    const urgClass  = urg==='high' ? 'ai-low' : urg==='medium' ? 'ai-mid' : 'ai-ok';
+    const emailTxt = p.email_sent ? '<span class="email ok">Email Sent</span>' : '<span class="email no">No Email</span>';
+    return `
+      <div class="post">
+        <div class="post-hdr">
+          <div class="flex">
+            <span class="badge">Post #${p.id || '-'}</span>
+            ${hasKW ? '<span class="badge">Keyword Hit</span>' : ''}
+            ${confirm ? '<span class="badge" style="background-color: #bfdbfe; color: #1e40af;">CONFIRMED</span>' : ''} <!-- Blue badge for confirmed -->
+          </div>
+          <div class="muted mono">Group: ${escapeHtml(p.group_link || 'N/A')}</div>
+        </div>
+        <div style="margin-top:6px; white-space: pre-wrap;">${escapeHtml(p.text || '')}</div> <!-- pre-wrap for text formatting -->
+
+        ${hasKW ? `
+          <div style="margin-top:6px">${(p.found_keywords||[]).map(k=>`<span class="kw">${escapeHtml(k)}</span>`).join('')}</div>
+        ` : ''}
+
+        ${ai ? `
+          <div class="ai">
+            <div class="ai-row">
+              <span class="ai-pill ${confClass}">Confidence: ${(ai.confidence||'').toUpperCase()}</span>
+              <span class="ai-pill ${urgClass}">Urgency: ${(ai.urgency_level||'').toUpperCase()}</span>
+              ${emailTxt}
+            </div>
+            <div><b>Summary:</b> ${escapeHtml(ai.medical_summary || '')}</div>
+            <div style="margin-top:4px"><b>Analysis:</b> ${escapeHtml(ai.analysis || '')}</div>
+            ${Array.isArray(ai.suggested_services) && ai.suggested_services.length ? `
+              <div style="margin-top:4px"><b>Suggested Services:</b> ${(ai.suggested_services||[]).map(s=>`<span class="badge" style="margin-right:6px">${escapeHtml(s)}</span>`).join('')}</div>
+            `:''}
+            <div style="margin-top:6px">
+              <button class="btn gray" onclick="toggleReason(${p.id})">Show Reasoning</button>
+              <div id="reason-${p.id}" class="reason">${escapeHtml(ai.reasoning || 'No reasoning provided.')}</div>
+            </div>
+          </div>
+        ` : '<div class="muted">Awaiting AI analysis...</div>'}
+      </div>
+    `;
+  }).join('');
+  document.getElementById('posts').innerHTML = html || `<div class="muted">No posts match the current filters.</div>`;
+}
+
+function toggleReason(id){
+  const el = document.getElementById(`reason-${id}`);
+  if(!el) return;
+  el.classList.toggle('show');
+}
+
+// auto poll live state
+async function pollLive(){
+  await refreshLive();
+  setTimeout(pollLive, 1200); // Poll live view every 1.2 seconds
+}
+
+// ---------------- Summary ----------------
+async function loadSummary(){
+  try {
+    const r = await fetch(`${API}/results/summary`);
+    const el = document.getElementById('summary');
+    if(r.status!==200){
+        el.innerHTML = `<div class="muted">Processing complete. Summary will appear here shortly.</div>`;
+        // Retry once after a short delay if not found immediately
+        setTimeout(async () => {
+             const retry = await fetch(`${API}/results/summary`);
+             if(retry.status === 200) {
+                 const j = await retry.json();
+                 el.innerHTML = `<pre class="mono">${JSON.stringify(j.data,null,2)}</pre>`;
+             }
+        }, 2000);
+        return;
+    }
+    const j = await r.json();
+    // Check if data exists and format nicely
+    if (j.success && j.data) {
+         // You can create a more user-friendly summary view here instead of raw JSON
+         // For now, we'll keep the JSON view
+         el.innerHTML = `<pre class="mono">${JSON.stringify(j.data,null,2)}</pre>`;
+    } else {
+        el.innerHTML = `<div class="muted">Summary data unavailable.</div>`;
+    }
+  } catch (error) {
+      console.error("Error loading summary:", error);
+      document.getElementById('summary').innerHTML = `<div class="muted">Error loading summary.</div>`;
+  }
+}
+
+// ---------------- Tabs ----------------
+function switchSection(name){
+  const sec = (id,show)=>document.getElementById(id).style.display = show?'block':'none';
+  document.getElementById('tab-groups').classList.toggle('active', name==='groups');
+  document.getElementById('tab-analysis').classList.toggle('active', name==='analysis');
+  document.getElementById('tab-summary').classList.toggle('active', name==='summary');
+  sec('section-groups', name==='groups');
+  sec('section-analysis', name==='analysis');
+  sec('section-summary', name==='summary');
+  if(name==='analysis') refreshLive();
+  if(name==='summary') loadSummary();
+}
+
+// ---------------- Boot ----------------
+// Load initial data and start polling
+refreshSystem();
+loadGroups();
+loadRecipients(); // Load recipients on startup
+loadSummary();
+pollStatus(); // Start status polling
+pollLogs();   // Start log polling
+pollLive();   // Start live analysis polling
+
+</script>
+</body>
+</html>

recipients.json

@@ -0,0 +1,6 @@
+[
+  {"name": "Dr. Patel", "email": ""},
+  {"name": "Bruno", "email": "admin@hillsidemedicalgroup.com"},
+  {"name": "Afren", "email": "reports@hillsidemedicalgroup.com"},
+  {"name": "Sonu", "email": "smahato@hillsidemedicalgroup.com"}
+]

requirements.txt

@@ -0,0 +1,10 @@
+selenium
+google-api-python-client
+google-auth-oauthlib
+google-auth-httplib2
+google-generativeai
+Flask
+Flask-Cors
+python-dotenv
+requests
+webdriver-manager