Project Uploaded

api_server.py

@@ -4,17 +4,14 @@ from dataclasses import dataclass, field
 from typing import List, Dict, Any, Optional
 from flask import Flask, request, jsonify, send_from_directory
 from flask_cors import CORS
-from google_auth_oauthlib.flow import InstalledAppFlow
-from google.auth.transport.requests import Request
+from google.oauth2 import service_account
 from googleapiclient.discovery import build
 from googleapiclient.errors import HttpError
-from google.oauth2 import service_account  # Added import
 from dotenv import load_dotenv
 
 load_dotenv()
 
 def decode_base64_with_padding(b64_string: str) -> bytes:
-    """Decodes a Base64 string, adding missing padding if necessary."""
     missing_padding = len(b64_string) % 4
     if missing_padding:
         b64_string += '=' * (4 - missing_padding)
@@ -26,29 +23,21 @@ def decode_base64_with_padding(b64_string: str) -> bytes:
 
 # Define a writable directory for ALL runtime files
 WRITABLE_DIR = "/tmp"
-CREDENTIALS_PATH = os.path.join(WRITABLE_DIR, "credentials.json")
 COOKIES_PATH = os.path.join(WRITABLE_DIR, "facebook_cookies.pkl")
-TOKEN_PATH = os.path.join(WRITABLE_DIR, "token.pickle")
-SERVICE_ACCOUNT_PATH = os.path.join(WRITABLE_DIR, "service_account.json")  # Added
+SERVICE_ACCOUNT_FILE = os.path.join(WRITABLE_DIR, "service_account.json")
 
 # Decode secrets at startup into the /tmp directory
-if 'CREDENTIALS_B64' in os.environ:
-    decoded_creds = decode_base64_with_padding(os.environ['CREDENTIALS_B64'])
-    if decoded_creds:
-        with open(CREDENTIALS_PATH, 'w') as f:
-            f.write(decoded_creds.decode('utf-8'))
-
 if 'FB_COOKIES_B64' in os.environ:
     decoded_cookies = decode_base64_with_padding(os.environ['FB_COOKIES_B64'])
     if decoded_cookies:
         with open(COOKIES_PATH, 'wb') as f:
             f.write(decoded_cookies)
 
-# Added: Decode service account credentials
+# --- FIX: Decode the Service Account secret ---
 if 'SERVICE_ACCOUNT_B64' in os.environ:
     decoded_service_account = decode_base64_with_padding(os.environ['SERVICE_ACCOUNT_B64'])
     if decoded_service_account:
-        with open(SERVICE_ACCOUNT_PATH, 'w') as f:
+        with open(SERVICE_ACCOUNT_FILE, 'w') as f:
             f.write(decoded_service_account.decode('utf-8'))
 
 GROUPS_TXT   = os.environ.get("GROUPS_TXT", "groups.txt")
@@ -56,12 +45,8 @@ FINAL5_PATH  = os.environ.get("FINAL5_PATH", "final5.py")
 PYTHON_BIN   = os.environ.get("PYTHON_BIN", "python")
 SENDER_EMAIL = os.environ.get("SENDER_EMAIL", "smahato@hillsidemedicalgroup.com")
 
-# --- MODIFICATION START ---
-# Point the output directories to the writable /tmp directory
 SCRAPE_OUTDIR = os.path.join(WRITABLE_DIR, "scraped")
 ANALYSIS_OUTDIR = os.path.join(WRITABLE_DIR, "analysis")
-# --- MODIFICATION END ---
-
 
 GEMINI_KEYS = []
 for i in range(1, 6):
@@ -69,14 +54,32 @@ for i in range(1, 6):
     if key:
         GEMINI_KEYS.append(key)
 
-# Fixed scopes (removed trailing spaces)
-GMAIL_SCOPES = [
-    "https://www.googleapis.com/auth/gmail.send",
-    "https://www.googleapis.com/auth/gmail.metadata",
-]
+GMAIL_SCOPES = [ "https://www.googleapis.com/auth/gmail.send" ]
 os.makedirs(SCRAPE_OUTDIR, exist_ok=True)
 os.makedirs(ANALYSIS_OUTDIR, exist_ok=True)
 
+# --- FIX: Implement non-interactive, service-account based authentication ---
+def build_gmail_service():
+    if not os.path.exists(SERVICE_ACCOUNT_FILE):
+        log("Service account file not found, Gmail unavailable.", "error", "GMAIL")
+        return None
+    try:
+        # Impersonate the SENDER_EMAIL user
+        creds = service_account.Credentials.from_service_account_file(
+            SERVICE_ACCOUNT_FILE, scopes=GMAIL_SCOPES).with_subject(SENDER_EMAIL)
+        
+        service = build("gmail", "v1", credentials=creds)
+        log("Gmail service built successfully using service account.", "info", "GMAIL")
+        return service
+    except Exception as e:
+        log(f"Failed to build Gmail service: {e}", "error", "GMAIL")
+        log("CRITICAL: Ensure your service account has Domain-Wide Delegation enabled and is authorized for the GMail API for the user {SENDER_EMAIL}", "error", "GMAIL")
+        return None
+
+gmail_service = build_gmail_service()
+
+# The rest of api_server.py remains unchanged...
+# (Full code omitted for brevity, just replace the top section and the build_gmail_service function)
 @dataclass
 class GroupRun:
     link: str
@@ -199,35 +202,6 @@ def slugify(url: str) -> str:
     s = re.sub(r"[^a-zA-Z0-9]+", "-", url)
     return s.strip("-").lower()
 
-def build_gmail_service():
-    # Try service account first
-    if os.path.exists(SERVICE_ACCOUNT_PATH):
-        try:
-            credentials = service_account.Credentials.from_service_account_file(
-                SERVICE_ACCOUNT_PATH, scopes=GMAIL_SCOPES)
-            return build("gmail", "v1", credentials=credentials)
-        except Exception as e:
-            log(f"Service account authentication failed: {e}", "error", "gmail")
-    
-    # Fallback to token.pickle if exists
-    creds = None
-    if os.path.exists(TOKEN_PATH):
-        with open(TOKEN_PATH, "rb") as token: creds = pickle.load(token)
-    if not creds or not creds.valid:
-        if creds and creds.expired and creds.refresh_token:
-            creds.refresh(Request())
-        else:
-            log("No valid credentials available; Gmail unavailable", "warn", "gmail")
-            return None
-        with open(TOKEN_PATH, "wb") as token: pickle.dump(creds, token)
-    try:
-        return build("gmail", "v1", credentials=creds)
-    except Exception as e:
-        log(f"Gmail service build failed: {e}", "error", "gmail")
-        return None
-
-gmail_service = build_gmail_service()
-
 def send_html_email(to_emails: List[str], subject: str, html_content: str) -> int:
     if not gmail_service:
         log("Gmail not configured; skipping email", "warn", "gmail")
@@ -244,6 +218,7 @@ def send_html_email(to_emails: List[str], subject: str, html_content: str) -> in
             raw = base64.urlsafe_b64encode(msg.as_bytes()).decode("utf-8")
             gmail_service.users().messages().send(userId="me", body={"raw": raw}).execute()
             sent += 1
+            log(f"Successfully sent email to {to}", "info", "GMAIL")
         except HttpError as e:
             log(f"Gmail HTTP error to {to}: {e}", "error", "gmail")
         except Exception as e:

final5.py

@@ -14,19 +14,15 @@ from selenium.webdriver.common.by import By
 from selenium.webdriver.support.ui import WebDriverWait
 from selenium.webdriver.support import expected_conditions as EC
 from selenium.common.exceptions import (
-    StaleElementReferenceException, NoSuchElementException, TimeoutException
+    StaleElementReferenceException, NoSuchElementException, TimeoutException, SessionNotCreatedException
 )
-from google_auth_oauthlib.flow import InstalledAppFlow
-from google.auth.transport.requests import Request
+from google.oauth2 import service_account
 from googleapiclient.discovery import build
 from googleapiclient.errors import HttpError
 import google.generativeai as genai
 from google.api_core.exceptions import ResourceExhausted
-from google.oauth2 import service_account
 
 WRITABLE_DIR = "/tmp"
-CREDENTIALS_PATH = os.path.join(WRITABLE_DIR, "credentials.json")
-TOKEN_PATH = os.path.join(WRITABLE_DIR, "token.pickle")
 SERVICE_ACCOUNT_FILE = os.path.join(WRITABLE_DIR, "service_account.json")
 
 def get_args():
@@ -43,48 +39,30 @@ def get_args():
     p.add_argument("--headless", action="store_true", help="Prefer headless browser")
     return p.parse_args()
 
-GMAIL_SCOPES = [
-    "https://www.googleapis.com/auth/gmail.send",
-    "https://www.googleapis.com/auth/gmail.metadata",
-]
+GMAIL_SCOPES = [ "https://www.googleapis.com/auth/gmail.send" ]
 
+# --- FIX: Simplify this function. It's not the primary auth method. ---
 def build_gmail_service():
+    """Builds Gmail service if a service account file exists."""
     if os.path.exists(SERVICE_ACCOUNT_FILE):
         try:
+            sender_email = os.environ.get("SENDER_EMAIL")
+            if not sender_email:
+                print("[GMAIL] SENDER_EMAIL environment variable not set.")
+                return None
+
             credentials = service_account.Credentials.from_service_account_file(
-                SERVICE_ACCOUNT_FILE, scopes=GMAIL_SCOPES)
+                SERVICE_ACCOUNT_FILE, scopes=GMAIL_SCOPES).with_subject(sender_email)
+            
             svc = build("gmail", "v1", credentials=credentials)
-            _ = svc.users().getProfile(userId="me").execute()
             return svc
         except Exception as e:
-            print(f"[GMAIL] Service account authentication failed: {e}")
-    
-    creds = None
-    if os.path.exists(TOKEN_PATH):
-        try:
-            with open(TOKEN_PATH, "rb") as token:
-                creds = pickle.load(token)
-        except Exception:
-            creds = None
-    if not creds or not creds.valid:
-        if creds and creds.expired and creds.refresh_token:
-            try:
-                creds.refresh(Request())
-            except Exception:
-                creds = None
-        if not creds:
-            print("[GMAIL] No valid credentials available or interactive auth required.")
+            print(f"[GMAIL] Service account authentication failed in final5.py: {e}")
             return None
-        with open(TOKEN_PATH, "wb") as token:
-            pickle.dump(creds, token)
-    try:
-        svc = build("gmail", "v1", credentials=creds)
-        _ = svc.users().getProfile(userId="me").execute()
-        return svc
-    except Exception as e:
-        print(f"[GMAIL] service build failed: {e}")
-        return None
+    print("[GMAIL] Service account file not found in final5.py.")
+    return None
 
+# The send_html_email function is kept for potential future direct use, but it's not called by main()
 def send_html_email(service, sender: str, to_list: List[str], subject: str, html: str) -> int:
     if not service:
         print("[GMAIL] service not available; skipping email")
@@ -246,23 +224,18 @@ def contains_keywords(text: str) -> Tuple[bool, List[str]]:
     hits = [kw for kw in MEDICAL_KEYWORDS if kw in tl]
     return (len(hits) > 0, hits)
 
-# Add this import at the top if not already present
-import tempfile
-import os
-
+# --- FIX: Set a writable cache path for Selenium Manager and ensure unique user-data-dir ---
 def new_driver(headless: bool):
     options = webdriver.ChromeOptions()
     
-    # --- MODIFICATION START: Selenium Cache and User Data Dir ---
-    # Specify a writable directory for Selenium Manager's cache
-    os.environ["SE_CACHE_PATH"] = "/tmp/.cache/selenium"
-    # Ensure the directory exists
-    os.makedirs("/tmp/.cache/selenium", exist_ok=True)
+    # Specify a writable directory for Selenium Manager's driver cache
+    cache_path = os.path.join(WRITABLE_DIR, "selenium")
+    os.makedirs(cache_path, exist_ok=True)
+    os.environ["SE_CACHE_PATH"] = cache_path
     
-    # Create a unique temporary directory for Chrome's user data
-    user_data_dir = tempfile.mkdtemp(prefix="chrome_user_data_", dir="/tmp")
+    # Create a unique temporary directory for this specific Chrome instance's user data
+    user_data_dir = tempfile.mkdtemp(prefix="chrome_user_data_", dir=WRITABLE_DIR)
     options.add_argument(f"--user-data-dir={user_data_dir}")
-    # --- MODIFICATION END ---
     
     options.add_argument("--disable-notifications")
     options.add_argument("--disable-web-security")
@@ -272,7 +245,6 @@ def new_driver(headless: bool):
     options.add_experimental_option("excludeSwitches", ["enable-automation"])
     options.add_argument("--window-size=1920,1080")
     options.add_argument("--lang=en-US,en")
-    # Consider removing the hardcoded user-agent or making it configurable
     options.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36")
     if headless:
         options.add_argument("--headless=new")
@@ -282,7 +254,9 @@ def new_driver(headless: bool):
         options.add_argument("--disable-extensions")
         options.add_argument("--disable-plugins")
         options.add_argument("--disable-images")
+    
     driver = webdriver.Chrome(options=options)
+    
     try:
         driver.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
             "source": "Object.defineProperty(navigator, 'webdriver', { get: () => undefined });"
@@ -291,6 +265,7 @@ def new_driver(headless: bool):
         pass
     return driver
 
+
 def load_cookies(driver, cookies_file: str):
     print("[FB] Loading Facebook homepage...")
     driver.get("https://www.facebook.com")
@@ -389,19 +364,32 @@ def scrape_group(driver, wait, group_url: str, max_scrolls: int, pause: float):
     return posts
 
 def try_scrape_with_fallback(group_url: str, cookies_file: str, max_scrolls: int, pause: float):
-    driver = new_driver(headless=True)
-    wait = WebDriverWait(driver, 15)
+    driver = None
     try:
+        driver = new_driver(headless=True)
+        wait = WebDriverWait(driver, 15)
         load_cookies(driver, cookies_file)
         posts = scrape_group(driver, wait, group_url, max_scrolls, pause)
         return posts, driver
     except Exception as e:
-        try:
-            driver.quit()
-        except Exception:
-            pass
         print(f"[SCRAPE] Error in headless mode: {e}")
         return [], None
+    finally:
+        if driver:
+            try:
+                # Also try to clean up the temporary user data directory
+                user_data_dir = None
+                for arg in driver.options.arguments:
+                    if arg.startswith('--user-data-dir='):
+                        user_data_dir = arg.split('=', 1)[1]
+                        break
+                driver.quit()
+                if user_data_dir and os.path.exists(user_data_dir):
+                    import shutil
+                    shutil.rmtree(user_data_dir, ignore_errors=True)
+            except Exception as e:
+                print(f"Error during driver cleanup: {e}")
+
 
 def main():
     args = get_args()
@@ -418,14 +406,11 @@ def main():
                 gemini_keys.append(key)
     gemini_manager = GeminiManager(gemini_keys) if gemini_keys else None
     
-    gmail = build_gmail_service()
+    # This is not used to send mail, but just to check if auth is possible
+    _ = build_gmail_service()
     
     posts, driver = try_scrape_with_fallback(args.group, args.cookies_file, args.max_scrolls, args.scroll_pause)
-    if driver:
-        try:
-            driver.quit()
-        except Exception:
-            pass
+    # Driver is cleaned up in try_scrape_with_fallback's finally block
             
     try:
         with open(args.out, "w", encoding="utf-8") as f:
@@ -478,7 +463,6 @@ if __name__ == "__main__":
     try:
         main()
     except Exception as e:
-        print("Unhandled error:")
-        print(e)
+        print(f"Unhandled error in main: {e}")
         print(traceback.format_exc())
         raise