Add logging to app.py

app.py

@@ -1,5 +1,8 @@
 
+import os
 import time
+import tempfile
+import logging
 from selenium import webdriver
 from selenium.webdriver.common.by import By
 from selenium.webdriver.common.by import By
@@ -14,53 +17,70 @@ import gradio as gr
 # from selenium.webdriver.firefox.service import Service
 # from selenium.webdriver.firefox.options import Options
 # from selenium.webdriver.common.by import By
-import os
-import tempfile
+
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(levelname)s - %(message)s',
+    handlers=[
+        logging.FileHandler('/tmp/webscraper.log'),
+        logging.StreamHandler()
+    ]
+)
+logger = logging.getLogger(__name__)
 
 # Configure environment for matplotlib
 os.environ['MPLCONFIGDIR'] = tempfile.mkdtemp()
+logger.info(f"Matplotlib config directory set to: {os.environ['MPLCONFIGDIR']}")
 
 def setup_selenium():
-    options = Options()
-    options.headless = True
-    options.binary_location = '/usr/bin/firefox-esr'
-    options.add_argument("--headless")
-    options.add_argument("--disable-gpu")
-    options.add_argument("--no-sandbox")
-    options.add_argument("--disable-dev-shm-usage")
-
-    options.add_argument("--headless")  # Run in headless mode (optional)
-    options.add_argument("--start-maximized")  # Start the browser maximized
-    options.add_argument("--disable-blink-features=AutomationControlled")  # Disable automation flag
-    options.add_argument("--disable-notifications")  # Disable notifications
-    options.set_preference("dom.webnotifications.enabled", False)  # Disable web notifications
-    options.set_preference("general.useragent.override", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.82 Safari/537.36")  # Mimic a real browser's user agent
-
-
-    # service = Service(
-    #     executable_path=os.path.join(os.getcwd(), 'geckodriver'),
-    #     log_path=os.path.join(tempfile.gettempdir(), 'geckodriver.log')
-    #     )
-
-    # Ensure geckodriver is executable
-    geckodriver_path = os.path.join(os.getcwd(), 'geckodriver')
-    os.chmod(geckodriver_path, 0o755)  # Make sure it's executable
-
-    # Configure service with log_path
-    service = Service(
-        executable_path=geckodriver_path,
-        log_path=os.path.join(tempfile.gettempdir(), 'geckodriver.log')
-    )
-
-    driver = webdriver.Firefox(
-        options=options,
-        service=service
-        # executable_path=os.path.join(os.getcwd(), 'geckodriver')
-    )
-    return driver
+    try:
+        logger.info("Initializing Selenium Firefox driver...")
+        options = Options()
+        options.headless = True
+        options.binary_location = '/usr/bin/firefox-esr'
+        options.add_argument("--headless")
+        options.add_argument("--disable-gpu")
+        options.add_argument("--no-sandbox")
+        options.add_argument("--disable-dev-shm-usage")
+
+        options.add_argument("--headless")  # Run in headless mode (optional)
+        options.add_argument("--start-maximized")  # Start the browser maximized
+        options.add_argument("--disable-blink-features=AutomationControlled")  # Disable automation flag
+        options.add_argument("--disable-notifications")  # Disable notifications
+        options.set_preference("dom.webnotifications.enabled", False)  # Disable web notifications
+        options.set_preference("general.useragent.override", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.82 Safari/537.36")  # Mimic a real browser's user agent
+
+
+        # service = Service(
+        #     executable_path=os.path.join(os.getcwd(), 'geckodriver'),
+        #     log_path=os.path.join(tempfile.gettempdir(), 'geckodriver.log')
+        #     )
+
+        # Ensure geckodriver is executable
+        geckodriver_path = os.path.join(os.getcwd(), 'geckodriver')
+        os.chmod(geckodriver_path, 0o755)  # Make sure it's executable
+
+        # Configure service with log_path
+        service = Service(
+            executable_path=geckodriver_path,
+            log_path=os.path.join(tempfile.gettempdir(), 'geckodriver.log')
+        )
+
+        driver = webdriver.Firefox(
+            options=options,
+            service=service
+            # executable_path=os.path.join(os.getcwd(), 'geckodriver')
+        )
+        return driver
+    except Exception as e:
+        logger.error(f"Failed to initialize Selenium: {str(e)}")
+        raise
 
 def form_input_text(word):
     # word = 'go'
+    logger.info(f"Generating input text for word: {word}")
     return f"use the word '{word}' based on the following sentences types, give output in a table format, no quotes around sentences, no repeating the same sentence: Simple Sentence, Compound Sentence"
     
     # Complex Sentence, Compound-Complex Sentence, Declarative Sentence, Interrogative Sentence, Imperative Sentence, Exclamatory Sentence, Optative Sentence, Parallel Sentences, Balanced Sentences, Loose Sentence, Periodic Sentence, Elliptical Sentence, Inverted Sentence, Chiasmus, Anaphora, Minor Sentence, Run-on Sentence, Sentence Fragments, Conditional Sentence, Cleft Sentence, Existential Sentence, Tautology, Paradoxical Sentence, Formal Sentence, Colloquial Sentence, Poetic Sentence, Humorous or Ironic Sentence, Everyday Sentence, Technical Sentence, Standard Negation (Using 'not' or contractions), Negative Words (Indefinite pronouns/adverbs), Double Negation (Non-standard or emphatic), Affixal Negation (Prefixes like un-, in-, dis-), Implied Negation (No direct negative word), Negative Questions, Negative Imperatives, Negative interrogative, Active Sentence, Full Passive Sentence, Agentless Passive Sentence, Get-Passive (Informal), Passive with Modal Verbs, Impersonal Passive, Interrogative Passive, Negative Passive, Combined Cases: Negative + Passive, Gerundive Sentences, Indicative (States facts)"
@@ -69,12 +89,17 @@ def fetch_sentences(input_text):
     driver = None
     generated_sentences = []
     try:
+        logger.info("Starting sentence fetching process...")
+        logger.debug(f"Input text: {input_text[:100]}...")  # Log first 100 chars
+
         # Set up the WebDriver (e.g., Chrome)
         # driver = webdriver.Chrome(options=options)  # Ensure ChromeDriver is in PATH
         # driver = webdriver.Firefox(options=options)
 
         driver = setup_selenium()
         driver.implicitly_wait(5)
+
+        logger.info("Navigating to target URL...")
         # Open a website
         driver.get("https://copilot.microsoft.com/chat")
         print(f"...after: driver.get('https://copilot.microsoft.com/chat')")