afa

Dockerfile

@@ -1,61 +1,51 @@
-# # Use the official Python image as the base image
-# FROM python:3.10-slim
-
-# # Install dependencies for Selenium and Chrome
-# RUN apt-get update && apt-get install -y \
-#     wget \
-#     unzip \
-#     curl \
-#     gnupg \
-#     libnss3 \
-#     libgconf-2-4 \
-#     libxi6 \
-#     libxcursor1 \
-#     libxrandr2 \
-#     libxss1 \
-#     libxtst6 \
-#     fonts-liberation \
-#     xdg-utils \
-#     libatk-bridge2.0-0 \
-#     libgtk-3-0 \
-#     --no-install-recommends && \
-#     apt-get clean && \
-#     rm -rf /var/lib/apt/lists/*
+# Use the official Python image as the base image
+FROM python:3.10-slim
+
+# Install dependencies for Selenium and Chrome
+RUN apt-get update && apt-get install -y \
+    wget \
+    unzip \
+    curl \
+    gnupg \
+    libnss3 \
+    libgconf-2-4 \
+    libxi6 \
+    libxcursor1 \
+    libxrandr2 \
+    libxss1 \
+    libxtst6 \
+    fonts-liberation \
+    xdg-utils \
+    libatk-bridge2.0-0 \
+    libgtk-3-0 \
+    --no-install-recommends && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
 
 # RUN apt-get update && apt-get install -y wget unzip && \
 #     wget https://dl.google.com/Linux/direct/google-chrome-stable_current_amd64.deb && \
 #     apt install -y ./google-chrome-stable_current_amd64.deb && \
 #     rm google-chrome-stable_current_amd64.deb && \
 #     apt-get clean
-# RUN useradd -m -u 1000 user
-# USER user
-# ENV HOME=/home/user \
-# 	PATH=/home/user/.local/bin:$PATH
-
-# WORKDIR $HOME/app
-# # WORKDIR /app
-
-# COPY --chown=user . $HOME/app
-# # COPY . /app
-# # Install Python dependencies
-# RUN pip install --no-cache-dir -r requirements.txt
-
-# # Run the Selenium script
-# # CMD ["gunicorn", "-b", "0.0.0.0:7860","app:app"]
-# CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]
-ARG PORT=443
-FROM cypress/browsers:latest
-
-RUN apt-get install python3 -y
-
-RUN echo $(python3 -m site --user-base)
 
-COPY requirements.txt  .
+# Update the package list and install wget, unzip, and Firefox
+RUN apt-get update && apt-get install -y wget unzip \
+    && apt-get install -y firefox-esr \
+    && apt-get clean
 
-ENV PATH /home/root/.local/bin:${PATH}
+RUN useradd -m -u 1000 user
+USER user
+ENV HOME=/home/user \
+	PATH=/home/user/.local/bin:$PATH
 
-RUN  apt-get update && apt-get install -y python3-pip && pip install -r requirements.txt
+WORKDIR $HOME/app
+# WORKDIR /app
 
-COPY . .
+COPY --chown=user . $HOME/app
+# COPY . /app
+# Install Python dependencies
+RUN pip install --no-cache-dir -r requirements.txt
 
-CMD uvicorn main:app --host 0.0.0.0 --port $PORT
+# Run the Selenium script
+# CMD ["gunicorn", "-b", "0.0.0.0:7860","app:app"]
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]

app.py

@@ -1,174 +1,175 @@
-# import os
-# import random
-# import time
-# import pandas as pd
-# from fastapi import FastAPI, HTTPException
-# from selenium import webdriver
-# from selenium.webdriver.chrome.service import Service
-# from selenium.webdriver.common.action_chains import ActionChains
-# from selenium.webdriver.chrome.options import Options
-# from webdriver_manager.chrome import ChromeDriverManager
-# from webdriver_manager.core.driver_cache import DriverCacheManager
-# from selenium.webdriver.common.by import By
-# from fake_headers import Headers
-# from fastapi.middleware.cors import CORSMiddleware
-# import logging
-# from selenium_driverless import webdriver as webdriverless
-
-
-# # Initialize FastAPI
-# app = FastAPI(
-#     debug=True,
-#     title="NextAnalytics Server",
-#     consumes=["application/x-www-form-urlencoded", "multipart/form-data"],
-#     docs_url='/swagger'
-# )
-
-# # Configure CORS
-# app.add_middleware(
-#     CORSMiddleware,
-#     allow_origins=["*"],
-#     allow_credentials=True,
-#     allow_methods=["*"],
-#     allow_headers=["*"],
-# )
-
-# # Setup ChromeDriver and Selenium
-# # custom_wdm_cache = os.path.join(os.getcwd(), 'custom_wdm_cache')
-# # os.environ['WDM_LOCAL'] = custom_wdm_cache
-
-# # Setup logging
-# logging.basicConfig(level=logging.INFO)
-
-# def setup_chromedriver():
-#     logging.info("Setting up ChromeDriver...")
-#     # custom_wdm_cache = os.path.join(os.getcwd(), 'custom_wdm_cache')
-#     # os.environ['WDM_LOCAL'] = custom_wdm_cache
-#     # cache_manager = DriverCacheManager(custom_wdm_cache)
-#     # os.chmod(custom_wdm_cache, 0o755)  # Ensure proper permissions
-#     # path = ChromeDriverManager(cache_manager=cache_manager).install()
-#     path = ChromeDriverManager().install()
-#     os.chmod(path, 0o755)  # Ensure ChromeDriver is executable
-#     logging.info(f"ChromeDriver path: {path}")
-#     return path
-
-# # Setup headless Chrome options
-# # Define a custom user agent
-# my_user_agent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36"
-
-# # my_user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36"
-
-# # proxy = None
-# browser_option = Options()
-# browser_option.add_argument("--headless")  # Running in headless mode (no GUI)
-# browser_option.add_argument("--no-sandbox")
-# browser_option.add_argument("--disable-dev-shm-usage")
-# browser_option.add_argument("--ignore-certificate-errors")
-# # browser_option.add_argument("--disable-gpu")
-# # browser_option.add_argument("--log-level=3")
-# # browser_option.add_argument("--disable-notifications")
-# # browser_option.add_argument("--disable-popup-blocking")
-# browser_option.add_argument(f"--user-agent={my_user_agent}")
-
-# # if proxy:
-# #     browser_option.add_argument(f"--proxy-server={proxy}")
-
-
-# # Setup WebDriver
-# driver_path = setup_chromedriver()
-# service = Service(executable_path=driver_path)
-# driver = webdriver.Chrome(service=service, options=browser_option)
-# # actions = ActionChains(driver)
-
-# def getSearchPostData(search_keyword, index, name="", forCompetitorAnalysis=False):
-#     # Navigate to the search results page
-#     url = f'https://www.reddit.com/search/?q={search_keyword}'
-#     driver.get(url)
-#     time.sleep(3)  # Consider using WebDriverWait instead of sleep for better reliability
-#     logging.info("Navigated to search page.")
-
-#     posts_data = []
-#     list_length = 0  # posts count
-#     try:
-#         if forCompetitorAnalysis:
-#             driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
-#             time.sleep(5)
-#         post_cards = driver.find_elements(By.CSS_SELECTOR, 'a[data-testid="post-title-text"]')
-#         post_cards_1 = driver.find_elements(By.CSS_SELECTOR, 'div[data-testid="search-counter-row"]')
-#         post_cards_2 = driver.find_elements(By.CSS_SELECTOR, 'faceplate-timeago')
-#         logging.info(f"Found {len(post_cards)} post cards.")
-
-#         idx = list_length
-#         for card in post_cards_1:
-#             try:
-#                 votes_count = card.find_element(By.XPATH, './/faceplate-number').text
-#                 comments_count = card.find_element(By.XPATH,
-#                     './/span[contains(text(), "comment") or contains(text(), "comments")]/preceding-sibling::faceplate-number'
-#                 ).text
-#                 posts_data.append({
-#                     "index": idx,
-#                     "comment_count": comments_count,
-#                     "votes_count": votes_count
-#                 })
-#                 idx += 1
-#             except Exception as e:
-#                 logging.error(f"Error processing post_card_1: {e}")
-
-#         idx = list_length
-#         for card in post_cards:
-#             try:
-#                 url = card.get_attribute("href")
-#                 title = card.text
-#                 posts_data[idx]["title"] = title
-#                 posts_data[idx]["url"] = url
-#                 idx += 1
-#             except Exception as e:
-#                 logging.error(f"Error processing post_cards: {e}")
+import os
+import random
+import time
+import pandas as pd
+from fastapi import FastAPI, HTTPException
+from selenium import webdriver
+from selenium.webdriver.firefox.service import Service
+from selenium.webdriver.common.action_chains import ActionChains
+from selenium.webdriver.firefox.options import Options
+from webdriver_manager.firefox import GeckoDriverManager
+from webdriver_manager.core.driver_cache import DriverCacheManager
+from selenium.webdriver.common.by import By
+from fake_headers import Headers
+from fastapi.middleware.cors import CORSMiddleware
+import logging
+from selenium_driverless import webdriver as webdriverless
+
+
+# Initialize FastAPI
+app = FastAPI(
+    debug=True,
+    title="NextAnalytics Server",
+    consumes=["application/x-www-form-urlencoded", "multipart/form-data"],
+    docs_url='/swagger'
+)
+
+# Configure CORS
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# Setup ChromeDriver and Selenium
+# custom_wdm_cache = os.path.join(os.getcwd(), 'custom_wdm_cache')
+# os.environ['WDM_LOCAL'] = custom_wdm_cache
+
+# Setup logging
+logging.basicConfig(level=logging.INFO)
+
+def setup_chromedriver():
+    logging.info("Setting up ChromeDriver...")
+    # custom_wdm_cache = os.path.join(os.getcwd(), 'custom_wdm_cache')
+    # os.environ['WDM_LOCAL'] = custom_wdm_cache
+    # cache_manager = DriverCacheManager(custom_wdm_cache)
+    # os.chmod(custom_wdm_cache, 0o755)  # Ensure proper permissions
+    # path = ChromeDriverManager(cache_manager=cache_manager).install()
+    path = GeckoDriverManager().install()
+    os.chmod(path, 0o755)  # Ensure ChromeDriver is executable
+    logging.info(f"ChromeDriver path: {path}")
+    return path
+
+# Setup headless Chrome options
+# Define a custom user agent
+my_user_agent = "Mozilla/5.0 (X11; Linux x86_64; rv:92.0) Gecko/20100101 Firefox/92.0"
+
+# my_user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36"
+
+# proxy = None
+browser_option = Options()
+browser_option.add_argument("--headless")  # Running in headless mode (no GUI)
+browser_option.add_argument("--no-sandbox")
+browser_option.add_argument("--disable-dev-shm-usage")
+browser_option.add_argument("--ignore-certificate-errors")
+# browser_option.binary_location = r'C:\Users\HP\.cache\selenium\firefox\win64\133.0\firefox.exe'
+# browser_option.add_argument("--disable-gpu")
+# browser_option.add_argument("--log-level=3")
+# browser_option.add_argument("--disable-notifications")
+# browser_option.add_argument("--disable-popup-blocking")
+browser_option.add_argument(f"--user-agent={my_user_agent}")
+
+# if proxy:
+#     browser_option.add_argument(f"--proxy-server={proxy}")
+
+
+# Setup WebDriver
+driver_path = setup_chromedriver()
+service = Service(executable_path=driver_path)
+driver = webdriver.Firefox(service=service, options=browser_option)
+# actions = ActionChains(driver)
+
+def getSearchPostData(search_keyword, index, name="", forCompetitorAnalysis=False):
+    # Navigate to the search results page
+    url = f'https://www.reddit.com/search/?q={search_keyword}'
+    driver.get(url)
+    time.sleep(3)  # Consider using WebDriverWait instead of sleep for better reliability
+    logging.info("Navigated to search page.")
+
+    posts_data = []
+    list_length = 0  # posts count
+    try:
+        if forCompetitorAnalysis:
+            driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
+            time.sleep(5)
+        post_cards = driver.find_elements(By.CSS_SELECTOR, 'a[data-testid="post-title-text"]')
+        post_cards_1 = driver.find_elements(By.CSS_SELECTOR, 'div[data-testid="search-counter-row"]')
+        post_cards_2 = driver.find_elements(By.CSS_SELECTOR, 'faceplate-timeago')
+        logging.info(f"Found {len(post_cards)} post cards.")
+
+        idx = list_length
+        for card in post_cards_1:
+            try:
+                votes_count = card.find_element(By.XPATH, './/faceplate-number').text
+                comments_count = card.find_element(By.XPATH,
+                    './/span[contains(text(), "comment") or contains(text(), "comments")]/preceding-sibling::faceplate-number'
+                ).text
+                posts_data.append({
+                    "index": idx,
+                    "comment_count": comments_count,
+                    "votes_count": votes_count
+                })
+                idx += 1
+            except Exception as e:
+                logging.error(f"Error processing post_card_1: {e}")
+
+        idx = list_length
+        for card in post_cards:
+            try:
+                url = card.get_attribute("href")
+                title = card.text
+                posts_data[idx]["title"] = title
+                posts_data[idx]["url"] = url
+                idx += 1
+            except Exception as e:
+                logging.error(f"Error processing post_cards: {e}")
         
-#         idx = list_length
-#         for card in post_cards_2:
-#             try:
-#                 time_element = card.find_element(By.XPATH, './time')
-#                 post_time = time_element.get_attribute('datetime')
-#                 posts_data[idx]["time"] = post_time
-#                 idx += 1
-#             except Exception as e:
-#                 logging.error(f"Error processing post_cards_2: {e}")
-#     except Exception as e:
-#         logging.error(f"Error in scrolling or extracting data: {e}")
-
-#     df = pd.DataFrame(posts_data)
-#     df.to_csv(f'posts_data_{index}.csv', index=False)
-#     logging.info(f"Data saved to posts_data_{index}.csv")
-#     return df
-
-# def get_webpage_title(url: str) -> str:
-#     try:
-#         getSearchPostData(search_keyword="migraine", index=0)
-#         url="https://www.reddit.com"
-#         driver.get(url)
-#         title = driver.title
-#         logging.info(f"Page title: {title}")
-#         return title
-#     except Exception as e:
-#         logging.error(f"Error fetching webpage title: {e}")
-#         return str(e)
-
-# @app.get("/")
-# async def home():
-#     return {"message": "Hello"}
-
-# @app.get("/get-title/")
-# async def fetch_title(url: str):
-#     """
-#     Fetch the title of a webpage by URL.
-#     Example: /get-title/?url=https://www.reddit.com
-#     """
-#     try:
-#         title = get_webpage_title(url)
-#         return {"url": url, "title": title}
-#     except Exception as e:
-#         raise HTTPException(status_code=500, detail=str(e))
+        idx = list_length
+        for card in post_cards_2:
+            try:
+                time_element = card.find_element(By.XPATH, './time')
+                post_time = time_element.get_attribute('datetime')
+                posts_data[idx]["time"] = post_time
+                idx += 1
+            except Exception as e:
+                logging.error(f"Error processing post_cards_2: {e}")
+    except Exception as e:
+        logging.error(f"Error in scrolling or extracting data: {e}")
+
+    df = pd.DataFrame(posts_data)
+    df.to_csv(f'posts_data_{index}.csv', index=False)
+    logging.info(f"Data saved to posts_data_{index}.csv")
+    return df
+
+def get_webpage_title(url: str) -> str:
+    try:
+        getSearchPostData(search_keyword="migraine", index=0)
+        url="https://www.reddit.com"
+        driver.get(url)
+        title = driver.title
+        logging.info(f"Page title: {title}")
+        return title
+    except Exception as e:
+        logging.error(f"Error fetching webpage title: {e}")
+        return str(e)
+
+@app.get("/")
+async def home():
+    return {"message": "Hello"}
+
+@app.get("/get-title/")
+async def fetch_title(url: str):
+    """
+    Fetch the title of a webpage by URL.
+    Example: /get-title/?url=https://www.reddit.com
+    """
+    try:
+        title = get_webpage_title(url)
+        return {"url": url, "title": title}
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
 # @app.get("/get-reddit/")
 # async def getReddit(url: str):
 #     """
@@ -199,43 +200,55 @@
 #     except Exception as e:
 #         raise HTTPException(status_code=500, detail=str(e))
 
-# # Run the app
-# # if __name__ == "__main__":
-# #     import uvicorn
-# #     uvicorn.run(app, host="127.0.0.1", port=7860)
-
-from selenium import webdriver
-from flask import Flask, request
-from selenium.webdriver.chrome.service import Service
-from webdriver_manager.chrome import ChromeDriverManager
-from selenium.webdriver.common.by import By
-
-app = Flask(__name__)
-
+# Run the app
+# if __name__ == "__main__":
+#     import uvicorn
+#     uvicorn.run(app, host="127.0.0.1", port=7860)
 
-def download_selenium():
-    chrome_options = webdriver.ChromeOptions()
-    driver_agent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36"
-
-    chrome_options.add_argument("--headless")
-    chrome_options.add_argument("--no-sandbox")
-    chrome_options.add_argument("--disable-dev-shm-usage")
-    prefs = {"profile.managed_default_content_settings.images": 2}
-    chrome_options.add_experimental_option("prefs", prefs)
-    chrome_options.add_argument(f"--user-agent={driver_agent}")
-    driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=chrome_options)
-    driver.get("https://reddit.com")
-    title = driver.title
-    # language = driver.find_element(By.XPATH, "//div[@id='SIvCob']").text
-    data = {'Page Title': title}
-    return data
-
-
-@app.route('/', methods = ['GET','POST'])
-def home():
-    if (request.method == 'GET'):
-        return download_selenium()
-
-
-if __name__ == "__main__":
-    app.run(debug=True, port=3000)
+# from selenium import webdriver
+# from flask import Flask, request
+# from selenium.webdriver.chrome.service import Service
+# from webdriver_manager.chrome import ChromeDriverManager
+# from selenium.webdriver.common.by import By
+# from selenium.webdriver.common.proxy import Proxy, ProxyType
+# app = Flask(__name__)
+
+
+# def download_selenium():
+#     prox = Proxy()
+#     prox.proxy_type = ProxyType.MANUAL
+#     prox.http_proxy = "ip_addr:port"
+#     prox.socks_proxy = "ip_addr:port"
+#     prox.ssl_proxy = "ip_addr:port"
+#     chrome_options = webdriver.ChromeOptions()
+#     driver_agent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36"
+#     capabilities = webdriver.DesiredCapabilities.CHROME
+#     prox.to_capabilities(capabilities)
+#     chrome_options.add_argument("--headless=new")
+#     # chrome_options.add_argument(f"--proxy-server={proxy}")
+#     chrome_options.add_argument("--no-sandbox")
+#     chrome_options.add_argument("--disable-dev-shm-usage")
+#     # chrome_options.add_argument("--ignore-certificate-errors")
+#     # chrome_options.add_argument("--disable-gpu")
+#     # chrome_options.add_argument("--log-level=3")
+#     # chrome_options.add_argument("--disable-notifications")
+#     # chrome_options.add_argument("--disable-popup-blocking")
+#     prefs = {"profile.managed_default_content_settings.images": 2}
+#     # chrome_options.add_experimental_option("prefs", prefs)
+#     chrome_options.add_argument(f"--user-agent={driver_agent}")
+#     driver = webdriver.Chrome(service=Service(ChromeDriverManager().install(),desired_capabilities=capabilities), options=chrome_options)
+#     driver.get("https://reddit.com")
+#     title = driver.title
+#     # language = driver.find_element(By.XPATH, "//div[@id='SIvCob']").text
+#     data = {'Page Title': title}
+#     return data
+
+
+# @app.route('/', methods = ['GET','POST'])
+# def home():
+#     if (request.method == 'GET'):
+#         return download_selenium()
+
+
+# if __name__ == "__main__":
+#     app.run(debug=True, port=3000)