ass

Dockerfile

@@ -22,16 +22,16 @@ RUN apt-get update && apt-get install -y \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
-# RUN apt-get update && apt-get install -y wget unzip && \
-#     wget https://dl.google.com/Linux/direct/google-chrome-stable_current_amd64.deb && \
-#     apt install -y ./google-chrome-stable_current_amd64.deb && \
-#     rm google-chrome-stable_current_amd64.deb && \
-#     apt-get clean
+RUN apt-get update && apt-get install -y wget unzip && \
+    wget https://dl.google.com/Linux/direct/google-chrome-stable_current_amd64.deb && \
+    apt install -y ./google-chrome-stable_current_amd64.deb && \
+    rm google-chrome-stable_current_amd64.deb && \
+    apt-get clean
 
-# Update the package list and install wget, unzip, and Firefox
-RUN apt-get update && apt-get install -y wget unzip \
-    && apt-get install -y firefox-esr \
-    && apt-get clean
+# # Update the package list and install wget, unzip, and Firefox
+# RUN apt-get update && apt-get install -y wget unzip \
+#     && apt-get install -y firefox-esr \
+#     && apt-get clean
 RUN which firefox
 RUN useradd -m -u 1000 user
 USER user

app.py

@@ -4,10 +4,10 @@ import time
 import pandas as pd
 from fastapi import FastAPI, HTTPException
 from selenium import webdriver
-from selenium.webdriver.firefox.service import Service
+from selenium.webdriver.chrome.service import Service
 from selenium.webdriver.common.action_chains import ActionChains
-from selenium.webdriver.firefox.options import Options
-from webdriver_manager.firefox import GeckoDriverManager
+from selenium.webdriver.chrome.options import Options
+from webdriver_manager.chrome import ChromeDriverManager
 from webdriver_manager.core.driver_cache import DriverCacheManager
 from selenium.webdriver.common.by import By
 from fake_headers import Headers
@@ -41,13 +41,13 @@ app.add_middleware(
 logging.basicConfig(level=logging.INFO)
 
 def setup_chromedriver():
-    logging.info("Setting up ChromeDriver...")
+    # logging.info("Setting up ChromeDriver...")
     custom_wdm_cache = os.path.join(os.getcwd(), 'custom_wdm_cache')
     os.environ['WDM_LOCAL'] = custom_wdm_cache
     cache_manager = DriverCacheManager(custom_wdm_cache)
     os.chmod(custom_wdm_cache, 0o755)  # Ensure proper permissions
-    # path = ChromeDriverManager(cache_manager=cache_manager).install()
-    path = GeckoDriverManager(cache_manager=cache_manager).install()
+    path = ChromeDriverManager(cache_manager=cache_manager).install()
+    # path = GeckoDriverManager(cache_manager=cache_manager).install()
     os.chmod(path, 0o755)  # Ensure ChromeDriver is executable
     logging.info(f"ChromeDriver path: {path}")
     return path
@@ -58,18 +58,23 @@ def setup_chromedriver():
 header = Headers().generate()["User-Agent"]
 # my_user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36"
 
-capabilities = webdriver.DesiredCapabilities.FIREFOX
+# capabilities = webdriver.DesiredCapabilities.CHROME
 # proxy = None
 browser_option = Options()
-browser_option.log.level = 'trace'
+# browser_option..level = 'trace'
 browser_option.add_argument("--headless")  # Running in headless mode (no GUI)
 browser_option.add_argument("--no-sandbox")
 browser_option.add_argument("--disable-dev-shm-usage")
 browser_option.add_argument("--ignore-certificate-errors")
-profile = webdriver.FirefoxProfile()
-profile.set_preference("general.useragent.override", "Your User Agent String")
-browser_option.profile = profile
+# profile = webdriver.FirefoxProfile()
+# profile.set_preference("general.useragent.override", "Your User Agent String")
+# browser_option.profile = profile
 logging.info(f"browser_version: {browser_option.browser_version}")
+# browser_option.set_capability(
+    
+# )
+#     name="",
+#     value=capabilities)
 # browser_option.capabilities = {
 #     "moz:firefoxOptions": {
 #         "args": [
@@ -81,7 +86,7 @@ logging.info(f"browser_version: {browser_option.browser_version}")
 #         ]
 #     }
 # }
-browser_option.binary_location = '/usr/bin/firefox'
+# browser_option.binary_location = '/usr/bin/firefox'
 # browser_option.binary_location = r'C:\Users\HP\.cache\selenium\firefox\win64\133.0\firefox.exe'
 # browser_option.add_argument("--disable-gpu")
 # browser_option.add_argument("--log-level=3")
@@ -97,7 +102,7 @@ browser_option.add_argument("--user-agent={}".format(header))
 # Setup WebDriver
 driver_path = setup_chromedriver()
 service = Service(executable_path=driver_path)
-driver = webdriver.Firefox(service=service, options=browser_option)
+driver = webdriver.Chrome(service=service, options=browser_option)
 # actions = ActionChains(driver)
 
 def getSearchPostData(search_keyword, index, name="", forCompetitorAnalysis=False):
@@ -162,10 +167,76 @@ def getSearchPostData(search_keyword, index, name="", forCompetitorAnalysis=Fals
     logging.info(f"Data saved to posts_data_{index}.csv")
     return df
 
+def getPinterestSearchPostData(search_keyword, index, name="", forCompetitorAnalysis=False):
+    # Navigate to the search results page
+    url = f'https://www.pinterest.com/search/pins?q={search_keyword}'
+    driver.get(url)
+    time.sleep(3)  # Consider using WebDriverWait instead of sleep for better reliability
+    logging.info("Navigated to search page.")
+    # links = driver.find_elements(By.CSS_SELECTOR, "div[data-test-id='related-pins-title']")
+    # print("links", links)
+    posts_data = []
+    list_length = 0  # posts count
+
+    try:
+        if forCompetitorAnalysis:
+            driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
+            time.sleep(5)
+        post_cards = driver.find_elements(By.CSS_SELECTOR, "div[data-test-id='related-pins-title']")
+        # post_cards_1 = driver.find_elements(By.CSS_SELECTOR, 'div[data-testid="search-counter-row"]')
+        # post_cards_2 = driver.find_elements(By.CSS_SELECTOR, 'faceplate-timeago')
+        logging.info(f"Found {len(post_cards)} post cards.")
+
+        # idx = list_length
+        # for card in post_cards_1:
+        #     try:
+        #         votes_count = card.find_element(By.XPATH, './/faceplate-number').text
+        #         comments_count = card.find_element(By.XPATH,
+        #             './/span[contains(text(), "comment") or contains(text(), "comments")]/preceding-sibling::faceplate-number'
+        #         ).text
+        #         posts_data.append({
+        #             "index": idx,
+        #             "comment_count": comments_count,
+        #             "votes_count": votes_count
+        #         })
+        #         idx += 1
+        #     except Exception as e:
+        #         logging.error(f"Error processing post_card_1: {e}")
+
+        idx = 0
+        for card in post_cards:
+            try:
+                title = card.find_element(By.XPATH,
+                    './/div'
+                ).text
+                # posts_data[idx]["title"] = title
+                print("title", title)
+                # idx += 1
+            except Exception as e:
+                logging.error(f"Error processing post_cards: {e}")
+        
+        # idx = list_length
+        # for card in post_cards_2:
+        #     try:
+        #         time_element = card.find_element(By.XPATH, './time')
+        #         post_time = time_element.get_attribute('datetime')
+        #         posts_data[idx]["time"] = post_time
+        #         idx += 1
+        #     except Exception as e:
+        #         logging.error(f"Error processing post_cards_2: {e}")
+   
+    except Exception as e:
+        logging.error(f"Error in scrolling or extracting data: {e}")
+
+    df = pd.DataFrame(posts_data)
+    df.to_csv(f'posts_data_{index}.csv', index=False)
+    logging.info(f"Data saved to posts_data_{index}.csv")
+    return df
+
 def get_webpage_title(url: str) -> str:
     try:
         # getSearchPostData(search_keyword="migraine", index=0)
-        
+        getPinterestSearchPostData(search_keyword="watercolor art",index=0)
         driver.get(url)
         time.sleep(3)
         title = driver.title

posts_data_0.csv

@@ -1,26 +1 @@
-index,comment_count,votes_count,title,url,time
-0,26,6,Migraine DBQ,https://www.reddit.com/r/VeteransBenefits/comments/1cogdv4/migraine_dbq/,2024-05-10T03:51:24.726Z
-1,84,651,A cool guide to migraine symptoms!,https://www.reddit.com/r/coolguides/comments/1f7wymg/a_cool_guide_to_migraine_symptoms/,2024-09-03T10:56:10.230Z
-2,321,141,What makes you sure it's a migraine?,https://www.reddit.com/r/migraine/comments/1cth4kw/what_makes_you_sure_its_a_migraine/,2024-05-16T16:33:05.274Z
-3,217,100,Biggest migraine life hacks?,https://www.reddit.com/r/migraine/comments/1902t0p/biggest_migraine_life_hacks/,2024-01-06T15:49:12.132Z
-4,112,40,What finally stopped your migraine?,https://www.reddit.com/r/migraine/comments/1e7wvsc/what_finally_stopped_your_migraine/,2024-07-20T14:27:22.529Z
-5,228,222,ELI5: What causes Migraines?,https://www.reddit.com/r/explainlikeimfive/comments/1e84z59/eli5_what_causes_migraines/,2024-07-20T20:35:12.699Z
-6,1K,2.9K,Most people use the word 'migraine' wrong.,https://www.reddit.com/r/unpopularopinion/comments/1884d1g/most_people_use_the_word_migraine_wrong/,2023-12-01T05:45:16.842Z
-7,236,1.9K,Wife’s migraines reduced by 90% and I feel like a jackass,https://www.reddit.com/r/migraine/comments/1g1dxw5/wifes_migraines_reduced_by_90_and_i_feel_like_a/,2024-10-11T16:31:11.670Z
-8,257,1.1K,It's just a migraine,https://www.reddit.com/r/Radiology/comments/1cisw7o/its_just_a_migraine/,2024-05-02T22:33:13.773Z
-9,239,660,"The cure for migraines! I've got it now, you can all stop searching! /s",https://www.reddit.com/r/migraine/comments/1ebkkff/the_cure_for_migraines_ive_got_it_now_you_can_all/,2024-07-25T03:09:38.426Z
-10,40,10,What are your migraine relief tricks?,https://www.reddit.com/r/AskReddit/comments/15oald2/what_are_your_migraine_relief_tricks/,2023-08-11T14:34:43.326Z
-11,400,1.7K,Some migraine symptoms you might not expect,https://www.reddit.com/r/migraine/comments/1e8zlkv/some_migraine_symptoms_you_might_not_expect/,2024-07-21T23:20:45.826Z
-12,158,3.1K,I painted this based on the visuals I get from migraines.,https://www.reddit.com/r/AbstractArt/comments/1gojgxz/i_painted_this_based_on_the_visuals_i_get_from/,2024-11-11T04:02:54.421Z
-13,185,698,migraine suffers are born with defective nervous systems,https://www.reddit.com/r/migraine/comments/1bs1g6c/migraine_suffers_are_born_with_defective_nervous/,2024-03-31T04:09:32.745Z
-14,911,25K,I’ve had constant migraines for the past week thanks to Matlab.,https://www.reddit.com/r/ProgrammerHumor/comments/xbmg98/ive_had_constant_migraines_for_the_past_week/,2022-09-11T16:10:36.852Z
-15,65,41,ELI5: what is the difference between a Headache vs a migraine?,https://www.reddit.com/r/explainlikeimfive/comments/1ccqoaj/eli5_what_is_the_difference_between_a_headache_vs/,2024-04-25T12:28:12.099Z
-16,243,6.5K,Need to share this. I got a migraine from laughing so hard.,https://www.reddit.com/r/BaldursGate3/comments/1butvvc/need_to_share_this_i_got_a_migraine_from_laughing/,2024-04-03T14:23:48.741Z
-17,54,6,What type of Migraine Logs does VA accept?,https://www.reddit.com/r/VeteransBenefits/comments/1fmdy2v/what_type_of_migraine_logs_does_va_accept/,2024-09-21T21:54:44.854Z
-18,231,1.9K,Raise your hand if you’re an American with a stress induced migraine today,https://www.reddit.com/r/migraine/comments/1gk64ce/raise_your_hand_if_youre_an_american_with_a/,2024-11-05T13:01:07.737Z
-19,240,316,Sign a migraine is coming,https://www.reddit.com/r/migraine/comments/1b90est/sign_a_migraine_is_coming/,2024-03-07T17:33:14.935Z
-20,1.1K,24K,TIL: Migraines are 3 times more common in women than in men.,https://www.reddit.com/r/todayilearned/comments/t0jzlf/til_migraines_are_3_times_more_common_in_women/,2022-02-24T20:03:31.456Z
-21,16,6,Migraine or MS?,https://www.reddit.com/r/MultipleSclerosis/comments/1fu5xnr/migraine_or_ms/,2024-10-02T02:31:44.292Z
-22,644,3.4K,This is the first time I’ve had someone tell me what I have isn’t a migraine. I mentioned having to grocery shop during a migraine and got this as a response.,https://www.reddit.com/r/migraine/comments/1780gdm/this_is_the_first_time_ive_had_someone_tell_me/,2023-10-14T22:12:03.962Z
-23,91,46,How do you manage your migraines?,https://www.reddit.com/r/POTS/comments/1duim0f/how_do_you_manage_your_migraines/,2024-07-03T16:27:50.806Z
-24,928,10K,10 minutes video call healthcare for $300 ...and just for migraine. Wow.,https://www.reddit.com/r/facepalm/comments/w6t020/10_minutes_video_call_healthcare_for_300_and_just/,2022-07-24T10:59:04.264Z
+