added code

Dockerfile

@@ -0,0 +1,31 @@
+# Use an official Python runtime as a parent image
+FROM python:3.11-slim
+
+# Set the working directory to /app
+WORKDIR /app
+
+# Install required C++11 libraries and ca-certificates
+RUN apt-get update -qq \
+      && apt-get install -y \
+      build-essential \
+      python3-dev \
+      ca-certificates \
+      && apt-get clean \
+      && rm -rf /var/lib/apt/lists/*
+
+# Copy the current directory contents into the container at /app
+COPY . /app
+
+# Create and set up a virtual environment inside the container
+RUN python -m venv /app/venv \
+      && /app/venv/bin/pip install --no-cache-dir --upgrade pip \
+      && /app/venv/bin/pip install --no-cache-dir -r requirements.txt
+
+# Ensure the virtual environment's Python and pip are used by default
+ENV PATH="/app/venv/bin:$PATH"
+
+# Make port 8000 available to the world outside this container
+EXPOSE 8000
+
+# Run the application when the container launches
+CMD ["uvicorn", "main:app", "--host", "127.0.0.23", "--port", "7860"]

venv/.gitignore

@@ -0,0 +1,54 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# Virtual environment
+venv/
+ENV/
+env/
+.venv/
+.ENV/
+
+# Jupyter Notebook checkpoints
+.ipynb_checkpoints
+
+# VS Code files
+.vscode/
+
+# PyCharm files
+.idea/
+
+# Distribution / Packaging
+build/
+dist/
+Lib/
+Include/
+Scripts/
+*.egg-info/
+*.egg
+*.whl
+
+# Logs and debug files
+*.log
+
+# Test results
+*.out
+*.coverage
+.coverage.*
+
+# Environment variables and settings
+.env
+*.env
+
+# Cache files
+*.cache
+.cache/
+pip-log.txt
+pip-delete-this-directory.txt
+
+# macOS files
+.DS_Store
+
+# Windows files
+Thumbs.db

venv/app.py

@@ -0,0 +1,156 @@
+from collections import deque
+from fastapi import FastAPI, HTTPException
+from fastapi.middleware.cors import CORSMiddleware
+from models.reddit_models import RedditPostDataModel
+from reddit.reddit_functions import getRedditData
+from reddit.reddit_gemini import getKeywords
+from reddit.api_keys import api_key,api_key2
+import google.generativeai as genai
+
+from reddit.reddit_pain_point_analysis import pain_point_analysis
+from reddit.reddit_search_scrapper import getCompetitorAnalysisData
+from utils import time_execution
+app = FastAPI()
+import asyncio
+from fastapi import HTTPException, APIRouter
+from asyncio import TimeoutError
+
+# Assuming you have defined the necessary imports, e.g., config, getKeywords, api_key, api_key2
+
+# CORS configuration
+origins = [
+    "*",
+    # Add more origins as needed
+]
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=origins,  # Allows specified origins
+    allow_credentials=False,
+    allow_methods=["*"],     # Allows all HTTP methods
+    allow_headers=["*"],     # Allows all headers
+)
+from fastapi import FastAPI
+from functools import wraps
+
+app = FastAPI()
+
+class Config:
+    def __init__(self):
+        self.called = False
+        self.retry=False
+        self.chat_session=None
+        self.queue=deque([])
+
+    def configure(self, api_key):
+        genai.configure(api_key=api_key)
+
+
+config = Config()
+
+@app.get("/")
+@time_execution
+def read_root():
+    if not config.called:
+        print("called", config.called)
+        if config.retry:
+            config.configure(api_key=api_key2)
+            config.called = True
+        else:
+            config.configure(api_key=api_key)
+            config.called = True
+    return {"message": "Hello, World!"}
+
+# Timeout handler: check if getKeywords takes too long
+async def fetch_keywords_with_timeout(user_query: str, timeout: int = 60, retry: bool = True):
+    try:
+        # Simulate the getKeywords function with a timeout using asyncio.wait_for
+        keywords = await asyncio.wait_for(asyncio.to_thread(getKeywords, user_query), timeout=timeout)
+        return keywords
+    except TimeoutError:
+        print("Timeout exceeded, switching to api_key2")
+        if retry:
+            config.called = False
+            config.retry=True
+            # Timeout exceeded, switch to api_key2 and retry fetching keywords
+            read_root()  # Switch API key
+            # Retry fetching keywords
+            return await fetch_keywords_with_timeout(user_query, timeout, retry=False)  # Set retry to False to prevent infinite loop
+        else:
+            # If we already tried once, handle as a failure or return a fallback response
+            raise HTTPException(status_code=504, detail="Request timed out even after retrying")
+
+@app.get("/keywords")
+@time_execution
+async def fetch_keywords(user_query: str):
+    if not user_query:
+        raise HTTPException(status_code=400, detail="User query must not be empty")
+
+    # Fetch keywords with a 10-second timeout and retry mechanism
+    keywords = await fetch_keywords_with_timeout(user_query=user_query)
+    return keywords
+
+
+@app.post("/getRedditPostsData")
+@time_execution
+def getRedditPostsData(request: RedditPostDataModel):
+    """Requires user_query and search_keywords as arguments.
+        Steps involved in this api:
+        1. get posts data from reddit
+        2. filter top 18 posts
+        3. get comments data
+        4. get sentiment data
+    """
+    try:
+        # Extract user_query and search_keywords from the request body
+        user_query = request.user_query
+        search_keywords = request.search_keywords
+        
+        if not user_query:
+            raise HTTPException(status_code=400, detail="User  query must not be empty")
+        
+        if not search_keywords:
+            raise HTTPException(status_code=400, detail="Search keywords must not be empty")
+        print("user_query",user_query,"search_keywords",search_keywords)
+        result = getRedditData(user_query=user_query, search_keywords=search_keywords)
+        return result
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(f"Failed to run getRedditPostsData : {e}"))
+
+# pain point analysis api which takes user_query and fileName as arguments
+@app.get("/getPainPointAnalysis")
+@time_execution
+def getPainPointAnalysis(user_query: str, fileName: str, uniqueFileId: str):
+    try:
+        # Extract user_query and fileName
+        if not user_query:
+            raise HTTPException(status_code=400, detail="User  query must not be empty")
+        
+        if not fileName:
+            raise HTTPException(status_code=400, detail="fileName must not be empty")
+        print("user_query",user_query,"fileName",fileName)
+        result=pain_point_analysis(user_query=user_query,fileName=fileName,uniqueFileId=uniqueFileId)
+        config.chat_session= result[1]
+        return result[0]
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(f"Failed to run getPainPointAnalysis : {e}"))
+    
+# pain point analysis api which takes user_query and fileName as arguments
+@app.get("/getCompetitorAnalysis")
+@time_execution
+def getCompetitorAnalysis(user_query: str, fileName: str,isSolo=True):
+    try:
+        # Extract user_query and fileName
+        if not user_query:
+            raise HTTPException(status_code=400, detail="User  query must not be empty")
+        
+        if not fileName:
+            raise HTTPException(status_code=400, detail="fileName must not be empty")
+        print("user_query",user_query,"isSolo",isSolo,"fileName",fileName)
+        result  = getCompetitorAnalysisData(user_query=user_query,isSolo=isSolo,chat_session=config.chat_session,fileName=fileName)
+        return result
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(f"Failed to run getCompetitorAnalysis : {e}"))
+# if __name__ == "__main__":
+#     import uvicorn
+#     uvicorn.run("main:app", host="127.0.0.23", workers=1,reload=True,port=786)

venv/models/reddit_models.py

@@ -0,0 +1,6 @@
+from typing import List
+from pydantic import BaseModel
+
+class RedditPostDataModel(BaseModel):
+    user_query: str
+    search_keywords: List[str]

venv/pyvenv.cfg

@@ -0,0 +1,5 @@
+home = C:\Users\HP\AppData\Local\Microsoft\WindowsApps\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0
+include-system-site-packages = false
+version = 3.11.9
+executable = C:\Users\HP\AppData\Local\Microsoft\WindowsApps\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\python.exe
+command = C:\Users\HP\AppData\Local\Microsoft\WindowsApps\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\python.exe -m venv D:\development\nextAnalytics\venv

venv/reddit/api_keys.py

@@ -0,0 +1,13 @@
+import os
+from dotenv import load_dotenv
+
+load_dotenv()
+
+api_key = os.getenv('GEMINI_API_KEY')
+api_key2 = os.getenv('GEMINI_SECOND_API_KEY')
+api_key3 = os.getenv('GEMINI_THIRD_API_KEY')
+api_key4 = os.getenv('GEMINI_FOURTH_API_KEY')
+api_key5 = os.getenv('GEMINI_FIVE_API_KEY')
+api_key6 = os.getenv('GEMINI_SIX_API_KEY')
+api_key7 = os.getenv('GEMINI_SEVEN_API_KEY')
+api_key8 = os.getenv('GEMINI_EIGHT_API_KEY')

venv/reddit/prompts.py

@@ -0,0 +1,340 @@
+def featureAnalysisPrompt():
+    return f'''
+{{
+  "competitor": "COMPETITOR_NAME",
+  "user_query": "USER_QUERY",
+  "competitor_data_source": "DATA_SOURCE",
+  "overall_sentiment": {{
+    "positive_percentage": "PERCENTAGE",
+    "negative_percentage": "PERCENTAGE",
+    "neutral_percentage": "PERCENTAGE"
+  }},
+  "features": [
+    {{
+      "feature": "FEATURE_NAME",
+      "feature_description": "FEATURE_DESCRIPTION",
+      "sentiment_analysis": {{
+        "positive": "PERCENTAGE",
+        "negative": "PERCENTAGE",
+        "neutral": "PERCENTAGE"
+      }},
+      "emotion_analysis": {{
+        "joy": "PERCENTAGE",
+        "anger": "PERCENTAGE",
+        "disappointment": "PERCENTAGE",
+        "surprise": "PERCENTAGE"
+      }},
+      "user_personas": {{
+        "age_groups": [
+          {{
+            "age_range": "AGE_RANGE",
+            "positive_sentiment_count": "COUNT",
+            "negative_sentiment_count": "COUNT",
+            "neutral_sentiment_count": "COUNT"
+          }},
+          {{
+            "age_range": "AGE_RANGE",
+            "positive_sentiment_count": "COUNT",
+            "negative_sentiment_count": "COUNT",
+            "neutral_sentiment_count": "COUNT"
+          }}
+        ],
+        "gender": [
+          {{
+            "gender": "GENDER_TYPE",
+            "positive_sentiment_count": "COUNT",
+            "negative_sentiment_count": "COUNT",
+            "neutral_sentiment_count": "COUNT"
+          }}
+        ]
+      }},
+      "adoption_rate": {{
+        "early_adopters": "PERCENTAGE",
+        "mainstream_users": "PERCENTAGE",
+        "dissatisfied_users": "PERCENTAGE"
+      }},
+      "usage_behavior": {{
+        "frequency_of_use": {{
+          "daily": "COUNT",
+          "weekly": "COUNT",
+          "occasionally": "COUNT",
+          "never": "COUNT"
+        }},
+        "engagement_level": {{
+          "high": "COUNT",
+          "medium": "COUNT",
+          "low": "COUNT"
+        }}
+      }},
+      "feature_comparison": {{
+        "strengths": [
+          "STRENGTH_1",
+          "STRENGTH_2"
+        ],
+        "weaknesses": [
+          "WEAKNESS_1",
+          "WEAKNESS_2"
+        ],
+        "opportunities_for_improvement": [
+          "OPPORTUNITY_1",
+          "OPPORTUNITY_2"
+        ],
+        "threats": [
+          "THREAT_1",
+          "THREAT_2"
+        ]
+      }},
+      "pain_points": [
+        {{
+          "issue": "ISSUE_DESCRIPTION",
+          "impact": "ISSUE_IMPACT",
+          "mentions": "COUNT",
+          "user_sentiment": "SENTIMENT"
+        }}
+      ],
+      "feature_improvements_suggestions": [
+        {{
+          "suggestion": "SUGGESTION_DESCRIPTION",
+          "priority": "PRIORITY"
+        }}
+      ]
+    }}
+  ],
+  "conclusion": {{
+    "summary": "SUMMARY_OF_KEY_FINDINGS",
+    "recommendations": [
+      "RECOMMENDATION_1",
+      "RECOMMENDATION_2"
+    ]
+  }}
+}}
+'''
+
+def getPainPointAnalysisPrompt(user_query):
+    return f"""
+    1. analyze the given csv data of reddit posts with sentiments and provide a list of pain point analysis categories for the user query ={user_query}. categories in pain point analysis should be set of 2 keyword only which emphasis the abstract of pain point and also rank them based on importance and relevancy to user query
+    return categories title only which attracts the readers . get your data in json format {{
+    "pain_point_categories": [
+    "category1",
+    "category2",
+    "category3",
+    .
+    .
+    .
+    ]
+    }}
+    2. For all the pain points categories that you got from first step, analyze the file_with_sentiment.csv data and peform pain point analysis on it  and  return the response in JSON format provided below for all categories, nothing else.
+    {{
+    "pain_point_analysis": {{
+      "key_insights": ["insight1", "insight2",...],
+      "pain_points": [
+        {{
+          "category1": "Category of Pain Point (e.g., Product Issues, Customer Service, Pricing)",
+          "pain_point": "Brief description of the issue (e.g., Slow Performance)",
+          "frequency": "number",
+          "sentiment_analysis": {{
+            "positive": "number",
+            "neutral": "number",
+            "negative": "number"
+          }},
+          "related_features": ["feature1", "feature2",...],
+          "examples": [
+            {{
+              "post_title": "Title of the post/comment",
+              "comment": "Description or excerpt of user comment",
+              "upvotes": "number",
+              "post_url": "URL of the original post/comment"
+            }},
+            .
+            .
+            .
+          ],
+          "recommended_actions": [
+            "Recommended solution/action 1",
+            "Recommended solution/action 2",
+            .
+            .
+            .
+        ]
+        }},
+      similarly for remaining categories
+      .
+      .
+      .
+      ],
+      "overall_insights": {{
+        "top_pain_points": ["pain_point1", "pain_point2",...],
+        "user_segments_most_affected": ["segment1", "segment2",...],
+        "impact_on_product_development": [
+        "Insight for development 1",
+        "Insight for development 2",
+        .
+        .
+        .
+        ]
+      }}
+    }}
+}}
+""" 
+   
+
+def getKeywordsPrompt(user_query):
+    return f"""1. Enhance the user query ="{user_query}" for better relevance to the main intent of query
+   get the enhanced query in JSON:
+   {{
+     "query": "enhanced_user_query"
+   }}.
+2. Extract keywords from the enhanced query and get them in JSON:
+   {{
+     "keywords": ["keyword1", ...]
+   }}.
+3. Create relevant combinations of 2-3 keywords for the enhanced query context.
+4. Get 3 keyword combinations with 95% relevancy to the original user query in JSON:
+   {{
+     "top_3_combinations": ["combination phrase 1", "combination phrase 2", "combination phrase 3"]
+   }}.
+5. Return the final output in JSON format:
+   {{
+     "query": "enhanced_user_query",
+     "keywords": ["keyword1", ...],
+     "top_3_combinations": ["combination phrase 1", "combination phrase 2", "combination phrase 3"]
+   }}.
+ """
+
+def getCompetitorPrompt(user_query):
+  pain_point_prompt =getPainPointAnalysisPrompt(user_query = user_query)
+  return f'''
+  1.{pain_point_prompt}
+  2. Perform competitor analysis on the given csv file and return your final output in JSON format:
+  {{
+  "competitor_analysis": {{
+    "competitor_name": "<CompetitorName>",
+    "overview": {{
+      "date_range": "<Start_Date> to <End_Date>",
+      "total_posts_analyzed": "<Total_Posts>",
+      "total_comments_analyzed": "<Total_Comments>"
+    }},
+    "market_sentiment": {{
+      "overall": {{
+        "positive": "<Positive_Percentage>",
+        "neutral": "<Neutral_Percentage>",
+        "negative": "<Negative_Percentage>"
+      }},
+      "trend_over_time": {{
+        "<Year-Month>": {{
+          "positive": "<Positive_Percentage>",
+          "neutral": "<Neutral_Percentage>",
+          "negative": "<Negative_Percentage>"
+        }},
+        "<Year-Month>": {{
+          "positive": "<Positive_Percentage>",
+          "neutral": "<Neutral_Percentage>",
+          "negative": "<Negative_Percentage>"
+        }}
+      }}
+    }},
+    "pain_points": {{pain point from step 1}},
+    "features_and_differentiators": [
+      {{
+        "feature": "<Feature_Name>",
+        "sentiment": "<Sentiment_Type>",
+        "mentions": "<Mentions>",
+        "related_comments": [
+          {{
+            "comment": "<Comment_Text>",
+            "upvotes": "<Upvotes>",
+            "post_url": "<Post_URL>"
+          }}
+          .
+          .
+          .
+        ]
+      }}
+      .
+      .
+      .
+    ],
+    "sentiment_by_feature": {{
+      "<Feature_Name>": {{
+        "positive": "<Positive_Percentage>",
+        "neutral": "<Neutral_Percentage>",
+        "negative": "<Negative_Percentage>"
+      }},
+      "<Feature_Name>": {{
+        "positive": "<Positive_Percentage>",
+        "neutral": "<Neutral_Percentage>",
+        "negative": "<Negative_Percentage>"
+      }}
+    }},
+    "audience_analysis": {{
+      "popular_subreddits": [
+        "<Subreddit_1>",
+        "<Subreddit_2>"
+      ],
+      "user_segments": [
+        "<User_Segment_1>",
+        "<User_Segment_2>"
+      ]
+    }},
+    "pricing_feedback": {{
+      "value_perception": {{
+        "positive": "<Positive_Percentage>",
+        "neutral": "<Neutral_Percentage>",
+        "negative": "<Negative_Percentage>"
+      }},
+      "related_comments": [
+        {{
+          "comment": "<Comment_Text>",
+          "upvotes": "<Upvotes>",
+          "post_url": "<Post_URL>"
+        }}
+      ]
+    }},
+    "competitor_strengths": [
+      "<Strength_1>",
+      "<Strength_2>",
+      .
+      .
+      .
+    ],
+    "competitor_weaknesses": [
+      "<Weakness_1>",
+      "<Weakness_2>",
+      .
+      .
+      .
+    ],
+    "user_recommendations": [
+      "<Recommendation_1>",
+      "<Recommendation_2>",
+      .
+      .
+      .
+    ],
+    "competitive_strategy": {{
+      "pricing_strategy": "<Pricing_Strategy>",
+      "feature_improvement": "<Feature_Strategy>",
+      .
+      .
+      .
+    }}
+  }}
+}}
+
+'''
+
+def getTop10CompetitorPrompt(reddit_data,gemini_data,user_query):
+    return f'''
+Competitor names from reddit are:
+{reddit_data}
+
+Competitor name from gemini are:
+{gemini_data}
+
+both are for the user query: "{user_query}"
+
+get me 10 most relevant competitors for the given user query from both the data and return a combined list in following json format, nothing else and try not to include very general competitors into the list which are not directly related to user query, be specific according to the user query. competitor1 ,2,..10 should be the details given in the json data:
+{{
+list:[competitor1,competitor2,....competitor10]
+}}
+'''

venv/reddit/reddit_call.py

@@ -0,0 +1,42 @@
+# Reddit Data Scrapper
+from reddit_utils import get_microseconds_list
+from reddit_pain_point_analysis import pain_point_analysis
+from reddit_sentiment_analysis import SentimentAnalysis
+from reddit_gemini import getKeywords
+from reddit_search_scrapper import getCompetitorAnalysisData, getFinalData
+import google.generativeai as genai
+from scraping import driver, getPostComments, getSearchPostData
+from api_keys import api_key
+genai.configure(api_key=api_key)
+
+user_query = "AI image"
+
+def redditScrapper(user_query):
+    search_keywords=getKeywords(user_query=user_query)
+    # unique_list = get_microseconds_list()
+    # for i in range(len(search_keywords["top_3_combinations"])):
+    #     getSearchPostData( search_keyword=search_keywords['top_3_combinations'][i],index=unique_list[i])
+   
+    # getFinalData(user_query=user_query)
+    # getPostComments()
+    sentiment_instance= SentimentAnalysis()
+    sentiment_instance.generate_sentiment_and_emotion_from_data(fileName='posts_data_1732105228633815.csv')
+    # sentiment_data.to_csv("file_with_sentiment.csv", index=False)
+    # # Specify the file path
+    # file_path = 'posts_data.csv'
+
+    # # Check if the file exists before attempting to delete
+    # if os.path.exists(file_path):
+    #     os.remove(file_path)
+    #     print("File deleted successfully")
+    # else:
+    #     print("File does not exist")
+    # pain_point_analysis(user_query = "AI image generation techniques and applications", fileName="posts_data.csv")
+    getCompetitorAnalysisData(user_query="social media analytics tools and strategies")
+    
+    driver.quit()
+
+
+# Call the function
+redditScrapper(user_query)
+

venv/reddit/reddit_community_post_scraper.py

@@ -0,0 +1,66 @@
+import praw
+
+reddit = praw.Reddit(
+    client_id="yjGfys3QZPpdCpNZl25Kig",
+    client_secret="dqoc8LrQBQhB_IgjV-lKyrD9lBPftg",
+    password="&honeyB90",
+    user_agent="Curious",
+    username="Final-Difference7055",
+)
+
+subRed = reddit.subreddit("SkincareAddiction")
+data=[]
+for posts in subRed.hot(limit=25):
+    data.append([posts.title,posts.selftext,f"https://www.reddit.com/r/{subRed}/comments/{posts.id}"])
+
+def process_comment(comment):
+    # Prepare the comment data
+    comment_data = {
+        "user": comment.author.name if comment.author else "Unknown",
+        "comment": comment.body,
+        "replies": []  # Initialize replies list
+    }
+    
+    # Process replies recursively if any
+    if comment.replies:
+        for reply in comment.replies:
+            reply_data = process_comment(reply)  # Recursive call for replies
+            comment_data["replies"].append(reply_data)
+    
+    return comment_data
+for url in range(len(data)):
+    # comment_data_sub=[]
+    submission = reddit.submission(url=data[url][2])
+
+# Fetch and process comments
+    submission.comments.replace_more(limit=2)  # Use limit=0 to get all comments
+    comments_data = []
+
+    # Function to process a comment and its replies
+    # Seed with top-level comments
+    comment_queue = list(submission.comments)
+
+    while comment_queue:
+        comment = comment_queue.pop(0)
+        comment_data = process_comment(comment)  # Process each comment
+        comments_data.append(comment_data)
+
+    # Now, structure the data into the desired JSON format
+    json_output = {
+        "comments": comments_data
+    }
+    data[url].append(json_output)
+
+new_data=[]
+for i in range(len(data)):
+    new_data.append([data[i][0],data[i][1],data[i][3]])
+import pandas as pd
+
+
+# Convert the list to a DataFrame and specify column names
+df = pd.DataFrame(new_data, columns=["Title", "Description", "Comments"])
+
+# Save the DataFrame as a CSV file
+df.to_csv('output.csv', index=False)
+
+print("Data saved to output.csv")

venv/reddit/reddit_competitor_analysis.py

@@ -0,0 +1,217 @@
+import json
+import os
+import google.generativeai as genai
+
+from reddit.prompts import getCompetitorPrompt, getTop10CompetitorPrompt
+from reddit.reddit_utils import get_microseconds_list
+from reddit.scraping import getPostComments, getSearchPostData
+from reddit.reddit_gemini import getModelAndGenerationConfigCommon
+from reddit.api_keys import  api_key3,api_key4,api_key5,api_key6,api_key7,api_key8
+
+def getCompetitorNames(user_query ): #model, generation_config
+  prompt=f"""Extract a list of product names, alternatives, and competitors relevant to the query: {user_query}. Ensure that the results focus on tools, platforms, or services explicitly aligned with the domain and purpose of the query. Avoid including general or loosely related products unless they directly offer features tailored to the query's intent.
+
+Additionally, provide the platform(s) (e.g., web, apps, integrations) on which each competitor operates and categorize their functionality. Include a frequency count indicating the number of times each entry is mentioned or relevant to the query. Also, aggregate the total frequency of each platform across all entries. Also give provide popularity score for each competitor out of 100.
+give top 6 competitors details only.
+return in given json format only:
+{{
+"competitors":[{{"name":"","platform":[],"category":"","count":number,"popularity":number}}],
+"platforms":[{{"platform":name,count:number}}]
+}}"""
+  generation_config = {
+      "temperature": 1,
+      "top_p": 0.95,
+      "top_k": 40,
+      "max_output_tokens": 8192,
+      "response_mime_type": "application/json",
+    }
+   
+  model = genai.GenerativeModel(
+    model_name="gemini-1.5-pro-002",
+    generation_config=generation_config,
+  )
+  try:
+    response = model.generate_content(prompt)
+    data = response.text
+    print("getCompetitorNames",data)
+    return json.loads(data)
+  except:
+    try:
+      # retry
+      response = model.generate_content(prompt)
+      data = response.text
+      print("retry getCompetitorNames",data)
+      return json.loads(data)
+    except Exception as e:
+      return {"details": str(e)}
+
+
+def getCompetitorNamesFromReddit(user_query,fileName,isSolo=True,last_chat_session=None ): #model, generation_config
+  prompt=f"""Extract a list of product names, alternatives, and competitors relevant to the query: {user_query} from the given csv data. Ensure that the results focus on tools, platforms, or services explicitly aligned with the domain and purpose of the query and do not include very general competitors into the list which are not directly related to user query use case or intent.
+
+Additionally, provide the platform(s) (e.g., web, apps, integrations) on which each competitor operates and categorize their functionality.Category should be string. Include a frequency count indicating the number of times each entry is mentioned or relevant to the query. Also, aggregate the total frequency of each platform across all entries. Also give provide popularity score for each competitor out of 100.
+give top 6 competitors details only.
+return in given json format only:
+{{
+"competitors":[{{"name":"","platform":[],"category":"","count":number,"popularity":number}}],
+"platforms":[{{"platform":name,count:number}}]
+}}"""
+  chat_session=None
+  if isSolo:
+    data=getModelAndGenerationConfigCommon(fileName=fileName,isFlash=False)
+    model = data[0]
+    chat_session = model.start_chat(
+        history=[
+          {
+            "role": "user",
+            "parts": [
+              data[1],
+              prompt
+            ],
+          }
+        ]
+      )
+  else:
+    chat_session = last_chat_session
+  try:
+    response = chat_session.send_message("give your last response of competitor names")
+
+    data = response.text
+    print("getCompetitorNames",data)
+    return json.loads(data)
+  except:
+    try:
+      # retry
+      response = chat_session.send_message("give your last response of competitor names")
+      data = response.text
+      print("retry getCompetitorNames",data)
+      return json.loads(data)
+    except Exception as e:
+      return {"details": str(e)}
+
+
+# def top10competitor(reddit_data,gemini_data):
+#   # ensure path 1  is reddit and path 2 is gemini
+#   d1 = reddit_data
+#   d2 = gemini_data
+#   popularity = {}
+#   count = {}
+#   for x in d1['competitors']:count[x['name']] = x['count']
+#   for x in d2['competitors']:
+#     if x['name'] in count.keys(): continue
+#     else:popularity[x['name']] = x['popularity']
+#   ma = sum(count.values())/len(count.values())
+#   mb = sum(popularity.values())/len(popularity.values())
+#   df = pd.DataFrame(d1['competitors'])
+#   df['extracted_by'] = 'reddit'
+#   df2 = pd.DataFrame(d2['competitors'])
+#   df2['extracted_by'] = 'gemini'
+#   df2['count'] = df2['popularity']/(mb/ma)
+#   df = pd.concat([df, df2], axis=0)
+#   df = df.drop_duplicates('name')
+#   df = df.reset_index(drop=True)
+#   df = df.sort_values(by ='count', ascending=False)
+#   df = df.reset_index(drop=True)
+#   df = df.head(10)
+#   df = df.drop('extracted_by', axis=1)
+#   return df
+
+def getTop10Competitors(user_query,reddit_data,gemini_data):
+  prompt = getTop10CompetitorPrompt(user_query=user_query,reddit_data=reddit_data,gemini_data=gemini_data)
+  model = genai.GenerativeModel("gemini-exp-1114")
+
+  generation_config = genai.GenerationConfig(response_mime_type="application/json")
+
+  try:
+      response = model.generate_content(prompt, generation_config=generation_config)  # Adjust if the library supports async
+      data = response.text
+      print("getTop10Competitors:", data)
+      return json.loads(data)
+  except Exception as e:
+      print("Error while fetching getTop10Competitors: %s", e)
+
+def getPostDataofCompetitor(fileName,user_query):
+  df = fileName
+  unique_list = get_microseconds_list(length=len(df))
+  actual_list=[]
+  for i in range(len(df)):
+    index=getSearchPostData(forCompetitorAnalysis=True,search_keyword=f"{df.iloc[i]['name']} {df.iloc[i]['category']}",name=df.iloc[i]['name'] ,index=unique_list[i])
+    if index is not None:
+      actual_list.append(index)
+  print("actual_list",actual_list)
+  print("Fetched data for competitors")
+  fileNames = [f"posts_data_{actual_list[i]}.csv" for i in range(len(actual_list))]
+  return preprocessingCompetitorsData(user_query=user_query,fileNames=fileNames,fileUniqueIds=actual_list)
+
+
+def preprocessingCompetitorsData(user_query,fileNames,fileUniqueIds):
+  c=0
+  competitors_json_data = []
+  for i in range(len(fileUniqueIds)):
+    if c==6:break
+    print(f"Processing file {fileNames[i]}")
+    # get posts comments data
+    getPostComments(fileName=fileNames[i])
+    json_data = getCompetitorAnalysisReport(user_query=user_query,fileName=fileNames[i],count=c)
+    c+=1
+    # if json_data does  contain "details" field, then skip this file
+    if "details" in json_data.keys():
+      continue
+    # save json_data to json file
+    with open(f"competitor_analysis_report_{fileUniqueIds[i]}.json", "w") as outfile:
+      json.dump(json_data, outfile)
+    print("Competitor Analysis Report",f"competitor_analysis_report_{fileUniqueIds[i]}.json")
+    competitors_json_data.append(json_data)
+    
+  for file_path in fileNames:
+    # Check if the file exists before attempting to delete
+    if os.path.exists(file_path):
+        os.remove(file_path)
+        print("File deleted successfully")
+    else:
+        print("File does not exist")
+  return competitors_json_data
+
+def getCompetitorAnalysisReport(user_query,fileName,count=0):
+  prompt = getCompetitorPrompt(user_query=user_query)
+  api_key_map = {
+        0: api_key3,
+        1: api_key4,
+        2: api_key5,
+        3: api_key6,
+        4: api_key7,
+        5: api_key8
+    }
+
+  selected_api_key = api_key_map.get(count, api_key8)  # Default to api_key8 if count > 5
+  genai.configure(api_key=selected_api_key)
+  data = getModelAndGenerationConfigCommon(fileName=fileName,isFlash=False)
+  model = data[0]
+  chat_session = model.start_chat(
+      history=[
+        {
+          "role": "user",
+          "parts": [
+            data[1],
+             prompt
+          ],
+        }
+      ]
+    )
+
+
+  try:
+    response = chat_session.send_message("give your last response of competitor analysis")
+    data = response.text
+    print("getCompetitorNames",data)
+    return json.loads(data)
+  except:
+    try:
+      # retry
+      response = chat_session.send_message("give your last response of competitor analysis")
+      data = response.text
+      print("retry getCompetitorNames",data)
+      return json.loads(data)
+    except Exception as e:
+      print("competitor analysis error",str(e))
+      return {"details": str(e)}

venv/reddit/reddit_functions.py

@@ -0,0 +1,49 @@
+
+from reddit.reddit_search_scrapper import getFinalData
+from reddit.reddit_sentiment_analysis import SentimentAnalysis
+from reddit.reddit_utils import get_microseconds_list
+from reddit.scraping import getPostComments, getSearchPostData
+
+
+def getRedditData(user_query, search_keywords):
+    unique_list = get_microseconds_list()
+
+    successful_steps = []
+
+    # Step 1: Get search post data
+    for i in range(len(search_keywords)):
+        try:
+            getSearchPostData(search_keyword=search_keywords[i], index=unique_list[i])
+            successful_steps.append(('getSearchPostData', i))  # Mark this step as successful
+        except Exception as e:
+            print(f"Failed at getSearchPostData for keyword {search_keywords[i]}: {e}")
+
+    # Step 2: Generate file names
+    fileNames = [f"posts_data_{unique_list[i]}.csv" for i in range(len(unique_list))]
+
+    # Step 3: Get final data
+    try:
+        getFinalData(user_query=user_query, filesNames=fileNames)
+        successful_steps.append(('getFinalData',))  # Mark this step as successful
+    except Exception as e:
+        print(f"Failed at getFinalData: {e}")
+
+    # Step 4: Get post comments
+    try:
+        getPostComments(fileName=fileNames[0])
+        successful_steps.append(('getPostComments',))  # Mark this step as successful
+    except Exception as e:
+        print(f"Failed at getPostComments: {e}")
+    # Step 5: Get sentiment of post comments
+    try:
+        sentiment_instance = SentimentAnalysis()
+        sentiment_instance.generate_sentiment_and_emotion_from_data(fileName=fileNames[0])
+        successful_steps.append(('getPostSentiment',))  # Mark this step as successful
+    except Exception as e:
+        print(f"Failed at getPostSentiment: {e}")
+    # Optionally, return the successful steps for logging or further processing
+    return {
+        "fileName":fileNames[0],
+        "fileUniqueId":str(unique_list[0]),
+        "successful_steps": successful_steps
+    }

venv/reddit/reddit_gemini.py

@@ -0,0 +1,70 @@
+import json
+import logging
+import time
+import google.generativeai as genai
+
+from reddit.prompts import getKeywordsPrompt
+
+def getKeywords(user_query: str):
+    prompt = getKeywordsPrompt(user_query)
+    model = genai.GenerativeModel("gemini-exp-1114")
+
+    generation_config = genai.GenerationConfig(response_mime_type="application/json")
+
+    try:
+        response = model.generate_content(prompt, generation_config=generation_config)  # Adjust if the library supports async
+        data = response.text
+        logging.info("Enhanced user query: %s", data)
+        return json.loads(data)
+    except Exception as e:
+        logging.error("Error while fetching keywords: %s", e)
+        # raise HTTPException(status_code=500, detail="Error processing request")
+
+
+def upload_to_gemini(path, mime_type=None):
+  file = genai.upload_file(path, mime_type=mime_type)
+  print(f"Uploaded file '{file.display_name}' as: {file.uri}")
+  return file
+
+def wait_for_files_active(files):
+  """Waits for the given files to be active.
+
+  Some files uploaded to the Gemini API need to be processed before they can be
+  used as prompt inputs. The status can be seen by querying the file's "state"
+  field.
+
+  This implementation uses a simple blocking polling loop. Production code
+  should probably employ a more sophisticated approach.
+  """
+  print("Waiting for file processing...")
+  for name in (file.name for file in files):
+    file = genai.get_file(name)
+    while file.state.name == "PROCESSING":
+      print(".", end="", flush=True)
+      time.sleep(10)
+      file = genai.get_file(name)
+    print("file.state.name", file.state.name)
+    if file.state.name != "ACTIVE":
+      raise Exception(f"File {file.name} failed to process")
+  print("...all files ready")
+  print()
+def getModelAndGenerationConfigCommon(fileName, isFlash=True):
+    generation_config = {
+      "temperature": 1,
+      "top_p": 0.95,
+      "top_k": 40,
+      "max_output_tokens": 8192,
+      "response_mime_type": "application/json",
+    }
+   
+    model = genai.GenerativeModel(
+      model_name="gemini-1.5-flash" if isFlash else "gemini-1.5-pro-002",
+      generation_config=generation_config,
+    )
+    files = [
+      upload_to_gemini(fileName, mime_type="text/csv"),
+    ]
+
+    # Some files have a processing delay. Wait for them to be ready.
+    wait_for_files_active(files)
+    return [model,files[0]]

venv/reddit/reddit_pain_point_analysis.py

@@ -0,0 +1,51 @@
+import json
+import google.generativeai as genai
+
+from reddit.prompts import getPainPointAnalysisPrompt
+from reddit.reddit_gemini import upload_to_gemini, wait_for_files_active
+
+
+def pain_point_analysis(user_query,fileName,uniqueFileId):
+    pain_point_prompt = getPainPointAnalysisPrompt(user_query = user_query)
+    generation_config = genai.GenerationConfig(response_mime_type="application/json") # Request JSON response
+    model = genai.GenerativeModel(
+      model_name="gemini-1.5-pro-002",
+      generation_config=generation_config,
+    )
+    
+    files = [
+          upload_to_gemini(fileName, mime_type="text/csv"),
+        ]
+    # Some files have a processing delay. Wait for them to be ready.
+    wait_for_files_active(files)
+    chat_session = model.start_chat(
+    )
+
+    chat_session.history = [
+        {
+            "role": "user",
+            "parts": [
+                files[0],
+                pain_point_prompt
+            ]
+        }
+    ]
+    response = chat_session.send_message("give your pain point analysis output json as it is.")
+    data = response.text
+    try:
+        print("pain point analysis output",data)
+        json_data = json.loads(data)
+        with open(f'pain_point_analysis_{uniqueFileId}.json', 'w') as json_file:
+          json.dump(json_data, json_file, indent=4) 
+        return [json_data,chat_session]
+    except:
+      try:
+        response = chat_session.send_message("give your pain point analysis output json as it is.")
+        print("retried pain point analysis output",data)
+        json_data = json.loads(data)
+        with open(f'pain_point_analysis_{uniqueFileId}.json', 'w') as json_file:
+          json.dump(json_data, json_file, indent=4)
+        return [json_data,chat_session]
+      except:
+        json_data = {"details": "something went wrong"}
+        return [json_data,chat_session]      

venv/reddit/reddit_search_scrapper.py

@@ -0,0 +1,55 @@
+'''
+Reddit data pre processing code only.
+'''
+
+import os
+import pandas as pd
+from reddit.reddit_competitor_analysis import getCompetitorNames, getCompetitorNamesFromReddit, getPostDataofCompetitor, getTop10Competitors
+from reddit.reddit_utils import topic_sort
+
+def preProcessPostData(filesNames):
+    for i in filesNames:
+        df=pd.read_csv(i)
+        df.drop_duplicates(subset=["title"],inplace=True)
+        # drop rows with comment_count =0
+        df=df[df["comment_count"]!=0]
+        # drop index column
+        df.drop(columns=["index"], inplace=True)
+        indexes=[i for i in range(len(df))]
+        # insert index column
+        df.insert(0, "index", indexes)
+        df.to_csv(i, index=False)
+
+def getFinalData(user_query,filesNames):
+    preProcessPostData(filesNames=filesNames)
+    # files_name=["posts_data_0.csv","posts_data_1.csv","posts_data_2.csv"]
+    final_df = topic_sort(filesNames[0], filesNames[1], filesNames[2], user_query)
+    for file_path in filesNames:
+    # Check if the file exists before attempting to delete
+        if os.path.exists(file_path):
+            os.remove(file_path)
+            print("File deleted successfully")
+        else:
+            print("File does not exist")
+    final_df.to_csv(filesNames[0], index=False)
+    
+    print("Data saved to ",filesNames[0])
+
+
+def getCompetitorAnalysisData(user_query,fileName,isSolo=True,chat_session=None):
+    json_data_gemini = getCompetitorNames(user_query=user_query)
+    # with open('competitors_names_gemini.json', 'w') as json_file:
+    #     json.dump(json_data, json_file, indent=4)
+    json_data_reddit = getCompetitorNamesFromReddit(user_query=user_query,isSolo=isSolo,last_chat_session=chat_session,fileName=fileName)
+    # with open('competitors_names_reddit.json', 'w') as json_file:
+    #     json.dump(json_data, json_file, indent=4)
+    data = getTop10Competitors(gemini_data=json_data_gemini,reddit_data=json_data_reddit,user_query=user_query)
+    df = pd.DataFrame(data["list"])
+    # print("Data saved to competitors_names.csv")
+    competitors_data = getPostDataofCompetitor(user_query=user_query,fileName=df)
+
+    return {
+        "competitors_data": competitors_data,
+        "all_competitor_data": data["list"]
+    }
+

venv/reddit/reddit_sentiment_analysis.py

@@ -0,0 +1,53 @@
+import pandas as pd 
+import ast
+from transformers import BertTokenizer, BertForSequenceClassification
+from transformers import pipeline
+output=pd.DataFrame()
+class SentimentAnalysis:
+  def __init__(self):
+    self.classifier = pipeline(task="text-classification", model="SamLowe/roberta-base-go_emotions", top_k=None)
+    self.finbert = BertForSequenceClassification.from_pretrained('yiyanghkust/finbert-tone',num_labels=3)
+    self.tokenizer = BertTokenizer.from_pretrained('yiyanghkust/finbert-tone')
+  def process_comment(self,comment):
+      
+      sentence=[comment['comment'][:512]]
+      model_outputs = self.classifier(sentence)
+      # Prepare the comment data
+      comment_data = {
+          "comment": comment['comment'],
+          'emotion':model_outputs[0][:3],
+          "replies": []  # Initialize replies list
+      }
+      
+      # Process replies recursively if any
+      if comment['replies']:
+          for reply in comment['replies']:
+              
+              reply_data = self.process_comment(reply)  # Recursive call for replies
+              comment_data["replies"].append(reply_data)
+      
+      return comment_data
+  def generate_sentiment_and_emotion_from_data(self,fileName):    
+    df = pd.read_csv(fileName)
+    comments_data=[]
+    for i in range(df.shape[0]):
+      row=df.iloc[i]
+      commentary=(ast.literal_eval(row['comments']))
+      commentary=commentary['comments']
+      while commentary:
+        comment = commentary.pop(0)
+        comment_data = self.process_comment(comment)
+        comments_data.append(comment_data)
+      json_output = {
+            "comments": comments_data
+        }
+      subset_data = df.iloc[i].copy()
+
+      # Modify the subset
+      subset_data['comments'] = json_output
+
+      # Assign back if needed
+      df.iloc[i] = subset_data
+    df.to_csv(fileName, index=False)
+    print("Sentiment Data saved to",fileName)
+      

venv/reddit/reddit_utils.py

@@ -0,0 +1,38 @@
+import time
+
+import pandas as pd
+from sentence_transformers import SentenceTransformer
+
+def get_microseconds_list(length=3):
+    # Get the current time in microseconds
+    microseconds = int(time.time() * 1_000_000)
+    
+    # Create a list with three microseconds
+    return [microseconds + i for i in range(length)]
+
+
+def topic_sort(path1,query, path2='', path3='',isForCompetitorAnalysis=False):
+  sentence_model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")
+  if isForCompetitorAnalysis==True:
+      df=pd.read_csv(path1)
+  else:
+    df0 = pd.read_csv(path1)
+    df1 = pd.read_csv(path2)
+    df2 = pd.read_csv(path3)
+    df = pd.concat([df0, df1, df2],axis=0)
+    df = df.drop_duplicates("title")
+  df = df.reset_index(drop=True)
+  df = df.drop("index", axis = 1)
+  title = df["title"]
+  sentences = [query] + list(title)
+  embeddings = sentence_model.encode(sentences)
+  similarities = sentence_model.similarity(embeddings[0], embeddings)
+  print(len(similarities[0]))
+  df["similarity"] = similarities[0][1:]
+  df = df.sort_values(by='similarity', ascending=False)
+  df = df.reset_index(drop=True)
+  df = df.head(30)
+  df = df.sort_values(by=['comment_count','votes_count'], ascending=False)
+  df = df.reset_index(drop=True)
+  df = df.head(18)
+  return df

venv/reddit/scraping.py

@@ -0,0 +1,223 @@
+'''
+Only Scraping related code.
+'''
+from selenium import webdriver
+from selenium.webdriver.common.action_chains import ActionChains
+from selenium.webdriver.common.by import By
+from selenium.webdriver.firefox.options import Options as FirefoxOptions
+import time
+from fake_headers import Headers
+import pandas as pd
+import praw
+import re
+
+from reddit.reddit_utils import topic_sort
+# # Set up WebDriver
+header = Headers().generate()["User-Agent"]
+proxy=None
+browser_option = FirefoxOptions()
+browser_option.add_argument("--no-sandbox")
+browser_option.add_argument("--disable-dev-shm-usage")
+browser_option.add_argument("--ignore-certificate-errors")
+browser_option.add_argument("--disable-gpu")
+browser_option.add_argument("--log-level=3")
+browser_option.add_argument("--disable-notifications")
+browser_option.add_argument("--disable-popup-blocking")
+browser_option.add_argument("--user-agent={}".format(header))
+if proxy is not None:
+    browser_option.add_argument("--proxy-server=%s" % proxy)
+
+# For Hiding Browser
+browser_option.add_argument("--headless")
+
+driver = webdriver.Firefox(options=browser_option)
+actions = ActionChains(driver)
+reddit = praw.Reddit(
+    client_id="yjGfys3QZPpdCpNZl25Kig",
+    client_secret="dqoc8LrQBQhB_IgjV-lKyrD9lBPftg",
+    password="&honeyB90",
+    user_agent="Curious",
+    username="Final-Difference7055",
+)
+
+
+def extract_post_id(url):
+    # Use regex to match the post ID pattern
+    match = re.search(r'/comments/([^/]+)/', url)
+    if match:
+        return match.group(1)
+    return None
+# 1. Get Search Post Data
+def getSearchPostData( search_keyword,index, name="",forCompetitorAnalysis=False):
+        # Navigate to the search results page
+    
+    url = f'https://www.reddit.com/search/?q={search_keyword}'
+    driver.get(url)
+    time.sleep(3)
+    print("reached this step")
+
+    # Scroll and extract data
+    posts_data = [
+    #     {
+    #     "index":0,
+    #     "title":"",
+    #     "description":"",
+    #     "comment_count":0,
+    #     "votes_count":0,
+    #     "url":"",
+    #     "time":""
+    # }
+    ]
+    list_length=0 # posts count
+    # Scroll down and wait for content to load
+    try:
+        driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
+        time.sleep(5)
+        if forCompetitorAnalysis:
+            driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
+            time.sleep(5)
+        # Find post cards
+        post_cards = driver.find_elements(By.CSS_SELECTOR, 'a[data-testid="post-title-text"]')
+        post_cards_1 = driver.find_elements(By.CSS_SELECTOR, 'div[data-testid="search-counter-row"]')
+        post_cards_2 = driver.find_elements(By.CSS_SELECTOR, 'faceplate-timeago')
+        idx=list_length
+        # Loop through each post card and extract required information
+        for card in post_cards_1:
+            try:
+                # Extract votes count
+                votes_count = card.find_element(By.XPATH, './/faceplate-number').text
+            
+                # Extract the comments count, checking for both "comment" and "comments"
+                comments_count = card.find_element(By.XPATH,
+                    './/span[contains(text(), "comment") or contains(text(), "comments")]/preceding-sibling::faceplate-number'
+                ).text
+                posts_data.append({
+                    "index":idx,
+                    "comment_count":comments_count,
+                    "votes_count":votes_count
+                })
+                idx+=1
+            except Exception as e:
+                print("Error in post_card_1:", e)
+
+        
+        idx=list_length
+        for card in post_cards:
+            try:
+                url=card.get_attribute("href")
+                title=card.text
+                posts_data[idx]["title"]=title
+                posts_data[idx]["url"]=url
+                idx+=1
+            except Exception as e:
+                print("Error in post_cards:", e)
+        idx=list_length
+        
+        for card in post_cards_2:
+            try:
+                time_element = card.find_element(By.XPATH,'./time')
+                post_time=time_element.get_attribute('datetime')
+                posts_data[idx]["time"]=post_time
+                idx+=1
+            except Exception as e:
+                print("Error in post_cards_2:", e)
+    except Exception as e:
+        print("Error in scrolling:", e)
+    
+    df = pd.DataFrame(posts_data)
+    if name!="":
+        df = df.drop_duplicates("title")
+        df = df[df["title"].str.contains(name)]
+        if len(df) >=6:
+            df.to_csv(f'posts_data_{index}.csv', index=False)
+            if len(df)>18:
+                getFinalDataOfCompetitor(user_query=search_keyword,fileName=f'posts_data_{index}.csv')
+            print(f"Data saved to posts_data_{index}.csv")
+            return index
+    else:
+        df.to_csv(f'posts_data_{index}.csv', index=False)
+        print(f"Data saved to posts_data_{index}.csv")
+def getFinalDataOfCompetitor(user_query,fileName):
+    final_df = topic_sort(path1=fileName,isForCompetitorAnalysis=True,query= user_query)
+    final_df.to_csv(fileName, index=False)
+    print("getFinalDataOfCompetitor Data saved to ",fileName)
+def getSearchPostDescription(url):
+    id=extract_post_id(url)
+    # Navigate to the search results page
+    driver.get(url)
+    time.sleep(0.5)
+
+    # Scroll and extract data
+    description=""
+    try:
+        post_data = driver.find_element(By.CSS_SELECTOR, f'div[id="t3_{id}-post-rtjson-content"]')
+        description = post_data.find_element(By.XPATH, './p').text
+    except:
+        description=""
+    return description
+
+
+
+
+def process_comment(comment,reply_limit):
+    # Prepare the comment data
+    comment_data = {
+        "user": comment.author.name if comment.author else "Unknown",
+        "comment": comment.body,
+        "replies": []  # Initialize replies list
+    }
+    
+    # Process replies recursively if any
+    if comment.replies:
+        for reply in comment.replies:
+            if reply_limit==0:
+                break
+            reply_data = process_comment(reply,reply_limit=reply_limit-1)  # Recursive call for replies
+            comment_data["replies"].append(reply_data)
+    
+    return comment_data
+
+# 3. get post comments data
+def getPostComments(fileName,isForCompetitorAnalysis=False):
+    data= pd.DataFrame(pd.read_csv(fileName))
+    data["comments"]=""
+    for i in range(len(data)):
+    # comment_data_sub=[]
+        try:
+            submission = reddit.submission(url=data.iloc[i]['url'])
+
+            # Fetch and process comments
+            submission.comments.replace_more(limit=2)  # Use limit=0 to get all comments
+        except Exception as e:
+            print("skipping due to error",data.iloc[i]['url'], e)
+            continue
+        comments_data = []
+
+        # Function to process a comment and its replies
+        # Seed with top-level comments
+        comment_queue = list(submission.comments)
+        comment_count=0
+        threshold=20 if isForCompetitorAnalysis else 40
+        while comment_queue:
+            if(comment_count>=threshold):
+                break
+            comment = comment_queue.pop(0)
+            comment_data = process_comment(comment,reply_limit=2 if isForCompetitorAnalysis else 3)  # Process each comment
+            comments_data.append(comment_data)
+            comment_count+=1
+        # Now, structure the data into the desired JSON format
+        json_output = {
+            "comments": comments_data
+        }
+        subset_data = data.iloc[i].copy()
+
+        # Modify the subset
+        subset_data['comments'] = json_output
+
+        # Assign back if needed
+        data.iloc[i] = subset_data
+        # Remove rows where 'comments' is an empty string
+    data = data[data['comments'] != ""]
+    data["descriptions"] = data["url"].apply(getSearchPostDescription)
+    data.to_csv(fileName, index=False)
+    print("Data saved to",fileName)

venv/test.py

@@ -0,0 +1,200 @@
+# # import pandas as pd
+# # import json
+# # import ast
+# # # Load the CSV file
+# # df = pd.read_csv('file_with_sentiment.csv')
+
+# # # Convert the 'comments' column to a list
+# # comments_list = []
+# # for i in df['descriptions']:
+# #     # json_data= ast.literal_eval(i)
+# #     comments_list.append(i)
+
+# # # print("comments_list",len())
+# # # Recursive function to count non-empty replies
+# # def count_non_empty_replies(comments):
+# #     count = 1
+# #     for comment in comments:
+# #         if comment.get("replies"):
+# #             count += 1  # Increment for this non-empty replies list
+# #             count += count_non_empty_replies(comment["replies"])  # Recursively count nested replies
+# #     return count
+
+# # # Example usage
+# # # total_non_empty_replies = count_non_empty_replies(comments_list[0]['comments'][0]['replies'])
+# # # print("Total non-empty replies:", total_non_empty_replies)
+
+# # # Sample data structure
+# # # comments_list = [
+# # #     {
+# # #         "comments": [
+# # #             # Each post contains a list of comments with nested replies
+# # #         ]
+# # #     }
+# # #     # More comments_list here
+# # # ]
+
+# # # Recursive function to limit replies in a comment tree
+# # def limit_replies(comment, reply_limit=3):
+# #     limited_comment = {
+# #         "user": comment["user"],
+# #         "comment": comment["comment"],
+# #         "replies": []
+# #     }
+# #     if reply_limit == 0:
+# #         return limited_comment
+    
+# #     # Get up to reply_limit replies, recursively applying the function
+# #     if "replies" in comment:
+# #         for reply in comment["replies"][:reply_limit-1]:
+# #             limited_comment["replies"].append(limit_replies(reply, reply_limit-2))
+    
+# #     return limited_comment
+
+# # # Function to process each post, extracting 10 comments with limited replies
+# # def get_limited_comments(comments_list, comment_limit=10, reply_limit=7):
+# #     limited_comments_list = []
+    
+# #     for post in comments_list:
+# #         limited_post = {"comments": []}
+        
+# #         # Get up to comment_limit comments for each post
+# #         for comment in post["comments"][:comment_limit]:
+# #             limited_comment = limit_replies(comment, reply_limit)
+# #             limited_post["comments"].append(limited_comment)
+        
+# #         limited_comments_list.append(limited_post)
+    
+# #     return limited_comments_list
+
+# # Example usage
+# # limited_comments_data = get_limited_comments(comments_list)
+# # total_non_empty_replies = count_non_empty_replies(limited_comments_data[0]['comments'][0]['replies'])
+
+
+# # Save the list to a JSON file
+# # with open('comments2.json', 'w') as json_file:
+# #     json.dump(comments_list, json_file,indent=4)
+
+# # # from reddit.scraping import getPostComments
+
+
+# # # getPostComments(fileName="posts_data_1732105228633815.csv")
+# # import time
+
+# # from reddit.reddit_sentiment_analysis import SentimentAnalysis
+
+# # # Create an instance of the SentimentAnalysis class
+# # sentiment_instance = SentimentAnalysis()
+
+# # # Record the start time
+# # start_time = time.time()
+
+# # # Call the method to generate sentiment and emotion
+# # sentiment_instance.generate_sentiment_and_emotion_from_data(fileName='posts_data.csv')
+
+# # # Record the end time
+# # end_time = time.time()
+
+# # # Calculate and print the processing time
+# # process_time = end_time - start_time
+# # print(f"Processing time: {process_time:.2f} seconds")
+# # from reddit.reddit_pain_point_analysis import pain_point_analysis
+
+
+# # pain_point_analysis(user_query="artificial intelligence applications in skincare and cosmetic industry",fileName="file_with_sentiment.csv")
+
+# # import google.generativeai as genai
+# # genai.configure(api_key='AIzaSyBtHE4Bg2ERWsKeGLxGPOSmtZeWRD6nNr0')
+# # model = genai.GenerativeModel("gemini-1.5-flash")
+
+# # generation_config = genai.GenerationConfig(response_mime_type="application/json")
+# # response = model.generate_content("skin care ai ", generation_config=generation_config)  # Adjust if the library supports async
+# # data = response.text
+# # print(data)
+
+# '''
+# Only Scraping related code.
+# '''
+# from selenium import webdriver
+# from selenium.webdriver.common.action_chains import ActionChains
+# from selenium.webdriver.common.by import By
+# from selenium.webdriver.firefox.options import Options as FirefoxOptions
+# import time
+# from fake_headers import Headers
+# import pandas as pd
+# import praw
+# # from reddit_call import sentence_model
+# import re
+# # # Set up WebDriver
+# header = Headers().generate()["User-Agent"]
+# proxy=None
+# browser_option = FirefoxOptions()
+# browser_option.add_argument("--no-sandbox")
+# browser_option.add_argument("--disable-dev-shm-usage")
+# browser_option.add_argument("--ignore-certificate-errors")
+# browser_option.add_argument("--disable-gpu")
+# browser_option.add_argument("--log-level=3")
+# browser_option.add_argument("--disable-notifications")
+# browser_option.add_argument("--disable-popup-blocking")
+# browser_option.add_argument("--user-agent={}".format(header))
+# if proxy is not None:
+#     browser_option.add_argument("--proxy-server=%s" % proxy)
+
+# # For Hiding Browser
+# browser_option.add_argument("--headless")
+
+# driver = webdriver.Firefox(options=browser_option)
+# actions = ActionChains(driver)
+# reddit = praw.Reddit(
+#     client_id="yjGfys3QZPpdCpNZl25Kig",
+#     client_secret="dqoc8LrQBQhB_IgjV-lKyrD9lBPftg",
+#     password="&honeyB90",
+#     user_agent="Curious",
+#     username="Final-Difference7055",
+# )
+# fileName="posts_data_1732244765294548.csv"
+# data= pd.DataFrame(pd.read_csv(fileName))
+# data["comments"]=""
+# for i in range(len(data)):
+# # comment_data_sub=[]
+#     submission = reddit.submission(url=data.iloc[i]['url'])
+
+#     # Fetch and process comments
+#     submission.comments.replace_more(limit=2)  # Use limit=0 to get all comments
+#     comments_data = []
+
+#     # Function to process a comment and its replies
+#     # Seed with top-level comments
+#     comment_queue = list(submission.comments)
+#     comment_count=0
+#     threshold=20
+#     while comment_queue:
+#         if(comment_count>=threshold):
+#             break
+#         comment = comment_queue.pop(0)
+#         comment_data = process_comment(comment,reply_limit=2 if isForCompetitorAnalysis else 3)  # Process each comment
+#         comments_data.append(comment_data)
+#         comment_count+=1
+#     # Now, structure the data into the desired JSON format
+#     json_output = {
+#         "comments": comments_data
+#     }
+#     subset_data = data.iloc[i].copy()
+
+#     # Modify the subset
+#     subset_data['comments'] = json_output
+
+#     # Assign back if needed
+#     data.iloc[i] = subset_data
+#     # Remove rows where 'comments' is an empty string
+# data = data[data['comments'] != ""]
+# data["descriptions"] = data["url"].apply(getSearchPostDescription)
+# data.to_csv(fileName, index=False)
+# print("Data saved to",fileName)
+
+from reddit.reddit_competitor_analysis import getCompetitorNamesFromReddit
+from reddit.api_keys import api_key,api_key2, api_key3
+import google.generativeai as genai
+genai.configure(api_key=api_key3)
+getCompetitorNamesFromReddit(user_query='AI powered personalized skin care recommendations',isSolo=True,fileName='posts_data_1732244547776200.csv',last_chat_session=None)

venv/utils.py

@@ -0,0 +1,26 @@
+import asyncio
+from functools import wraps
+import time
+
+def time_execution(func):
+    if asyncio.iscoroutinefunction(func):
+        @wraps(func)
+        async def async_wrapper(*args, **kwargs):
+            start_time = time.time()
+            result = await func(*args, **kwargs)
+            end_time = time.time()
+            execution_time = end_time - start_time
+            print(f"Function '{func.__name__}' executed in {execution_time:.4f} seconds")
+            return result
+        return async_wrapper
+    else:
+        @wraps(func)
+        def sync_wrapper(*args, **kwargs):
+            start_time = time.time()
+            result = func(*args, **kwargs)
+            end_time = time.time()
+            execution_time = end_time - start_time
+            print(f"Function '{func.__name__}' executed in {execution_time:.4f} seconds")
+            return result
+        return sync_wrapper
+