Update app.py

app.py

@@ -8,7 +8,8 @@ import json
 import logging
 import sys
 import spacy
-#sys.setrecursionlimit(20000)
+
+# sys.setrecursionlimit(20000)
 import pandas as pd
 import numpy as np
 import os
@@ -22,17 +23,21 @@ from fastapi.staticfiles import StaticFiles
 from fastapi.templating import Jinja2Templates
 
 from rouge_score import rouge_scorer
+# Scripts
 import scripts.sentiment as sentiment
 import scripts.twitter_scraper as ts
 from scripts import sentiment
 from scripts.summarization import bert_summarization
 from scripts.twitter_scraper import get_latest_account_tweets
-from scripts import twitter_scraper as ts 
+from scripts.sentiment import twitter_sentiment_api_score
+from scripts import twitter_scraper as ts
 import scripts.utils as utils
+from scripts import translation
 from scripts import generative
 import nltk
 
 logging.basicConfig(level=logging.INFO)
+pd.set_option('display.max_colwidth', 20)
 
 app = FastAPI()
 templates = Jinja2Templates(directory="templates")
@@ -82,35 +87,34 @@ async def get_accounts() -> List[dict]:
 
 @app.get("/tweets/{username}")
 def get_tweets_username(username: str) -> dict:
-    # if username in username_list:
-    #     query = f"from:{username} since:{start_date} until:{end_date}"
-    #     return ts.get_tweets(query=query)
-    # else:
-    #     return {"detail": "Account not in scope of project."}
-
-    # Method 1: Using Tweepy method
-    # df_tweets = get_latest_account_tweets(username)
-
     # Method 2: Use Snscrape
     df_tweets = ts.get_tweets(handle=username)
 
     if isinstance(df_tweets, pd.DataFrame):
-        print(df_tweets.head(2))
-        print(df_tweets.shape)
         df_tweets = df_tweets[["handle", "created_at","retweet_count","view_count","like_count", "full_text"]]
-        df_tweets["created_at"] = df_tweets["created_at"].dt.strftime("%Y-%m-%d %H:%M:%S")
-        df_tweets = df_tweets.sort_values("created_at", ascending=False)#.tail(10)
+        df_tweets["created_at"] = df_tweets["created_at"].dt.strftime(
+            "%Y-%m-%d %H:%M:%S"
+        )
+        df_tweets = df_tweets.sort_values("created_at", ascending=False)  
+
+        # Additional processing
+        logging.info("Running sentiment on tweets")
+        sentiments = twitter_sentiment_api_score(
+            df_tweets['full_text'].to_list(), use_api=False
+        )
+        df_tweets["sentiment"] = [s['argmax'] for s in sentiments]
+        if username == "alikarimi_ak8": 
+            p = translation.PersianTextProcessor()
+            df_tweets['full_text_translated'] = df_tweets["full_text"].apply(lambda c: p.translate_text(persian_text = c))
+
+
         df_tweets_html = df_tweets.to_html(classes="center", index=False, escape=False)
-        df_tweets.to_html(open('df_tweets_html.html', 'w'))
+        df_tweets.to_html(open("df_tweets_html.html", "w"))
         df_tweets_data = df_tweets.to_dict(orient="records")
-        
-        response_data = {
-            "html": df_tweets_html,
-            "data": df_tweets_data
-        }
+
+        response_data = {"html": df_tweets_html, "data": df_tweets_data}
 
         return JSONResponse(content=response_data, status_code=200)
-        # return HTMLResponse(content=df_tweets_html, status_code=200)
     else:
         print("Error: Failed to retrieve tweets.")
         return df_tweets
@@ -214,6 +218,7 @@ async def get_sentiment(username: str) -> Dict[str, Dict[str, float]]:
     }
 
 
+## APIs: Primarily called by the index page
 @app.post("/api/generate")
 async def generate_text(request: Request):
     """Generate text from a prompt.
@@ -269,10 +274,9 @@ async def generate_summary(request: Request):
 
     print("*" * 50)
     data = await request.json()
-    print('data',data['tweetsData'])
+    print("data", data["tweetsData"])
     # Get the list of text
-    tweets = [t['full_text'] for t in data["tweetsData"]]
-
+    tweets = [t["full_text"] for t in data["tweetsData"]]
 
     # Concatenate tweets into a single string
     text = " .".join(tweets)
@@ -281,35 +285,25 @@ async def generate_summary(request: Request):
     nlp.add_pipe("sentencizer")
 
     sentences = nlp(text).sents
-    # sentences = Text8Corpus(text)
-    # phrases = Phrases(
-    #     sentences, min_count=1, threshold=1, connector_words=ENGLISH_CONNECTOR_WORDS
-    # )
-    # first_sentence = next(iter(sentences))
-    # first_sentence
+
     sentences = list(sentences)
-    # # Shuffle the list
-    # random.shuffle(sentences)
-    # Option 1
-    # sampled_tweets = random.sample(tweets, int(0.1 * len(tweets)))
 
     # Option 2
     sampled_sentences = random.sample(sentences, int(0.1 * len(sentences)))
-    
+
     sampled_sentences = [sentiment.tweet_cleaner(s.text) for s in sampled_sentences]
 
     # Join the strings into one text blob
     tweet_blob = " ".join(sampled_sentences)
 
     # Generate the summary
-    summary = bert_summarization(
-       tweet_blob
-    )
-    print("Summary:",summary)
+    summary = bert_summarization(tweet_blob)
+    print("Summary:", summary)
     # Return the summary
     return {"tweets_summary": summary}
 
 
+## Historical Tweets pages
 @app.get("/examples1")
 async def read_examples():
     with open("templates/charts/handle_sentiment_breakdown.html") as f:
@@ -322,3 +316,9 @@ async def read_examples():
     with open("templates/charts/handle_sentiment_timesteps.html") as f:
         html = f.read()
     return HTMLResponse(content=html)
+
+
+# uvicorn --workers=2 app:app
+if __name__ == "__main__":
+    # uvicorn.run(app, host="0.0.0.0", port=8000)
+    uvicorn.run("app:app", host="127.0.0.1", port=5050, reload=True)