Spaces:
Sleeping
Sleeping
File size: 5,088 Bytes
b7b041e |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 |
import pandas as pd
from bertopic import BERTopic
from bertopic.representation import KeyBERTInspired
def analyze_general_topic_evolution(topic_model, docs, timestamps):
"""
Analyzes general topic evolution over time.
Args:
topic_model: Trained BERTopic model.
docs (list): List of documents.
timestamps (list): List of timestamps corresponding to the documents.
Returns:
pd.DataFrame: DataFrame with topic evolution information.
"""
try:
topics_over_time = topic_model.topics_over_time(docs, timestamps, global_tuning=True)
return topics_over_time
except Exception:
# Fallback for small datasets or cases where evolution can't be computed
return pd.DataFrame(columns=['Topic', 'Words', 'Frequency', 'Timestamp'])
def analyze_user_topic_evolution(df: pd.DataFrame, topic_model):
"""
Analyzes topic evolution per user.
Args:
df (pd.DataFrame): DataFrame with (
"user_id", "post_content", "timestamp", and "topic_id" columns.
topic_model: Trained BERTopic model.
Returns:
dict: A dictionary where keys are user_ids and values are DataFrames of topic evolution for that user.
"""
user_topic_evolution = {}
for user_id in df["user_id"].unique():
user_df = df[df["user_id"] == user_id].copy()
if not user_df.empty and len(user_df) > 1:
try:
# Ensure timestamps are sorted for topics_over_time
user_df = user_df.sort_values(by="timestamp")
docs = user_df["post_content"].tolist()
timestamps = user_df["timestamp"].tolist()
selected_topics = user_df["topic_id"].tolist() # Get topic_ids for the user's posts
topics_over_time = topic_model.topics_over_time(docs, timestamps, topics=selected_topics, global_tuning=True)
user_topic_evolution[user_id] = topics_over_time
except Exception:
user_topic_evolution[user_id] = pd.DataFrame(columns=['Topic', 'Words', 'Frequency', 'Timestamp'])
else:
user_topic_evolution[user_id] = pd.DataFrame(columns=['Topic', 'Words', 'Frequency', 'Timestamp'])
return user_topic_evolution
if __name__ == "__main__":
# Example Usage:
data = {
"user_id": ["user1", "user2", "user1", "user3", "user2", "user1", "user4", "user3", "user2", "user1", "user5", "user4", "user3", "user2", "user1"],
"post_content": [
"This is a great movie, I loved the acting and the plot. It was truly captivating.",
"The new phone has an amazing camera and long battery life. Highly recommend it.",
"I enjoyed the film, especially the special effects and the soundtrack. A must-watch.",
"Learning about AI and machine learning is fascinating. The future is here.",
"My old phone is so slow, I need an upgrade soon. Thinking about the latest model.",
"The best part of the movie was the soundtrack and the stunning visuals. Very immersive.",
"Exploring the vastness of space is a lifelong dream. Astronomy is amazing.",
"Data science is revolutionizing industries. Predictive analytics is key.",
"I need a new laptop for work. Something powerful and portable.",
"Just finished reading a fantastic book on quantum physics. Mind-blowing concepts.",
"Cooking new recipes is my passion. Today, I tried a spicy Thai curry.",
"The universe is full of mysteries. Black holes and dark matter are intriguing.",
"Deep learning models are becoming incredibly sophisticated. Image recognition is impressive.",
"My current laptop is crashing frequently. Time for an upgrade.",
"Science fiction movies always make me think about the future of humanity."
],
"timestamp": [
"2023-01-01 10:00:00", "2023-01-01 11:00:00", "2023-01-02 10:30:00",
"2023-01-02 14:00:00", "2023-01-03 09:00:00", "2023-01-03 16:00:00",
"2023-01-04 08:00:00", "2023-01-04 12:00:00", "2023-01-05 10:00:00",
"2023-01-05 15:00:00", "2023-01-06 09:30:00", "2023-01-06 13:00:00",
"2023-01-07 11:00:00", "2023-01-07 14:30:00", "2023-01-08 10:00:00"
]
}
df = pd.DataFrame(data)
df["timestamp"] = pd.to_datetime(df["timestamp"])
print("Performing topic modeling (English)...")
model_en, topics_en, probs_en = perform_topic_modeling(df, language="english")
df["topic_id"] = topics_en
print("\nAnalyzing general topic evolution...")
general_evolution_df = analyze_general_topic_evolution(model_en, df["post_content"].tolist(), df["timestamp"].tolist())
print(general_evolution_df.head())
print("\nAnalyzing per user topic evolution...")
user_evolution_dict = analyze_user_topic_evolution(df, model_en)
for user_id, evolution_df in user_evolution_dict.items():
print(f"\nTopic evolution for {user_id}:")
print(evolution_df.head()) |