File size: 1,060 Bytes
de99ac3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 | import chromadb
client = chromadb.PersistentClient(path="./my_mastodon_db")
collection = client.get_collection(name="mastodon_posts")
# This might take a moment.
print("Fetching all data from ChromaDB...")
all_data = collection.get(
# Ask for 750k (or more) to get everything
limit=750000,
include=["documents", "embeddings"]
)
documents = all_data['documents']
embeddings = all_data['embeddings']
from bertopic import BERTopic
print("Data fetched. Starting topic modeling...")
# We tell BERTopic to use our pre-made embeddings
topic_model = BERTopic(embedding_model="disable")
# This one-line command does it all:
# 1. Runs UMAP to reduce dimensions
# 2. Runs HDBSCAN to find clusters
# 3. Runs a c-TF-IDF to find topic words
topics, probs = topic_model.fit_transform(documents, embeddings)
print("Topic modeling complete!")
# 4. See your topics!
# This shows the most common topics
print(topic_model.get_topic_info())
# This shows the keywords for a specific topic
print(topic_model.get_topic(0))
topic_model.visualize_topics()
|