Pushkar Niroula commited on
Commit
f7f7385
·
unverified ·
2 Parent(s): 64ace6c929258f

Merge pull request #7 from Pushkar222-n/feature/ProdDeploy_Supabase_Qdrant

Browse files
config.py CHANGED
@@ -9,6 +9,8 @@ class Settings(BaseSettings):
9
  postgres_password: str
10
  postgres_db: str
11
  database_url: str
 
 
12
 
13
  class Config:
14
  env_file = '.env'
 
9
  postgres_password: str
10
  postgres_db: str
11
  database_url: str
12
+ qdrant_url: str
13
+ qdrant_api_key: str
14
 
15
  class Config:
16
  env_file = '.env'
pyproject.toml CHANGED
@@ -8,6 +8,8 @@ dependencies = [
8
  "chromadb>=1.5.2",
9
  "fastapi[all]>=0.122.0",
10
  "gradio>=6.2.0",
 
 
11
  "langchain>=1.1.3",
12
  "langchain-groq>=1.1.1",
13
  "pandas>=2.3.3",
 
8
  "chromadb>=1.5.2",
9
  "fastapi[all]>=0.122.0",
10
  "gradio>=6.2.0",
11
+ "grpcio>=1.78.0",
12
+ "grpcio-tools>=1.78.0",
13
  "langchain>=1.1.3",
14
  "langchain-groq>=1.1.1",
15
  "pandas>=2.3.3",
src/data_ingestion/create_embeddings.py CHANGED
@@ -1,8 +1,10 @@
1
  from sentence_transformers import SentenceTransformer
2
- import chromadb
3
- from chromadb.config import Settings
 
4
  from sqlmodel import Session, select
5
  import logging
 
6
 
7
  from src.database.session import engine
8
  from src.database.models import Animes
@@ -25,42 +27,74 @@ class EmbeddingPipeline:
25
  """
26
  logger.info(f"Loading embedding model: {model_name}")
27
  self.model = SentenceTransformer(model_name)
 
28
 
29
- self.chroma_client = chromadb.PersistentClient(
30
- path="data/embeddings/chroma_db")
 
 
 
31
 
32
  self.use_existing_embeddings = False
33
  print("ChromaDB initialized at data/embeddings/chroma_db")
34
 
35
  def create_or_get_collection(self, collection_name: str = "anime_collection"):
36
  """Create or get existing collection"""
37
- try:
38
- collection = self.chroma_client.get_collection(collection_name)
39
  logger.info(f"Found existing collection: {collection_name}")
40
- logger.info(f"Current count: {collection.count()} documents")
 
 
 
41
 
42
  user_input = input("Reset collection? (y/n): ")
43
  if user_input.lower() == "y":
44
- self.chroma_client.delete_collection(collection_name)
45
- collection = self.chroma_client.create_collection(
46
- collection_name)
47
  logger.info("Collection reset")
48
  else:
49
  self.use_existing_embeddings = True
 
 
 
 
 
 
 
 
 
50
 
51
- except:
52
- collection = self.chroma_client.create_collection(collection_name)
53
- logger.info(f"Created new collection: {collection_name}")
54
 
55
- return collection
 
 
 
56
 
57
- def fetch_data_from_postgres(self):
58
- """Fetch all anime records from PostgreSQL"""
59
- logger.info("Fetching data from PostgreSQL...")
60
  with Session(engine) as session:
61
- statement = select(Animes).where(Animes.searchable_text != None)
62
- results = session.exec(statement).all()
63
- return results
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
64
 
65
  def embed_texts(self, texts: list[str], batch_size: int = 32) -> list[list[float]] | None:
66
  """
@@ -86,55 +120,56 @@ class EmbeddingPipeline:
86
  else:
87
  logger.info(f"Using existing stored embeddings.")
88
 
89
- def store_in_chromadb(self, collection, db_records: list[Animes], embeddings: list[list[float]]):
90
  """
91
- Store embeddings and metadata in ChromaDB
92
 
93
  Args:
94
- collection: ChromaDB collection,
 
95
  db_records: List of Anime data retrieved from PostgreSQL database,
96
  embeddings: Pre_commputed embeddings
97
  """
98
 
99
- logger.info("Storing in ChromaDB...")
100
-
101
- ids = []
102
- documents = []
103
- metadatas = []
104
 
105
- for row in db_records:
106
- ids.append(str(row.mal_id))
107
- documents.append(row.searchable_text)
108
 
109
- # genres_str = ", ".join(row.genres) if isinstance(
110
- # row.genres, list) else str(row.genres or "")
111
  genres_list = row.genres if isinstance(row.genres, list) else []
112
  if len(genres_list) == 0:
113
  genres_list = ["Unknown"]
114
 
115
- metadatas.append({
116
- "title": row.title,
117
- "genres": genres_list,
118
- "score": float(row.score) if row.score else 0.0,
119
- "type": row.type if row.type else "Unknown",
120
- "scored_by": row.scored_by if row.scored_by else 0
121
- })
122
- print(f"Genre List saved: {", ".join(genres_list)}")
 
 
 
 
 
 
 
 
123
 
124
  chunk_size = 500
125
- total_chunks = (len(ids) // chunk_size) + 1
126
- logger.info(f"Inserting into ChromaDB in {total_chunks} batches...")
127
-
128
- for i in range(0, len(ids), chunk_size):
129
- collection.add(
130
- ids=ids[i: i + chunk_size],
131
- embeddings=embeddings[i: i + chunk_size],
132
- documents=documents[i: i + chunk_size],
133
- metadatas=metadatas[i: i + chunk_size]
134
  )
135
  logger.info(f"Inserted batch {(i//chunk_size)+1}/{total_chunks}")
136
 
137
- logger.info(f"Successfully stored {len(ids)} animes in ChromaDB")
138
 
139
  def run_pipeline(self):
140
  """Run complete embedding pipeline"""
@@ -143,34 +178,55 @@ class EmbeddingPipeline:
143
  db_records = self.fetch_data_from_postgres()
144
  logger.info(f"Loaded {len(db_records)} animes from Postgres")
145
 
146
- collection = self.create_or_get_collection()
147
 
148
  if not self.use_existing_embeddings:
149
- texts_to_embed = [record.searchable_text for record in db_records]
 
 
 
 
 
 
 
150
  embeddings = self.embed_texts(texts_to_embed)
151
 
152
  if embeddings:
153
- self.store_in_chromadb(collection, db_records, embeddings)
 
154
 
155
  logger.info("Embedding pipeline complete!")
156
- return collection
157
 
158
 
159
  if __name__ == "__main__":
160
- pipeline = EmbeddingPipeline()
161
- collection = pipeline.run_pipeline()
162
-
163
- print("\n--- Testing vector search ---")
164
- query = "Anime similar to Attack Titan"
165
-
166
- print(f"Query: {query}")
167
-
168
- results = collection.query(query_texts=[query], n_results=15)
169
-
170
- print("\n--- TOP 15 RESULTS ---")
171
-
172
- for i, (title, distance) in enumerate(zip(
173
- [m["title"] for m in results["metadatas"][0]],
174
- results["distances"][0]
175
- )):
176
- print(f"{i+1}. {title} (distance: {distance:.3f})")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  from sentence_transformers import SentenceTransformer
2
+ # import chromadb
3
+ # from chromadb.config import Settings
4
+ from qdrant_client import QdrantClient, models
5
  from sqlmodel import Session, select
6
  import logging
7
+ from config import settings
8
 
9
  from src.database.session import engine
10
  from src.database.models import Animes
 
27
  """
28
  logger.info(f"Loading embedding model: {model_name}")
29
  self.model = SentenceTransformer(model_name)
30
+ self.vector_size = self.model.get_sentence_embedding_dimension() or 0
31
 
32
+ # self.chroma_client = chromadb.PersistentClient(
33
+ # path="data/embeddings/chroma_db")
34
+ self.client = QdrantClient(url=settings.qdrant_url,
35
+ api_key=settings.qdrant_api_key,
36
+ cloud_inference=True)
37
 
38
  self.use_existing_embeddings = False
39
  print("ChromaDB initialized at data/embeddings/chroma_db")
40
 
41
  def create_or_get_collection(self, collection_name: str = "anime_collection"):
42
  """Create or get existing collection"""
43
+ if self.client.collection_exists(collection_name=collection_name):
 
44
  logger.info(f"Found existing collection: {collection_name}")
45
+
46
+ collection = self.client.get_collection(collection_name)
47
+ logger.info(f"Found existing collection: {collection_name}")
48
+ logger.info(f"Current count: {collection.points_count} points")
49
 
50
  user_input = input("Reset collection? (y/n): ")
51
  if user_input.lower() == "y":
52
+ self.client.delete_collection(collection_name)
 
 
53
  logger.info("Collection reset")
54
  else:
55
  self.use_existing_embeddings = True
56
+ return collection_name
57
+
58
+ if not self.use_existing_embeddings:
59
+ is_collection_created = self.client.create_collection(collection_name=collection_name,
60
+ vectors_config=models.VectorParams(
61
+ size=self.vector_size,
62
+ distance=models.Distance.COSINE
63
+ ))
64
+ logger.info(f"Created new collection: {collection_name}: {is_collection_created}")
65
 
66
+ return collection_name
 
 
67
 
68
+ def fetch_data_from_postgres(self, batch_size: int = 2000):
69
+ """Fetch anime records from PostgreSQL in batches to avoid timeouts"""
70
+ logger.info("Fetching data from PostgreSQL in batches...")
71
+ all_results = []
72
 
 
 
 
73
  with Session(engine) as session:
74
+ offset = 0
75
+ while True:
76
+ # order_by is strictly required when using offset/limit to guarantee no duplicates
77
+ statement = (
78
+ select(Animes)
79
+ .where(Animes.searchable_text != None)
80
+ .order_by(Animes.id)
81
+ .offset(offset)
82
+ .limit(batch_size)
83
+ )
84
+
85
+ batch = session.exec(statement).all()
86
+
87
+ if not batch:
88
+ break # Break the loop when no more rows are returned
89
+
90
+ all_results.extend(batch)
91
+ offset += len(batch)
92
+ logger.info(
93
+ f"Downloaded {offset} rows from Supabase so far...")
94
+
95
+ logger.info(
96
+ f"Successfully fetched a total of {len(all_results)} records.")
97
+ return all_results
98
 
99
  def embed_texts(self, texts: list[str], batch_size: int = 32) -> list[list[float]] | None:
100
  """
 
120
  else:
121
  logger.info(f"Using existing stored embeddings.")
122
 
123
+ def store_in_QdrantDB(self, client: QdrantClient, collection_name, db_records: list[Animes], final_texts: list[str], embeddings: list[list[float]]):
124
  """
125
+ Store embeddings and metadata in QdrantDB
126
 
127
  Args:
128
+ client: QdrantDB Client
129
+ collection_name: QdrantDB collection name,
130
  db_records: List of Anime data retrieved from PostgreSQL database,
131
  embeddings: Pre_commputed embeddings
132
  """
133
 
134
+ logger.info("Storing in QdrantDB...")
 
 
 
 
135
 
136
+ points = []
 
 
137
 
138
+ for i, row in enumerate(db_records):
 
139
  genres_list = row.genres if isinstance(row.genres, list) else []
140
  if len(genres_list) == 0:
141
  genres_list = ["Unknown"]
142
 
143
+ # Qdrant uses 'PointStruct' which holds the ID, Vector, and Payload (metadata + document)
144
+ point = models.PointStruct(
145
+ # Qdrant requires IDs to be integers or UUIDs
146
+ id=int(row.mal_id),
147
+ vector=embeddings[i],
148
+ payload={
149
+ # Store the text here since Qdrant doesn't separate docs from metadata
150
+ "document": final_texts[i],
151
+ "title": row.title,
152
+ "genres": genres_list,
153
+ "score": float(row.score) if row.score else 0.0,
154
+ "type": row.type if row.type else "Unknown",
155
+ "scored_by": row.scored_by if row.scored_by else 0
156
+ }
157
+ )
158
+ points.append(point)
159
 
160
  chunk_size = 500
161
+ total_chunks = (len(points) // chunk_size) + 1
162
+ logger.info(f"Inserting into Qdrant in {total_chunks} batches...")
163
+
164
+ for i in range(0, len(points), chunk_size):
165
+ batch = points[i: i + chunk_size]
166
+ self.client.upsert(
167
+ collection_name=collection_name,
168
+ points=batch
 
169
  )
170
  logger.info(f"Inserted batch {(i//chunk_size)+1}/{total_chunks}")
171
 
172
+ logger.info(f"Successfully stored {len(points)} animes in Qdrant")
173
 
174
  def run_pipeline(self):
175
  """Run complete embedding pipeline"""
 
178
  db_records = self.fetch_data_from_postgres()
179
  logger.info(f"Loaded {len(db_records)} animes from Postgres")
180
 
181
+ collection_name = self.create_or_get_collection()
182
 
183
  if not self.use_existing_embeddings:
184
+ texts_to_embed = []
185
+ for row in db_records:
186
+ text = row.searchable_text if row.searchable_text else ""
187
+ if hasattr(row, 'studios') and row.studios:
188
+ text += f" Studio: {', '.join(row.studios)}"
189
+ texts_to_embed.append(text)
190
+
191
+ print(texts_to_embed[0])
192
  embeddings = self.embed_texts(texts_to_embed)
193
 
194
  if embeddings:
195
+ self.store_in_QdrantDB(
196
+ self.client, collection_name, db_records, texts_to_embed, embeddings)
197
 
198
  logger.info("Embedding pipeline complete!")
199
+ return collection_name
200
 
201
 
202
  if __name__ == "__main__":
203
+ pass
204
+ # pipeline = EmbeddingPipeline()
205
+ # collection_name = pipeline.run_pipeline()
206
+
207
+ # client = QdrantClient(
208
+ # url=settings.qdrant_url,
209
+ # api_key=settings.qdrant_api_key,
210
+ # cloud_inference=True
211
+ # )
212
+
213
+ # print("\n--- Testing vector search ---")
214
+ # query = "Attack Titan"
215
+
216
+ # print(f"Query: {query}")
217
+
218
+ # search_results = client.search(
219
+ # collection_name=collection_name,
220
+ # query_vector=query_vector,
221
+ # limit=limit,
222
+ # # We want Qdrant to return the payload (metadata) so we can see the titles
223
+ # with_payload=True
224
+ # )
225
+
226
+ # print("\n--- TOP 15 RESULTS ---")
227
+
228
+ # for i, (title, distance) in enumerate(zip(
229
+ # [m["title"] for m in results["metadatas"][0]],
230
+ # results["distances"][0]
231
+ # )):
232
+ # print(f"{i+1}. {title} (distance: {distance:.3f})")
src/database/session.py CHANGED
@@ -2,4 +2,13 @@ from sqlmodel import create_engine
2
  import os
3
  from config import settings
4
 
5
- engine = create_engine(settings.database_url, echo=True)
 
 
 
 
 
 
 
 
 
 
2
  import os
3
  from config import settings
4
 
5
+ engine = create_engine(settings.database_url,
6
+ echo=True,
7
+ pool_size=10, # Keep 10 connections permanently open and ready
8
+ max_overflow=20, # Allow up to 20 temporary extra connections during traffic spikes
9
+ # How long to wait for an available connection before throwing an error
10
+ pool_timeout=30,
11
+ # Pings the DB slightly before executing a query to ensure the connection didn't drop
12
+ pool_pre_ping=True,
13
+ pool_recycle=1800 # Refresh connections every 30 minutes to prevent stale timeouts
14
+ )
src/llm/anime_reranker.py CHANGED
@@ -15,7 +15,7 @@ class AnimeReranker:
15
  # ----Postgres Global Truth
16
  self.C = 7.676 # The true weighted average score
17
  # 50th percentile of votes (Confidence threshold)
18
- self.bayesian_m = 10875
19
 
20
  # Hyperparameters for scoring
21
  self.passion_weight = 50 # Scaling factor for favorites/scored_by
 
15
  # ----Postgres Global Truth
16
  self.C = 7.676 # The true weighted average score
17
  # 50th percentile of votes (Confidence threshold)
18
+ self.bayesian_m = 44000
19
 
20
  # Hyperparameters for scoring
21
  self.passion_weight = 50 # Scaling factor for favorites/scored_by
src/retrieval/vector_search.py CHANGED
@@ -1,9 +1,12 @@
1
- import chromadb
 
 
 
 
2
  from sentence_transformers import SentenceTransformer
3
  from src.database.session import engine
4
  from src.database.models import Animes
5
  from sqlmodel import Session, select
6
- import logging
7
 
8
 
9
  logger = logging.getLogger(__name__)
@@ -15,14 +18,20 @@ class AnimeRetriever:
15
  """Handles anime retrieval from ChromaDB"""
16
 
17
  def __init__(self,
18
- chroma_path: str = "data/embeddings/chroma_db",
19
- collection_name: str = "anime_collection",
20
- model: str = "all-MiniLM-L6-v2"):
21
- self.client = chromadb.PersistentClient(chroma_path)
22
- self.collection = self.client.get_collection(collection_name)
23
- self.model = SentenceTransformer(model)
24
-
25
- print(f"Loaded collection with {self.collection.count()} anime")
 
 
 
 
 
 
26
 
27
  def fetch_anime_batch_from_postgres(self, mal_ids: list[int]) -> dict[int, Animes]:
28
  """Fetch multiple animes at once and return a dictionary mapped by mal_id"""
@@ -54,62 +63,78 @@ class AnimeRetriever:
54
  Returns:
55
  List of dicts with anime info
56
  """
57
- # 1. Build chromadb WHERE clause for filtering
58
- conditions = [{"scored_by": {"$gte": 9000}}]
 
 
 
 
 
 
59
  if min_score:
60
  logger.info(f"SCORE: Filtered based on min_score: {min_score}")
61
- conditions.append({"score": {"$gte": min_score}})
 
 
 
 
 
62
 
63
  if anime_type:
64
  logger.info(
65
  f"ANIME TYPE: Filtered based on anime_type: {anime_type}")
66
- conditions.append({"type": {"$eq": anime_type}})
 
 
 
 
 
67
 
68
  if genre_filter:
69
  logger.info(
70
  f"GENRE: Pre-filtering (OR) for genres: {', '.join(genre_filter)}")
71
-
72
- genre_or_conditions = [
73
- {"genres": {"$contains": genre}} for genre in genre_filter]
74
-
75
- if len(genre_or_conditions) == 1:
76
- conditions.append(genre_or_conditions[0])
77
- else:
78
- conditions.append({"$or": genre_or_conditions})
79
-
80
- where_clause = None
81
- if len(conditions) == 1:
82
- where_clause = conditions[0]
83
- elif len(conditions) > 1:
84
- where_clause = {"$and": conditions}
85
-
86
- # 2. Query ChromaDB
87
- results = self.collection.query(
88
- query_texts=[query],
89
- n_results=n_results,
90
- where=where_clause
91
- )
92
- if not results["ids"][0]:
 
93
  return []
94
 
95
- # 3. Batch fetch from PostgreSQL
96
- retrieved_ids = [int(id_str) for id_str in results["ids"][0]]
97
  postgres_data_map = self.fetch_anime_batch_from_postgres(retrieved_ids)
98
 
99
  anime_list = []
100
- for i, id_str in enumerate(results["ids"][0]):
101
- mal_id = int(id_str)
102
- distance = results["distances"][0][i]
103
 
104
  # Get the rich data from our Postgres map
105
  pg_anime = postgres_data_map.get(mal_id)
106
 
107
  if not pg_anime:
108
  logger.warning(
109
- f"Anime ID {mal_id} found in Chroma but missing in Postgres!")
110
  continue
111
 
112
- # Merge Chroma math with Postgres truths
113
  anime_info = {
114
  "mal_id": pg_anime.mal_id,
115
  "mal_url": pg_anime.url,
@@ -119,7 +144,7 @@ class AnimeRetriever:
119
  "scored_by": pg_anime.scored_by,
120
  "type": pg_anime.type,
121
  "year": pg_anime.year,
122
- "genres": pg_anime.genres, # Now an actual list again!
123
  "studios": pg_anime.studios,
124
  "themes": pg_anime.themes,
125
  "demographics": pg_anime.demographics,
@@ -131,7 +156,7 @@ class AnimeRetriever:
131
  "favorites": pg_anime.favorites,
132
  "images": pg_anime.images,
133
  "synopsis": pg_anime.synopsis,
134
- "searchable_text": pg_anime.searchable_text
135
  }
136
  anime_list.append(anime_info)
137
 
@@ -147,12 +172,12 @@ class AnimeRetriever:
147
  if __name__ == "__main__":
148
  retriever = AnimeRetriever()
149
 
150
- # # Test queries
151
- # print("=== Test 1: Basic Search ===")
152
- # results = retriever.search("dark psychological anime", n_results=15)
153
- # for anime in results:
154
- # print(
155
- # f"- {anime['title']} (score: {anime['score']}, relevance: {anime['relevance_score']:.3f})")
156
 
157
  # print("\n=== Test 2: Genre Filter ===")
158
  # results = retriever.search(
 
1
+ # import chromadb
2
+ import logging
3
+ import grpc
4
+ from config import settings
5
+ from qdrant_client import QdrantClient, models
6
  from sentence_transformers import SentenceTransformer
7
  from src.database.session import engine
8
  from src.database.models import Animes
9
  from sqlmodel import Session, select
 
10
 
11
 
12
  logger = logging.getLogger(__name__)
 
18
  """Handles anime retrieval from ChromaDB"""
19
 
20
  def __init__(self,
21
+ collection_name: str = "anime_collection"):
22
+ self.client = QdrantClient(url=settings.qdrant_url,
23
+ api_key=settings.qdrant_api_key,
24
+ cloud_inference=True,
25
+ prefer_grpc=True,
26
+ timeout=10)
27
+ self.collection_name = collection_name
28
+ self.points_count = self.client.count(
29
+ collection_name=self.collection_name, exact=False).count
30
+ # self.model = SentenceTransformer(model)
31
+ self.model = "sentence-transformers/all-minilm-l6-v2"
32
+
33
+ print(
34
+ f"Loaded collection with {self.points_count} anime approximately")
35
 
36
  def fetch_anime_batch_from_postgres(self, mal_ids: list[int]) -> dict[int, Animes]:
37
  """Fetch multiple animes at once and return a dictionary mapped by mal_id"""
 
63
  Returns:
64
  List of dicts with anime info
65
  """
66
+ must_conditions = [
67
+ # Base condition: scored_by >= 9000
68
+ models.FieldCondition(
69
+ key="scored_by",
70
+ range=models.Range(gte=20000)
71
+ )
72
+ ]
73
+
74
  if min_score:
75
  logger.info(f"SCORE: Filtered based on min_score: {min_score}")
76
+ must_conditions.append(
77
+ models.FieldCondition(
78
+ key="score",
79
+ range=models.Range(gte=min_score)
80
+ )
81
+ )
82
 
83
  if anime_type:
84
  logger.info(
85
  f"ANIME TYPE: Filtered based on anime_type: {anime_type}")
86
+ must_conditions.append(
87
+ models.FieldCondition(
88
+ key="type",
89
+ match=models.MatchValue(value=anime_type)
90
+ )
91
+ )
92
 
93
  if genre_filter:
94
  logger.info(
95
  f"GENRE: Pre-filtering (OR) for genres: {', '.join(genre_filter)}")
96
+ # Qdrant's MatchAny automatically acts as an OR condition against list fields!
97
+ must_conditions.append(
98
+ models.FieldCondition(
99
+ key="genres",
100
+ match=models.MatchAny(any=genre_filter)
101
+ )
102
+ )
103
+
104
+ # Wrap all conditions in a Filter object
105
+ query_filter = models.Filter(
106
+ must=must_conditions) if must_conditions else None
107
+
108
+ search_results = self.client.query_points(
109
+ collection_name=self.collection_name,
110
+ query=models.Document(
111
+ text=query,
112
+ model=self.model
113
+ ),
114
+ query_filter=query_filter,
115
+ limit=n_results
116
+ ).points
117
+
118
+ if not search_results:
119
  return []
120
 
121
+ retrieved_ids = [hit.id for hit in search_results]
 
122
  postgres_data_map = self.fetch_anime_batch_from_postgres(retrieved_ids)
123
 
124
  anime_list = []
125
+ for hit in search_results:
126
+ mal_id = hit.id
127
+ similarity_score = hit.score # Qdrant returns cosine similarity here
128
 
129
  # Get the rich data from our Postgres map
130
  pg_anime = postgres_data_map.get(mal_id)
131
 
132
  if not pg_anime:
133
  logger.warning(
134
+ f"Anime ID {mal_id} found in Qdrant but missing in Postgres!")
135
  continue
136
 
137
+ # Merge Vector Search results with Postgres truths
138
  anime_info = {
139
  "mal_id": pg_anime.mal_id,
140
  "mal_url": pg_anime.url,
 
144
  "scored_by": pg_anime.scored_by,
145
  "type": pg_anime.type,
146
  "year": pg_anime.year,
147
+ "genres": pg_anime.genres,
148
  "studios": pg_anime.studios,
149
  "themes": pg_anime.themes,
150
  "demographics": pg_anime.demographics,
 
156
  "favorites": pg_anime.favorites,
157
  "images": pg_anime.images,
158
  "synopsis": pg_anime.synopsis,
159
+ "searchable_text": pg_anime.searchable_text,
160
  }
161
  anime_list.append(anime_info)
162
 
 
172
  if __name__ == "__main__":
173
  retriever = AnimeRetriever()
174
 
175
+ # Test queries
176
+ print("=== Test 1: Basic Search ===")
177
+ results = retriever.search("dark psychological anime", n_results=15)
178
+ for anime in results:
179
+ print(
180
+ f"- {anime['title']} (score: {anime['score']})")
181
 
182
  # print("\n=== Test 2: Genre Filter ===")
183
  # results = retriever.search(
tests/add_index_to_qdrant.py ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from qdrant_client import QdrantClient, models
2
+ from config import settings
3
+ import logging
4
+
5
+ logging.basicConfig(level=logging.INFO)
6
+
7
+ # Connect to your Qdrant Cloud instance
8
+ client = QdrantClient(
9
+ url=settings.qdrant_url,
10
+ api_key=settings.qdrant_api_key,
11
+ )
12
+
13
+ collection_name = "anime_collection"
14
+
15
+ logging.info("Creating payload indices...")
16
+
17
+ # 1. Index for scored_by (Integer)
18
+ client.create_payload_index(
19
+ collection_name=collection_name,
20
+ field_name="scored_by",
21
+ field_schema=models.PayloadSchemaType.INTEGER,
22
+ )
23
+
24
+ # 2. Index for score (Float)
25
+ client.create_payload_index(
26
+ collection_name=collection_name,
27
+ field_name="score",
28
+ field_schema=models.PayloadSchemaType.FLOAT,
29
+ )
30
+
31
+ # 3. Index for type (String/Keyword)
32
+ client.create_payload_index(
33
+ collection_name=collection_name,
34
+ field_name="type",
35
+ field_schema=models.PayloadSchemaType.KEYWORD,
36
+ )
37
+
38
+ # 4. Index for genres (List of Strings -> Keyword)
39
+ # Note: Qdrant automatically handles arrays of strings as Keyword indices!
40
+ client.create_payload_index(
41
+ collection_name=collection_name,
42
+ field_name="genres",
43
+ field_schema=models.PayloadSchemaType.KEYWORD,
44
+ )
45
+
46
+ logging.info("✅ All payload indices created successfully!")
ui/gradio_app.py CHANGED
@@ -16,8 +16,6 @@ GENRES = [
16
  ]
17
 
18
  SUGGESTIONS = [
19
- {"label": "🌸 Similar to Naruto",
20
- "query": "Action packed themed anime similar to Naruto"},
21
  {"label": "⚔️ Dark fantasy",
22
  "query": "Dark fantasy with an unreliable narrator"},
23
  {"label": "🤖 Cyberpunk action",
@@ -25,6 +23,8 @@ SUGGESTIONS = [
25
  {"label": "💀 Psychological",
26
  "query": "Psychological thriller that messes with your head"},
27
  {"label": "💘 Romance drama", "query": "Bittersweet romance that makes you cry"},
 
 
28
  ]
29
 
30
  # ─── HTML builders ────────────────────────────────────────────────────────────
@@ -41,6 +41,7 @@ def build_spotlight_card(anime: dict) -> str:
41
  title = anime.get("title", "Unknown")
42
  eng = anime.get("title_english") or ""
43
  score = anime.get("score") or "—"
 
44
  year = anime.get("year") or "—"
45
  kind = anime.get("type") or "—"
46
  eps = anime.get("episodes") or "—"
@@ -71,7 +72,7 @@ def build_spotlight_card(anime: dict) -> str:
71
  f"</div>"
72
  f"<div class='sc-body'>"
73
  f"<div class='sc-chips'>"
74
- f"<span class='mc'>{kind}</span><span class='mc'>{year}</span><span class='mc'>{eps} ep</span>"
75
  f"</div>"
76
  f"<h3 class='sc-title'>{title}</h3>"
77
  f"{eng_html}"
@@ -762,7 +763,7 @@ if __name__ == "__main__":
762
  demo.launch(
763
  server_name="0.0.0.0",
764
  server_port=7860,
765
- share=True,
766
  css=CSS,
767
  theme=theme,
768
  )
 
16
  ]
17
 
18
  SUGGESTIONS = [
 
 
19
  {"label": "⚔️ Dark fantasy",
20
  "query": "Dark fantasy with an unreliable narrator"},
21
  {"label": "🤖 Cyberpunk action",
 
23
  {"label": "💀 Psychological",
24
  "query": "Psychological thriller that messes with your head"},
25
  {"label": "💘 Romance drama", "query": "Bittersweet romance that makes you cry"},
26
+ {"label": "🌸 Similar to Naruto",
27
+ "query": "Action packed themed anime similar to Naruto"}
28
  ]
29
 
30
  # ─── HTML builders ────────────────────────────────────────────────────────────
 
41
  title = anime.get("title", "Unknown")
42
  eng = anime.get("title_english") or ""
43
  score = anime.get("score") or "—"
44
+ scored_by = anime.get("scored_by") or "—"
45
  year = anime.get("year") or "—"
46
  kind = anime.get("type") or "—"
47
  eps = anime.get("episodes") or "—"
 
72
  f"</div>"
73
  f"<div class='sc-body'>"
74
  f"<div class='sc-chips'>"
75
+ f"<span class='mc'>{kind}</span><span class='mc'>{year}</span><span class='mc'>{eps} ep</span><span class='mc'>Scored by:{scored_by}</span>"
76
  f"</div>"
77
  f"<h3 class='sc-title'>{title}</h3>"
78
  f"{eng_html}"
 
763
  demo.launch(
764
  server_name="0.0.0.0",
765
  server_port=7860,
766
+ share=False,
767
  css=CSS,
768
  theme=theme,
769
  )
uv.lock CHANGED
@@ -30,6 +30,8 @@ dependencies = [
30
  { name = "chromadb" },
31
  { name = "fastapi", extra = ["all"] },
32
  { name = "gradio" },
 
 
33
  { name = "langchain" },
34
  { name = "langchain-groq" },
35
  { name = "pandas" },
@@ -58,6 +60,8 @@ requires-dist = [
58
  { name = "chromadb", specifier = ">=1.5.2" },
59
  { name = "fastapi", extras = ["all"], specifier = ">=0.122.0" },
60
  { name = "gradio", specifier = ">=6.2.0" },
 
 
61
  { name = "langchain", specifier = ">=1.1.3" },
62
  { name = "langchain-groq", specifier = ">=1.1.1" },
63
  { name = "pandas", specifier = ">=2.3.3" },
@@ -1017,6 +1021,49 @@ wheels = [
1017
  { url = "https://files.pythonhosted.org/packages/4d/27/d86b89e36de8a951501fb06a0f38df19853210f341d0b28f83f4aa0ffa08/grpcio-1.78.0-cp313-cp313-win_amd64.whl", hash = "sha256:f2d4e43ee362adfc05994ed479334d5a451ab7bc3f3fee1b796b8ca66895acb4", size = 4797393, upload-time = "2026-02-06T09:56:17.882Z" },
1018
  ]
1019
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1020
  [[package]]
1021
  name = "h11"
1022
  version = "0.16.0"
 
30
  { name = "chromadb" },
31
  { name = "fastapi", extra = ["all"] },
32
  { name = "gradio" },
33
+ { name = "grpcio" },
34
+ { name = "grpcio-tools" },
35
  { name = "langchain" },
36
  { name = "langchain-groq" },
37
  { name = "pandas" },
 
60
  { name = "chromadb", specifier = ">=1.5.2" },
61
  { name = "fastapi", extras = ["all"], specifier = ">=0.122.0" },
62
  { name = "gradio", specifier = ">=6.2.0" },
63
+ { name = "grpcio", specifier = ">=1.78.0" },
64
+ { name = "grpcio-tools", specifier = ">=1.78.0" },
65
  { name = "langchain", specifier = ">=1.1.3" },
66
  { name = "langchain-groq", specifier = ">=1.1.1" },
67
  { name = "pandas", specifier = ">=2.3.3" },
 
1021
  { url = "https://files.pythonhosted.org/packages/4d/27/d86b89e36de8a951501fb06a0f38df19853210f341d0b28f83f4aa0ffa08/grpcio-1.78.0-cp313-cp313-win_amd64.whl", hash = "sha256:f2d4e43ee362adfc05994ed479334d5a451ab7bc3f3fee1b796b8ca66895acb4", size = 4797393, upload-time = "2026-02-06T09:56:17.882Z" },
1022
  ]
1023
 
1024
+ [[package]]
1025
+ name = "grpcio-tools"
1026
+ version = "1.78.0"
1027
+ source = { registry = "https://pypi.org/simple" }
1028
+ dependencies = [
1029
+ { name = "grpcio" },
1030
+ { name = "protobuf" },
1031
+ { name = "setuptools" },
1032
+ ]
1033
+ sdist = { url = "https://files.pythonhosted.org/packages/8b/d1/cbefe328653f746fd319c4377836a25ba64226e41c6a1d7d5cdbc87a459f/grpcio_tools-1.78.0.tar.gz", hash = "sha256:4b0dd86560274316e155d925158276f8564508193088bc43e20d3f5dff956b2b", size = 5393026, upload-time = "2026-02-06T09:59:59.53Z" }
1034
+ wheels = [
1035
+ { url = "https://files.pythonhosted.org/packages/75/78/280184d19242ed6762bf453c47a70b869b3c5c72a24dc5bf2bf43909faa3/grpcio_tools-1.78.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:6a8b8b7b49f319d29dbcf507f62984fa382d1d10437d75c3f26db5f09c4ac0af", size = 2545904, upload-time = "2026-02-06T09:57:52.769Z" },
1036
+ { url = "https://files.pythonhosted.org/packages/5b/51/3c46dea5113f68fe879961cae62d34bb7a3c308a774301b45d614952ee98/grpcio_tools-1.78.0-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:d62cf3b68372b0c6d722a6165db41b976869811abeabc19c8522182978d8db10", size = 5709078, upload-time = "2026-02-06T09:57:56.389Z" },
1037
+ { url = "https://files.pythonhosted.org/packages/e0/2c/dc1ae9ec53182c96d56dfcbf3bcd3e55a8952ad508b188c75bf5fc8993d4/grpcio_tools-1.78.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fa9056742efeaf89d5fe14198af71e5cbc4fbf155d547b89507e19d6025906c6", size = 2591744, upload-time = "2026-02-06T09:57:58.341Z" },
1038
+ { url = "https://files.pythonhosted.org/packages/04/63/9b53fc9a9151dd24386785171a4191ee7cb5afb4d983b6a6a87408f41b28/grpcio_tools-1.78.0-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:e3191af125dcb705aa6bc3856ba81ba99b94121c1b6ebee152e66ea084672831", size = 2905113, upload-time = "2026-02-06T09:58:00.38Z" },
1039
+ { url = "https://files.pythonhosted.org/packages/96/b2/0ad8d789f3a2a00893131c140865605fa91671a6e6fcf9da659e1fabba10/grpcio_tools-1.78.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:283239ddbb67ae83fac111c61b25d8527a1dbd355b377cbc8383b79f1329944d", size = 2656436, upload-time = "2026-02-06T09:58:03.038Z" },
1040
+ { url = "https://files.pythonhosted.org/packages/09/4d/580f47ce2fc61b093ade747b378595f51b4f59972dd39949f7444b464a03/grpcio_tools-1.78.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ac977508c0db15301ef36d6c79769ec1a6cc4e3bc75735afca7fe7e360cead3a", size = 3106128, upload-time = "2026-02-06T09:58:05.064Z" },
1041
+ { url = "https://files.pythonhosted.org/packages/c9/29/d83b2d89f8d10e438bad36b1eb29356510fb97e81e6a608b22ae1890e8e6/grpcio_tools-1.78.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:4ff605e25652a0bd13aa8a73a09bc48669c68170902f5d2bf1468a57d5e78771", size = 3654953, upload-time = "2026-02-06T09:58:07.15Z" },
1042
+ { url = "https://files.pythonhosted.org/packages/08/71/917ce85633311e54fefd7e6eb1224fb780ef317a4d092766f5630c3fc419/grpcio_tools-1.78.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0197d7b561c79be78ab93d0fe2836c8def470683df594bae3ac89dd8e5c821b2", size = 3322630, upload-time = "2026-02-06T09:58:10.305Z" },
1043
+ { url = "https://files.pythonhosted.org/packages/b2/55/3fbf6b26ab46fc79e1e6f7f4e0993cf540263dad639290299fad374a0829/grpcio_tools-1.78.0-cp311-cp311-win32.whl", hash = "sha256:28f71f591f7f39555863ced84fcc209cbf4454e85ef957232f43271ee99af577", size = 993804, upload-time = "2026-02-06T09:58:13.698Z" },
1044
+ { url = "https://files.pythonhosted.org/packages/73/86/4affe006d9e1e9e1c6653d6aafe2f8b9188acb2b563cd8ed3a2c7c0e8aec/grpcio_tools-1.78.0-cp311-cp311-win_amd64.whl", hash = "sha256:5a6de495dabf86a3b40b9a7492994e1232b077af9d63080811838b781abbe4e8", size = 1158566, upload-time = "2026-02-06T09:58:15.721Z" },
1045
+ { url = "https://files.pythonhosted.org/packages/0c/ae/5b1fa5dd8d560a6925aa52de0de8731d319f121c276e35b9b2af7cc220a2/grpcio_tools-1.78.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:9eb122da57d4cad7d339fc75483116f0113af99e8d2c67f3ef9cae7501d806e4", size = 2546823, upload-time = "2026-02-06T09:58:17.944Z" },
1046
+ { url = "https://files.pythonhosted.org/packages/a7/ed/d33ccf7fa701512efea7e7e23333b748848a123e9d3bbafde4e126784546/grpcio_tools-1.78.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:d0c501b8249940b886420e6935045c44cb818fa6f265f4c2b97d5cff9cb5e796", size = 5706776, upload-time = "2026-02-06T09:58:20.944Z" },
1047
+ { url = "https://files.pythonhosted.org/packages/c6/69/4285583f40b37af28277fc6b867d636e3b10e1b6a7ebd29391a856e1279b/grpcio_tools-1.78.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:77e5aa2d2a7268d55b1b113f958264681ef1994c970f69d48db7d4683d040f57", size = 2593972, upload-time = "2026-02-06T09:58:23.29Z" },
1048
+ { url = "https://files.pythonhosted.org/packages/d7/eb/ecc1885bd6b3147f0a1b7dff5565cab72f01c8f8aa458f682a1c77a9fb08/grpcio_tools-1.78.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:8e3c0b0e6ba5275322ba29a97bf890565a55f129f99a21b121145e9e93a22525", size = 2905531, upload-time = "2026-02-06T09:58:25.406Z" },
1049
+ { url = "https://files.pythonhosted.org/packages/ae/a9/511d0040ced66960ca10ba0f082d6b2d2ee6dd61837b1709636fdd8e23b4/grpcio_tools-1.78.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:975d4cb48694e20ebd78e1643e5f1cd94cdb6a3d38e677a8e84ae43665aa4790", size = 2656909, upload-time = "2026-02-06T09:58:28.022Z" },
1050
+ { url = "https://files.pythonhosted.org/packages/06/a3/3d2c707e7dee8df842c96fbb24feb2747e506e39f4a81b661def7fed107c/grpcio_tools-1.78.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:553ff18c5d52807dedecf25045ae70bad7a3dbba0b27a9a3cdd9bcf0a1b7baec", size = 3109778, upload-time = "2026-02-06T09:58:30.091Z" },
1051
+ { url = "https://files.pythonhosted.org/packages/1f/4b/646811ba241bf05da1f0dc6f25764f1c837f78f75b4485a4210c84b79eae/grpcio_tools-1.78.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:8c7f5e4af5a84d2e96c862b1a65e958a538237e268d5f8203a3a784340975b51", size = 3658763, upload-time = "2026-02-06T09:58:32.875Z" },
1052
+ { url = "https://files.pythonhosted.org/packages/45/de/0a5ef3b3e79d1011375f5580dfee3a9c1ccb96c5f5d1c74c8cee777a2483/grpcio_tools-1.78.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:96183e2b44afc3f9a761e9d0f985c3b44e03e8bb98e626241a6cbfb3b6f7e88f", size = 3325116, upload-time = "2026-02-06T09:58:34.894Z" },
1053
+ { url = "https://files.pythonhosted.org/packages/95/d2/6391b241ad571bc3e71d63f957c0b1860f0c47932d03c7f300028880f9b8/grpcio_tools-1.78.0-cp312-cp312-win32.whl", hash = "sha256:2250e8424c565a88573f7dc10659a0b92802e68c2a1d57e41872c9b88ccea7a6", size = 993493, upload-time = "2026-02-06T09:58:37.242Z" },
1054
+ { url = "https://files.pythonhosted.org/packages/7c/8f/7d0d3a39ecad76ccc136be28274daa660569b244fa7d7d0bbb24d68e5ece/grpcio_tools-1.78.0-cp312-cp312-win_amd64.whl", hash = "sha256:217d1fa29de14d9c567d616ead7cb0fef33cde36010edff5a9390b00d52e5094", size = 1158423, upload-time = "2026-02-06T09:58:40.072Z" },
1055
+ { url = "https://files.pythonhosted.org/packages/53/ce/17311fb77530420e2f441e916b347515133e83d21cd6cc77be04ce093d5b/grpcio_tools-1.78.0-cp313-cp313-linux_armv7l.whl", hash = "sha256:2d6de1cc23bdc1baafc23e201b1e48c617b8c1418b4d8e34cebf72141676e5fb", size = 2546284, upload-time = "2026-02-06T09:58:43.073Z" },
1056
+ { url = "https://files.pythonhosted.org/packages/1d/d3/79e101483115f0e78223397daef71751b75eba7e92a32060c10aae11ca64/grpcio_tools-1.78.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:2afeaad88040894c76656202ff832cb151bceb05c0e6907e539d129188b1e456", size = 5705653, upload-time = "2026-02-06T09:58:45.533Z" },
1057
+ { url = "https://files.pythonhosted.org/packages/8b/a7/52fa3ccb39ceeee6adc010056eadfbca8198651c113e418dafebbdf2b306/grpcio_tools-1.78.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:33cc593735c93c03d63efe7a8ba25f3c66f16c52f0651910712490244facad72", size = 2592788, upload-time = "2026-02-06T09:58:48.918Z" },
1058
+ { url = "https://files.pythonhosted.org/packages/68/08/682ff6bb548225513d73dc9403742d8975439d7469c673bc534b9bbc83a7/grpcio_tools-1.78.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:2921d7989c4d83b71f03130ab415fa4d66e6693b8b8a1fcbb7a1c67cff19b812", size = 2905157, upload-time = "2026-02-06T09:58:51.478Z" },
1059
+ { url = "https://files.pythonhosted.org/packages/b2/66/264f3836a96423b7018e5ada79d62576a6401f6da4e1f4975b18b2be1265/grpcio_tools-1.78.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e6a0df438e82c804c7b95e3f311c97c2f876dcc36376488d5b736b7bcf5a9b45", size = 2656166, upload-time = "2026-02-06T09:58:54.117Z" },
1060
+ { url = "https://files.pythonhosted.org/packages/f3/6b/f108276611522e03e98386b668cc7e575eff6952f2db9caa15b2a3b3e883/grpcio_tools-1.78.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e9c6070a9500798225191ef25d0055a15d2c01c9c8f2ee7b681fffa99c98c822", size = 3109110, upload-time = "2026-02-06T09:58:56.891Z" },
1061
+ { url = "https://files.pythonhosted.org/packages/6f/c7/cf048dbcd64b3396b3c860a2ffbcc67a8f8c87e736aaa74c2e505a7eee4c/grpcio_tools-1.78.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:394e8b57d85370a62e5b0a4d64c96fcf7568345c345d8590c821814d227ecf1d", size = 3657863, upload-time = "2026-02-06T09:58:59.176Z" },
1062
+ { url = "https://files.pythonhosted.org/packages/b6/37/e2736912c8fda57e2e57a66ea5e0bc8eb9a5fb7ded00e866ad22d50afb08/grpcio_tools-1.78.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a3ef700293ab375e111a2909d87434ed0a0b086adf0ce67a8d9cf12ea7765e63", size = 3324748, upload-time = "2026-02-06T09:59:01.242Z" },
1063
+ { url = "https://files.pythonhosted.org/packages/1c/5d/726abc75bb5bfc2841e88ea05896e42f51ca7c30cb56da5c5b63058b3867/grpcio_tools-1.78.0-cp313-cp313-win32.whl", hash = "sha256:6993b960fec43a8d840ee5dc20247ef206c1a19587ea49fe5e6cc3d2a09c1585", size = 993074, upload-time = "2026-02-06T09:59:03.085Z" },
1064
+ { url = "https://files.pythonhosted.org/packages/c5/68/91b400bb360faf9b177ffb5540ec1c4d06ca923691ddf0f79e2c9683f4da/grpcio_tools-1.78.0-cp313-cp313-win_amd64.whl", hash = "sha256:275ce3c2978842a8cf9dd88dce954e836e590cf7029649ad5d1145b779039ed5", size = 1158185, upload-time = "2026-02-06T09:59:05.036Z" },
1065
+ ]
1066
+
1067
  [[package]]
1068
  name = "h11"
1069
  version = "0.16.0"