Shreekant Kalwar (Nokia) commited on
Commit
28b14ff
Β·
1 Parent(s): 28d4382

major changess

Browse files
app.py CHANGED
@@ -3,7 +3,12 @@ from fastapi import FastAPI
3
  from pydantic import BaseModel
4
  from fastapi.middleware.cors import CORSMiddleware
5
  from bot_instance import gemini_bot, llama_bot # singleton ErrorBot instance
6
- from typing import List, Optional
 
 
 
 
 
7
 
8
  app = FastAPI(title="ErrorBot API")
9
 
@@ -24,6 +29,7 @@ class MessageItem(BaseModel):
24
  class ChatRequest(BaseModel):
25
  message: str
26
  history: Optional[List[MessageItem]] = [] # optional conversation history
 
27
 
28
  # ---------------- Endpoints ----------------
29
  @app.get("/")
@@ -46,15 +52,42 @@ def root():
46
 
47
  # return {"reply": answer}
48
 
 
 
 
 
 
 
49
 
50
  @app.post("/gemini/chat")
51
  def gemini_chat(request: ChatRequest):
52
  history_list = [{"role": msg.role, "content": msg.content} for msg in request.history]
53
- answer = gemini_bot.ask(request.message, history=history_list)
54
- return {"reply": answer}
 
 
 
 
 
 
 
 
 
 
 
55
 
56
  @app.post("/llama/chat")
57
  def llama_chat(request: ChatRequest):
58
  history_list = [{"role": msg.role, "content": msg.content} for msg in request.history]
59
- answer = llama_bot.ask(request.message, history=history_list)
60
- return {"reply": answer}
 
 
 
 
 
 
 
 
 
 
 
3
  from pydantic import BaseModel
4
  from fastapi.middleware.cors import CORSMiddleware
5
  from bot_instance import gemini_bot, llama_bot # singleton ErrorBot instance
6
+ from typing import List, Optional,Any
7
+
8
+ import os
9
+ from dotenv import load_dotenv
10
+ from util import ErrorBot
11
+
12
 
13
  app = FastAPI(title="ErrorBot API")
14
 
 
29
  class ChatRequest(BaseModel):
30
  message: str
31
  history: Optional[List[MessageItem]] = [] # optional conversation history
32
+ lastContext: List[Any] = None
33
 
34
  # ---------------- Endpoints ----------------
35
  @app.get("/")
 
52
 
53
  # return {"reply": answer}
54
 
55
+ load_dotenv()
56
+ GOOGLE_API_KEY = os.environ.get("GOOGLE_API_KEY")
57
+
58
+ GROQ_API_KEY = os.getenv("GROQ_API_KEY")
59
+
60
+ EMBEDDING_MODEL = "BAAI/bge-base-en-v1.5"
61
 
62
  @app.post("/gemini/chat")
63
  def gemini_chat(request: ChatRequest):
64
  history_list = [{"role": msg.role, "content": msg.content} for msg in request.history]
65
+ gemini_bot = ErrorBot(
66
+ embedding_model_name=EMBEDDING_MODEL,
67
+ llm_model_name="gemini-2.5-flash",
68
+ google_api_key=GOOGLE_API_KEY,
69
+ llm_provider="gemini",
70
+ last_context = request.lastContext
71
+ )
72
+ print("In App.py")
73
+ print(request.lastContext)
74
+ answer, last_context = gemini_bot.ask(request.message, history=history_list)
75
+ print(answer)
76
+ print(last_context)
77
+ return {"reply": answer, "last_context": last_context}
78
 
79
  @app.post("/llama/chat")
80
  def llama_chat(request: ChatRequest):
81
  history_list = [{"role": msg.role, "content": msg.content} for msg in request.history]
82
+ llama_bot = ErrorBot(
83
+ embedding_model_name=EMBEDDING_MODEL,
84
+ llm_model_name="llama-3.3-70b-versatile",
85
+ groq_api_key=GROQ_API_KEY,
86
+ llm_provider="groq",
87
+ last_context = request.lastContext
88
+
89
+ )
90
+ answer, last_context = llama_bot.ask(request.message, history=history_list)
91
+ print(answer)
92
+ print(last_context)
93
+ return {"reply": answer, "last_context": last_context}
embedding_model_instance.py ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from sentence_transformers import SentenceTransformer, CrossEncoder
3
+
4
+
5
+ # --- Embedding model
6
+
7
+
8
+ EMBEDDING_MODEL = "BAAI/bge-base-en-v1.5"
9
+
10
+ device = "cuda" if torch.cuda.is_available() else "cpu"
11
+ print(f"Using device: {device}")
12
+ embedding_model = SentenceTransformer(EMBEDDING_MODEL, device=device)
13
+ embedding_dim = embedding_model.get_sentence_embedding_dimension()
14
+
15
+ reranker = CrossEncoder("cross-encoder/ms-marco-MiniLM-L-6-v2")
llm.py ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ import google.generativeai as genai
3
+ from groq import Groq
4
+ import os
5
+ from dotenv import load_dotenv
6
+
7
+
8
+ load_dotenv()
9
+ GOOGLE_API_KEY = os.environ.get("GOOGLE_API_KEY")
10
+
11
+ GROQ_API_KEY = os.getenv("GROQ_API_KEY")
12
+
13
+ genai.configure(api_key=GOOGLE_API_KEY)
14
+
15
+
16
+ gemini = genai.GenerativeModel("gemini-2.5-flash")
17
+
18
+
19
+ groq = Groq(api_key=GROQ_API_KEY)
mongo_instance.py ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ from pymongo import MongoClient
2
+
3
+ # Connect to MongoDB
4
+ client = MongoClient("mongodb+srv://dhaval:Dhaval15@cluster0.rwu1ze6.mongodb.net/prontoDB?retryWrites=true&w=majority&appName=Cluster0") # replace with your URI
5
+ db = client["prontoDB"]
6
+
qdrant_instance.py ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from qdrant_client import QdrantClient, models
2
+ import os
3
+ from dotenv import load_dotenv
4
+
5
+
6
+ load_dotenv()
7
+
8
+
9
+ print("Connecting to Qdrant...")
10
+ qdrant = QdrantClient(
11
+ url=os.getenv("QDRANT_URL"),
12
+ api_key=os.getenv("QDRANT_API_KEY"),
13
+ )
requirements.txt CHANGED
Binary files a/requirements.txt and b/requirements.txt differ
 
util.py CHANGED
@@ -8,6 +8,13 @@ from typing import List, Dict
8
  import google.generativeai as genai
9
  from groq import Groq
10
 
 
 
 
 
 
 
 
11
  def build_content(doc: dict, entity_type: str) -> str:
12
  """Convert MongoDB document into natural text for embeddings."""
13
  parts = [f"{entity_type} ID: {doc.get('id', str(doc.get('_id', '')))}"]
@@ -28,40 +35,42 @@ def build_content(doc: dict, entity_type: str) -> str:
28
  class ErrorBot:
29
  """Chatbot using RAG (Qdrant + Gemini API)."""
30
 
31
- def __init__(self, embedding_model_name: str, llm_model_name: str, google_api_key: str = None, groq_api_key: str = None, llm_provider: str = "gemini"):
32
  print("πŸš€ Initializing ErrorBot...")
 
33
 
 
34
  # --- Embedding model
35
- self.device = "cuda" if torch.cuda.is_available() else "cpu"
36
- print(f"Using device: {self.device}")
37
- self.embedding_model = SentenceTransformer(embedding_model_name, device=self.device)
38
- self.embedding_dim = self.embedding_model.get_sentence_embedding_dimension()
39
 
 
40
  # --- Qdrant client
41
- print("Connecting to Qdrant...")
42
- self.qdrant = QdrantClient(
43
- url=os.getenv("QDRANT_URL"),
44
- api_key=os.getenv("QDRANT_API_KEY"),
45
- )
46
  self.collection_name = "technical_errors"
47
- self._setup_collection()
48
 
49
  # --- LLM setup
50
  self.llm_provider = llm_provider.lower()
51
  self.llm_model_name = llm_model_name
52
 
53
  if self.llm_provider == "gemini":
54
- genai.configure(api_key=google_api_key)
55
- self.llm = genai.GenerativeModel(llm_model_name)
56
 
57
  elif self.llm_provider == "groq":
58
- self.llm = Groq(api_key=groq_api_key)
 
59
 
60
  else:
61
  raise ValueError(f"Unsupported LLM provider: {self.llm_provider}")
62
 
63
  # --- Cross encoder reranker
64
- self.reranker = CrossEncoder("cross-encoder/ms-marco-MiniLM-L-6-v2")
 
65
  print(f"βœ… ErrorBot ready with {self.llm_provider.upper()}")
66
 
67
  def _setup_collection(self):
@@ -147,20 +156,108 @@ class ErrorBot:
147
 
148
  return candidates[:top_k]
149
 
150
- def generate_answer(self, query: str, context: List[Dict], history: list = None):
151
- context_str = "\n---\n".join(
152
- [f"{c['entity_type']} (Score: {c['score']:.2f}):\n{c['content']}" for c in context]
153
- )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
154
 
155
  # --- System prompt
156
- system_prompt = f"""
157
- You are a technical assistant. You have access to Problem Reports (PR), Fault Analyses (FA), and Corrections (CR).
158
- Use the provided context and conversation history to answer the question clearly and concisely.
159
- If context is not relevant, say you do not have enough information.
 
 
 
 
160
 
161
- ### Context
162
- {context_str}
163
- """
 
 
 
 
 
 
 
 
 
 
 
 
164
 
165
  # --- Conversation history in list-of-dicts format
166
  convo = []
@@ -187,20 +284,247 @@ class ErrorBot:
187
  messages=[{"role": "system", "content": system_prompt}] + convo
188
  )
189
  return completion.choices[0].message.content.strip()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
190
 
 
 
 
 
 
 
 
 
191
 
192
  def ask(self, query: str, history: list = None):
193
  print(f"\n❓ Query: {query}")
194
- retrieved_context = self.retrieve(query)
195
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
196
  if not retrieved_context:
197
  print("πŸ’¬ No relevant context found.")
198
- return "I could not find any relevant information."
199
 
200
- print(f"βœ… Retrieved {len(retrieved_context)} documents.")
201
- for i, doc in enumerate(retrieved_context):
202
- print(f" - Context {i+1} ({doc['entity_type']}, ID: {doc['id']}, Score: {doc['score']:.2f})")
203
-
204
- answer = self.generate_answer(query, retrieved_context, history)
205
  print(f"\nπŸ€– Answer: {answer}")
206
- return answer
 
 
 
 
 
8
  import google.generativeai as genai
9
  from groq import Groq
10
 
11
+ from embedding_model_instance import embedding_model, embedding_dim, reranker
12
+ from qdrant_instance import qdrant
13
+ from llm import gemini, groq
14
+ from mongo_instance import db
15
+ import json
16
+ from bson import ObjectId
17
+
18
  def build_content(doc: dict, entity_type: str) -> str:
19
  """Convert MongoDB document into natural text for embeddings."""
20
  parts = [f"{entity_type} ID: {doc.get('id', str(doc.get('_id', '')))}"]
 
35
  class ErrorBot:
36
  """Chatbot using RAG (Qdrant + Gemini API)."""
37
 
38
+ def __init__(self, embedding_model_name: str, llm_model_name: str, google_api_key: str = None, groq_api_key: str = None, llm_provider: str = "gemini", last_context: list = None):
39
  print("πŸš€ Initializing ErrorBot...")
40
+ self.last_context = last_context
41
 
42
+ print("last_context", last_context)
43
  # --- Embedding model
44
+ # self.device = "cuda" if torch.cuda.is_available() else "cpu"
45
+
46
+ self.embedding_model = embedding_model
47
+ self.embedding_dim = embedding_dim
48
 
49
+ self.db = db
50
  # --- Qdrant client
51
+
52
+ self.qdrant = qdrant
 
 
 
53
  self.collection_name = "technical_errors"
54
+ #self._setup_collection()
55
 
56
  # --- LLM setup
57
  self.llm_provider = llm_provider.lower()
58
  self.llm_model_name = llm_model_name
59
 
60
  if self.llm_provider == "gemini":
61
+
62
+ self.llm = gemini
63
 
64
  elif self.llm_provider == "groq":
65
+
66
+ self.llm = groq
67
 
68
  else:
69
  raise ValueError(f"Unsupported LLM provider: {self.llm_provider}")
70
 
71
  # --- Cross encoder reranker
72
+
73
+ self.reranker = reranker
74
  print(f"βœ… ErrorBot ready with {self.llm_provider.upper()}")
75
 
76
  def _setup_collection(self):
 
156
 
157
  return candidates[:top_k]
158
 
159
+ def generate_answer(self, query: str, context: List[Dict], history: list = None, is_followup: bool = False ):
160
+ """
161
+ Generates an answer using the LLM, guiding it to identify which context is useful.
162
+ """
163
+ context_str=""
164
+
165
+ if(is_followup):
166
+ pass
167
+
168
+ # Aggregation pipeline
169
+ # pipeline = [
170
+ # # Start with problemReports
171
+ # {"$match": {"_id": {"$in": self.last_context}}},
172
+
173
+ # # Add faultAnalysis
174
+ # {"$unionWith": {
175
+ # "coll": "faultanalysis",
176
+ # "pipeline": [{"$match": {"id": {"$in": self.last_context}}}]
177
+ # }},
178
+
179
+ # # Add corrections
180
+ # {"$unionWith": {
181
+ # "coll": "corrections",
182
+ # "pipeline": [{"$match": {"id": {"$in": self.last_context}}}]
183
+ # }}
184
+ # ]
185
+
186
+ pipeline = [
187
+ # Start with problemReports
188
+ {
189
+ "$match": {"_id": {"$in": self.last_context}}
190
+ },
191
+ {
192
+ "$addFields": {"entity_type": "ProblemReport"}
193
+ },
194
+
195
+ # Add faultAnalysis
196
+ {
197
+ "$unionWith": {
198
+ "coll": "faultanalysis",
199
+ "pipeline": [
200
+ {"$match": {"id": {"$in": self.last_context}}},
201
+ {"$addFields": {"entity_type": "FaultAnalysis"}}
202
+ ]
203
+ }
204
+ },
205
+
206
+ # Add corrections
207
+ {
208
+ "$unionWith": {
209
+ "coll": "corrections",
210
+ "pipeline": [
211
+ {"$match": {"id": {"$in": self.last_context}}},
212
+ {"$addFields": {"entity_type": "Correction"}}
213
+ ]
214
+ }
215
+ }
216
+ ]
217
+
218
+ # Run aggregation on problemReports
219
+ context_docs = list(db.problemReports.aggregate(pipeline))
220
+ # Serialize full documents as text for LLM
221
+ #print(context_docs)
222
+ context_str = "\n---\n".join(
223
+ [f"{c['entity_type']} (ID: {c['_id']}):\n{json.dumps(c, default=str)}"
224
+ for c in context_docs]
225
+ )
226
+ print("Context String in Follow Up:")
227
+ #print(context_str)
228
+
229
+
230
+ else:
231
+
232
+ context_str = "\n---\n".join(
233
+ [f"{c['entity_type']} (Score: {c['score']:.2f}):\n{c['content']}" for c in context]
234
+ )
235
 
236
  # --- System prompt
237
+ # system_prompt = f"""
238
+ # You are a technical assistant. You have access to Problem Reports (PR), Fault Analyses (FA), and Corrections (CR).
239
+ # Use the provided context and conversation history to answer the question clearly and concisely.
240
+ # If context is not relevant, say you do not have enough information.
241
+
242
+ # ### Context
243
+ # {context_str}
244
+ # """
245
 
246
+ system_prompt = f"""
247
+ You are a technical assistant. A user may ask questions about Problem Reports (PR), Fault Analyses (FA), and Corrections (CR).
248
+ Your task is to:
249
+ 1. Identify which information (PR, FA, CR) is relevant to answering the user's question.
250
+ 2. Explain the solution in simple, clear, actionable language.
251
+ 3. Do not just repeat the content; summarize and explain.
252
+
253
+ ### User Question:
254
+
255
+
256
+ ### Context:
257
+ {context_str}
258
+
259
+ Provide a concise, step-by-step explanation if applicable.
260
+ """
261
 
262
  # --- Conversation history in list-of-dicts format
263
  convo = []
 
284
  messages=[{"role": "system", "content": system_prompt}] + convo
285
  )
286
  return completion.choices[0].message.content.strip()
287
+
288
+ def fetch_problem_report_with_links(self, pr_id: str):
289
+
290
+ # --- Fetch Problem Report
291
+ pr_doc = db["problemReports"].find_one({"id": pr_id})
292
+ if not pr_doc:
293
+ return None, [], [], [], []
294
+
295
+ if "_id" in pr_doc and isinstance(pr_doc["_id"], ObjectId):
296
+ pr_doc["_id"] = str(pr_doc["_id"])
297
+
298
+ # --- Extract linked IDs
299
+ cr_ids = pr_doc.get("correctionIds", [])
300
+ fa_ids = pr_doc.get("faultAnalysisId", [])
301
+
302
+ # ensure both are lists
303
+ if isinstance(cr_ids, str):
304
+ cr_ids = [cr_ids]
305
+ elif cr_ids is None:
306
+ cr_ids = []
307
+
308
+ if isinstance(fa_ids, str):
309
+ fa_ids = [fa_ids]
310
+ elif fa_ids is None:
311
+ fa_ids = []
312
+
313
+ # --- Fetch Correction Reports
314
+ cr_docs = list(db["corrections"].find({"id": {"$in": cr_ids}})) if cr_ids else []
315
+ for doc in cr_docs:
316
+ if "_id" in doc and isinstance(doc["_id"], ObjectId):
317
+ doc["_id"] = str(doc["_id"])
318
+
319
+ # --- Fetch Fault Analysis Reports
320
+ fa_docs = list(db["faultanalysis"].find({"id": {"$in": fa_ids}})) if fa_ids else []
321
+ for doc in fa_docs:
322
+ if "_id" in doc and isinstance(doc["_id"], ObjectId):
323
+ doc["_id"] = str(doc["_id"])
324
+
325
+ return pr_doc, cr_ids, fa_ids, cr_docs, fa_docs
326
+
327
+
328
+ def is_technical_query(self, query: str) -> bool:
329
+ """
330
+ Classify query as TECHNICAL or NON-TECHNICAL.
331
+ """
332
+ classification_prompt = f"""
333
+ You are a classifier. Determine if the following query is TECHNICAL
334
+ (related to software, debugging, errors, troubleshooting, fault analysis,
335
+ corrections, technical problem reports) or NON-TECHNICAL
336
+ (general questions, greetings, chit-chat, unrelated topics).
337
+
338
+ Query: "{query}"
339
+
340
+ Respond with exactly one word: "TECHNICAL" or "NON-TECHNICAL".
341
+ """
342
+
343
+ if self.llm_provider == "gemini":
344
+ response = self.llm.generate_content(classification_prompt)
345
+ result = response.text.strip().upper()
346
+
347
+ elif self.llm_provider == "groq":
348
+ completion = self.llm.chat.completions.create(
349
+ model=self.llm_model_name,
350
+ messages=[{"role": "system", "content": classification_prompt}]
351
+ )
352
+ result = completion.choices[0].message.content.strip().upper()
353
+
354
+ return result == "TECHNICAL"
355
+
356
+ def is_followup_query(self, query: str, history: list = None) -> bool:
357
+ """
358
+ Detect if query is a follow-up based on conversation history.
359
+ """
360
+ if not history:
361
+ return False
362
+
363
+ classification_prompt = f"""
364
+ You are a classifier. Determine if the following user query
365
+ is a FOLLOW-UP (depends on the previous conversation)
366
+ or a NEW QUERY (can be answered independently).
367
+
368
+ Previous conversation:
369
+ { [msg['content'] for msg in history][-3:] }
370
+
371
+ Current query: "{query}"
372
+
373
+ Respond with exactly one word: "FOLLOW-UP" or "NEW".
374
+ """
375
+
376
+ if self.llm_provider == "gemini":
377
+ response = self.llm.generate_content(classification_prompt)
378
+ result = response.text.strip().upper()
379
 
380
+ elif self.llm_provider == "groq":
381
+ completion = self.llm.chat.completions.create(
382
+ model=self.llm_model_name,
383
+ messages=[{"role": "system", "content": classification_prompt}]
384
+ )
385
+ result = completion.choices[0].message.content.strip().upper()
386
+ print("Follow up: ", result)
387
+ return result == "FOLLOW-UP"
388
 
389
  def ask(self, query: str, history: list = None):
390
  print(f"\n❓ Query: {query}")
 
391
 
392
+ # Step 1: Classify
393
+ is_technical = self.is_technical_query(query)
394
+ is_followup = self.is_followup_query(query, history)
395
+
396
+ # Step 2: Non-technical standalone
397
+ #if not is_technical:
398
+ if not is_technical and not is_followup:
399
+ print("⚠️ Non-technical standalone query β†’ skipping Qdrant.")
400
+ system_prompt = "You are a helpful assistant. Answer clearly and concisely."
401
+ convo = [{"role": "system", "content": system_prompt},
402
+ {"role": "user", "content": query}]
403
+
404
+ if self.llm_provider == "gemini":
405
+ convo_str = "\n".join([f"{m['role'].capitalize()}: {m['content']}" for m in convo])
406
+ response = self.llm.generate_content(convo_str)
407
+ return response.text.strip(), []
408
+
409
+ elif self.llm_provider == "groq":
410
+ completion = self.llm.chat.completions.create(
411
+ model=self.llm_model_name,
412
+ messages=convo
413
+ )
414
+ return completion.choices[0].message.content.strip(), []
415
+
416
+ # Step 3: Technical or follow-up
417
+ print("is_followup", is_followup)
418
+ print("last_context", self.last_context)
419
+ print("is_technical", is_technical)
420
+
421
+ if is_followup and self.last_context:
422
+ if not is_technical:
423
+ print("⚠️ Non-technical followup β†’ skipping Qdrant.")
424
+ system_prompt = "You are a helpful assistant. Answer clearly and concisely."
425
+ convo = [{"role": "system", "content": system_prompt},
426
+ {"role": "user", "content": query}]
427
+
428
+ if self.llm_provider == "gemini":
429
+ convo_str = "\n".join([f"{m['role'].capitalize()}: {m['content']}" for m in convo])
430
+ response = self.llm.generate_content(convo_str)
431
+ return response.text.strip(), []
432
+
433
+ elif self.llm_provider == "groq":
434
+ completion = self.llm.chat.completions.create(
435
+ model=self.llm_model_name,
436
+ messages=convo
437
+ )
438
+ return completion.choices[0].message.content.strip(), []
439
+ else:
440
+ print("πŸ”„ Follow-up query β†’ reusing previous context.")
441
+ retrieved_context = self.last_context
442
+ context_docs = retrieved_context
443
+
444
+ else:
445
+ print("πŸ“₯ New technical query β†’ retrieving from Qdrant.")
446
+ retrieved_context = self.retrieve(query)
447
+ last_context = []
448
+ for i, doc in enumerate(retrieved_context):
449
+ last_context.append(doc['id'])
450
+ print(f" - Context {i+1} ({doc['entity_type']}, ID: {doc['id']}, Score: {doc['score']:.2f})")
451
+
452
+ first_doc = retrieved_context[0]
453
+ context_docs = []
454
+
455
+ # Step 2: Determine starting point based on entity type
456
+ pr_docs_to_use = []
457
+
458
+ if first_doc["entity_type"] == "ProblemReport":
459
+ pr_id = first_doc["id"]
460
+ print(f"πŸ“Œ Using PR from context1: {pr_id}")
461
+ pr_doc, cr_ids, fa_ids, cr_docs, fa_docs = self.fetch_problem_report_with_links(pr_id)
462
+ pr_docs_to_use.append((pr_doc, cr_docs, fa_docs))
463
+
464
+ elif first_doc["entity_type"] == "Correction":
465
+ cr_id = first_doc["id"]
466
+ print(f"πŸ“Œ Using CR from context1: {cr_id}")
467
+ cr_doc = self.db["corrections"].find_one({"id": cr_id})
468
+ pr_ids = cr_doc.get("problemReportIds", []) if cr_doc else []
469
+
470
+ if isinstance(pr_ids, str):
471
+ pr_ids = [pr_ids]
472
+ for pr_id in pr_ids:
473
+ pr_doc, cr_ids, fa_ids, cr_docs, fa_docs = self.fetch_problem_report_with_links(pr_id)
474
+ pr_docs_to_use.append((pr_doc, cr_docs, fa_docs))
475
+
476
+ elif first_doc["entity_type"] == "FaultAnalysis":
477
+ fa_id = first_doc["id"]
478
+ print(f"πŸ“Œ Using FA from context1: {fa_id}")
479
+ fa_doc = self.db["faultanalysis"].find_one({"id": fa_id})
480
+ pr_ids = fa_doc.get("problemReportIds", []) if fa_doc else []
481
+
482
+ if isinstance(pr_ids, str):
483
+ pr_ids = [pr_ids]
484
+ for pr_id in pr_ids:
485
+ pr_doc, cr_ids, fa_ids, cr_docs, fa_docs = self.fetch_problem_report_with_links(pr_id)
486
+ pr_docs_to_use.append((pr_doc, cr_docs, fa_docs))
487
+
488
+ # Step 3: Build context documents for LLM, prioritize CR and FA
489
+ for pr_doc, cr_docs, fa_docs in pr_docs_to_use:
490
+ # Include FA first (analysis of problem)
491
+ for fa in fa_docs:
492
+ context_docs.append({
493
+ "entity_type": "FaultAnalysis",
494
+ "content": build_content(fa, "FaultAnalysis"),
495
+ "score": 1.0
496
+ })
497
+ # Include CR next (solutions/corrections)
498
+ for cr in cr_docs:
499
+ context_docs.append({
500
+ "entity_type": "Correction",
501
+ "content": build_content(cr, "Correction"),
502
+ "score": 1.0
503
+ })
504
+ # PR last (problem description)
505
+ if pr_doc:
506
+ context_docs.append({
507
+ "entity_type": "ProblemReport",
508
+ "content": build_content(pr_doc, "ProblemReport"),
509
+ "score": 0.9
510
+ })
511
+
512
+ print(f"βœ… Total documents for LLM context: {len(context_docs)}")
513
+
514
+ if(len(last_context)>0):
515
+ self.last_context = context_docs # save for future follow-ups
516
  if not retrieved_context:
517
  print("πŸ’¬ No relevant context found.")
518
+ return "I could not find any relevant information.", []
519
 
520
+ print(f"βœ… Using {len(retrieved_context)} documents as context.")
521
+ #answer = self.generate_answer(query, retrieved_context, history, is_followup)
522
+
523
+ answer = self.generate_answer(query, context_docs, history, is_followup)
524
+ last_context = self.last_context
525
  print(f"\nπŸ€– Answer: {answer}")
526
+ return (answer, last_context)
527
+
528
+
529
+
530
+
util_backup.py ADDED
@@ -0,0 +1,387 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import torch
3
+ from qdrant_client import QdrantClient, models
4
+ from sentence_transformers import SentenceTransformer, CrossEncoder
5
+ from pymongo import MongoClient
6
+ from bson import ObjectId
7
+ from typing import List, Dict
8
+ import google.generativeai as genai
9
+ from groq import Groq
10
+
11
+ def build_content(doc: dict, entity_type: str) -> str:
12
+ """Convert MongoDB document into natural text for embeddings."""
13
+ parts = [f"{entity_type} ID: {doc.get('id', str(doc.get('_id', '')))}"]
14
+ for k, v in doc.items():
15
+ if k in ["_id"]: # skip ObjectId
16
+ continue
17
+ if isinstance(v, list):
18
+ parts.append(f"{k}: {', '.join(map(str, v))}")
19
+ elif isinstance(v, dict):
20
+ nested = "; ".join([f"{nk}: {nv}" for nk, nv in v.items() if nv])
21
+ parts.append(f"{k}: {nested}")
22
+ else:
23
+ if v:
24
+ parts.append(f"{k}: {v}")
25
+ return "\n".join(parts)
26
+
27
+
28
+ class ErrorBot:
29
+ """Chatbot using RAG (Qdrant + Gemini API)."""
30
+
31
+ def __init__(self, embedding_model_name: str, llm_model_name: str, google_api_key: str = None, groq_api_key: str = None, llm_provider: str = "gemini"):
32
+ print("πŸš€ Initializing ErrorBot...")
33
+ self.last_context = None
34
+ # --- Embedding model
35
+ self.device = "cuda" if torch.cuda.is_available() else "cpu"
36
+ print(f"Using device: {self.device}")
37
+ self.embedding_model = SentenceTransformer(embedding_model_name, device=self.device)
38
+ self.embedding_dim = self.embedding_model.get_sentence_embedding_dimension()
39
+
40
+ # --- Qdrant client
41
+ print("Connecting to Qdrant...")
42
+ self.qdrant = QdrantClient(
43
+ url=os.getenv("QDRANT_URL"),
44
+ api_key=os.getenv("QDRANT_API_KEY"),
45
+ )
46
+ self.collection_name = "technical_errors"
47
+ self._setup_collection()
48
+
49
+ # --- LLM setup
50
+ self.llm_provider = llm_provider.lower()
51
+ self.llm_model_name = llm_model_name
52
+
53
+ if self.llm_provider == "gemini":
54
+ genai.configure(api_key=google_api_key)
55
+ self.llm = genai.GenerativeModel(llm_model_name)
56
+
57
+ elif self.llm_provider == "groq":
58
+ self.llm = Groq(api_key=groq_api_key)
59
+
60
+ else:
61
+ raise ValueError(f"Unsupported LLM provider: {self.llm_provider}")
62
+
63
+ # --- Cross encoder reranker
64
+ self.reranker = CrossEncoder("cross-encoder/ms-marco-MiniLM-L-6-v2")
65
+ print(f"βœ… ErrorBot ready with {self.llm_provider.upper()}")
66
+
67
+ def _setup_collection(self):
68
+ if not self.qdrant.collection_exists(self.collection_name):
69
+ self.qdrant.create_collection(
70
+ collection_name=self.collection_name,
71
+ vectors_config=models.VectorParams(
72
+ size=self.embedding_dim,
73
+ distance=models.Distance.COSINE,
74
+ ),
75
+ )
76
+
77
+ def ingest_from_mongodb(self, mongo_uri: str, db_name: str, batch_size: int = 32):
78
+ client = MongoClient(mongo_uri)
79
+ db = client[db_name]
80
+
81
+ collections = {
82
+ "ProblemReport": db["problemReports"],
83
+ "FaultAnalysis": db["faultanalysis"],
84
+ "Correction": db["corrections"],
85
+ }
86
+
87
+ docs = []
88
+ for entity_type, coll in collections.items():
89
+ for doc in coll.find():
90
+ if "_id" in doc and isinstance(doc["_id"], ObjectId):
91
+ doc["_id"] = str(doc["_id"])
92
+ docs.append({"entity_type": entity_type, "data": doc})
93
+
94
+ contents = [build_content(d["data"], d["entity_type"]) for d in docs]
95
+
96
+ all_embeddings = []
97
+ for i in range(0, len(contents), batch_size):
98
+ batch_contents = contents[i:i + batch_size]
99
+ embeddings = self.embedding_model.encode(batch_contents, show_progress_bar=True).tolist()
100
+ all_embeddings.extend(embeddings)
101
+
102
+ self.qdrant.upsert(
103
+ collection_name=self.collection_name,
104
+ points=[
105
+ models.PointStruct(
106
+ id=i,
107
+ vector=emb,
108
+ payload={
109
+ "id": d["data"].get("id", str(d["data"].get("_id", i))),
110
+ "entity_type": d["entity_type"],
111
+ "raw": d["data"],
112
+ "content": c,
113
+ },
114
+ )
115
+ for i, (d, emb, c) in enumerate(zip(docs, all_embeddings, contents))
116
+ ],
117
+ wait=True,
118
+ )
119
+ print(f"βœ… Ingested {len(docs)} documents into '{self.collection_name}'")
120
+
121
+ def retrieve(self, query: str, top_k: int = 5, score_threshold: float = 0.3, rerank: bool = True):
122
+ query_embedding = self.embedding_model.encode(query).tolist()
123
+ hits = self.qdrant.query_points(
124
+ collection_name=self.collection_name,
125
+ query=query_embedding,
126
+ limit=top_k * 3 if rerank else top_k,
127
+ with_payload=True,
128
+ score_threshold=score_threshold,
129
+ ).points
130
+
131
+ candidates = [
132
+ {
133
+ "id": hit.payload.get("id"),
134
+ "entity_type": hit.payload.get("entity_type", ""),
135
+ "content": hit.payload.get("content", ""),
136
+ "score": hit.score,
137
+ }
138
+ for hit in hits
139
+ ]
140
+
141
+ if rerank and candidates:
142
+ pairs = [(query, c["content"]) for c in candidates]
143
+ scores = self.reranker.predict(pairs)
144
+ for i, score in enumerate(scores):
145
+ candidates[i]["rerank_score"] = float(score)
146
+ candidates = sorted(candidates, key=lambda x: x["rerank_score"], reverse=True)
147
+
148
+ return candidates[:top_k]
149
+
150
+ def generate_answer(self, query: str, context: List[Dict], history: list = None):
151
+ context_str = "\n---\n".join(
152
+ [f"{c['entity_type']} (Score: {c['score']:.2f}):\n{c['content']}" for c in context]
153
+ )
154
+
155
+ # --- System prompt
156
+ system_prompt = f"""
157
+ You are a technical assistant. You have access to Problem Reports (PR), Fault Analyses (FA), and Corrections (CR).
158
+ Use the provided context and conversation history to answer the question clearly and concisely.
159
+ If context is not relevant, say you do not have enough information.
160
+
161
+ ### Context
162
+ {context_str}
163
+ """
164
+
165
+ # --- Conversation history in list-of-dicts format
166
+ convo = []
167
+ if history:
168
+ for msg in history:
169
+ convo.append({
170
+ "role": "user" if msg["role"] == "user" else "assistant",
171
+ "content": msg["content"],
172
+ })
173
+
174
+ convo.append({"role": "user", "content": query})
175
+
176
+ # --- Gemini flow
177
+ if self.llm_provider == "gemini":
178
+ convo_str = "\n".join([f"{m['role'].capitalize()}: {m['content']}" for m in convo])
179
+ prompt = system_prompt + "\n\n" + convo_str + "\nAssistant:"
180
+ response = self.llm.generate_content(prompt)
181
+ return response.text.strip()
182
+
183
+ # --- Groq flow
184
+ elif self.llm_provider == "groq":
185
+ completion = self.llm.chat.completions.create(
186
+ model=self.llm_model_name,
187
+ messages=[{"role": "system", "content": system_prompt}] + convo
188
+ )
189
+ return completion.choices[0].message.content.strip()
190
+
191
+
192
+ # def ask(self, query: str, history: list = None):
193
+ # print(f"\n❓ Query: {query}")
194
+ # retrieved_context = self.retrieve(query)
195
+
196
+ # if not retrieved_context:
197
+ # print("πŸ’¬ No relevant context found.")
198
+ # return "I could not find any relevant information."
199
+
200
+ # print(f"βœ… Retrieved {len(retrieved_context)} documents.")
201
+ # for i, doc in enumerate(retrieved_context):
202
+ # print(f" - Context {i+1} ({doc['entity_type']}, ID: {doc['id']}, Score: {doc['score']:.2f})")
203
+
204
+ # answer = self.generate_answer(query, retrieved_context, history)
205
+ # print(f"\nπŸ€– Answer: {answer}")
206
+ # return answer
207
+
208
+ # def is_technical_query(self, query: str) -> bool:
209
+ # """
210
+ # Ask the LLM to classify whether a query is technical or not.
211
+ # Returns True if technical, False otherwise.
212
+ # """
213
+ # classification_prompt = f"""
214
+ # You are a classifier. Determine if the following query is TECHNICAL
215
+ # (related to software, debugging, errors, troubleshooting, fault analysis,
216
+ # corrections, technical problem reports) or NON-TECHNICAL
217
+ # (general questions, greetings, chit-chat, unrelated topics).
218
+
219
+ # Query: "{query}"
220
+
221
+ # Respond with exactly one word: "TECHNICAL" or "NON-TECHNICAL".
222
+ # """
223
+
224
+ # if self.llm_provider == "gemini":
225
+ # response = self.llm.generate_content(classification_prompt)
226
+ # result = response.text.strip().upper()
227
+
228
+ # elif self.llm_provider == "groq":
229
+ # completion = self.llm.chat.completions.create(
230
+ # model=self.llm_model_name,
231
+ # messages=[{"role": "system", "content": classification_prompt}]
232
+ # )
233
+ # result = completion.choices[0].message.content.strip().upper()
234
+
235
+ # else:
236
+ # raise ValueError(f"Unsupported LLM provider: {self.llm_provider}")
237
+
238
+ # return result == "TECHNICAL"
239
+
240
+
241
+ # def ask(self, query: str, history: list = None):
242
+ # print(f"\n❓ Query: {query}")
243
+
244
+ # # --- Step 1: Check if query is technical
245
+ # if not self.is_technical_query(query):
246
+ # print("⚠️ Non-technical query detected β†’ skipping Qdrant.")
247
+
248
+ # # Minimal system prompt for non-technical queries
249
+ # system_prompt = "You are a helpful assistant. Answer clearly and concisely."
250
+ # convo = [{"role": "system", "content": system_prompt},
251
+ # {"role": "user", "content": query}]
252
+
253
+ # if self.llm_provider == "gemini":
254
+ # convo_str = "\n".join([f"{m['role'].capitalize()}: {m['content']}" for m in convo])
255
+ # response = self.llm.generate_content(convo_str)
256
+ # return response.text.strip()
257
+
258
+ # elif self.llm_provider == "groq":
259
+ # completion = self.llm.chat.completions.create(
260
+ # model=self.llm_model_name,
261
+ # messages=convo
262
+ # )
263
+ # return completion.choices[0].message.content.strip()
264
+
265
+ # # --- Step 2: If technical, go through retrieval
266
+ # retrieved_context = self.retrieve(query)
267
+
268
+ # if not retrieved_context:
269
+ # print("πŸ’¬ No relevant context found.")
270
+ # return "I could not find any relevant information."
271
+
272
+ # print(f"βœ… Retrieved {len(retrieved_context)} documents.")
273
+ # for i, doc in enumerate(retrieved_context):
274
+ # print(f" - Context {i+1} ({doc['entity_type']}, ID: {doc['id']}, Score: {doc['score']:.2f})")
275
+
276
+ # answer = self.generate_answer(query, retrieved_context, history)
277
+ # print(f"\nπŸ€– Answer: {answer}")
278
+ # return answer
279
+
280
+ def is_technical_query(self, query: str) -> bool:
281
+ """
282
+ Classify query as TECHNICAL or NON-TECHNICAL.
283
+ """
284
+ classification_prompt = f"""
285
+ You are a classifier. Determine if the following query is TECHNICAL
286
+ (related to software, debugging, errors, troubleshooting, fault analysis,
287
+ corrections, technical problem reports) or NON-TECHNICAL
288
+ (general questions, greetings, chit-chat, unrelated topics).
289
+
290
+ Query: "{query}"
291
+
292
+ Respond with exactly one word: "TECHNICAL" or "NON-TECHNICAL".
293
+ """
294
+
295
+ if self.llm_provider == "gemini":
296
+ response = self.llm.generate_content(classification_prompt)
297
+ result = response.text.strip().upper()
298
+
299
+ elif self.llm_provider == "groq":
300
+ completion = self.llm.chat.completions.create(
301
+ model=self.llm_model_name,
302
+ messages=[{"role": "system", "content": classification_prompt}]
303
+ )
304
+ result = completion.choices[0].message.content.strip().upper()
305
+
306
+ return result == "TECHNICAL"
307
+
308
+ def is_followup_query(self, query: str, history: list = None) -> bool:
309
+ """
310
+ Detect if query is a follow-up based on conversation history.
311
+ """
312
+ if not history:
313
+ return False
314
+
315
+ classification_prompt = f"""
316
+ You are a classifier. Determine if the following user query
317
+ is a FOLLOW-UP (depends on the previous conversation)
318
+ or a NEW QUERY (can be answered independently).
319
+
320
+ Previous conversation:
321
+ { [msg['content'] for msg in history][-3:] }
322
+
323
+ Current query: "{query}"
324
+
325
+ Respond with exactly one word: "FOLLOW-UP" or "NEW".
326
+ """
327
+
328
+ if self.llm_provider == "gemini":
329
+ response = self.llm.generate_content(classification_prompt)
330
+ result = response.text.strip().upper()
331
+
332
+ elif self.llm_provider == "groq":
333
+ completion = self.llm.chat.completions.create(
334
+ model=self.llm_model_name,
335
+ messages=[{"role": "system", "content": classification_prompt}]
336
+ )
337
+ result = completion.choices[0].message.content.strip().upper()
338
+
339
+ return result == "FOLLOW-UP"
340
+
341
+ def ask(self, query: str, history: list = None):
342
+ print(f"\n❓ Query: {query}")
343
+
344
+ # Step 1: Classify
345
+ is_technical = self.is_technical_query(query)
346
+ is_followup = self.is_followup_query(query, history)
347
+
348
+ # Step 2: Non-technical standalone
349
+ if not is_technical and not is_followup:
350
+ print("⚠️ Non-technical standalone query β†’ skipping Qdrant.")
351
+ system_prompt = "You are a helpful assistant. Answer clearly and concisely."
352
+ convo = [{"role": "system", "content": system_prompt},
353
+ {"role": "user", "content": query}]
354
+
355
+ if self.llm_provider == "gemini":
356
+ convo_str = "\n".join([f"{m['role'].capitalize()}: {m['content']}" for m in convo])
357
+ response = self.llm.generate_content(convo_str)
358
+ return response.text.strip()
359
+
360
+ elif self.llm_provider == "groq":
361
+ completion = self.llm.chat.completions.create(
362
+ model=self.llm_model_name,
363
+ messages=convo
364
+ )
365
+ return completion.choices[0].message.content.strip()
366
+
367
+ # Step 3: Technical or follow-up
368
+ if is_followup and self.last_context:
369
+ print("πŸ”„ Follow-up query β†’ reusing previous context.")
370
+ retrieved_context = self.last_context
371
+ else:
372
+ print("πŸ“₯ New technical query β†’ retrieving from Qdrant.")
373
+ retrieved_context = self.retrieve(query)
374
+ self.last_context = retrieved_context # save for future follow-ups
375
+
376
+ if not retrieved_context:
377
+ print("πŸ’¬ No relevant context found.")
378
+ return "I could not find any relevant information."
379
+
380
+ print(f"βœ… Using {len(retrieved_context)} documents as context.")
381
+ answer = self.generate_answer(query, retrieved_context, history)
382
+ print(f"\nπŸ€– Answer: {answer}")
383
+ return answer
384
+
385
+
386
+
387
+
util_backup_29_09_2025.py ADDED
@@ -0,0 +1,413 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import torch
3
+ from qdrant_client import QdrantClient, models
4
+ from sentence_transformers import SentenceTransformer, CrossEncoder
5
+ from pymongo import MongoClient
6
+ from bson import ObjectId
7
+ from typing import List, Dict
8
+ import google.generativeai as genai
9
+ from groq import Groq
10
+
11
+ from embedding_model_instance import embedding_model, embedding_dim, reranker
12
+ from qdrant_instance import qdrant
13
+ from llm import gemini, groq
14
+ from mongo_instance import db
15
+ import json
16
+ from bson import ObjectId
17
+
18
+ def build_content(doc: dict, entity_type: str) -> str:
19
+ """Convert MongoDB document into natural text for embeddings."""
20
+ parts = [f"{entity_type} ID: {doc.get('id', str(doc.get('_id', '')))}"]
21
+ for k, v in doc.items():
22
+ if k in ["_id"]: # skip ObjectId
23
+ continue
24
+ if isinstance(v, list):
25
+ parts.append(f"{k}: {', '.join(map(str, v))}")
26
+ elif isinstance(v, dict):
27
+ nested = "; ".join([f"{nk}: {nv}" for nk, nv in v.items() if nv])
28
+ parts.append(f"{k}: {nested}")
29
+ else:
30
+ if v:
31
+ parts.append(f"{k}: {v}")
32
+ return "\n".join(parts)
33
+
34
+
35
+ class ErrorBot:
36
+ """Chatbot using RAG (Qdrant + Gemini API)."""
37
+
38
+ def __init__(self, embedding_model_name: str, llm_model_name: str, google_api_key: str = None, groq_api_key: str = None, llm_provider: str = "gemini", last_context: list = None):
39
+ print("πŸš€ Initializing ErrorBot...")
40
+ self.last_context = last_context
41
+
42
+ print("last_context", last_context)
43
+ # --- Embedding model
44
+ # self.device = "cuda" if torch.cuda.is_available() else "cpu"
45
+
46
+ self.embedding_model = embedding_model
47
+ self.embedding_dim = embedding_dim
48
+
49
+ self.db = db
50
+ # --- Qdrant client
51
+
52
+ self.qdrant = qdrant
53
+ self.collection_name = "technical_errors"
54
+ #self._setup_collection()
55
+
56
+ # --- LLM setup
57
+ self.llm_provider = llm_provider.lower()
58
+ self.llm_model_name = llm_model_name
59
+
60
+ if self.llm_provider == "gemini":
61
+
62
+ self.llm = gemini
63
+
64
+ elif self.llm_provider == "groq":
65
+
66
+ self.llm = groq
67
+
68
+ else:
69
+ raise ValueError(f"Unsupported LLM provider: {self.llm_provider}")
70
+
71
+ # --- Cross encoder reranker
72
+
73
+ self.reranker = reranker
74
+ print(f"βœ… ErrorBot ready with {self.llm_provider.upper()}")
75
+
76
+ def _setup_collection(self):
77
+ if not self.qdrant.collection_exists(self.collection_name):
78
+ self.qdrant.create_collection(
79
+ collection_name=self.collection_name,
80
+ vectors_config=models.VectorParams(
81
+ size=self.embedding_dim,
82
+ distance=models.Distance.COSINE,
83
+ ),
84
+ )
85
+
86
+ def ingest_from_mongodb(self, mongo_uri: str, db_name: str, batch_size: int = 32):
87
+ client = MongoClient(mongo_uri)
88
+ db = client[db_name]
89
+
90
+ collections = {
91
+ "ProblemReport": db["problemReports"],
92
+ "FaultAnalysis": db["faultanalysis"],
93
+ "Correction": db["corrections"],
94
+ }
95
+
96
+ docs = []
97
+ for entity_type, coll in collections.items():
98
+ for doc in coll.find():
99
+ if "_id" in doc and isinstance(doc["_id"], ObjectId):
100
+ doc["_id"] = str(doc["_id"])
101
+ docs.append({"entity_type": entity_type, "data": doc})
102
+
103
+ contents = [build_content(d["data"], d["entity_type"]) for d in docs]
104
+
105
+ all_embeddings = []
106
+ for i in range(0, len(contents), batch_size):
107
+ batch_contents = contents[i:i + batch_size]
108
+ embeddings = self.embedding_model.encode(batch_contents, show_progress_bar=True).tolist()
109
+ all_embeddings.extend(embeddings)
110
+
111
+ self.qdrant.upsert(
112
+ collection_name=self.collection_name,
113
+ points=[
114
+ models.PointStruct(
115
+ id=i,
116
+ vector=emb,
117
+ payload={
118
+ "id": d["data"].get("id", str(d["data"].get("_id", i))),
119
+ "entity_type": d["entity_type"],
120
+ "raw": d["data"],
121
+ "content": c,
122
+ },
123
+ )
124
+ for i, (d, emb, c) in enumerate(zip(docs, all_embeddings, contents))
125
+ ],
126
+ wait=True,
127
+ )
128
+ print(f"βœ… Ingested {len(docs)} documents into '{self.collection_name}'")
129
+
130
+ def retrieve(self, query: str, top_k: int = 5, score_threshold: float = 0.3, rerank: bool = True):
131
+ query_embedding = self.embedding_model.encode(query).tolist()
132
+ hits = self.qdrant.query_points(
133
+ collection_name=self.collection_name,
134
+ query=query_embedding,
135
+ limit=top_k * 3 if rerank else top_k,
136
+ with_payload=True,
137
+ score_threshold=score_threshold,
138
+ ).points
139
+
140
+ candidates = [
141
+ {
142
+ "id": hit.payload.get("id"),
143
+ "entity_type": hit.payload.get("entity_type", ""),
144
+ "content": hit.payload.get("content", ""),
145
+ "score": hit.score,
146
+ }
147
+ for hit in hits
148
+ ]
149
+
150
+ if rerank and candidates:
151
+ pairs = [(query, c["content"]) for c in candidates]
152
+ scores = self.reranker.predict(pairs)
153
+ for i, score in enumerate(scores):
154
+ candidates[i]["rerank_score"] = float(score)
155
+ candidates = sorted(candidates, key=lambda x: x["rerank_score"], reverse=True)
156
+
157
+ return candidates[:top_k]
158
+
159
+ def generate_answer(self, query: str, context: List[Dict], history: list = None, is_followup: bool = False ):
160
+ """
161
+ Generates an answer using the LLM, guiding it to identify which context is useful.
162
+ """
163
+ context_str=""
164
+
165
+ if(is_followup):
166
+ pass
167
+
168
+ # Aggregation pipeline
169
+ pipeline = [
170
+ # Start with problemReports
171
+ {"$match": {"_id": {"$in": self.last_context}}},
172
+
173
+ # Add faultAnalysis
174
+ {"$unionWith": {
175
+ "coll": "faultanalysis",
176
+ "pipeline": [{"$match": {"id": {"$in": self.last_context}}}]
177
+ }},
178
+
179
+ # Add corrections
180
+ {"$unionWith": {
181
+ "coll": "corrections",
182
+ "pipeline": [{"$match": {"id": {"$in": self.last_context}}}]
183
+ }}
184
+ ]
185
+
186
+ # Run aggregation on problemReports
187
+ context_docs = list(db.problemReports.aggregate(pipeline))
188
+ # Serialize full documents as text for LLM
189
+ #print(context_docs)
190
+ context_str = "\n---\n".join(
191
+ [f"{c.get('entity_type', 'Unknown')} (ID: {c['_id']}):\n{json.dumps(c, default=str)}"
192
+ for c in context_docs]
193
+ )
194
+ print("Context String in Follow Up:")
195
+ #print(context_str)
196
+
197
+
198
+ else:
199
+
200
+ context_str = "\n---\n".join(
201
+ [f"{c['entity_type']} (Score: {c['score']:.2f}):\n{c['content']}" for c in context]
202
+ )
203
+
204
+ # --- System prompt
205
+ # system_prompt = f"""
206
+ # You are a technical assistant. You have access to Problem Reports (PR), Fault Analyses (FA), and Corrections (CR).
207
+ # Use the provided context and conversation history to answer the question clearly and concisely.
208
+ # If context is not relevant, say you do not have enough information.
209
+
210
+ # ### Context
211
+ # {context_str}
212
+ # """
213
+
214
+ system_prompt = f"""
215
+ You are a technical assistant. A user may ask questions about Problem Reports (PR), Fault Analyses (FA), and Corrections (CR).
216
+ Your task is to:
217
+ 1. Identify which information (PR, FA, CR) is relevant to answering the user's question.
218
+ 2. Explain the solution in simple, clear, actionable language.
219
+ 3. Do not just repeat the content; summarize and explain.
220
+
221
+ ### User Question:
222
+
223
+
224
+ ### Context:
225
+ {context_str}
226
+
227
+ Provide a concise, step-by-step explanation if applicable.
228
+ """
229
+
230
+ # --- Conversation history in list-of-dicts format
231
+ convo = []
232
+ if history:
233
+ for msg in history:
234
+ convo.append({
235
+ "role": "user" if msg["role"] == "user" else "assistant",
236
+ "content": msg["content"],
237
+ })
238
+
239
+ convo.append({"role": "user", "content": query})
240
+
241
+ # --- Gemini flow
242
+ if self.llm_provider == "gemini":
243
+ convo_str = "\n".join([f"{m['role'].capitalize()}: {m['content']}" for m in convo])
244
+ prompt = system_prompt + "\n\n" + convo_str + "\nAssistant:"
245
+ response = self.llm.generate_content(prompt)
246
+ return response.text.strip()
247
+
248
+ # --- Groq flow
249
+ elif self.llm_provider == "groq":
250
+ completion = self.llm.chat.completions.create(
251
+ model=self.llm_model_name,
252
+ messages=[{"role": "system", "content": system_prompt}] + convo
253
+ )
254
+ return completion.choices[0].message.content.strip()
255
+
256
+ def fetch_problem_report_with_links(self, pr_id: str):
257
+
258
+ # --- Fetch Problem Report
259
+ pr_doc = db["problemReports"].find_one({"id": pr_id})
260
+ if not pr_doc:
261
+ return None, [], [], [], []
262
+
263
+ if "_id" in pr_doc and isinstance(pr_doc["_id"], ObjectId):
264
+ pr_doc["_id"] = str(pr_doc["_id"])
265
+
266
+ # --- Extract linked IDs
267
+ cr_ids = pr_doc.get("correctionIds", [])
268
+ fa_ids = pr_doc.get("faultAnalysisId", [])
269
+
270
+ # ensure both are lists
271
+ if isinstance(cr_ids, str):
272
+ cr_ids = [cr_ids]
273
+ elif cr_ids is None:
274
+ cr_ids = []
275
+
276
+ if isinstance(fa_ids, str):
277
+ fa_ids = [fa_ids]
278
+ elif fa_ids is None:
279
+ fa_ids = []
280
+
281
+ # --- Fetch Correction Reports
282
+ cr_docs = list(db["corrections"].find({"id": {"$in": cr_ids}})) if cr_ids else []
283
+ for doc in cr_docs:
284
+ if "_id" in doc and isinstance(doc["_id"], ObjectId):
285
+ doc["_id"] = str(doc["_id"])
286
+
287
+ # --- Fetch Fault Analysis Reports
288
+ fa_docs = list(db["faultanalysis"].find({"id": {"$in": fa_ids}})) if fa_ids else []
289
+ for doc in fa_docs:
290
+ if "_id" in doc and isinstance(doc["_id"], ObjectId):
291
+ doc["_id"] = str(doc["_id"])
292
+
293
+ return pr_doc, cr_ids, fa_ids, cr_docs, fa_docs
294
+
295
+
296
+ def is_technical_query(self, query: str) -> bool:
297
+ """
298
+ Classify query as TECHNICAL or NON-TECHNICAL.
299
+ """
300
+ classification_prompt = f"""
301
+ You are a classifier. Determine if the following query is TECHNICAL
302
+ (related to software, debugging, errors, troubleshooting, fault analysis,
303
+ corrections, technical problem reports) or NON-TECHNICAL
304
+ (general questions, greetings, chit-chat, unrelated topics).
305
+
306
+ Query: "{query}"
307
+
308
+ Respond with exactly one word: "TECHNICAL" or "NON-TECHNICAL".
309
+ """
310
+
311
+ if self.llm_provider == "gemini":
312
+ response = self.llm.generate_content(classification_prompt)
313
+ result = response.text.strip().upper()
314
+
315
+ elif self.llm_provider == "groq":
316
+ completion = self.llm.chat.completions.create(
317
+ model=self.llm_model_name,
318
+ messages=[{"role": "system", "content": classification_prompt}]
319
+ )
320
+ result = completion.choices[0].message.content.strip().upper()
321
+
322
+ return result == "TECHNICAL"
323
+
324
+ def is_followup_query(self, query: str, history: list = None) -> bool:
325
+ """
326
+ Detect if query is a follow-up based on conversation history.
327
+ """
328
+ if not history:
329
+ return False
330
+
331
+ classification_prompt = f"""
332
+ You are a classifier. Determine if the following user query
333
+ is a FOLLOW-UP (depends on the previous conversation)
334
+ or a NEW QUERY (can be answered independently).
335
+
336
+ Previous conversation:
337
+ { [msg['content'] for msg in history][-3:] }
338
+
339
+ Current query: "{query}"
340
+
341
+ Respond with exactly one word: "FOLLOW-UP" or "NEW".
342
+ """
343
+
344
+ if self.llm_provider == "gemini":
345
+ response = self.llm.generate_content(classification_prompt)
346
+ result = response.text.strip().upper()
347
+
348
+ elif self.llm_provider == "groq":
349
+ completion = self.llm.chat.completions.create(
350
+ model=self.llm_model_name,
351
+ messages=[{"role": "system", "content": classification_prompt}]
352
+ )
353
+ result = completion.choices[0].message.content.strip().upper()
354
+ print("Follow up: ", result)
355
+ return result == "FOLLOW-UP"
356
+
357
+ def ask(self, query: str, history: list = None):
358
+ print(f"\n❓ Query: {query}")
359
+
360
+ # Step 1: Classify
361
+ is_technical = self.is_technical_query(query)
362
+ is_followup = self.is_followup_query(query, history)
363
+
364
+ # Step 2: Non-technical standalone
365
+ if not is_technical and not is_followup:
366
+ print("⚠️ Non-technical standalone query β†’ skipping Qdrant.")
367
+ system_prompt = "You are a helpful assistant. Answer clearly and concisely."
368
+ convo = [{"role": "system", "content": system_prompt},
369
+ {"role": "user", "content": query}]
370
+
371
+ if self.llm_provider == "gemini":
372
+ convo_str = "\n".join([f"{m['role'].capitalize()}: {m['content']}" for m in convo])
373
+ response = self.llm.generate_content(convo_str)
374
+ return response.text.strip(), []
375
+
376
+ elif self.llm_provider == "groq":
377
+ completion = self.llm.chat.completions.create(
378
+ model=self.llm_model_name,
379
+ messages=convo
380
+ )
381
+ return completion.choices[0].message.content.strip(), []
382
+
383
+ # Step 3: Technical or follow-up
384
+ print("is_followup", is_followup)
385
+ print("last_context", self.last_context)
386
+ if is_followup and self.last_context:
387
+ print("πŸ”„ Follow-up query β†’ reusing previous context.")
388
+ retrieved_context = self.last_context
389
+ else:
390
+ print("πŸ“₯ New technical query β†’ retrieving from Qdrant.")
391
+ retrieved_context = self.retrieve(query)
392
+ last_context = []
393
+ for i, doc in enumerate(retrieved_context):
394
+ last_context.append(doc['id'])
395
+ print(f" - Context {i+1} ({doc['entity_type']}, ID: {doc['id']}, Score: {doc['score']:.2f})")
396
+
397
+
398
+
399
+ if(len(last_context)>0):
400
+ self.last_context = last_context # save for future follow-ups
401
+ if not retrieved_context:
402
+ print("πŸ’¬ No relevant context found.")
403
+ return "I could not find any relevant information.", []
404
+
405
+ print(f"βœ… Using {len(retrieved_context)} documents as context.")
406
+ answer = self.generate_answer(query, retrieved_context, history, is_followup)
407
+ last_context = self.last_context
408
+ print(f"\nπŸ€– Answer: {answer}")
409
+ return (answer, last_context)
410
+
411
+
412
+
413
+