Add 899999999999999999999999

check_agentic_docs.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+"""
+Check documents in the agentic_chatbot LanceDB database
+"""
+import os
+import sys
+
+# Add the path to access lancedb_service
+sys.path.append('/Users/abhishekchoudhary/Abhi Project/Voice_Bot/PensionBot')
+
+# Change to agentic_chatbot directory to use its database
+os.chdir('/Users/abhishekchoudhary/Abhi Project/Voice_Bot/agentic_chatbot')
+
+import asyncio
+import logging
+import lancedb
+
+# Setup logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger("check_agentic_docs")
+
+async def check_agentic_documents():
+    """Check how many documents are in the agentic_chatbot database"""
+    try:
+        logger.info("🔍 Checking agentic_chatbot LanceDB document count...")
+        
+        # Connect to the local LanceDB
+        db = lancedb.connect("lancedb_data")
+        table_names = db.table_names()
+        
+        logger.info(f"📊 Found {len(table_names)} tables in agentic_chatbot database:")
+        
+        total_documents = 0
+        for table_name in table_names:
+            try:
+                table = db.open_table(table_name)
+                count = table.count_rows()
+                logger.info(f"  📄 {table_name}: {count} documents")
+                total_documents += count
+            except Exception as e:
+                logger.error(f"  ❌ Error checking table {table_name}: {e}")
+        
+        logger.info(f"📈 Total documents in agentic_chatbot: {total_documents}")
+            
+    except Exception as e:
+        logger.error(f"❌ Error checking documents: {e}")
+
+if __name__ == "__main__":
+    asyncio.run(check_agentic_documents())

check_documents.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+"""
+Check the number of documents in the LanceDB database
+"""
+import asyncio
+import logging
+from lancedb_service import lancedb_service
+
+# Setup logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger("check_docs")
+
+async def check_document_count():
+    """Check how many documents are in the database"""
+    try:
+        logger.info("🔍 Checking LanceDB document count...")
+        
+        # Get all tables in the database
+        db = lancedb_service.db
+        table_names = await asyncio.to_thread(lambda: db.table_names())
+        
+        logger.info(f"📊 Found {len(table_names)} tables in database:")
+        
+        total_documents = 0
+        for table_name in table_names:
+            try:
+                table = await asyncio.to_thread(lambda: db.open_table(table_name))
+                count = await asyncio.to_thread(lambda: table.count_rows())
+                logger.info(f"  📄 {table_name}: {count} documents")
+                total_documents += count
+            except Exception as e:
+                logger.error(f"  ❌ Error checking table {table_name}: {e}")
+        
+        logger.info(f"📈 Total documents across all tables: {total_documents}")
+        
+        # Also check if we can search the main documents table
+        logger.info("\n🔍 Testing document search...")
+        from rag_service import search_documents_async
+        
+        test_queries = ["pension", "salary", "leave", "training"]
+        for query in test_queries:
+            docs = await search_documents_async(query, limit=3)
+            logger.info(f"  Query '{query}': Found {len(docs) if docs else 0} documents")
+            
+    except Exception as e:
+        logger.error(f"❌ Error checking documents: {e}")
+
+if __name__ == "__main__":
+    asyncio.run(check_document_count())

config.py

@@ -44,7 +44,7 @@ CHUNK_OVERLAP = int(os.environ.get("CHUNK_OVERLAP", "200"))
 ALLOWED_ORIGINS = os.environ.get("ALLOWED_ORIGINS", "*").split(",") if os.environ.get("ALLOWED_ORIGINS") != "*" else ["*"]
 
 # LanceDB Configuration
-LANCEDB_PATH = os.environ.get("LANCEDB_PATH", "./lancedb_data")
+LANCEDB_PATH = os.environ.get("LANCEDB_PATH", "../agentic_chatbot/lancedb_data")
 
 # JWT Configuration
 JWT_SECRET_KEY = os.environ.get("JWT_SECRET_KEY")

reset_database.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+"""
+Reset the LanceDB database and reload with new diverse content
+"""
+import asyncio
+import logging
+import shutil
+import os
+from pathlib import Path
+
+# Setup logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger("reset_db")
+
+async def reset_database():
+    """Reset the database and reload with diverse content"""
+    try:
+        # Path to the database directory
+        db_path = Path("lancedb_data")
+        
+        if db_path.exists():
+            logger.info("🗑️ Removing existing database...")
+            shutil.rmtree(db_path)
+            logger.info("✅ Existing database removed")
+        
+        # Recreate the database directory
+        db_path.mkdir(exist_ok=True)
+        logger.info("📁 Created new database directory")
+        
+        # Now run the setup documents script
+        logger.info("📚 Loading new diverse documents...")
+        from setup_documents import setup_sample_documents
+        await setup_sample_documents()
+        
+        logger.info("🎉 Database reset complete with diverse content!")
+        
+    except Exception as e:
+        logger.error(f"❌ Error resetting database: {e}")
+
+if __name__ == "__main__":
+    asyncio.run(reset_database())

setup_documents.py

@@ -8,74 +8,238 @@ import logging
 
 logger = logging.getLogger("voicebot")
 
-# Sample government documents content
+# Sample government documents content - Diverse content for different queries
 SAMPLE_DOCUMENTS = [
     {
-        "content": """Pension is a regular payment made during a person's retirement from an investment fund to which that person or their employer has contributed during their working career. In the context of government employees in India, pension refers to the retirement benefits provided to civil servants after completing their service period.
-
-The pension system for government employees in India includes:
-1. Basic Pension: Calculated based on the last drawn salary and years of service
-2. Dearness Relief (DR): Additional amount to counter inflation
-3. Medical Benefits: Healthcare coverage post-retirement
-4. Family Pension: Benefits provided to family members after the employee's death
-
-Key features:
-- Minimum service period: Usually 10 years for qualifying for pension
-- Calculation formula: (Last drawn basic salary × years of service) / 70 (for pre-2016 employees)
-- Monthly payment: Pension is paid monthly to the retired employee
-- Indexation: Pension amount is revised periodically to account for inflation""",
-        "filename": "pension_basics.txt",
-        "source": "Government Pension Manual"
+        "content": """Government employees are entitled to various types of leave during their service period. The leave rules are governed by the Central Civil Services (Leave) Rules and state-specific adaptations.
+
+Types of Leave Available:
+1. Casual Leave (CL): 8 days per calendar year
+   - Can be taken for urgent personal work
+   - No medical certificate required
+   - Cannot be carried forward to next year
+
+2. Earned Leave (EL): 30 days per year
+   - Can be accumulated up to 300 days
+   - Encashment allowed at retirement
+   - Advance grant possible for valid reasons
+
+3. Medical Leave: As per medical requirements
+   - Medical certificate mandatory
+   - Can be combined with EL
+   - Special provisions for chronic illnesses
+
+4. Maternity/Paternity Leave:
+   - Maternity: 180 days (6 months)
+   - Paternity: 15 days within 6 months of child birth
+
+Application Process:
+- Submit leave application in advance
+- Get approval from competent authority
+- Maintain proper leave records
+- Emergency leave can be regularized later""",
+        "filename": "leave_rules.txt",
+        "source": "Government Leave Manual 2024"
+    },
+    {
+        "content": """Government employee salary structure consists of multiple components designed to provide comprehensive compensation and benefits.
+
+Salary Components:
+1. Basic Pay: Core salary amount based on pay scale and grade
+2. Dearness Allowance (DA): Currently 50% of basic pay (as of 2024)
+3. House Rent Allowance (HRA): Varies by city classification
+   - X Class cities: 24% of basic pay
+   - Y Class cities: 16% of basic pay  
+   - Z Class cities: 8% of basic pay
+
+4. Transport Allowance: ₹3,600 per month for most employees
+5. Medical Allowance: ₹1,000 per month
+
+Special Allowances:
+- City Compensatory Allowance (CCA)
+- Special Allowance for difficult postings
+- Overtime Allowance (where applicable)
+
+Deductions:
+- Income Tax (as per IT rules)
+- Provident Fund: 12% of basic pay
+- Group Insurance Scheme (GIS)
+- Professional Tax (state-specific)
+
+Pay Revision:
+- Pay Commission recommendations every 10 years
+- Annual increment: Usually 3% of basic pay
+- Promotion-based pay upgrades as per rules""",
+        "filename": "salary_structure.txt",
+        "source": "Pay Commission Guidelines 2024"
+    },
+    {
+        "content": """Transfer and posting policies for government employees are designed to ensure fair distribution of work, prevent corruption, and provide career development opportunities.
+
+Transfer Rules:
+1. Tenure-based Transfers:
+   - Minimum tenure: 3 years in sensitive posts
+   - Maximum tenure: 5 years in one location (general rule)
+   - Cooling off period: 2 years before returning to same post
+
+2. Request Transfers:
+   - Can apply after completing minimum tenure
+   - Valid grounds: medical, family, educational needs
+   - Subject to administrative approval and replacement availability
+
+3. Administrative Transfers:
+   - Based on service requirements
+   - Immediate transfer in case of disciplinary issues
+   - Mutual transfers allowed with proper approvals
+
+Posting Guidelines:
+- Home district posting: After 5 years of service
+- Difficult area posting: Incentives and allowances provided
+- Border area posting: Special security clearance required
+
+Transfer Process:
+1. Issue of transfer order by competent authority
+2. Relieving formalities at current posting
+3. Journey time allowance as per rules
+4. Joining at new posting within prescribed time
+5. Handing/taking over of charge properly
+
+Benefits:
+- Transfer TA/DA as per entitlement
+- Family accommodation assistance
+- School transfer certificates for children""",
+        "filename": "transfer_posting_rules.txt",
+        "source": "Administrative Transfer Policy 2024"
+    },
+    {
+        "content": """Training and skill development programs are essential for government employees to enhance their capabilities and stay updated with modern administrative practices.
+
+Training Categories:
+1. Induction Training:
+   - Mandatory for all new recruits
+   - Duration: 3-6 months depending on service
+   - Covers service rules, conduct rules, and job-specific skills
+
+2. In-Service Training:
+   - Periodic skill upgradation programs
+   - Leadership development courses
+   - Technology and digital literacy training
+
+3. Specialized Training:
+   - Domain-specific technical training
+   - Foreign training opportunities for senior officers
+   - Research and innovation workshops
+
+Training Institutes:
+- National Academy of Administration (NAAN) - for IAS officers
+- State Administrative Training Institutes
+- Sector-specific training centers
+- Online learning platforms (iGOT Karmayogi)
+
+Benefits of Training:
+1. Career advancement opportunities
+2. Better performance and efficiency
+3. Exposure to best practices
+4. Networking with peers
+5. Personal and professional development
+
+Training Leave:
+- Special training leave with full pay
+- Study leave for higher education
+- Deputation opportunities to training institutes
+
+Digital Learning Initiative:
+- Mission Karmayogi for capacity building
+- Online certification courses
+- Competency-based training modules
+- Performance-linked training requirements
+
+Training is considered essential for promotion to higher grades and is often a mandatory requirement for career progression in government service.""",
+        "filename": "training_development.txt",
+        "source": "Government Training Policy 2024"
     },
     {
-        "content": """Dearness Allowance (DA) is a component of salary paid to government employees and pensioners to offset the impact of inflation. For government employees in Rajasthan and other states, DA is calculated as a percentage of basic salary.
-
-Current DA rates and impact:
-- DA is revised twice a year (January and July)
-- Based on All India Consumer Price Index (AICPI)
-- Recent DA increment: 6% increase effective from January 2024
-- Impact on salary: For an employee with basic salary of ₹50,000, a 6% DA increment adds ₹3,000 per month
-- Impact on pension: Pensioners also receive corresponding Dearness Relief (DR) increase
-
-Calculation method:
-- DA percentage = ((Average AICPI for 12 months - Base AICPI) / Base AICPI) × 100
-- Rounded to nearest 0.5%
-
-Benefits of 6% DA increment:
-1. Increased monthly income for serving employees
-2. Higher pension amounts for retirees
-3. Improved purchasing power to counter inflation
-4. Applicable to all government employees across pay scales""",
-        "filename": "dearness_allowance_impact.txt", 
-        "source": "DA Revision Circular 2024"
+        "content": """Retirement benefits for government employees include multiple components to ensure financial security post-retirement.
+
+Retirement Benefits Package:
+1. Pension: Monthly payment based on last drawn salary and service years
+   - Calculation: (Last drawn basic pay + DA) × service years ÷ 70
+   - Minimum pension: ₹9,000 per month
+   - Maximum pension: No upper limit
+
+2. Gratuity: Lump sum payment at retirement
+   - Formula: (Basic pay + DA) × 15/26 × years of service
+   - Maximum: ₹20 lakh (as of 2024)
+   - Tax exemption available
+
+3. Provident Fund (GPF/CPF):
+   - Employee + Government contribution throughout service
+   - Withdrawal allowed at retirement
+   - Interest rate: Currently 8% per annum
+
+4. Commutation of Pension:
+   - Option to convert part of pension to lump sum
+   - Up to 50% of pension can be commuted
+   - Restoration after 15 years
+
+5. Medical Benefits:
+   - Continued medical facility post-retirement
+   - Central Government Health Scheme (CGHS) coverage
+   - Reimbursement of medical expenses
+
+Retirement Process:
+- Apply 6 months before retirement date
+- Complete all service formalities
+- Obtain clearances from all departments
+- Submit required documents and forms
+
+The retirement package is designed to provide comprehensive financial support and maintain dignity of life for retired government employees.""",
+        "filename": "retirement_benefits.txt",
+        "source": "Retirement Benefits Manual 2024"
     },
     {
-        "content": """Government employees in Rajasthan are entitled to various retirement benefits including pension, gratuity, and provident fund. The pension system has undergone reforms with the introduction of the New Pension Scheme (NPS) for employees joining after 2004.
-
-Rajasthan Pension Rules:
-1. Old Pension Scheme (OPS): For employees joined before 2004
-   - Defined benefit scheme
-   - Pension = 50% of last drawn salary after 33 years of service
-   - Family pension available to spouse and children
-
-2. New Pension Scheme (NPS): For employees joined after 2004
-   - Defined contribution scheme
-   - Employee contribution: 10% of basic salary
-   - Government contribution: 14% of basic salary
-   - Market-linked returns
-
-Recent Developments:
-- Rajasthan government restored OPS for all employees in 2022
-- Employees under NPS can now opt for OPS benefits
-- Enhanced pension benefits for teachers and other government employees
-
-Pension Processing:
-- Application to be submitted 6 months before retirement
-- Required documents: Service records, medical fitness certificate, family details
-- Processing time: Usually 3-6 months
-- Monthly pension credit: Through NEFT to bank account""",
-        "filename": "rajasthan_pension_rules.txt",
-        "source": "Rajasthan Government Pension Manual 2024"
+        "content": """Government procurement policies and procedures ensure transparency, fairness, and value for money in public purchases.
+
+Procurement Methods:
+1. Open Tender:
+   - Public advertisement required
+   - Minimum 21 days for bid submission
+   - Used for high-value procurements above ₹25 lakh
+
+2. Limited Tender:
+   - Invitation to selected vendors
+   - For specialized items or urgent requirements
+   - Proper justification required
+
+3. Single Tender:
+   - Direct negotiation with one vendor
+   - Only in exceptional circumstances
+   - Requires special approval
+
+Key Procurement Rules:
+- Preference to Make in India products
+- MSME reservation: 25% of procurement
+- Minimum 50% local content requirement
+- GeM (Government e-Marketplace) mandatory for routine items
+
+Tender Process:
+1. Preparation of tender documents
+2. Advertisement and bidder registration
+3. Pre-bid meetings for clarifications
+4. Bid submission and opening
+5. Technical and financial evaluation
+6. Contract award to lowest compliant bidder
+
+Documentation Required:
+- Technical specifications
+- Terms and conditions
+- Evaluation criteria
+- Approval for procurement
+- Committee formations for evaluation
+
+This system ensures competitive pricing, quality products, and supports domestic manufacturing while maintaining complete transparency in government spending.""",
+        "filename": "procurement_policy.txt", 
+        "source": "Government Procurement Guidelines 2024"
     }
 ]
 

test_search.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+"""
+Test script to verify that diverse documents are being returned for different queries
+"""
+import asyncio
+import logging
+from rag_service import search_documents_async
+
+# Setup logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger("test_search")
+
+async def test_diverse_responses():
+    """Test that different queries return different, relevant documents"""
+    
+    test_queries = [
+        "pension rules",
+        "leave policy",
+        "salary structure", 
+        "training programs",
+        "retirement benefits",
+        "procurement policy",
+        "get out",  # This was giving generic response before
+        "what about transfers"
+    ]
+    
+    logger.info("🧪 Testing diverse document responses...")
+    
+    for query in test_queries:
+        logger.info(f"\n🔍 Testing query: '{query}'")
+        try:
+            docs = await search_documents_async(query, limit=2)
+            if docs:
+                for i, doc in enumerate(docs):
+                    title = doc.get('source_title', 'Unknown')
+                    content_preview = doc.get('clause_text', '')[:100] + "..."
+                    logger.info(f"  📄 Document {i+1}: {title}")
+                    logger.info(f"      Preview: {content_preview}")
+            else:
+                logger.warning(f"  ⚠️ No documents found for: {query}")
+        except Exception as e:
+            logger.error(f"  ❌ Error searching for '{query}': {e}")
+
+if __name__ == "__main__":
+    asyncio.run(test_diverse_responses())