setup_knowledge.py

"""
Enhanced setup script with more comprehensive knowledge base
"""

import os
import sys
from pathlib import Path
import numpy as np
import faiss
import pickle
from sentence_transformers import SentenceTransformer
from typing import List, Dict

sys.path.append(os.path.dirname(os.path.abspath(__file__)))

# Extended wisdom database with more texts
WISDOM_DATABASE = [
    # Bhagavad Gita
    {
        "text": "The mind is restless and difficult to restrain, but it is subdued by practice.",
        "source": "Bhagavad Gita",
        "chapter": "6.35",
        "tags": ["mind", "practice", "discipline"]
    },
    {
        "text": "Set thy heart upon thy work, but never on its reward.",
        "source": "Bhagavad Gita", 
        "chapter": "2.47",
        "tags": ["work", "detachment", "karma"]
    },
    {
        "text": "The soul is neither born, and nor does it die.",
        "source": "Bhagavad Gita",
        "chapter": "2.20", 
        "tags": ["soul", "eternal", "death"]
    },
    
    # Autobiography of a Yogi
    {
        "text": "The season of failure is the best time for sowing the seeds of success.",
        "source": "Autobiography of a Yogi",
        "tags": ["failure", "success", "growth"]
    },
    {
        "text": "Live quietly in the moment and see the beauty of all before you.",
        "source": "Autobiography of a Yogi",
        "tags": ["present", "beauty", "mindfulness"]
    },
    
    # The Power of Now
    {
        "text": "Realize deeply that the present moment is all you have.",
        "source": "The Power of Now",
        "tags": ["present", "awareness", "now"]
    },
    {
        "text": "The primary cause of unhappiness is never the situation but your thoughts about it.",
        "source": "The Power of Now",
        "tags": ["unhappiness", "thoughts", "perception"]
    },
    {
        "text": "Life will give you whatever experience is most helpful for the evolution of your consciousness.",
        "source": "The Power of Now",
        "tags": ["life", "experience", "consciousness"]
    },
    
    # Man's Search for Meaning
    {
        "text": "When we are no longer able to change a situation, we are challenged to change ourselves.",
        "source": "Man's Search for Meaning",
        "tags": ["change", "adaptation", "growth"]
    },
    {
        "text": "Those who have a 'why' to live, can bear with almost any 'how'.",
        "source": "Man's Search for Meaning",
        "tags": ["purpose", "meaning", "resilience"]
    },
    
    # Atomic Habits
    {
        "text": "You do not rise to the level of your goals. You fall to the level of your systems.",
        "source": "Atomic Habits",
        "tags": ["goals", "systems", "habits"]
    },
    {
        "text": "Every action you take is a vote for the type of person you wish to become.",
        "source": "Atomic Habits",
        "tags": ["identity", "actions", "becoming"]
    },
    {
        "text": "Habits are the compound interest of self-improvement.",
        "source": "Atomic Habits",
        "tags": ["habits", "improvement", "compound"]
    },
    
    # Meditations - Marcus Aurelius
    {
        "text": "You have power over your mind - not outside events. Realize this, and you will find strength.",
        "source": "Meditations",
        "tags": ["control", "mind", "strength"]
    },
    {
        "text": "The best revenge is not to be like your enemy.",
        "source": "Meditations",
        "tags": ["revenge", "character", "virtue"]
    },
    {
        "text": "What we cannot bear removes us from life; what remains can be borne.",
        "source": "Meditations",
        "tags": ["endurance", "life", "strength"]
    },
    
    # Tao Te Ching
    {
        "text": "When I let go of what I am, I become what I might be.",
        "source": "Tao Te Ching",
        "tags": ["letting go", "becoming", "potential"]
    },
    {
        "text": "The journey of a thousand miles begins with a single step.",
        "source": "Tao Te Ching",
        "tags": ["journey", "beginning", "action"]
    },
    {
        "text": "Knowing others is intelligence; knowing yourself is true wisdom.",
        "source": "Tao Te Ching",
        "tags": ["self-knowledge", "wisdom", "intelligence"]
    },
    
    # 7 Habits of Highly Effective People
    {
        "text": "Between stimulus and response there is a space. In that space is our power to choose our response.",
        "source": "7 Habits of Highly Effective People",
        "tags": ["choice", "response", "freedom"]
    },
    {
        "text": "Begin with the end in mind.",
        "source": "7 Habits of Highly Effective People",
        "tags": ["vision", "planning", "purpose"]
    },
    
    # Mindset
    {
        "text": "Becoming is better than being.",
        "source": "Mindset",
        "tags": ["growth", "becoming", "fixed mindset"]
    },
    {
        "text": "The view you adopt for yourself profoundly affects the way you lead your life.",
        "source": "Mindset",
        "tags": ["perspective", "self-view", "life"]
    },
    
    # Additional Universal Wisdom
    {
        "text": "The wound is the place where the Light enters you.",
        "source": "Rumi",
        "tags": ["pain", "growth", "transformation"]
    },
    {
        "text": "Yesterday I was clever, so I wanted to change the world. Today I am wise, so I am changing myself.",
        "source": "Rumi",
        "tags": ["change", "wisdom", "self"]
    },
    {
        "text": "Do not dwell in the past, do not dream of the future, concentrate the mind on the present moment.",
        "source": "Buddha",
        "tags": ["present", "past", "future"]
    }
]

def setup_comprehensive_knowledge_base():
    """Setup knowledge base with all wisdom texts"""
    print("Setting up comprehensive knowledge base...")
    
    # Create directories
    data_dir = Path("data")
    faiss_dir = data_dir / "faiss_index"
    books_dir = data_dir / "books"
    
    for dir_path in [data_dir, faiss_dir, books_dir]:
        dir_path.mkdir(exist_ok=True)
    
    # Initialize embedder
    print("Loading sentence transformer model...")
    embedder = SentenceTransformer('all-MiniLM-L6-v2')
    
    # Process wisdom texts
    texts = []
    metadata = []
    
    for item in WISDOM_DATABASE:
        texts.append(item["text"])
        metadata.append({
            "source": item["source"],
            "chapter": item.get("chapter", ""),
            "tags": item.get("tags", [])
        })
    
    # Create embeddings
    print(f"Creating embeddings for {len(texts)} wisdom passages...")
    embeddings = embedder.encode(texts, show_progress_bar=True)
    
    # Create FAISS index with cosine similarity
    dimension = embeddings.shape[1]
    
    # Normalize embeddings for cosine similarity
    faiss.normalize_L2(embeddings)
    
    # Use Inner Product (equivalent to cosine similarity after normalization)
    index = faiss.IndexFlatIP(dimension)
    index.add(embeddings.astype('float32'))
    
    # Save everything
    print("Saving knowledge base...")
    faiss.write_index(index, str(faiss_dir / "index.faiss"))
    
    with open(faiss_dir / "passages.pkl", 'wb') as f:
        pickle.dump(texts, f)
    
    with open(faiss_dir / "metadata.pkl", 'wb') as f:
        pickle.dump(metadata, f)
    
    print(f"✅ Knowledge base setup complete!")
    print(f"   - Total passages: {len(texts)}")
    print(f"   - Embedding dimension: {dimension}")
    print(f"   - Index type: Cosine similarity (normalized L2)")
    
    # Create placeholder book files
    print("\nCreating placeholder book files...")
    book_files = [
        "bhagavad_gita.txt",
        "autobiography_of_a_yogi.txt",
        "the_power_of_now.txt",
        "mans_search_for_meaning.txt",
        "atomic_habits.txt",
        "meditations.txt",
        "tao_te_ching.txt",
        "dhyana_vahini.txt",
        "7_habits.txt",
        "mindset.txt",
        "prema_vahini.txt",
        "prasnothara_vahini.txt"
    ]
    
    for book_file in book_files:
        filepath = books_dir / book_file
        if not filepath.exists():
            with open(filepath, 'w', encoding='utf-8') as f:
                f.write(f"Placeholder for {book_file}\nAdd full text here.")
    
    print("✅ Setup complete! You can now run the main application.")
    print("\nTo add full books:")
    print("1. Replace placeholder files in data/books/ with full text")
    print("2. Re-run this script to update the knowledge base")

if __name__ == "__main__":
    setup_comprehensive_knowledge_base()