Add FAISS index using Git LFS

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.index filter=lfs diff=lfs merge=lfs -text

README_MODULES.md

@@ -0,0 +1,16 @@
+Refactor notes
+
+- Created `backend/` package with wrapper modules that re-export names from
+  `server.py` to preserve runtime behavior while splitting the project into
+  modules.
+- The wrappers are temporary and intended to make incremental movement of
+  functions/classes from `server.py` into dedicated files safer.
+
+Next steps
+
+- Move related functions and classes from `server.py` into the appropriate
+  files in `backend/` (e.g. models into `backend/db.py`, routes into
+  `backend/routes.py`, matching logic into `backend/services.py`).
+- Update imports throughout the codebase to reference `backend.*` modules.
+- Remove re-export wrappers and keep the new modules as the single source of
+  truth.

app.py

@@ -0,0 +1,141 @@
+# app.py (updated with debug routes)
+import os
+import datetime
+from flask import Flask, jsonify, request
+from flask_cors import CORS
+
+from config import SQL_DRIVER, SQL_SERVER, SQL_DB, SQL_TRUSTED, SQL_USER, SQL_PASSWORD, SQL_PORT, SQL_ENCRYPT, SQL_TRUSTCERT, IS_HUGGING_FACE
+from models import db
+
+def create_app():
+    app = Flask(__name__)
+    CORS(app, resources={r"/*": {"origins": "*"}})
+    
+    # Debug: Print all imports
+    print(f"🚀 DEBUG: Starting app creation...")
+    print(f"🚀 DEBUG: SQL_SERVER = {SQL_SERVER}")
+    print(f"🚀 DEBUG: SQL_DB = {SQL_DB}")
+    
+    # Add request logging middleware
+    @app.before_request
+    def log_request_info():
+        print(f"\n{'='*60}")
+        print(f"📥 INCOMING REQUEST:")
+        print(f"   Time: {datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
+        print(f"   Method: {request.method}")
+        print(f"   Path: {request.path}")
+        print(f"   URL: {request.url}")
+        print(f"   Remote Address: {request.remote_addr}")
+        if request.user_agent:
+            print(f"   User Agent: {request.user_agent.string[:50]}...")
+        print(f"   Referrer: {request.referrer}")
+        print(f"{'='*60}")
+    
+    # Initialize database
+    try:
+        from database import init_database
+        init_database(app)
+        print("✅ DEBUG: Database initialized successfully")
+    except Exception as e:
+        print(f"❌ DEBUG: Failed to initialize database: {e}")
+    
+    # Try to import and register blueprints
+    try:
+        import routes
+        print("✅ DEBUG: Imported routes module")
+        print(f"✅ DEBUG: Available blueprints: {[bp.name for bp in [routes.auth_bp, routes.profiles_bp, routes.expectations_bp, routes.matching_bp, routes.llm_bp]]}")
+        
+        # Register blueprints with url_prefix
+        app.register_blueprint(routes.auth_bp, url_prefix='/api')
+        app.register_blueprint(routes.profiles_bp)
+        app.register_blueprint(routes.expectations_bp)
+        app.register_blueprint(routes.matching_bp, url_prefix='/api')
+        app.register_blueprint(routes.llm_bp)
+        print("✅ DEBUG: All blueprints registered")
+        
+    except ImportError as e:
+        print(f"❌ DEBUG: Failed to import routes: {e}")
+    except Exception as e:
+        print(f"❌ DEBUG: Failed to register blueprints: {e}")
+        import traceback
+        traceback.print_exc()
+    
+    # Add debug endpoint to list all routes
+    @app.route('/debug/routes')
+    def debug_routes():
+        routes_list = []
+        for rule in app.url_map.iter_rules():
+            routes_list.append({
+                'endpoint': rule.endpoint,
+                'methods': list(rule.methods),
+                'rule': str(rule)
+            })
+        return jsonify({'routes': routes_list})
+    
+    # Health endpoint
+    @app.get("/health")
+    def health():
+        try:
+            from faiss_service import knowledge, TEXT_CHUNKS
+            from llm_service import CHAIN_BATCH
+            faiss_loaded = len(TEXT_CHUNKS) > 0 if TEXT_CHUNKS else False
+            knowledge_loaded = knowledge is not None and hasattr(knowledge, 'indices') and len(knowledge.indices) > 0
+        except ImportError:
+            faiss_loaded = False
+            knowledge_loaded = False
+            CHAIN_BATCH = None
+            
+        return {
+            "status": "ok",
+            "llm": ("openai" if CHAIN_BATCH is not None else "offline-fallback"),
+            "has_openai_key": bool(os.getenv("OPENAI_API_KEY")),
+            "db": {"server": SQL_SERVER, "database": SQL_DB},
+            "faiss_loaded": faiss_loaded,
+            "faiss_chunks": len(TEXT_CHUNKS) if 'TEXT_CHUNKS' in locals() else 0,
+            "knowledge_base_loaded": knowledge_loaded,
+        }
+    
+    # Home endpoint
+    @app.get("/")
+    def home():
+        return {
+            "message": "Unified Py-Match Service (FAISS-enabled)",
+            "try": [
+                "POST /api/signup",
+                "POST /api/questions/select-role",
+                "GET  /api/questions/<role>",
+                "POST /api/questions/submit-answers/<role>",
+                "POST /llm/start   (body: { user_id, role, n_questions, batch_size })",
+                "POST /llm/next    (body: { session_id, selected_color })",
+                "GET  /api/match/<user_id> (query: ?role=<role>&limit=<num>)",
+                "GET  /debug/routes  (to see all registered routes)",
+            ]
+        }
+    
+    # Error handlers
+    @app.errorhandler(404)
+    def not_found(error):
+        return jsonify({"error": "Endpoint not found", "path": request.path}), 404
+    
+    @app.errorhandler(405)
+    def method_not_allowed(error):
+        return jsonify({
+            "error": "Method not allowed",
+            "message": f"Method {request.method} not allowed for {request.path}",
+            "allowed_methods": error.valid_methods if hasattr(error, 'valid_methods') else []
+        }), 405
+    
+    @app.errorhandler(500)
+    def internal_error(error):
+        return jsonify({"error": "Internal server error"}), 500
+    
+    return app
+
+if __name__ == "__main__":
+    app = create_app()
+    
+    print(f"\n{'='*60}")
+    print("🚀 Flask server starting...")
+    print(f"{'='*60}")
+    
+    app.run(host="0.0.0.0", port=5000, debug=True)

character_functions.py

@@ -0,0 +1,507 @@
+# character_functions.py
+import json
+import numpy as np
+from typing import Dict, List, Optional
+import os
+
+from config import COLOR_KEYS
+from models import LLMGeneratedQuestions, Users, Marriage
+
+# Try importing LLM libraries
+try:
+    from pydantic import BaseModel, Field
+    from langchain_core.prompts import ChatPromptTemplate
+    from langchain_core.output_parsers import PydanticOutputParser, StrOutputParser
+    from langchain_openai import ChatOpenAI
+    HAS_LLM_STACK = True
+    HAS_LLM = True
+except Exception:
+    HAS_LLM_STACK = False
+    HAS_LLM = False
+
+def cosine_sim(a: np.ndarray, b: np.ndarray) -> float:
+    na, nb = np.linalg.norm(a), np.linalg.norm(b)
+    if na == 0.0 or nb == 0.0: return 0.0
+    return float(np.dot(a, b) / (na * nb))
+
+def calculate_character_similarity(b, g, y, r):
+    # 🚨 FIX: Convert all inputs to numbers
+    try:
+        b = float(b) if b is not None else 0.0
+        g = float(g) if g is not None else 0.0
+        y = float(y) if y is not None else 0.0
+        r = float(r) if r is not None else 0.0
+    except (ValueError, TypeError) as e:
+        print(f"❌ Error converting character scores to numbers: {e}")
+        b, g, y, r = 0.0, 0.0, 0.0, 0.0
+    
+    total = b + g + y + r
+    
+    if total <= 0:
+        return 0.0
+    
+    # Normalize the values
+    b_norm = b / total
+    g_norm = g / total
+    y_norm = y / total
+    r_norm = r / total
+    
+    # Ideal distribution (you can adjust these weights based on your preference)
+    ideal_b = 0.4  # 40% blue (stable/calm)
+    ideal_g = 0.3  # 30% green (growth-oriented)  
+    ideal_y = 0.2  # 20% yellow (cautious)
+    ideal_r = 0.1  # 10% red (passionate)
+    
+    # Calculate similarity using cosine similarity or simple difference
+    # Using simple weighted difference for now
+    similarity = 1.0 - (
+        abs(b_norm - ideal_b) * 0.25 +
+        abs(g_norm - ideal_g) * 0.25 +
+        abs(y_norm - ideal_y) * 0.25 +
+        abs(r_norm - ideal_r) * 0.25
+    )
+    
+    # Ensure score is between 0 and 1
+    return max(0.0, min(1.0, similarity))
+
+def get_user_background(user_id: int) -> Dict:
+    """Get comprehensive user background for LLM analysis"""
+    background = {}
+
+    # Get basic user info
+    user = Users.query.filter_by(user_id=user_id).first()
+    if user:
+        background.update({
+            "name": user.name or "Unknown",
+            "email": user.email or "",
+        })
+
+    # Get marriage profile if exists
+    marriage_profile = Marriage.query.filter_by(user_id=user_id).first()
+    if marriage_profile:
+        background.update({
+            "current_location": marriage_profile.current_city or "",
+            "education": marriage_profile.education_level or "",
+            "employment": marriage_profile.employment_status or "",
+            "hobbies": marriage_profile.hobbies_interests or "",
+            "conflict_style": marriage_profile.conflict_approach or "",
+            "financial_style": marriage_profile.financial_style or "",
+            "family_type": marriage_profile.family_type or "",
+        })
+
+    return background
+
+def generate_character_llm_explanation(u_vec, v_vec):
+    """Character explanation using FAISS + LLM - NO FALLBACK"""
+    
+    print("🟢 Starting LLM character explanation...")
+    
+    if not HAS_LLM:
+        raise Exception("LLM service is currently unavailable. Please try again later.")
+    
+    # Import inside function to avoid circular imports
+    try:
+        from faiss_service import get_faiss_context
+        context = get_faiss_context(3)
+    except ImportError:
+        context = ""
+
+    data = {
+        "User1": [float(u_vec[0]), float(u_vec[1]), float(u_vec[2]), float(u_vec[3])],
+        "User2": [float(v_vec[0]), float(v_vec[1]), float(v_vec[2]), float(v_vec[3])]
+    }
+
+    json_data = json.dumps(data, indent=2)
+
+    prompt = ChatPromptTemplate.from_messages([
+        ("system", """
+    You are a personality and relationship compatibility expert.
+
+    Generate CHARACTER compatibility in EXACTLY 3 groups with these EXACT section headers:
+
+    1. Character Strengths
+    2. Character Risks  
+    3. Sacrifices Needed
+
+    CRITICAL RULES:
+    - Use ONLY these exact section headers: "Character Strengths", "Character Risks", "Sacrifices Needed"
+    - NO markdown formatting
+    - Each section should have 1-5 points based on actual needs
+    - Write only the points that are truly necessary
+    - If only one point is needed, write only one point
+    - If no points are needed in a section, write "None" for that section
+    - Maximum 5 points per section for very low compatibility cases
+    - Each point should be a complete sentence starting with a capital letter
+    - Separate sections with a blank line
+    - No color names, no trait labels, no percentages
+    - Use simple English that anyone can understand
+    - Don't use "User1" or "User2" - refer to them as "the two people" or "both persons"
+    - BE TRUTHFUL: Write only real strengths, risks, and sacrifices based on their actual compatibility
+    """),
+        ("human", """
+    ### PERSONALITY DATA
+    {json_data}
+
+    ### BOOK CONTEXT
+    {context}
+
+    Generate the character analysis in the exact format specified above.
+    Write only the points that are truly needed - no filler content.
+    Use simple language that everyone can understand.
+    """)
+    ])
+
+    try:
+        llm = ChatOpenAI(
+            model="gpt-4o-mini",
+            temperature=0.65,
+            api_key=os.getenv("OPENAI_API_KEY"),
+            timeout=60,
+            max_retries=2,
+            request_timeout=60
+        )
+        
+        chain = prompt | llm | StrOutputParser()
+
+        print("⏳ Generating AI-powered character analysis...")
+        res = chain.invoke({"json_data": json_data, "context": context})
+        
+        print(f"🔍 DEBUG: Raw LLM response = {res}")
+        
+        if not res or len(res.strip()) < 10:
+            raise Exception("AI analysis returned insufficient response")
+        
+        # Process the response into lines
+        lines = []
+        for line in res.split('\n'):
+            line = line.strip()
+            if line and not line.startswith('###'):  # Remove markdown headers
+                lines.append(line)
+        
+        # Ensure we have all three sections
+        if len(lines) < 3:  # At least headers for all three sections
+            print(f"⚠️ LLM returned insufficient lines: {len(lines)}")
+            # Fallback: use the original response but clean it up
+            lines = [line for line in res.split('\n') if line.strip() and not line.startswith('###')]
+            
+        print(f"✅ AI character analysis completed with {len(lines)} lines")
+        return lines[:15]  # Increased limit for flexible points
+        
+    except Exception as e:
+        print(f"🔴 LLM character explanation failed: {e}")
+        raise Exception(f"AI analysis failed: {str(e)}. Please try again.")
+
+def generate_character_fallback_explanation(u_vec, v_vec):
+    """Generate structured fallback explanation when LLM is unavailable"""
+    
+    # Calculate basic similarity
+    character_score = cosine_sim(u_vec, v_vec)
+    
+    # Analyze dominant colors from vectors
+    colors = ['Blue', 'Green', 'Yellow', 'Red']
+    u_dominant_idx = np.argmax(u_vec)
+    v_dominant_idx = np.argmax(v_vec)
+    
+    dominant_color_u = colors[u_dominant_idx]
+    dominant_color_v = colors[v_dominant_idx]
+    
+    # Generate sections based on color combinations
+    strengths = generate_fallback_strengths(dominant_color_u, dominant_color_v)
+    risks = generate_fallback_risks(dominant_color_u, dominant_color_v)
+    sacrifices = generate_fallback_sacrifices(dominant_color_u, dominant_color_v)
+    
+    # Build explanation in LLM-like format
+    explanation = []
+    explanation.append(f"Character Score: {round(character_score * 100, 1)}%")
+    explanation.append("")
+    explanation.append("Character Strengths")
+    explanation.extend([f"• {s}" for s in strengths])
+    explanation.append("")
+    explanation.append("Character Risks")
+    explanation.extend([f"• {r}" for r in risks])
+    explanation.append("")
+    explanation.append("Sacrifices Needed From Both Partners")
+    explanation.extend([f"• {s}" for s in sacrifices])
+    
+    return explanation
+
+def generate_fallback_strengths(color1, color2):
+    """Generate strengths based on color combination"""
+    combinations = {
+        ('Blue', 'Red'): [
+            "Analytical thinking complements decisive action",
+            "Thorough planning balances quick decision-making", 
+            "Data-driven approach supports confident leadership"
+        ],
+        ('Green', 'Yellow'): [
+            "Structured organization grounds creative ideas",
+            "Process-oriented approach gives vision practical form",
+            "Reliability provides stability for innovation"
+        ],
+        ('Blue', 'Green'): [
+            "Detailed analysis combines with systematic execution",
+            "Methodical approach ensures thorough implementation",
+            "Precision and organization create reliable outcomes"
+        ],
+        ('Red', 'Yellow'): [
+            "Action-oriented drive brings creative ideas to life",
+            "Bold decisions support visionary thinking",
+            "Energy and enthusiasm fuel innovative projects"
+        ],
+        ('Blue', 'Yellow'): [
+            "Analytical depth enhances creative problem-solving",
+            "Thorough research supports innovative approaches",
+            "Logical thinking balances imaginative ideas"
+        ],
+        ('Green', 'Red'): [
+            "Organized planning directs decisive action",
+            "Systematic approach channels energetic drive",
+            "Process efficiency supports quick implementation"
+        ]
+    }
+    
+    key = tuple(sorted([color1, color2]))
+    return combinations.get(key, [
+        "Complementary personality traits create balance",
+        "Different approaches bring diverse perspectives", 
+        "Varied strengths cover multiple relationship aspects"
+    ])
+
+def generate_fallback_risks(color1, color2):
+    """Generate risks based on color combination"""
+    combinations = {
+        ('Blue', 'Red'): [
+            "Over-analysis may frustrate action-oriented partner",
+            "Quick decisions might overlook important details",
+            "Direct communication may clash with thoughtful processing"
+        ],
+        ('Green', 'Yellow'): [
+            "Rigid routines may limit spontaneous creativity", 
+            "Unstructured ideas may disrupt organized systems",
+            "Process focus might slow down innovative thinking"
+        ],
+        ('Blue', 'Green'): [
+            "Excessive planning may delay actual progress",
+            "Over-caution might prevent necessary risks",
+            "Analysis paralysis in decision-making situations"
+        ],
+        ('Red', 'Yellow'): [
+            "Impulsive actions may lack long-term vision",
+            "Big ideas might overlook practical implementation",
+            "Enthusiasm may override careful consideration"
+        ],
+        ('Blue', 'Yellow'): [
+            "Over-thinking may dampen spontaneous creativity",
+            "Abstract ideas might lack practical grounding",
+            "Detail focus could miss the bigger picture"
+        ],
+        ('Green', 'Red'): [
+            "Bureaucratic processes may frustrate quick action",
+            "Impulsive decisions could disrupt careful planning",
+            "Directness may overwhelm methodical approach"
+        ]
+    }
+    
+    key = tuple(sorted([color1, color2]))
+    return combinations.get(key, [
+        "Different communication styles may cause misunderstandings",
+        "Varying energy levels could lead to timing conflicts",
+        "Contrasting approaches to problems may create tension"
+    ])
+
+def generate_fallback_sacrifices(color1, color2):
+    """Generate sacrifices based on color combination"""
+    combinations = {
+        ('Blue', 'Red'): [
+            "Analytical partner must accept quicker decisions sometimes",
+            "Action-oriented partner needs to allow time for reflection", 
+            "Both must find middle ground between speed and thoroughness"
+        ],
+        ('Green', 'Yellow'): [
+            "Organized partner should embrace some spontaneity",
+            "Creative partner needs to respect established routines",
+            "Both must balance structure with flexibility"
+        ],
+        ('Blue', 'Green'): [
+            "Need to move from planning to action more quickly",
+            "Must embrace some uncertainty in decision-making",
+            "Both should practice more direct communication"
+        ],
+        ('Red', 'Yellow'): [
+            "Need to ground big ideas with practical steps",
+            "Must balance enthusiasm with realistic planning",
+            "Both should develop more patience in execution"
+        ],
+        ('Blue', 'Yellow'): [
+            "Analytical thinker should embrace intuitive leaps",
+            "Creative partner needs to consider practical constraints",
+            "Both must balance imagination with reality checks"
+        ],
+        ('Green', 'Red'): [
+            "Structured partner should allow faster execution sometimes",
+            "Action-oriented partner needs to follow established processes",
+            "Both must compromise between speed and quality"
+        ]
+    }
+    
+    key = tuple(sorted([color1, color2]))
+    return combinations.get(key, [
+        "Both partners need to understand different communication styles",
+        "Compromise between individual preferences and shared needs",
+        "Balance personal approaches with relationship harmony"
+    ])
+
+def detailed_explanation(user1_id: int, user2_id: int, u_vec: np.ndarray, v_vec: np.ndarray) -> List[str]:
+    """Main function to generate detailed explanations"""
+    return generate_dynamic_explanation(user1_id, user2_id, u_vec, v_vec)
+
+def generate_dynamic_explanation(user1_id: int, user2_id: int, user1_vec: np.ndarray, user2_vec: np.ndarray) -> List[str]:
+    """Generate dynamic explanation using LLM and knowledge base"""
+    
+    # Get user backgrounds
+    user1_bg = get_user_background(user1_id)
+    user2_bg = get_user_background(user2_id)
+
+    # Create query for knowledge base
+    query = f"compatibility between personality types: {user1_bg.get('conflict_style', '')} and {user2_bg.get('conflict_style', '')}"
+    
+    # Import knowledge inside the function to avoid circular import
+    try:
+        from faiss_service import knowledge
+        context_chunks = knowledge.get_relevant_context(query, topk=2) if knowledge else []
+    except ImportError:
+        context_chunks = []
+    
+    context = "\n".join(context_chunks) if context_chunks else "No specific psychological context available."
+    # Try LLM first if available
+    if HAS_LLM and os.getenv("OPENAI_API_KEY"):
+        try:
+            llm = ChatOpenAI(
+                model="gpt-4o-mini",
+                temperature=0.7,
+                max_retries=2,
+                timeout=30
+            )
+
+            prompt_template = ChatPromptTemplate.from_messages([
+                ("system", """You are a relationship compatibility expert specializing in personality color analysis (Blue, Green, Yellow, Red).
+                Based on the personality profiles, relevant psychological context, and user profiles, provide a detailed compatibility analysis.
+
+                CRITICAL GUIDELINES:
+                1. Focus on practical relationship dynamics, not just theoretical compatibility
+                2. Use insights from the provided psychological context when relevant
+                3. Be specific about strengths and potential challenges
+                4. Provide actionable advice for the couple
+                5. Keep explanations natural and conversational, not robotic
+                6. Reference specific personality traits and how they interact
+                7. Consider cultural and personal background when relevant
+                8. Balance positivity with realistic expectations
+
+                Structure your response with:
+                - Compatibility overview (1-2 sentences)
+                - Key strengths of this pairing
+                - Potential challenges to be aware of
+                - Practical advice for success
+                - Daily life compatibility"""),
+                ("human", """Personality Profiles:
+User 1 ({user1_name}, {user1_gender}): Blue {user1_blue}%, Green {user1_green}%, Yellow {user1_yellow}%, Red {user1_red}%
+User 2 ({user2_name}, {user2_gender}): Blue {user2_blue}%, Green {user2_green}%, Yellow {user2_yellow}%, Red {user2_red}%
+
+User 1 Background: {user1_background}
+User 2 Background: {user2_background}
+
+Relevant Psychological Context:
+{context}
+
+Please provide a comprehensive compatibility analysis:""")
+            ])
+
+            chain = prompt_template | llm | StrOutputParser()
+            
+            response = chain.invoke({
+                "user1_name": user1_bg.get("name", "User 1"),
+                "user1_gender": user1_bg.get("gender", "Not specified"),
+                "user1_blue": round(user1_vec[0] * 100, 1),
+                "user1_green": round(user1_vec[1] * 100, 1),
+                "user1_yellow": round(user1_vec[2] * 100, 1),
+                "user1_red": round(user1_vec[3] * 100, 1),
+                "user2_name": user2_bg.get("name", "User 2"),
+                "user2_gender": user2_bg.get("gender", "Not specified"),
+                "user2_blue": round(user2_vec[0] * 100, 1),
+                "user2_green": round(user2_vec[1] * 100, 1),
+                "user2_yellow": round(user2_vec[2] * 100, 1),
+                "user2_red": round(user2_vec[3] * 100, 1),
+                "user1_background": json.dumps(user1_bg, indent=2),
+                "user2_background": json.dumps(user2_bg, indent=2),
+                "context": context
+            })
+            
+            # Parse LLM response into structured points
+            points = []
+            lines = response.split('\n')
+            for line in lines:
+                line = line.strip()
+                if line and not line.startswith(('- Compatibility', '- Key', '- Potential', '- Practical', '- Daily')):
+                    if line.startswith('•') or line.startswith('-'):
+                        points.append(line[1:].strip())
+                    elif len(line) > 20:  # Substantive lines
+                        points.append(line)
+
+            if points:
+                return points[:5]  # Return top 5 most relevant points
+        except Exception as e:
+            print(f"LLM explanation failed: {e}")
+
+    # Fallback to rule-based explanations
+    return generate_rule_based_explanation(user1_vec, user2_vec, user1_bg, user2_bg)
+
+def generate_rule_based_explanation(user1_vec: np.ndarray, user2_vec: np.ndarray, user1_bg: Dict, user2_bg: Dict) -> List[str]:
+    """Rule-based fallback explanation"""
+    labels = ["Blue", "Green", "Yellow", "Red"]
+    user1_dom = labels[int(np.argmax(user1_vec))]
+    user2_dom = labels[int(np.argmax(user2_vec))]
+
+    explanations = []
+
+    # Dominant trait analysis
+    if user1_dom == user2_dom:
+        explanations.append(f"Both share {user1_dom} dominance: Strong alignment in core approach and values.")
+    else:
+        explanations.append(f"{user1_dom}-{user2_dom} pairing: Complementary strengths create balanced dynamics.")
+
+    # Difference analysis
+    diffs = user2_vec - user1_vec
+    for idx, diff in enumerate(diffs):
+        color = labels[idx]
+        if abs(diff) > 0.15:
+            if diff > 0:
+                explanations.append(f"Higher {color} influence brings {get_color_strength(color)} to the relationship.")
+            else:
+                explanations.append(f"Lower {color} presence allows for more {get_color_balance(color)} in dynamics.")
+
+    # Background considerations
+    if user1_bg.get("hobbies") and user2_bg.get("hobbies"):
+        explanations.append("Shared interests and hobbies create strong bonding opportunities.")
+
+    if user1_bg.get("conflict_style") and user2_bg.get("conflict_style"):
+        explanations.append("Complementary conflict styles can lead to effective problem-solving.")
+
+    return explanations[:4]  # Limit to 4 points
+
+def get_color_strength(color: str) -> str:
+    strengths = {
+        "Blue": "analytical precision and structured thinking",
+        "Green": "emotional stability and patient understanding",
+        "Yellow": "creative energy and social connection",
+        "Red": "decisive action and goal orientation"
+    }
+    return strengths.get(color, "unique strengths")
+
+def get_color_balance(color: str) -> str:
+    balances = {
+        "Blue": "flexibility and spontaneity",
+        "Green": "directness and assertiveness",
+        "Yellow": "focus and routine",
+        "Red": "collaboration and patience"
+    }
+    return balances.get(color, "balanced approaches")

config.py

@@ -0,0 +1,48 @@
+# config.py
+import os
+from dotenv import load_dotenv
+
+# --- load .env so OPENAI_API_KEY (and others) are available ---
+BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+
+# Load environment variables - priority: Hugging Face secrets > .env file
+IS_HUGGING_FACE = os.environ.get('HUGGINGFACE_SPACES') == 'true' or os.environ.get('SPACE_ID') is not None
+if not IS_HUGGING_FACE:
+    # Only load from .env file when running locally
+    load_dotenv(os.path.join(BASE_DIR, ".env"))
+else:
+    # On Hugging Face, secrets are automatically available as environment variables
+    print("Running on Hugging Face Spaces - using secrets from environment variables")
+
+if IS_HUGGING_FACE:
+    # Hugging Face Spaces configuration
+    DEFAULT_SQL_SERVER = "pykara-sqlserver.c5aosm6ie5j3.eu-north-1.rds.amazonaws.com,1433"
+    DEFAULT_SQL_DB = "PyMatch"
+    DEFAULT_SQL_TRUSTED = "yes"  # Use SQL authentication on Hugging Face
+else:
+    # Local development configuration
+    DEFAULT_SQL_SERVER = "localhost\sqlexpress"
+    DEFAULT_SQL_DB = "Py_Match"
+    DEFAULT_SQL_TRUSTED = "yes"  # Use Windows authentication locally
+
+SQL_DRIVER   = os.getenv("PYMATCH_SQL_DRIVER", "ODBC Driver 17 for SQL Server")
+SQL_SERVER   = os.getenv("PYMATCH_SQL_SERVER", DEFAULT_SQL_SERVER)
+SQL_DB       = os.getenv("PYMATCH_SQL_DB", DEFAULT_SQL_DB)
+SQL_TRUSTED  = os.getenv("PYMATCH_SQL_TRUSTED", DEFAULT_SQL_TRUSTED)  # yes/no
+SQL_USER      = os.getenv("PYMATCH_SQL_USER", "")
+SQL_PASSWORD  = os.getenv("PYMATCH_SQL_PASSWORD", "")
+SQL_PORT      = os.getenv("PYMATCH_SQL_PORT", "")
+SQL_ENCRYPT   = os.getenv("PYMATCH_SQL_ENCRYPT", "no").lower().strip()
+SQL_TRUSTCERT = os.getenv("PYMATCH_SQL_TRUST_CERT", "yes").lower().strip()
+
+PROGRESS_TBL = os.getenv("PYMATCH_PROGRESS_TABLE", "LLMGeneratedQuestions")
+DEFAULT_N_QUESTIONS = int(os.getenv("PYMATCH_DEFAULT_N_QUESTIONS", "20"))
+DEFAULT_BATCH_SIZE = int(os.getenv("PYMATCH_DEFAULT_BATCH_SIZE", "10"))
+MAX_QUESTIONS = int(os.getenv("PYMATCH_MAX_QUESTIONS", "50"))
+
+# Some constants used across the app
+COLOR_KEYS = ["blue", "green", "red", "yellow"]
+DOMAINS = ["marriage", "interview", "partnership", "general"]
+
+# Faiss index / chunks defaults - user should update FAISS_INDEX_PATH or provide companion chunks file
+FAISS_INDEX_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "faiss_index_file.index")

database.py

@@ -0,0 +1,151 @@
+# database.py
+import pyodbc
+import urllib.parse
+import hashlib
+import json
+import pickle
+import random
+from typing import Dict, List
+from flask import Flask
+from config import SQL_DRIVER, SQL_SERVER, SQL_DB, SQL_TRUSTED, SQL_USER, SQL_PASSWORD, SQL_PORT, SQL_ENCRYPT, SQL_TRUSTCERT
+from models import db
+
+def get_db_connection():
+    """Get a raw pyodbc connection"""
+    return pyodbc.connect(
+        f"DRIVER={SQL_DRIVER};"
+        f"SERVER={SQL_SERVER};"
+        f"DATABASE={SQL_DB};"
+        f"Trusted_Connection={SQL_TRUSTED};"
+    )
+
+def row_to_dict(cursor, row) -> Dict:
+    """Convert a database row to dictionary"""
+    if row is None:
+        return {}
+    cols = [col[0] for col in cursor.description]
+    return {cols[i]: row[i] for i in range(len(cols))}
+
+def hash_password(password: str) -> str:
+    """Hash password using SHA256"""
+    return hashlib.sha256(password.encode("utf-8")).hexdigest()
+
+def init_database(app: Flask):
+    """Initialize database connection for Flask app"""
+    _server = SQL_SERVER
+    if SQL_PORT:
+        _server = f"{SQL_SERVER},{SQL_PORT}"
+
+    if SQL_TRUSTED == "yes":
+        raw = (
+            f"DRIVER={{{SQL_DRIVER}}};"
+            f"SERVER={_server};"
+            f"DATABASE={SQL_DB};"
+            f"Trusted_Connection=yes;"
+        )
+    else:
+        raw = (
+            f"DRIVER={{{SQL_DRIVER}}};"
+            f"SERVER={_server};"
+            f"DATABASE={SQL_DB};"
+            f"UID={SQL_USER};PWD={SQL_PASSWORD};"
+        )
+
+    if SQL_ENCRYPT == "yes":
+        raw += "Encrypt=yes;"
+    if SQL_TRUSTCERT == "yes":
+        raw += "TrustServerCertificate=yes;"
+
+    params = urllib.parse.quote_plus(raw)
+    SQLALCHEMY_DATABASE_URI = f"mssql+pyodbc:///?odbc_connect={params}"
+
+    app.config["SQLALCHEMY_DATABASE_URI"] = SQLALCHEMY_DATABASE_URI
+    app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
+    
+    db.init_app(app)
+    
+    return db
+
+def fetch_profile_for_role(user_id: str, role: str) -> Dict:
+    """Fetch profile from the correct table based on role"""
+    table = {
+        "marriage": "Marriage",
+        "interview": "Interview",
+        "partnership": "Partnership"
+    }.get(role.lower())
+
+    if not table:
+        return {}
+
+    try:
+        conn = get_db_connection()
+        cur = conn.cursor()
+        cur.execute(f"""
+            SELECT TOP 1 *
+            FROM {table}
+            WHERE user_id = ?
+            ORDER BY created_at DESC
+        """, (user_id,))
+        row = cur.fetchone()
+        if row is None:
+            return {}
+        prof = row_to_dict(cur, row)
+        # Normalize hobbies_interests if it exists
+        if "hobbies_interests" in prof and isinstance(prof["hobbies_interests"], str):
+            if prof["hobbies_interests"].strip().startswith("["):
+                try:
+                    prof["hobbies_interests"] = json.loads(prof["hobbies_interests"])
+                except Exception:
+                    prof["hobbies_interests"] = [s.strip() for s in prof["hobbies_interests"].split(",") if s.strip()]
+            else:
+                prof["hobbies_interests"] = [s.strip() for s in prof["hobbies_interests"].split(",") if s.strip()]
+        prof["user_id"] = str(user_id)
+        return prof
+    except pyodbc.Error as e:
+        print("Profile fetch error:", e)
+        return {}
+    finally:
+        try: conn.close()
+        except: pass
+
+def fetch_expectation_data(user_id: str) -> Dict:
+    """Fetch expectation data from ExpectationResponse table"""
+    try:
+        conn = get_db_connection()
+        cur = conn.cursor()
+        cur.execute("""
+            SELECT * FROM ExpectationResponse
+            WHERE user_id = ?
+            ORDER BY created_at DESC
+        """, (user_id,))
+        row = cur.fetchone()
+        if row is None:
+            return {}
+        return row_to_dict(cur, row)
+    except Exception as e:
+        print(f"Error fetching expectation data: {e}")
+        return {}
+    finally:
+        try: conn.close()
+        except: pass
+
+def fetch_marriage_profile_data(user_id: str) -> Dict:
+    """Fetch marriage profile data for comparison"""
+    try:
+        conn = get_db_connection()
+        cur = conn.cursor()
+        cur.execute("""
+            SELECT * FROM Marriage
+            WHERE user_id = ?
+            ORDER BY created_at DESC
+        """, (user_id,))
+        row = cur.fetchone()
+        if row is None:
+            return {}
+        return row_to_dict(cur, row)
+    except Exception as e:
+        print(f"Error fetching marriage profile data: {e}")
+        return {}
+    finally:
+        try: conn.close()
+        except: pass

faiss_Psychology2e_WEB.index

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c87e897737e6288e24e3c7bd7497b1fe320701f4c4e2671573a5126201b8cde9
+size 1536045

faiss_index_file.index

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d9e53182220a69e6dc00240a4a57e89d4bb8bbce525289677486b81ac800a755
+size 3575853

faiss_service.py

@@ -0,0 +1,168 @@
+# faiss_service.py
+import os
+import json
+import pickle
+import random
+from typing import Dict, List, Tuple, Optional
+
+# Try importing faiss
+try:
+    import faiss
+    HAS_FAISS = True
+except Exception as e:
+    print("faiss import failed:", e)
+    HAS_FAISS = False
+
+# Try importing sentence-transformers
+try:
+    from sentence_transformers import SentenceTransformer  # type: ignore
+    HAS_EMBEDDER = True
+except Exception:
+    SentenceTransformer = None
+    HAS_EMBEDDER = False
+
+from config import BASE_DIR, FAISS_INDEX_PATH
+
+BOOKS = [
+    {
+        "index": os.path.join(BASE_DIR, "psychology2e.index"),
+        "meta":  os.path.join(BASE_DIR, "psychology2e_meta.pkl"),
+        "name":  "Psychology 2e",
+    },
+    {
+        "index": os.path.join(BASE_DIR, "surrounded_by_idiots.index"),
+        "meta":  os.path.join(BASE_DIR, "surrounded_by_idiots_meta.pkl"),
+        "name":  "Surrounded by Idiots",
+    },
+]
+
+FAISS_INDEX = None
+TEXT_CHUNKS: List[str] = []
+
+class KnowledgeSource:
+    def __init__(self):
+        self.indices: List = []
+        self.metas: List[List[Dict]] = []
+        self.embedder = None
+        if not HAS_FAISS:
+            return
+        try:
+            if SentenceTransformer:
+                self.embedder = SentenceTransformer("all-MiniLM-L6-v2")
+        except Exception:
+            self.embedder = None
+        for b in BOOKS:
+            try:
+                idx_path = b["index"]
+                meta_path = b["meta"]
+                if os.path.exists(idx_path) and os.path.exists(meta_path):
+                    index = faiss.read_index(idx_path)
+                    with open(meta_path, "rb") as f:
+                        meta = pickle.load(f)
+                    self.indices.append(index)
+                    self.metas.append(meta)
+            except Exception:
+                continue
+
+    def get_relevant_context(self, query: str, topk: int = 3) -> List[str]:
+        """Get relevant context from knowledge base for relationship matching"""
+        if not self.indices or not self.embedder:
+            return []
+        try:
+            vec = self.embedder.encode([query]).astype("float32")
+            results: List[Tuple[float, str]] = []
+            for index, meta in zip(self.indices, self.metas):
+                D, I = index.search(vec, topk)
+                for d, i in zip(D[0], I[0]):
+                    if 0 <= i < len(meta):
+                        txt = meta[i].get("text", "")[:500]  # Increased length for better context
+                        results.append((float(d), txt))
+            results.sort(key=lambda x: x[0])
+            return [t for _, t in results[:topk]]
+        except Exception:
+            return []
+
+def try_load_chunks_from_disk(index_path: str) -> List[str]:
+    """Try several companion filenames for the chunk/text mapping."""
+    base = os.path.splitext(index_path)[0]
+    candidates = [base + ".chunks.json", base + "_chunks.json", base + ".chunks.pkl", base + "_chunks.pkl"]
+
+    for c in candidates:
+        if os.path.exists(c):
+            try:
+                if c.endswith(".json"):
+                    with open(c, "r", encoding="utf-8") as f:
+                        data = json.load(f)
+                        # expecting list of strings
+                        if isinstance(data, list):
+                            return data
+                        # sometimes stored as {"chunks": [...]}
+                        if isinstance(data, dict) and "chunks" in data:
+                            return data["chunks"]
+                else:
+                    with open(c, "rb") as f:
+                        data = pickle.load(f)
+                        if isinstance(data, list):
+                            return data
+            except Exception as e:
+                print(f"Failed to load chunks from {c}:", e)
+    return []
+
+def load_faiss_index(index_path: str):
+    global FAISS_INDEX, TEXT_CHUNKS
+    if not HAS_FAISS:
+        print("FAISS not installed. Skipping index load.")
+        return
+    if not os.path.exists(index_path):
+        print("Faiss index path does not exist:", index_path)
+        return
+    try:
+        FAISS_INDEX = faiss.read_index(index_path)
+        # try to load chunks from companion files
+        TEXT_CHUNKS = try_load_chunks_from_disk(index_path)
+        if not TEXT_CHUNKS:
+            print("Warning: Faiss index loaded but no companion text chunks found.")
+            print("Provide a companion .chunks.json or .chunks.pkl file with a list of text chunks.")
+    except Exception as e:
+        print("Failed to load faiss index:", e)
+        FAISS_INDEX = None
+
+def get_nearest_context(query_emb: List[float] = None, k: int = 5, query_vector: Optional[List[float]] = None):
+    """Return concatenated top-k chunks for a query."""
+    if FAISS_INDEX is None or not HAS_FAISS:
+        return ""
+    try:
+        # ... rest of the existing code remains the same
+        if query_vector is None:
+            # no embedding generation in this script: rely on an external embedding or LLM to provide a context id
+            return ""
+        import numpy as np
+        vec = np.array([query_vector], dtype='float32')
+        D, I = FAISS_INDEX.search(vec, k)
+        idxs = I[0].tolist()
+        texts = []
+        for idx in idxs:
+            if 0 <= idx < len(TEXT_CHUNKS):
+                texts.append(TEXT_CHUNKS[idx])
+        return "\n\n".join(texts)
+    except Exception as e:
+        print("Faiss search failed:", e)
+        return ""
+
+def get_faiss_context(k=3):
+    """Return k random FAISS text chunks for LLM context."""
+    if HAS_FAISS and TEXT_CHUNKS:
+        return "\n".join(random.sample(TEXT_CHUNKS, min(k, len(TEXT_CHUNKS))))
+    return ""
+
+# Initialize knowledge base only if FAISS is available
+knowledge = None
+if HAS_FAISS:
+    knowledge = KnowledgeSource()
+else:
+    print("FAISS not available - KnowledgeSource disabled")
+    # Create a dummy knowledge object with empty methods
+    class DummyKnowledge:
+        def get_relevant_context(self, *args, **kwargs):
+            return []
+    knowledge = DummyKnowledge()

llm_service.py

@@ -0,0 +1,948 @@
+# llm_service.py
+import pyodbc
+import os
+import json
+import uuid
+import random
+import threading
+from typing import Dict, List, Optional
+from datetime import datetime
+
+from config import DEFAULT_N_QUESTIONS, DEFAULT_BATCH_SIZE, MAX_QUESTIONS, COLOR_KEYS, DOMAINS
+
+# Try importing LLM libraries
+try:
+    from pydantic import BaseModel, Field
+    from langchain_core.prompts import ChatPromptTemplate
+    from langchain_core.output_parsers import PydanticOutputParser, StrOutputParser
+    from langchain_openai import ChatOpenAI
+    HAS_LLM_STACK = True
+    HAS_LLM = True
+except Exception:
+    HAS_LLM_STACK = False
+    HAS_LLM = False
+
+class Option(BaseModel):
+    text: str
+    color: str
+
+class QAItem(BaseModel):
+    question: str
+    options: List[Option] = Field(min_items=4, max_items=4)
+
+class BatchQA(BaseModel):
+    items: List[QAItem] = Field(..., min_items=1)
+SYSTEM_PROMPT = (
+    "You write marriage compatibility assessment questions that reveal four personality colors through forced choices:\n"
+    "- blue=analytical, fact-based (positive: thorough, precise | negative: overly critical, data-obsessed)\n"
+    "- green=organized, process-oriented (positive: systematic, reliable | negative: rigid, bureaucratic)\n"
+    "- red=decisive, action-oriented (positive: direct, results-driven | negative: impulsive, controlling)\n"
+    "- yellow=creative, big-picture (positive: innovative, visionary | negative: unrealistic, scattered)\n"
+    "\n"
+    "DISTRIBUTION REQUIREMENT:\n"
+    "For a 20-question assessment, you MUST generate:\n"
+    "1) 5 PROFILE-BASED questions (25%): Using ONLY user's current background (education, employment, hobbies, family)\n"
+    "2) 5 EXPECTATION-BASED questions (25%): Using ONLY user's relationship expectations (conflict style, financial preferences, values)\n"
+    "3) 10 CHARACTER-BASED questions (50%): About CURRENT behavior in various life situations\n"
+    "\n"
+    "CRITICAL RULES:\n"
+    "1) NO PREFIXES: Do NOT use phrases like 'Based on your profile' or 'Given your expectations' - embed naturally\n"
+    "2) Each option MUST clearly represent one color's typical behavior (include both positive and negative aspects)\n"
+    "3) Questions should force a choice that reveals CORE PERSONALITY TRAITS, not future plans\n"
+    "4) Use simple, everyday language but maintain situational specificity\n"
+    "5) Options should be practical actions someone would actually take (show both strengths and weaknesses)\n"
+    "6) Focus on CURRENT CHARACTER ANALYSIS, not future hypotheticals\n"
+    "7) Ensure each color option is distinctly different from others\n"
+    "8) Each option MUST be maximum 15 words - keep them concise and clear\n"
+    "9) AVOID future-focused questions about children, family planning, or long-term hypotheticals\n"
+    "10) Include some options that reveal potential negative/shadow aspects of each color\n"
+    "\n"
+    "QUESTION STYLE GUIDELINES:\n"
+    "- Profile questions: Embed background naturally (e.g., 'When learning something new, how do you approach it?' not 'Given your education...')\n"
+    "- Expectation questions: Embed values in scenarios (e.g., 'When managing shared expenses, what's your approach?')\n"
+    "- Character questions: Use everyday situations (e.g., 'When faced with an unexpected problem at work, what do you do first?')\n"
+)
+
+USER_PROMPT_BATCH = (
+    "Context (from Surrounded by Idiots or other corpus):\n{context}\n\n"
+    "Question Type: {question_type}\n\n"  # Add this line
+    
+    "User Profile (Current Background):\n"
+    "- Education: {education}\n"
+    "- Employment: {employment}\n"
+    "- Hobbies: {hobbies}\n"
+    "- Family Background: {family_type}\n"
+    "- Current Lifestyle: {current_lifestyle}\n"
+    "\n"
+    "User Relationship Expectations:\n"
+    "- Conflict Style: {conflict_style}\n"
+    "- Financial Style: {financial_style}\n"
+    "- Income Expectations: {income_range}\n"
+    "- Career Mobility: {relocation_willingness}\n"
+    "- Family Values: {family_values}\n"
+    "- Core Values: {core_values}\n"
+    "- Work-Life Balance: {lifestyle_pref}\n"
+    "- Social Preference: {social_pref}\n"
+    "- Ambition Level: {ambition_pref}\n"
+    "- Deal Breakers: {deal_breakers}\n"
+    "\n"
+    "Themes (array of short strings): {themes_json}\n"
+    "Previously asked questions: {previous_questions}\n\n"
+    "{format_instructions}\n\n"
+    
+    "Generate {question_type} questions:\n"
+    "- If QUESTION TYPE is 'profile': Generate 5 questions using ONLY profile data (education, employment, hobbies, family background, current lifestyle)\n"
+    "- If QUESTION TYPE is 'expectation': Generate 5 questions using ONLY expectation data (conflict style, financial preferences, values, deal breakers)\n"
+    "- If QUESTION TYPE is 'character': Generate 10 questions about CURRENT behavior in various life situations\n\n"
+    
+    "CRITICAL RULES:\n"
+    "1) DO NOT use prefixes like 'Based on your profile' or 'Considering your expectations'\n"
+    "2) Questions should be natural and flow conversationally\n"
+    "3) Focus on CURRENT traits and behaviors, not future plans\n"
+    "4) Each option must represent a clear personality color (blue=analytical, green=organized, red=decisive, yellow=creative)\n"
+    "5) Include both positive and negative aspects in options\n"
+    "6) Maximum 15 words per option\n"
+    "7) For profile questions: Reference background naturally without explicit labels\n"
+    "8) For expectation questions: Embed values naturally in the scenario\n"
+    "9) For character questions: Use everyday situations that reveal core personality\n"
+)
+
+PARSER_BATCH = None
+CHAIN_BATCH = None
+
+if HAS_LLM_STACK and os.getenv("OPENAI_API_KEY"):
+    try:
+        PARSER_BATCH = PydanticOutputParser(pydantic_object=BatchQA)
+        
+        def build_batch_chain():
+            llm = ChatOpenAI(
+                model="gpt-4o-mini",
+                temperature=0.7,
+                max_retries=2,
+                timeout=30,
+                model_kwargs={"response_format": {"type": "json_object"}},
+            )
+            prompt = ChatPromptTemplate.from_messages([
+                ("system", SYSTEM_PROMPT),
+                ("user", USER_PROMPT_BATCH),
+            ])
+            return prompt | llm | PARSER_BATCH
+
+        CHAIN_BATCH = build_batch_chain()
+    except Exception as e:
+        print("Failed to build CHAIN_BATCH:", e)
+        CHAIN_BATCH = None
+
+def ensure_valid_colors(options: List[Dict]) -> List[Dict]:
+    seen, fixed = set(), []
+    defaults = {
+        "blue": "Verify facts and numbers",
+        "green": "Outline a clear process",
+        "red": "Coordinate people and act",
+        "yellow": "Propose a fresh idea",
+    }
+    for o in options:
+        c = str(o.get("color", "")).lower()
+        t = str(o.get("text", "")).strip()
+        if c in COLOR_KEYS and c not in seen and t:
+            seen.add(c); fixed.append({"text": t[:80], "color": c})
+    for c in COLOR_KEYS:
+        if c not in seen:
+            fixed.append({"text": defaults[c], "color": c})
+    return fixed[:4]
+
+def summarize_profile(profile: Dict) -> Dict:
+    """Extract all non-PII columns from Marriage table for LLM context"""
+    out: Dict = {}
+    
+    # All columns from Marriage table (excluding PII where possible)
+    marriage_columns = [
+        "user_id", "full_name", "gender", "current_city", "marital_status",
+        "education_level", "employment_status", "number_of_siblings", "family_type",
+        "hobbies_interests", "conflict_approach", "financial_style", "income_range",
+        "relocation_willingness", "height", "skin_tone", "languages_spoken", "country",
+        "blood_group", "religion", "dual_citizenship", "siblings_position",
+        "parents_living_status", "live_with_parents", "support_parents_financially",
+        "family_communication_frequency", "food_preference", "smoking_habit",
+        "alcohol_habit", "daily_routine", "fitness_level", "own_pets", "travel_preference",
+        "relaxation_mode", "job_role", "work_experience_years", "career_aspirations",
+        "field_of_study", "remark", "children_timeline", "open_to_adoption",
+        "deal_breakers", "other_non_negotiables", "health_constraints", "live_with_inlaws"
+    ]
+    
+    for col in marriage_columns:
+        v = profile.get(col)
+        if v not in (None, "", []):
+            out[col] = v
+    
+    return out
+
+def offline_generate_batch(themes: List[str], state: Dict, context: str = "") -> List[Dict]:
+    prof = state.get("profile", {}) or {}
+    name = prof.get("full_name") or "Partner"
+    conflict = (prof.get("conflict_approach") or "").lower()
+    money = (prof.get("financial_style") or "").lower()
+    hobby = None
+    if isinstance(prof.get("hobbies_interests"), list) and prof["hobbies_interests"]:
+        hobby = random.choice(prof["hobbies_interests"])
+
+    def flavor():
+        bits = []
+        if conflict:
+            bits.append(f"{conflict} style")
+        if money:
+            bits.append(f"{money} finances")
+        if hobby:
+            bits.append(f"likes {hobby}")
+        return ", ".join(bits)
+
+    items = []
+    for theme in themes:
+        short = theme.split(" around ")[-1].strip()
+
+        # Simple fallback question generation without PROMPT_SCENARIOS_BY_THEME
+        tail = (", " + flavor()) if flavor() else ""
+        q = f"{name}, what would you do about {short}?".strip()
+
+        # incorporate small bit from context if available (first 120 chars)
+        if context:
+            ctx_snip = context.replace('\n', ' ')[:120]
+            q = f"{q} (Note: {ctx_snip})"
+
+        # Keep concise
+        if len(q.split()) > 20:
+            q = " ".join(q.split()[:20])
+
+        opts = [
+            {"text": "Check data and facts", "color": "blue"},
+            {"text": "Draft a step-by-step plan", "color": "green"},
+            {"text": "Align people and act", "color": "red"},
+            {"text": "Brainstorm bold ideas", "color": "yellow"},
+        ]
+        random.shuffle(opts)
+        items.append({"question": q, "options": opts, "source": "fallback"})
+    return items
+
+
+def generate_category_specific_options(question_type: str, question_text: str, profile_data: Dict = None, expectation_data: Dict = None) -> List[Dict]:
+    """
+    Generate options tailored to the question category and content
+    """
+    question_lower = question_text.lower()
+    
+    # Extract key themes from question for better contextualization
+    themes_in_question = []
+    for theme in ["learning", "problem", "conflict", "money", "family", "work", "social", "stress", "decision", "plan"]:
+        if theme in question_lower:
+            themes_in_question.append(theme)
+    
+    # Default option templates for each color
+    base_options = {
+        "blue": {
+            "profile": "Research thoroughly and analyze all available data",
+            "expectation": "Gather detailed information before forming an opinion",
+            "character": "Analyze the situation carefully with facts and logic"
+        },
+        "green": {
+            "profile": "Follow a structured, step-by-step approach",
+            "expectation": "Establish clear rules and procedures",
+            "character": "Create an organized plan and stick to it"
+        },
+        "red": {
+            "profile": "Take decisive action to address the situation",
+            "expectation": "Take charge and make things happen quickly",
+            "character": "Act immediately and coordinate people involved"
+        },
+        "yellow": {
+            "profile": "Explore creative possibilities and new approaches",
+            "expectation": "Consider innovative solutions and future potential",
+            "character": "Brainstorm creative ideas and possibilities"
+        }
+    }
+    
+    # Contextual variations based on question themes
+    contextual_variations = {
+        "learning": {
+            "blue": "Study methodically and verify all information",
+            "green": "Follow the curriculum in an organized manner",
+            "red": "Jump into practical application immediately",
+            "yellow": "Explore unconventional learning methods"
+        },
+        "problem": {
+            "blue": "Analyze root causes with data",
+            "green": "Systematically troubleshoot each component",
+            "red": "Take immediate corrective action",
+            "yellow": "Find innovative workarounds"
+        },
+        "conflict": {
+            "blue": "Analyze perspectives logically",
+            "green": "Establish fair mediation process",
+            "red": "Address it directly and decisively",
+            "yellow": "Find creative compromise"
+        },
+        "money": {
+            "blue": "Analyze financial data thoroughly",
+            "green": "Budget systematically and track expenses",
+            "red": "Make decisive investment choices",
+            "yellow": "Explore unconventional earning opportunities"
+        },
+        "family": {
+            "blue": "Analyze family dynamics logically",
+            "green": "Maintain family traditions and routines",
+            "red": "Take leadership in family matters",
+            "yellow": "Introduce new family activities"
+        }
+    }
+    
+    # Start with base options for the category
+    options = []
+    for color in COLOR_KEYS:
+        base_text = base_options[color][question_type]
+        
+        # Add contextual variation if theme matches
+        for theme, variations in contextual_variations.items():
+            if theme in themes_in_question:
+                base_text = variations[color]
+                break
+        
+        # Add shadow/negative aspects for realism
+        shadow_aspects = {
+            "blue": {
+                "profile": " (but can get stuck in analysis)",
+                "expectation": " (but may overanalyze)",
+                "character": " (but can be overly critical)"
+            },
+            "green": {
+                "profile": " (but can be too rigid)",
+                "expectation": " (but may create bureaucracy)",
+                "character": " (but can resist change)"
+            },
+            "red": {
+                "profile": " (but can be impulsive)",
+                "expectation": " (but may be controlling)",
+                "character": " (but can overlook details)"
+            },
+            "yellow": {
+                "profile": " (but can be unrealistic)",
+                "expectation": " (but may lack follow-through)",
+                "character": " (but can be scattered)"
+            }
+        }
+        
+        # Only add shadow aspects occasionally (30% chance) for variety
+        if random.random() < 0.3:
+            shadow = shadow_aspects[color][question_type]
+            # Ensure we don't exceed word limit
+            if len(base_text.split()) + len(shadow.split()) <= 15:
+                base_text += shadow
+        
+        options.append({
+            "text": base_text[:80],  # Limit length
+            "color": color
+        })
+    
+    return options
+
+
+
+
+
+
+def generate_batch_questions(themes: List[str], state: Dict, context: str = "", previous_questions: List[str] = None) -> List[Dict]:
+    # Extract ALL data from Marriage table
+    profile = state.get("profile", {})
+    user_id = profile.get("user_id")
+    
+    try:
+        from database import fetch_expectation_data
+        expectation_data = fetch_expectation_data(user_id) if user_id else {}
+    except ImportError:
+        expectation_data = {}
+    
+    # Extract ALL profile data from Marriage table
+    profile_data = {
+        # Personal Information
+        "full_name": profile.get("full_name", "Not specified"),
+        
+        "gender": profile.get("gender", "Not specified"),
+        "current_city": profile.get("current_city", "Not specified"),
+        "country": profile.get("country", "Not specified"),
+        "marital_status": profile.get("marital_status", "Not specified"),
+        
+        # Education & Career
+        "education_level": profile.get("education_level", "Not specified"),
+        "employment_status": profile.get("employment_status", "Not specified"),
+        "job_role": profile.get("job_role", "Not specified"),
+        "work_experience_years": profile.get("work_experience_years", "Not specified"),
+        "career_aspirations": profile.get("career_aspirations", "Not specified"),
+        "field_of_study": profile.get("field_of_study", "Not specified"),
+        "income_range": profile.get("income_range", "Not specified"),
+        
+        # Family & Background
+        "number_of_siblings": profile.get("number_of_siblings", "Not specified"),
+        "family_type": profile.get("family_type", "Not specified"),
+        "siblings_position": profile.get("siblings_position", "Not specified"),
+        "parents_living_status": profile.get("parents_living_status", "Not specified"),
+        "live_with_parents": profile.get("live_with_parents", "Not specified"),
+        "support_parents_financially": profile.get("support_parents_financially", "Not specified"),
+        "family_communication_frequency": profile.get("family_communication_frequency", "Not specified"),
+        
+        # Physical & Health
+        "height": profile.get("height", "Not specified"),
+        "skin_tone": profile.get("skin_tone", "Not specified"),
+        "blood_group": profile.get("blood_group", "Not specified"),
+        "health_constraints": profile.get("health_constraints", "Not specified"),
+        "fitness_level": profile.get("fitness_level", "Not specified"),
+        
+        # Lifestyle & Habits
+        "hobbies_interests": str(profile.get("hobbies_interests", "Not specified")),
+        "conflict_approach": profile.get("conflict_approach", "Not specified"),
+        "financial_style": profile.get("financial_style", "Not specified"),
+        "food_preference": profile.get("food_preference", "Not specified"),
+        "smoking_habit": profile.get("smoking_habit", "Not specified"),
+        "alcohol_habit": profile.get("alcohol_habit", "Not specified"),
+        "daily_routine": profile.get("daily_routine", "Not specified"),
+        "own_pets": profile.get("own_pets", "Not specified"),
+        "travel_preference": profile.get("travel_preference", "Not specified"),
+        "relaxation_mode": profile.get("relaxation_mode", "Not specified"),
+        
+        # Languages & Relocation
+        "languages_spoken": profile.get("languages_spoken", "Not specified"),
+        "relocation_willingness": profile.get("relocation_willingness", "Not specified"),
+        
+        # Religion & Citizenship
+        "religion": profile.get("religion", "Not specified"),
+        "dual_citizenship": profile.get("dual_citizenship", "Not specified"),
+        
+        # Relationship Preferences
+        "children_timeline": profile.get("children_timeline", "Not specified"),
+        "open_to_adoption": profile.get("open_to_adoption", "Not specified"),
+        "deal_breakers": profile.get("deal_breakers", "Not specified"),
+        "other_non_negotiables": profile.get("other_non_negotiables", "Not specified"),
+        "live_with_inlaws": profile.get("live_with_inlaws", "Not specified"),
+        
+        # Additional Info
+        "remark": profile.get("remark", "Not specified"),
+    }
+    
+    # Extract ALL expectation data from ExpectationResponse table
+    expectation_data_dict = {
+        # Basic Preferences
+        "pref_age_range": expectation_data.get("pref_age_range", "Not specified"),
+        "pref_height_range": expectation_data.get("pref_height_range", "Not specified"),
+        "pref_current_city": expectation_data.get("pref_current_city", "Not specified"),
+        "pref_countries": expectation_data.get("pref_countries", "Not specified"),
+        "pref_languages": expectation_data.get("pref_languages", "Not specified"),
+        "pref_education_level": expectation_data.get("pref_education_level", "Not specified"),
+        "pref_employment_status": expectation_data.get("pref_employment_status", "Not specified"),
+        
+        # Health & Lifestyle
+        "health_constraints": expectation_data.get("health_constraints", "Not specified"),
+        "pref_diet": expectation_data.get("pref_diet", "Not specified"),
+        "accept_smoking": expectation_data.get("accept_smoking", "Not specified"),
+        "accept_alcohol": expectation_data.get("accept_alcohol", "Not specified"),
+        "pref_fitness": expectation_data.get("pref_fitness", "Not specified"),
+        
+        # Family & Living
+        "pref_family_type": expectation_data.get("pref_family_type", "Not specified"),
+        "live_with_inlaws": expectation_data.get("live_with_inlaws", "Not specified"),
+        "children_timeline": expectation_data.get("children_timeline", "Not specified"),
+        "open_to_adoption": expectation_data.get("open_to_adoption", "Not specified"),
+        "pref_live_with_parents": expectation_data.get("pref_live_with_parents", "Not specified"),
+        "financial_support_to_parents": expectation_data.get("financial_support_to_parents", "Not specified"),
+        
+        # Conflict & Finance
+        "pref_conflict_approach": expectation_data.get("pref_conflict_approach", "Not specified"),
+        "pref_financial_style": expectation_data.get("pref_financial_style", "Not specified"),
+        "pref_income_range": expectation_data.get("pref_income_range", "Not specified"),
+        
+        # Values & Compatibility
+        "religion_alignment": expectation_data.get("religion_alignment", "Not specified"),
+        "pref_shared_hobbies": expectation_data.get("pref_shared_hobbies", "Not specified"),
+        "travel_pref": expectation_data.get("travel_pref", "Not specified"),
+        "pet_pref": expectation_data.get("pet_pref", "Not specified"),
+        
+        # Career & Relocation
+        "pref_partner_relocation": expectation_data.get("pref_partner_relocation", "Not specified"),
+        "pref_career_aspirations": expectation_data.get("pref_career_aspirations", "Not specified"),
+        
+        # Additional Preferences
+        "marital_status": expectation_data.get("marital_status", "Not specified"),
+        "skin_tone": expectation_data.get("skin_tone", "Not specified"),
+        "daily_routine": expectation_data.get("daily_routine", "Not specified"),
+        "family_communication_frequency": expectation_data.get("family_communication_frequency", "Not specified"),
+        "relaxation_mode": expectation_data.get("relaxation_mode", "Not specified"),
+        
+        # Non-negotiables
+        "deal_breakers": expectation_data.get("deal_breakers", "Not specified"),
+        "other_non_negotiables": expectation_data.get("other_non_negotiables", "Not specified"),
+        
+        # Summary
+        "expectation_summary": expectation_data.get("expectation_summary", "Not specified"),
+        "_mandatory_fields": expectation_data.get("_mandatory_fields", "Not specified"),
+    }
+    
+    if CHAIN_BATCH is not None and PARSER_BATCH is not None:
+        try:
+            items: List[Dict] = []
+            
+            # 1. PROFILE-BASED QUESTIONS (5 questions) - Using ALL Marriage table columns
+            profile_prompt = {
+                "state": json.dumps(state, ensure_ascii=False),
+                "themes_json": json.dumps(["profile"] * 5, ensure_ascii=False),
+                "previous_questions": json.dumps(previous_questions or [], ensure_ascii=False),
+                "format_instructions": PARSER_BATCH.get_format_instructions(),
+                "context": "Generate 5 PROFILE-BASED questions using ALL available user background information.",
+                "question_type": "profile",
+                
+                # Use ALL profile data
+                "education": f"{profile_data['education_level']} | {profile_data['field_of_study']}",
+                "employment": f"{profile_data['employment_status']} | {profile_data['job_role']} ({profile_data['work_experience_years']} years)",
+                "hobbies": profile_data['hobbies_interests'],
+                "family_type": f"{profile_data['family_type']} | Siblings: {profile_data['number_of_siblings']} | Position: {profile_data['siblings_position']}",
+                "current_lifestyle": f"City: {profile_data['current_city']}, {profile_data['country']} | Height: {profile_data['height']} | Languages: {profile_data['languages_spoken']} | Religion: {profile_data['religion']} | Fitness: {profile_data['fitness_level']} | Diet: {profile_data['food_preference']} | Habits: Smoking: {profile_data['smoking_habit']}, Alcohol: {profile_data['alcohol_habit']}",
+                
+                # Expectation data marked as irrelevant
+                "conflict_style": "IRRELEVANT_FOR_PROFILE_QUESTIONS",
+                "financial_style": "IRRELEVANT_FOR_PROFILE_QUESTIONS",
+                "income_range": "IRRELEVANT_FOR_PROFILE_QUESTIONS",
+                "relocation_willingness": "IRRELEVANT_FOR_PROFILE_QUESTIONS",
+                "family_values": "IRRELEVANT_FOR_PROFILE_QUESTIONS",
+                "core_values": "IRRELEVANT_FOR_PROFILE_QUESTIONS",
+                "lifestyle_pref": "IRRELEVANT_FOR_PROFILE_QUESTIONS",
+                "social_pref": "IRRELEVANT_FOR_PROFILE_QUESTIONS",
+                "ambition_pref": "IRRELEVANT_FOR_PROFILE_QUESTIONS",
+                "deal_breakers": "IRRELEVANT_FOR_PROFILE_QUESTIONS",
+            }
+            
+            result = CHAIN_BATCH.invoke(profile_prompt)
+            profile_items = get_items_from_result(result)
+            
+            for qa in profile_items[:5]:
+                out = qa.dict() if hasattr(qa, "dict") else dict(qa)
+                out["options"] = generate_category_specific_options(
+                    "profile", 
+                    out.get("question", ""),
+                    profile_data,
+                    None
+                )
+                out["source"] = "llm_profile"
+                out["question_type"] = "profile"
+                random.shuffle(out["options"])
+                items.append(out)
+            
+            # 2. EXPECTATION-BASED QUESTIONS (5 questions) - Using ALL ExpectationResponse columns
+            expectation_prompt = {
+                "state": json.dumps(state, ensure_ascii=False),
+                "themes_json": json.dumps(["expectation"] * 5, ensure_ascii=False),
+                "previous_questions": json.dumps([q["question"] for q in items] + (previous_questions or []), ensure_ascii=False),
+                "format_instructions": PARSER_BATCH.get_format_instructions(),
+                "context": "Generate 5 EXPECTATION-BASED questions using ALL relationship preferences and expectations.",
+                "question_type": "expectation",
+                
+                # Minimal profile context
+                "education": "Background context only",
+                "employment": "Background context only",
+                "hobbies": "Background context only",
+                "family_type": "Background context only",
+                "current_lifestyle": "General context",
+                
+                # Use ALL expectation data
+                "conflict_style": f"{expectation_data_dict['pref_conflict_approach']}",
+                "financial_style": f"{expectation_data_dict['pref_financial_style']} | Income: {expectation_data_dict['pref_income_range']}",
+                "income_range": expectation_data_dict['pref_income_range'],
+                "relocation_willingness": f"{expectation_data_dict['pref_partner_relocation']}",
+                "family_values": f"{expectation_data_dict['pref_family_type']} | Live with in-laws: {expectation_data_dict['live_with_inlaws']} | Children timeline: {expectation_data_dict['children_timeline']}",
+                "core_values": f"Religion: {expectation_data_dict['religion_alignment']} | Deal breakers: {expectation_data_dict['deal_breakers']}",
+                "lifestyle_pref": f"Fitness: {expectation_data_dict['pref_fitness']} | Diet: {expectation_data_dict['pref_diet']} | Daily routine: {expectation_data_dict['daily_routine']}",
+                "social_pref": f"Hobbies: {expectation_data_dict['pref_shared_hobbies']} | Travel: {expectation_data_dict['travel_pref']} | Pets: {expectation_data_dict['pet_pref']}",
+                "ambition_pref": f"Career: {expectation_data_dict['pref_career_aspirations']} | Education: {expectation_data_dict['pref_education_level']}",
+                "deal_breakers": f"{expectation_data_dict['deal_breakers']} | Other non-negotiables: {expectation_data_dict['other_non_negotiables']}",
+            }
+            
+            result = CHAIN_BATCH.invoke(expectation_prompt)
+            expectation_items = get_items_from_result(result)
+            
+            for qa in expectation_items[:5]:
+                out = qa.dict() if hasattr(qa, "dict") else dict(qa)
+                out["options"] = generate_category_specific_options(
+                    "expectation", 
+                    out.get("question", ""),
+                    None,
+                    expectation_data_dict
+                )
+                out["source"] = "llm_expectation"
+                out["question_type"] = "expectation"
+                random.shuffle(out["options"])
+                items.append(out)
+            
+            # 3. CHARACTER-BASED QUESTIONS (10 questions) - Using data from BOTH tables
+            character_prompt = {
+                "state": json.dumps(state, ensure_ascii=False),
+                "themes_json": json.dumps(themes[:10], ensure_ascii=False),
+                "previous_questions": json.dumps([q["question"] for q in items] + (previous_questions or []), ensure_ascii=False),
+                "format_instructions": PARSER_BATCH.get_format_instructions(),
+                "context": context[:2000] + "\n\nGenerate 10 CHARACTER-BASED questions using ALL available data.",
+                "question_type": "character",
+                
+                # All data from Marriage table
+                "education": profile_data['education_level'],
+                "employment": profile_data['employment_status'],
+                "hobbies": profile_data['hobbies_interests'],
+                "family_type": profile_data['family_type'],
+                "current_lifestyle": f"{profile_data['current_city']}, {profile_data['country']} | {profile_data['daily_routine']} | Relaxation: {profile_data['relaxation_mode']}",
+                
+                # All data from ExpectationResponse table
+                "conflict_style": expectation_data_dict['pref_conflict_approach'],
+                "financial_style": expectation_data_dict['pref_financial_style'],
+                "income_range": expectation_data_dict['pref_income_range'],
+                "relocation_willingness": expectation_data_dict['pref_partner_relocation'],
+                "family_values": expectation_data_dict['pref_family_type'],
+                "core_values": expectation_data_dict['religion_alignment'],
+                "lifestyle_pref": expectation_data_dict['pref_fitness'],
+                "social_pref": expectation_data_dict['pref_shared_hobbies'],
+                "ambition_pref": expectation_data_dict['pref_career_aspirations'],
+                "deal_breakers": expectation_data_dict['deal_breakers'],
+            }
+            
+            result = CHAIN_BATCH.invoke(character_prompt)
+            character_items = get_items_from_result(result)
+            
+            for qa in character_items[:10]:
+                out = qa.dict() if hasattr(qa, "dict") else dict(qa)
+                out["options"] = generate_category_specific_options(
+                    "character", 
+                    out.get("question", ""),
+                    profile_data,
+                    expectation_data_dict
+                )
+                out["source"] = "llm_character"
+                out["question_type"] = "character"
+                random.shuffle(out["options"])
+                items.append(out)
+            
+            # Verify we have exactly 20 questions
+            if len(items) == 20:
+                return items[:20]
+            else:
+                # If LLM didn't generate enough, fill with fallback
+                return fill_missing_questions(items, themes, state, profile_data, expectation_data_dict, context)
+                
+        except Exception as e:
+            print("LLM batch generation failed:", e)
+            return generate_fallback_with_distribution(themes, state, profile_data, expectation_data_dict, context)
+    else:
+        return generate_fallback_with_distribution(themes, state, profile_data, expectation_data_dict, context)
+
+
+
+
+def get_items_from_result(result):
+    """Helper to extract items from LLM result"""
+    if hasattr(result, "items"):
+        return result.items
+    elif isinstance(result, dict) and "items" in result:
+        return result["items"]
+    else:
+        return []
+
+def fill_missing_questions(current_items: List[Dict], themes: List[str], state: Dict, 
+                          profile_data: Dict, expectation_data: Dict, context: str = "") -> List[Dict]:
+    """Fill missing questions to reach 20 total"""
+    items = current_items.copy()
+    
+    # Count current distribution
+    profile_count = sum(1 for q in items if q.get("question_type") == "profile")
+    expectation_count = sum(1 for q in items if q.get("question_type") == "expectation")
+    character_count = sum(1 for q in items if q.get("question_type") == "character")
+    
+    # Fill profile questions if needed
+    while profile_count < 5:
+        profile_q = generate_profile_question(state, profile_data)
+        items.append(profile_q)
+        profile_count += 1
+    
+    # Fill expectation questions if needed
+    while expectation_count < 5:
+        expectation_q = generate_expectation_question(state, expectation_data)
+        items.append(expectation_q)
+        expectation_count += 1
+    
+    # Fill character questions if needed
+    while character_count < 10:
+        theme = themes[character_count % len(themes)] if themes else "daily situation"
+        character_q = generate_character_question(theme, state)
+        items.append(character_q)
+        character_count += 1
+    
+    return items[:20]
+
+def generate_profile_question(state: Dict, profile_data: Dict) -> Dict:
+    """Generate a single profile question"""
+    prof = state.get("profile", {})
+    name = prof.get("full_name") or "Partner"
+    
+    profile_topics = [
+        ("education", f"How does your educational background shape how you approach complex information?"),
+        ("employment", f"What methods from your professional life do you apply to personal challenges?"),
+        ("hobbies", f"How do your personal interests influence your approach to new experiences?"),
+        ("family", f"What communication patterns from your family background feel most natural to you?"),
+        ("background", f"How does your personal history affect your current decision-making style?")
+    ]
+    
+    topic_idx = len([q for q in state.get("history", []) if q.get("question_type") == "profile"])
+    if topic_idx >= len(profile_topics):
+        topic_idx = 0
+    
+    topic, question = profile_topics[topic_idx]
+    
+    # Generate contextual options
+    options = generate_category_specific_options("profile", question, profile_data, None)
+    random.shuffle(options)
+    
+    return {
+        "question": question,
+        "options": options,
+        "source": "fallback_profile",
+        "question_type": "profile"
+    }
+
+def generate_expectation_question(state: Dict, expectation_data: Dict) -> Dict:
+    """Generate a single expectation question"""
+    expectation_topics = [
+        ("conflict", f"When tensions arise, what's your instinctive approach to resolution?"),
+        ("values", f"How do your core principles guide your everyday choices?"),
+        ("finance", f"What mindset drives your approach to shared financial decisions?"),
+        ("balance", f"How do you navigate between personal needs and relationship commitments?"),
+        ("dealbreakers", f"What boundaries are non-negotiable for you in close relationships?")
+    ]
+    
+    topic_idx = len([q for q in state.get("history", []) if q.get("question_type") == "expectation"])
+    if topic_idx >= len(expectation_topics):
+        topic_idx = 0
+    
+    topic, question = expectation_topics[topic_idx]
+    
+    # Generate contextual options
+    options = generate_category_specific_options("expectation", question, None, expectation_data)
+    random.shuffle(options)
+    
+    return {
+        "question": question,
+        "options": options,
+        "source": "fallback_expectation",
+        "question_type": "expectation"
+    }
+
+
+def generate_character_question(theme: str, state: Dict) -> Dict:
+    """Generate a single character question"""
+    prof = state.get("profile", {})
+    name = prof.get("full_name") or "Partner"
+    
+    short_theme = theme.split(" around ")[-1].strip()[:50]
+    question = f"When {short_theme}, what's your typical response?"
+    
+    # Generate contextual options
+    options = generate_category_specific_options("character", question, None, None)
+    random.shuffle(options)
+    
+    return {
+        "question": question,
+        "options": options,
+        "source": "fallback_character",
+        "question_type": "character"
+    }
+
+
+
+
+def generate_fallback_with_distribution(themes: List[str], state: Dict, profile_data: Dict, expectation_data: Dict, context: str = "") -> List[Dict]:
+    """
+    Fallback generator that enforces the 5-5-10 distribution
+    """
+    items = []
+    prof = state.get("profile", {}) or {}
+    name = prof.get("full_name") or "Partner"
+    
+    # 1. Generate 5 PROFILE-BASED questions
+    profile_sources = [
+        ("education", f"How does your {profile_data['education']} background influence your approach to learning new things?"),
+        ("employment", f"Given your work as {profile_data['employment']}, what problem-solving methods do you typically use?"),
+        ("hobbies", f"When engaging in {profile_data['hobbies']}, how do you typically organize your activity?"),
+        ("family_type", f"Growing up in a {profile_data['family_type']} family, what communication patterns feel most natural to you?"),
+        ("current_city", f"Living in {profile_data['current_city']}, how do you adapt to your daily environment?")
+    ]
+    
+    for source, question in profile_sources:
+        opts = [
+            {"text": "Analyze data and research thoroughly before deciding", "color": "blue"},
+            {"text": "Create a structured plan and follow established procedures", "color": "green"},
+            {"text": "Take immediate action and coordinate with people involved", "color": "red"},
+            {"text": "Brainstorm creative approaches and explore possibilities", "color": "yellow"},
+        ]
+        random.shuffle(opts)
+        items.append({
+            "question": question,
+            "options": opts,
+            "source": "fallback_profile",
+            "question_type": "profile"
+        })
+    
+    # 2. Generate 5 EXPECTATION-BASED questions
+    expectation_sources = [
+        ("conflict_style", f"When facing disagreement ({expectation_data['conflict_style']}), how do you typically respond?"),
+        ("financial_style", f"Regarding money matters ({expectation_data['financial_style']}), what's your immediate reaction to financial decisions?"),
+        ("family_values", f"Considering your family values ({expectation_data['family_values']}), how do you approach family-related decisions?"),
+        ("work_life", f"With your work-life preference ({expectation_data['lifestyle_pref']}), how do you manage daily priorities?"),
+        ("deal_breakers", f"Given your deal breakers ({expectation_data['deal_breakers'][:50] if expectation_data['deal_breakers'] else 'certain boundaries'}), how do you establish personal limits?")
+    ]
+    
+    for source, question in expectation_sources:
+        opts = [
+            {"text": "Gather all relevant information and analyze carefully", "color": "blue"},
+            {"text": "Follow a systematic process to evaluate options", "color": "green"},
+            {"text": "Make a quick decision and implement immediately", "color": "red"},
+            {"text": "Consider innovative solutions and future possibilities", "color": "yellow"},
+        ]
+        random.shuffle(opts)
+        items.append({
+            "question": question,
+            "options": opts,
+            "source": "fallback_expectation",
+            "question_type": "expectation"
+        })
+    
+    # 3. Generate 10 CHARACTER-BASED questions from themes
+    for i, theme in enumerate(themes[:10]):  # Use first 10 themes
+        short = theme.split(" around ")[-1].strip()
+        question = f"When dealing with {short}, what is your typical approach?"
+        
+        opts = [
+            {"text": "Research facts and analyze details before acting", "color": "blue"},
+            {"text": "Develop a step-by-step plan and follow it", "color": "green"},
+            {"text": "Take charge and coordinate people to solve it", "color": "red"},
+            {"text": "Explore creative ideas and unconventional solutions", "color": "yellow"},
+        ]
+        random.shuffle(opts)
+        items.append({
+            "question": question,
+            "options": opts,
+            "source": "fallback_character",
+            "question_type": "character"
+        })
+    
+    # Ensure we have exactly 20 questions
+    return items[:20]
+
+
+class SessionState:
+    def __init__(self, n_questions: int, batch_size: int, domain: str = "general", role: Optional[str] = None, profile: Optional[Dict] = None):
+        domain = (domain or role or "general").lower()
+        self.domain = domain if domain in DOMAINS else "general"
+        self.role = (role or self.domain)
+        self.profile = profile or {}
+        self.n_questions = max(1, min(n_questions, MAX_QUESTIONS))
+        self.batch_size = max(1, batch_size)
+        self.asked = 0
+        self.color_counts = {c: 0 for c in COLOR_KEYS}
+        self.history: List[Dict] = []
+        self.queue: List[Dict] = []
+        self.finished = False
+        self.used_topics: List[str] = []
+        self.history_of_questions: List[str] = []  # Add this line to track question texts
+
+    def to_min_state(self) -> Dict:
+        total = sum(self.color_counts.values()) or 1
+        mix_percentages = {k: round((v / total) * 100, 2) for k, v in self.color_counts.items()}
+        dominant = max(self.color_counts, key=self.color_counts.get) if total else None
+        return {
+            "asked": self.asked,
+            "dominant": dominant,
+            "mix": mix_percentages,
+            "domain": self.domain,
+            "role": self.role,
+            "profile": summarize_profile(self.profile),
+        }
+
+    def remaining(self) -> int:
+        return self.n_questions - self.asked
+
+SESSIONS_FILE = os.getenv("PYMATCH_SESSIONS_FILE", "sessions.json")
+_sessions_lock = threading.Lock()
+SESSIONS: Dict[str, SessionState] = {}
+
+def save_sessions():
+    try:
+        with _sessions_lock:
+            serializable = {sid: s.__dict__ for sid, s in SESSIONS.items()}
+            tmp = SESSIONS_FILE + ".tmp"
+            with open(tmp, "w", encoding="utf-8") as f:
+                json.dump(serializable, f, ensure_ascii=False, indent=2, default=str)
+            os.replace(tmp, SESSIONS_FILE)
+    except Exception as e:
+        print("Failed to save sessions:", e)
+
+def persist_final_progress(user_id: Optional[str], role: str, mix: Dict[str, float]) -> bool:
+    from database import get_db_connection
+    from config import PROGRESS_TBL
+    
+    llm_id = str(uuid.uuid4())
+    blue   = float(mix.get("blue", 0.0))
+    green  = float(mix.get("green", 0.0))
+    yellow = float(mix.get("yellow", 0.0))
+    red    = float(mix.get("red", 0.0))
+    try:
+        conn = get_db_connection()
+        cur = conn.cursor()
+        # Try with llm_id; if identity error, retry without it
+        try:
+            cur.execute(f"""
+                INSERT INTO [dbo].[{PROGRESS_TBL}]
+                  ([llm_id],[user_id],[role],[blue],[green],[yellow],[red],[created_at])
+                VALUES (?,?,?,?,?,?,?,SYSUTCDATETIME())
+            """, (llm_id, str(user_id) if user_id is not None else None, role, blue, green, yellow, red))
+            conn.commit()
+            return True
+        except pyodbc.Error as e:
+            if "IDENTITY_INSERT" in str(e) or "(544)" in str(e):
+                cur.execute(f"""
+                    INSERT INTO [dbo].[{PROGRESS_TBL}]
+                      ([user_id],[role],[blue],[green],[yellow],[red],[created_at])
+                    VALUES (?,?,?,?,?,?,SYSUTCDATETIME())
+                """, (str(user_id) if user_id is not None else None, role, blue, green, yellow, red))
+                conn.commit()
+                return True
+            else:
+                print("Persist failed:", e)
+                return False
+    except Exception as ex:
+        print("Persist final progress failed:", ex)
+        return False
+    finally:
+        try: conn.close()
+        except: pass
+
+def choose_themes(sess, k: int) -> List[str]:
+    """
+    Instead of generic topic banks, use FAISS to retrieve text chunks from the document.
+    """
+    try:
+        from faiss_service import HAS_FAISS, FAISS_INDEX, TEXT_CHUNKS
+        
+        if HAS_FAISS and FAISS_INDEX is not None and TEXT_CHUNKS:
+            # Just grab k random chunks from the indexed document
+            selected = random.sample(TEXT_CHUNKS, min(k, len(TEXT_CHUNKS)))
+            # Wrap them as "themes" but really they're just context
+            return selected
+    except ImportError:
+        pass
+    
+    # fallback: use generic themes
+    fallback_themes = [
+        "communication style", "conflict resolution", "decision making",
+        "problem solving", "team collaboration", "personal values",
+        "work habits", "social interaction", "stress management",
+        "goal setting", "time management", "relationship dynamics"
+    ]
+    return random.sample(fallback_themes, min(k, len(fallback_themes)))

matching_functions.py

@@ -0,0 +1,1112 @@
+# matching_functions.py
+import json
+import random
+import pandas as pd
+import numpy as np
+from datetime import date, datetime
+from difflib import SequenceMatcher
+from typing import Dict, List, Optional, Tuple
+from sqlalchemy import func
+from models import ExpectationResponse, Marriage, LLMGeneratedQuestions, Users, db
+from character_functions import calculate_character_similarity
+
+
+def values_match(expect_value, profile_value, field_name):
+    """Check if expectation value matches profile value with special handling for all fields"""
+    # Handle None values before using pd.isna
+    if expect_value is None or (hasattr(expect_value, 'isna') and pd.isna(expect_value)) or expect_value in ['', 'No preference', 'Any', 'All']:
+        return True  # No preference means match with any value
+    
+    # Convert to string safely
+    expect_str = str(expect_value).lower().strip() if expect_value is not None else ""
+    profile_str = str(profile_value).lower().strip() if profile_value is not None else ""
+    
+    # Handle empty profile values
+    if profile_value is None or (hasattr(profile_value, 'isna') and pd.isna(profile_value)) or profile_str in ['', 'none', 'null']:
+        return False
+    
+    # 🚨 FIELD-SPECIFIC MATCHING LOGIC
+        
+    if field_name == 'pref_age_range':
+        try:
+            if '-' in expect_str and profile_value:
+                min_age, max_age = map(int, expect_str.split('-'))
+                from datetime import date, datetime
+
+                # 🧠 Handle both string and datetime.date types
+                if isinstance(profile_value, date):
+                    birth_date = profile_value
+                elif isinstance(profile_value, str):
+                    # Try common formats
+                    try:
+                        birth_date = datetime.strptime(profile_value, "%Y-%m-%d").date()
+                    except ValueError:
+                        birth_date = datetime.strptime(profile_value, "%d-%m-%Y").date()
+                else:
+                    print(f"⚠️ Unsupported date type: {type(profile_value)}")
+                    return False
+
+                # ✅ Calculate age accurately
+                today = date.today()
+                age = today.year - birth_date.year - (
+                    (today.month, today.day) < (birth_date.month, birth_date.day)
+                )
+
+                # ✅ Inclusive range with ±1 tolerance
+                if (min_age - 1) <= age <= (max_age + 1):
+                    return True
+                else:
+                    return False
+            return True
+        except Exception as e:
+            print(f"⚠️ Age parsing error: {e} for {profile_value}")
+            return False
+
+    elif field_name == 'pref_height_range':
+        try:
+            cleaned = expect_str.replace('cm', '').replace(' ', '').lower()
+            profile_height = int(profile_str.replace('cm', '').replace(' ', ''))
+
+            # 190+
+            if cleaned.endswith('+'):
+                base = int(cleaned.replace('+', ''))
+                return profile_height >= base
+
+            # 181-189
+            if '-' in cleaned:
+                min_h, max_h = map(int, cleaned.split('-'))
+                return min_h <= profile_height <= max_h
+
+            # Single value
+            return profile_height == int(cleaned)
+
+        except:
+            return False
+    
+    # 3. City matching (pref_current_city vs current_city)
+    elif field_name == 'pref_current_city':
+        pref_cities = [city.strip().lower() for city in expect_str.split(',')]
+        return profile_str in pref_cities
+    
+    # 4. Country matching (pref_countries vs country)
+    elif field_name == 'pref_countries':
+        try:
+            # Handle None/empty values
+            if not expect_str or not profile_str:
+                return False
+                
+            # Normalise expectation values
+            pref_countries = [c.strip().lower() for c in str(expect_str).split(',') if c.strip()]
+            
+            # If user selected No Preference → auto match
+            if 'no preference' in pref_countries:
+                return True
+            
+            # Normalise profile value
+            profile_country = str(profile_str).lower().strip()
+            
+            return profile_country in pref_countries
+            
+        except Exception as e:
+            print(f"Error in country matching: {e}")
+            return False
+    
+    # 5. Languages matching (pref_languages vs languages_spoken)
+    elif field_name == 'pref_languages':
+        pref_langs = [lang.strip().lower() for lang in expect_str.split(',')]
+        profile_langs = [lang.strip().lower() for lang in profile_str.split(',')]
+        return any(lang in profile_langs for lang in pref_langs)
+    
+    # 6. Health Constraints matching
+    elif field_name == 'health_constraints':
+        health_mapping = {
+            'healthy': ['none', 'healthy'],
+            'minor': ['minor'],
+            'chronic': ['chronic'],
+            'allergies': ['allergies']
+        }
+        for exp_health, profile_options in health_mapping.items():
+            if expect_str == exp_health:
+                return profile_str in profile_options
+        return expect_str == profile_str
+    
+    # 7. Diet matching - STRICT EXACT MATCHING
+    elif field_name == 'pref_diet':
+        diet_options = {
+            'vegetarian': ['vegetarian'],
+            'non-vegetarian': ['non-vegetarian'],
+            'eggetarian': ['eggetarian']
+        }
+        
+        expect_clean = expect_str.replace('-', '').replace(' ', '')
+        profile_clean = profile_str.replace('-', '').replace(' ', '')
+        
+        if expect_clean == profile_clean:
+            return True
+        
+        for diet_type, variations in diet_options.items():
+            expect_variations = [v.replace('-', '').replace(' ', '') for v in variations]
+            profile_variations = [v.replace('-', '').replace(' ', '') for v in variations]
+            
+            if expect_clean in expect_variations:
+                return profile_clean in profile_variations
+        
+        return False
+    
+    # 8. Smoking matching
+    elif field_name == 'accept_smoking':
+        smoking_mapping = {
+            'never': ['no'],
+            'no preference': ['yes', 'no', 'occasionally'],
+            'occasionally': ['occasionally', 'yes']
+        }
+        for exp_option, profile_options in smoking_mapping.items():
+            if expect_str == exp_option:
+                return profile_str in profile_options
+        return expect_str == profile_str
+    
+    # 9. Alcohol matching
+    elif field_name == 'accept_alcohol':
+        alcohol_mapping = {
+            'never': ['no'],
+            'no preference': ['yes', 'no', 'occasionally'],
+            'occasionally': ['occasionally', 'yes']
+        }
+        for exp_option, profile_options in alcohol_mapping.items():
+            if expect_str == exp_option:
+                return profile_str in profile_options
+        return expect_str == profile_str
+    
+    # 10. Fitness matching
+    elif field_name == 'pref_fitness':
+        fitness_mapping = {
+            'low': ['low'],
+            'moderate': ['moderate'],
+            'high': ['high'],
+            'no preference': ['low', 'moderate', 'high']
+        }
+        for exp_level, profile_options in fitness_mapping.items():
+            if expect_str == exp_level:
+                return profile_str in profile_options
+        return expect_str == profile_str
+    
+    # 11. Family Type matching
+    elif field_name == 'pref_family_type':
+        family_mapping = {
+            'nuclear': ['nuclear'],
+            'joint': ['joint'],
+            'extended': ['extended'],
+            'no preference': ['nuclear', 'joint', 'extended']
+        }
+        for exp_type, profile_options in family_mapping.items():
+            if expect_str == exp_type:
+                return profile_str in profile_options
+        return expect_str == profile_str
+    
+    # 12. Live with In-laws matching
+    elif field_name == 'live_with_inlaws':
+        inlaw_mapping = {
+            'yes': ['yes'],
+            'no': ['no'],
+            'maybe': ['maybe'],
+            'no preference': ['yes', 'no', 'maybe']
+        }
+        for exp_option, profile_options in inlaw_mapping.items():
+            if expect_str == exp_option:
+                return profile_str in profile_options
+        return expect_str == profile_str
+    
+    # 13. Children Timeline matching
+    elif field_name == 'children_timeline':
+        timeline_mapping = {
+            'within 1 year': ['within 1 year'],
+            '1-3 years': ['1-3 years'],
+            'after 3 years': ['after 3 years'],
+            'not planning': ['not planning', 'no preference'],
+            'no preference': ['within 1 year', '1-3 years', 'after 3 years', 'not planning', 'no preference']
+        }
+        for exp_timeline, profile_options in timeline_mapping.items():
+            if expect_str == exp_timeline:
+                return profile_str in profile_options
+        return expect_str == profile_str
+    
+    # 14. Open to Adoption matching
+    elif field_name == 'open_to_adoption':
+        adoption_mapping = {
+            'yes': ['yes'],
+            'no': ['no'],
+            'maybe': ['maybe'],
+            'no preference': ['yes', 'no', 'maybe']
+        }
+        for exp_option, profile_options in adoption_mapping.items():
+            if expect_str == exp_option:
+                return profile_str in profile_options
+        return expect_str == profile_str
+    
+    # 15. Conflict Approach matching
+    elif field_name == 'pref_conflict_approach':
+        conflict_mapping = {
+            'discuss calmly': ['discuss calmly'],
+            'problem-solving': ['problem-solving'],
+            'compromise': ['compromise'],
+            'avoid': ['avoid'],
+            'decide fast': ['decide fast'],
+            'no preference': ['discuss calmly', 'problem-solving', 'compromise', 'avoid', 'decide fast']
+        }
+        for exp_approach, profile_options in conflict_mapping.items():
+            if expect_str == exp_approach:
+                return profile_str in profile_options
+        return expect_str == profile_str
+    
+    # 16. Financial Style matching
+    elif field_name == 'pref_financial_style':
+        financial_mapping = {
+            'budget-oriented': ['budget-oriented'],
+            'spend-oriented': ['spend-oriented'],
+            'balanced': ['balanced'],
+            'no preference': ['budget-oriented', 'spend-oriented', 'balanced']
+        }
+        for exp_style, profile_options in financial_mapping.items():
+            if expect_str == exp_style:
+                return profile_str in profile_options
+        return expect_str == profile_str
+    
+    # 17. Religion matching - Comprehensive version
+    elif field_name in ['pref_religion', 'religion_alignment', 'religion']:
+        # Handle "No preference" case
+        if expect_str in ['no preference', 'any', 'all']:
+            return True
+        
+        # Split expected religions (comma-separated)
+        expected_religions = [religion.strip().lower() for religion in expect_str.split(',')]
+        profile_religion = profile_str.lower().strip()
+        
+        # Handle cases where profile has multiple religions too
+        profile_religions = [religion.strip().lower() for religion in profile_str.split(',')]
+        
+        # Check if any profile religion matches any expected religion
+        return any(religion in expected_religions for religion in profile_religions)
+    
+    # 18. Income Range matching
+    elif field_name == 'pref_income_range':
+        if expect_str.lower() == 'prefer not to say' or profile_str.lower() == 'prefer not to say':
+            return True
+        if '-' in expect_str and '-' in profile_str:
+            try:
+                exp_min, exp_max = map(lambda x: int(x.replace('₹', '').replace(',', '').strip()), expect_str.split('-'))
+                prof_min, prof_max = map(lambda x: int(x.replace('₹', '').replace(',', '').strip()), profile_str.split('-'))
+                # Check if ranges overlap
+                return not (prof_max < exp_min or prof_min > exp_max)
+            except (ValueError, AttributeError):
+                pass
+        return True
+    
+    # 19. Education Level matching
+    elif field_name == 'pref_education_level':
+        education_mapping = {
+            'doctorate': ['doctorate', 'phd'],
+            'master': ['master', 'masters', 'postgraduate'],
+            'bachelor': ['bachelor', 'bachelors', 'undergraduate'],
+            'diploma': ['diploma', 'certificate'],
+            'school': ['school', 'secondary', 'higher secondary'],
+            'no preference': ['doctorate', 'master', 'bachelor', 'diploma', 'school']
+        }
+        for exp_level, profile_options in education_mapping.items():
+            if expect_str == exp_level:
+                return any(option in profile_str for option in profile_options)
+        return any(option in profile_str for option in education_mapping.get(expect_str, [expect_str]))
+    
+    # 20. Employment Status matching
+    elif field_name == 'pref_employment_status':
+        employment_mapping = {
+            'employed': ['Employed'],
+            'self-employed': ['Self-employed'],
+            'unemployed': ['Unemployed'], 
+            'freelancer': ['Freelancer'],
+            'government employee': ['Government employee'],
+            'no preference': ['Employed', 'Self-employed', 'Unemployed', 'Freelancer', 'Government employee']
+        }
+        
+        # Handle "no preference" case
+        if expect_str == 'no preference':
+            return True
+            
+        # Get expected options
+        expected_options = employment_mapping.get(expect_str, [expect_str])
+        
+        # Exact match comparison (case-insensitive)
+        profile_clean = profile_str.strip().lower()
+        return any(profile_clean == option.lower() for option in expected_options)
+    
+    # 21. Travel Preference matching
+    elif field_name == 'travel_pref':
+        travel_mapping = {
+            'frequent traveler': ['frequent traveler'],
+            'occasional traveler': ['occasional traveler'],
+            'homebody': ['homebody'],
+            'no preference': ['frequent traveler', 'occasional traveler', 'homebody']
+        }
+        for exp_travel, profile_options in travel_mapping.items():
+            if expect_str == exp_travel:
+                return profile_str in profile_options
+        return expect_str == profile_str
+    
+    # 22. Pet Preference matching
+    elif field_name == 'pet_pref':
+        pet_mapping = {
+            'open to pets': ['yes'],
+            'must like pets': ['yes'],
+            'no pets': ['no'],
+            'no preference': ['yes', 'no']
+        }
+        for exp_pet, profile_options in pet_mapping.items():
+            if expect_str == exp_pet:
+                return profile_str in profile_options
+        return expect_str == profile_str
+    
+    # 23. Deal Breakers - Complex logic (check if profile has any deal breakers)
+    elif field_name == 'deal_breakers':
+        if pd.isna(expect_value) or expect_str in ['', 'none']:
+            return True
+        
+        # What profiles actually track
+        PROFILE_DEAL_BREAKERS = {'smoking', 'different religion', 'alcohol', 
+                                'financial irresponsibility', 'no desire for children'}
+        
+        expect_breakers = {breaker.strip().lower() for breaker in expect_str.split(',')}
+        
+        # If expectation includes untrackable deal breakers → NO MATCH
+        if not expect_breakers.issubset(PROFILE_DEAL_BREAKERS):
+            return False
+        
+        # Check against actual profile data
+        if pd.isna(profile_value) or not str(profile_value).strip():
+            profile_breakers = set()
+        else:
+            profile_breakers = {breaker.strip().lower() for breaker in str(profile_value).split(',')}
+        
+        # No match if profile has any of the expected deal breakers
+        return len(expect_breakers.intersection(profile_breakers)) == 0
+    
+    # 24. Daily Routine matching
+    elif field_name == 'daily_routine':
+        routine_mapping = {
+            'early riser': ['early riser'],
+            'night owl': ['night owl'],
+            'balanced': ['balanced'],
+            'no preference': ['early riser', 'night owl', 'balanced']
+        }
+        for exp_routine, profile_options in routine_mapping.items():
+            if expect_str == exp_routine:
+                return profile_str in profile_options
+        return expect_str == profile_str
+    
+    # 25. Family Communication Frequency matching
+    elif field_name == 'family_communication_frequency':
+        comm_mapping = {
+            'daily': ['daily'],
+            'weekly': ['weekly'],
+            'monthly': ['monthly'],
+            'occasionally': ['occasionally'],
+            'no preference': ['daily', 'weekly', 'monthly', 'occasionally']
+        }
+        for exp_freq, profile_options in comm_mapping.items():
+            if expect_str == exp_freq:
+                return profile_str in profile_options
+        return expect_str == profile_str
+    
+    # 26. pref_shared_hobbies
+    elif field_name == "pref_shared_hobbies":
+        # Expectation list (split by comma)
+        expect_list = [x.strip().lower() for x in expect_str.split(",") if x.strip()]
+
+        # Profile list
+        profile_list = [x.strip().lower() for x in profile_str.split(",") if x.strip()]
+
+        # ANY overlap → MATCH
+        return any(h in profile_list for h in expect_list)
+    
+    # 27. pref_partner_relocation 
+    elif field_name == 'pref_partner_relocation':
+        relocation_mapping = {
+            'yes': ['yes'],
+            'no': ['no'],
+            'maybe': ['maybe'],
+            'no preference': ['yes', 'no', 'maybe']
+        }
+        for exp_option, profile_options in relocation_mapping.items():
+            if expect_str == exp_option:
+                return profile_str in profile_options
+        return expect_str == profile_str
+    
+    # 28. pref_live_with_parents 
+    elif field_name == 'pref_live_with_parents':
+        live_mapping = {
+            'yes': ['yes'],
+            'no': ['no'],
+            'maybe': ['maybe'],
+            'no preference': ['yes', 'no', 'maybe']
+        }
+        for exp_option, profile_options in live_mapping.items():
+            if expect_str == exp_option:
+                return profile_str in profile_options
+        return expect_str == profile_str
+    
+    # 29. financial_support_to_parents 
+    elif field_name == 'financial_support_to_parents':
+        support_mapping = {
+            'yes': ['yes'],
+            'no': ['no'],
+            'no preference': ['yes', 'no']
+        }
+        for exp_option, profile_options in support_mapping.items():
+            if expect_str == exp_option:
+                return profile_str in profile_options
+        return expect_str == profile_str
+    
+    # 30. other_non_negotiables 
+    elif field_name == 'other_non_negotiables':
+        expect_list = [x.strip().lower() for x in expect_str.split(',') if x.strip()]
+        profile_list = [x.strip().lower() for x in profile_str.split(',') if x.strip()]
+
+        # Match if ANY expected non-negotiable is found in profile
+        return any(item in profile_list for item in expect_list)
+    
+    # 31. skin_tone 
+    elif field_name == 'skin_tone':
+        tone_mapping = {
+            'fair': ['fair'],
+            'medium': ['medium'],
+            'dark': ['dark'],
+            'no preference': ['fair', 'medium', 'dark']
+        }
+        for exp_tone, profile_options in tone_mapping.items():
+            if expect_str == exp_tone:
+                return profile_str in profile_options
+        return expect_str == profile_str
+    
+    # 32. marital_status 
+    elif field_name == 'marital_status':
+        status_mapping = {
+            'single': ['single'],
+            'divorced': ['divorced'],
+            'widowed': ['widowed'],
+            'no preference': ['single', 'divorced', 'widowed']
+        }
+        for exp_status, profile_options in status_mapping.items():
+            if expect_str == exp_status:
+                return profile_str in profile_options
+        return expect_str == profile_str
+    
+    # 33. relaxation_mode 
+    elif field_name == 'relaxation_mode':
+        # No preference → always match
+        if expect_str in ['no preference', 'any']:
+            return True
+
+        expect_list = [x.strip().lower() for x in expect_str.split(',') if x.strip()]
+        profile_list = [x.strip().lower() for x in profile_str.split(',') if x.strip()]
+
+        # Any overlap = match
+        return any(item in profile_list for item in expect_list)
+    
+    elif field_name == 'expectation_summary':
+        if not expect_value or str(expect_value).strip().lower() in ['', 'no preference', 'any']:
+            return True
+        
+        if not profile_value or str(profile_value).strip().lower() in ['', 'none', 'null']:
+            return False
+        
+        return compare_expectation_with_remark(
+            str(expect_value).strip(), 
+            str(profile_value).strip()
+        )
+    
+    # 34. Career Aspirations matching
+    elif field_name == 'pref_career_aspirations':
+        career_mapping = {
+            'entrepreneurship': ['entrepreneurship', 'entrepreneur'],
+            'leadership': ['leadership'],
+            'stable job': ['stable job'],
+            'work-life balance': ['work-life balance'],
+            'research': ['research'],
+            'creativity': ['creativity'],
+            'social impact': ['social impact'],
+            'no preference': ['entrepreneurship', 'leadership', 'stable job', 'work-life balance', 'research', 'creativity', 'social impact']
+        }
+        for exp_career, profile_options in career_mapping.items():
+            if expect_str == exp_career:
+                return any(option in profile_str for option in profile_options)
+        return any(option in profile_str for option in career_mapping.get(expect_str, [expect_str]))
+    
+    # Default: Exact match for other fields
+    else:
+        return expect_str == profile_str
+
+def extract_key_concepts(text):
+    """Extract key concepts from text using NLP techniques"""
+    text = text.lower()
+    
+    # Remove common stop words
+    stop_words = {'the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 
+                  'for', 'of', 'with', 'by', 'is', 'are', 'was', 'were', 'be', 'been'}
+    
+    # Concept mapping - words that mean similar things
+    concept_groups = {
+        'family': ['family', 'parents', 'children', 'siblings', 'home', 'domestic'],
+        'career': ['career', 'job', 'work', 'profession', 'business', 'entrepreneur'],
+        'balance': ['balance', 'equilibrium', 'harmony', 'work-life'],
+        'growth': ['growth', 'development', 'improvement', 'learning', 'progress'],
+        'values': ['values', 'principles', 'ethics', 'morals', 'beliefs'],
+        'communication': ['communication', 'talking', 'discussing', 'expressing'],
+        'shared': ['shared', 'together', 'mutual', 'common', 'joint'],
+        'happiness': ['happiness', 'joy', 'fulfillment', 'contentment'],
+        'respect': ['respect', 'esteem', 'admiration', 'honor'],
+        'understanding': ['understanding', 'comprehension', 'insight', 'empathy'],
+        'goals': ['goals', 'objectives', 'aims', 'aspirations', 'ambitions'],
+        'compatibility': ['compatibility', 'harmony', 'agreement', 'fit'],
+        'lifestyle': ['lifestyle', 'way of life', 'routine', 'daily life'],
+        'culture': ['culture', 'cultural', 'tradition', 'heritage'],
+        'religion': ['religion', 'faith', 'spiritual', 'belief'],
+        'partner': ['partner', 'spouse', 'companion', 'mate'],
+        'love': ['love', 'affection', 'care', 'fondness'],
+        'trust': ['trust', 'confidence', 'reliance', 'faith'],
+        'support': ['support', 'encouragement', 'backing', 'assistance'],
+        'stability': ['stability', 'security', 'steadiness', 'reliability']
+    }
+    
+    concepts = set()
+    words = text.split()
+    
+    for word in words:
+        word_clean = ''.join(c for c in word if c.isalnum())  # Remove punctuation
+        
+        if word_clean in stop_words or len(word_clean) < 3:
+            continue
+            
+        # Check if word belongs to any concept group
+        for concept, related_words in concept_groups.items():
+            if word_clean in related_words:
+                concepts.add(concept)
+                break
+        else:
+            # Add the word itself if it's meaningful
+            if len(word_clean) > 4:  # Longer words are usually more meaningful
+                concepts.add(word_clean)
+    
+    return concepts
+
+def compare_expectation_with_remark(exp_summary, profile_remark):
+    """Compare expectation summary with profile remark using multiple strategies"""
+    
+    exp_summary_lower = exp_summary.lower()
+    profile_remark_lower = profile_remark.lower()
+    
+    print(f"🔍 Comparing expectation with remark:")
+    print(f"   Expectation: '{exp_summary}'")
+    print(f"   Remark (first 100 chars): '{profile_remark[:100]}...'")
+    
+    # Strategy 1: Check for exact phrase matching (for very specific expectations)
+    if len(exp_summary.split()) <= 6:  # Short expectations (6 words or less)
+        if exp_summary_lower in profile_remark_lower:
+            print(f"   ✅ Exact phrase found in remark")
+            return True
+    
+    # Strategy 2: Extract and compare key concepts
+    exp_concepts = extract_key_concepts(exp_summary)
+    remark_concepts = extract_key_concepts(profile_remark)
+    
+    print(f"   Expectation concepts: {exp_concepts}")
+    print(f"   Remark concepts found: {len(remark_concepts)} total")
+    
+    # Count overlapping concepts
+    overlapping = exp_concepts.intersection(remark_concepts)
+    if exp_concepts:
+        concept_overlap = len(overlapping) / len(exp_concepts)
+    else:
+        concept_overlap = 0
+    
+    print(f"   Concept overlap: {len(overlapping)}/{len(exp_concepts)} = {concept_overlap:.2f}")
+    
+    # Strategy 3: Use difflib for text similarity (fallback)
+    from difflib import SequenceMatcher
+    text_similarity = SequenceMatcher(None, exp_summary, profile_remark).ratio()
+    print(f"   Text similarity: {text_similarity:.2f}")
+    
+    # Strategy 4: Check for important keywords
+    important_keywords = ['family', 'career', 'balance', 'growth', 'values', 
+                         'communication', 'shared', 'respect', 'understanding',
+                         'partner', 'love', 'trust', 'support', 'happiness']
+    
+    keyword_matches = 0
+    for keyword in important_keywords:
+        if keyword in exp_summary_lower and keyword in profile_remark_lower:
+            keyword_matches += 1
+    
+    print(f"   Important keyword matches: {keyword_matches}")
+    
+    # Combined decision logic
+    # Match if ANY of these conditions are met:
+    # 1. Good concept overlap (> 40%)
+    # 2. Reasonable text similarity (> 25%)
+    # 3. At least 2 important keyword matches
+    # 4. Exact phrase match (already handled above)
+    
+    result = (concept_overlap > 0.4) or (text_similarity > 0.25) or (keyword_matches >= 2)
+    
+    print(f"   Final decision: {'✅ MATCH' if result else '❌ NO MATCH'}")
+    print(f"   Reasons: concept_overlap={concept_overlap:.2f}, "
+          f"text_similarity={text_similarity:.2f}, "
+          f"keyword_matches={keyword_matches}")
+    
+    return result
+
+def compute_expectation_score(expect, profile, mandatory_fields):
+    """Compute expectation match percentage based on satisfied fields"""
+    satisfied_fields = 0
+    total_fields_checked = 0
+    mandatory_violations = 0
+
+    print(f"🔍 COMPUTE_SCORE: Evaluating profile {profile.user_id} ({profile.full_name}) from {profile.current_city}")
+
+    # 🚨 UPDATED FIELD MAPPING - all expectation fields
+    field_mapping = {
+        'pref_age_range': 'date_of_birth',
+        'pref_height_range': 'height',
+        'pref_education_level': 'education_level',
+        'pref_employment_status': 'employment_status',
+        'expectation_summary': 'remark',  # Map expectation_summary to profile remark
+        
+        
+        'pref_current_city': 'current_city',
+        'pref_countries': 'country',
+        'pref_diet': 'food_preference',
+        'pref_fitness': 'fitness_level',
+        'pref_family_type': 'family_type',
+
+        'accept_smoking': 'smoking_habit',
+        'accept_alcohol': 'alcohol_habit',
+        'pref_languages': 'languages_spoken',
+        'religion_alignment': 'religion',
+        'pref_partner_relocation': 'relocation_willingness',
+
+        'pref_conflict_approach': 'conflict_approach',
+        'pref_financial_style': 'financial_style',
+        'pref_shared_hobbies': 'hobbies_interests',
+        'travel_pref': 'travel_preference',
+        'pet_pref': 'own_pets',
+        
+        'pref_income_range': 'income_range',
+        'live_with_inlaws': 'live_with_inlaws',
+        'pref_live_with_parents': 'live_with_parents',
+        'financial_support_to_parents': 'support_parents_financially',
+        'pref_career_aspirations': 'career_aspirations',
+
+        'children_timeline': 'children_timeline',
+        'open_to_adoption': 'open_to_adoption',
+        'deal_breakers': 'deal_breakers',
+        'other_non_negotiables': 'other_non_negotiables',
+        'health_constraints': 'health_constraints',
+
+        'skin_tone': 'skin_tone',
+        'marital_status': 'marital_status',
+        'daily_routine': 'daily_routine',
+        'family_communication_frequency': 'family_communication_frequency',
+        'relaxation_mode': 'relaxation_mode'
+    }
+
+    # 🚨 DEBUG: Track all field processing
+    field_details = []
+
+    # 🚨 CRITICAL FIX: Check ALL mandatory fields FIRST
+    print(f"🎯 COMPUTE_SCORE: CHECKING ALL MANDATORY FIELDS: {mandatory_fields}")
+    
+    for field_name, is_mandatory in mandatory_fields.items():
+        if is_mandatory:
+            print(f"🎯 COMPUTE_SCORE: Checking mandatory field: {field_name}")
+            
+            # Get expectation value
+            expect_value = getattr(expect, field_name, None)
+            
+            # Map expectation field to actual profile field
+            profile_field_name = field_mapping.get(field_name, field_name)
+            profile_value = getattr(profile, profile_field_name, None)
+            
+            # Special handling for location field
+            if field_name == 'pref_current_city' and not profile_value:
+                profile_value = profile.current_city
+            
+            print(f"   Expect: '{expect_value}', Profile: '{profile_value}' (mapped to: {profile_field_name})")
+
+            print(
+                f"[COMPARE] Expectation Field: {field_name} "
+                f"({expect_value})  ↔  Profile Field: {profile_field_name} "
+                f"({profile_value})"
+            )
+
+            # If expectation has a value for this mandatory field
+            if expect_value and str(expect_value).strip():
+                total_fields_checked += 1
+                # Profile must have a matching value
+                if not profile_value or not str(profile_value).strip():
+                    print(f"❌ COMPUTE_SCORE: Mandatory violation: {field_name} - Profile missing value")
+                    mandatory_violations += 1
+                    field_details.append(f"🚫 MANDATORY FAIL: {field_name}: {expect_value} -> MISSING")
+                elif not values_match(expect_value, profile_value, field_name):
+                    print(f"❌ COMPUTE_SCORE: Mandatory violation: {field_name} - Values don't match")
+                    print(f"   Expect: '{expect_value}', Profile: '{profile_value}'")
+                    mandatory_violations += 1
+                    field_details.append(f"🚫 MANDATORY FAIL: {field_name}: {expect_value} -> {profile_value}")
+                else:
+                    satisfied_fields += 1
+                    print(f"✅ COMPUTE_SCORE: Mandatory match: {field_name} - '{expect_value}'")
+                    field_details.append(f"✅ MANDATORY: {field_name}: {expect_value} -> {profile_value}")
+            else:
+                print(f"ℹ️ COMPUTE_SCORE: Mandatory field {field_name} has no expectation value, skipping")
+                field_details.append(f"➖ MANDATORY NO PREF: {field_name}")
+
+    # 🚨 CRITICAL FIX: REJECT if ANY mandatory violations
+    if mandatory_violations > 0:
+        print(f"🚫 COMPUTE_SCORE: Profile {profile.user_id} REJECTED due to {mandatory_violations} mandatory violations")
+        return 0  # Return 0 score to indicate rejection
+
+    print(f"✅ COMPUTE_SCORE: Profile {profile.user_id} passed ALL mandatory checks")
+
+    # 🚨 NOW CHECK ALL EXPECTATION FIELDS for percentage calculation
+    all_expectation_fields = [
+        'pref_age_range', 'pref_height_range', 'pref_education_level', 'pref_employment_status',
+        'pref_current_city', 'pref_countries', 'pref_diet', 'pref_fitness', 'pref_family_type',
+        'accept_smoking', 'accept_alcohol', 'pref_languages', 'religion_alignment',
+        'pref_partner_relocation', 'pref_conflict_approach', 'pref_financial_style',
+        'pref_shared_hobbies', 'travel_pref', 'pet_pref', 'pref_income_range',
+        'live_with_inlaws', 'pref_live_with_parents', 'financial_support_to_parents',
+        'pref_career_aspirations', 'children_timeline', 'open_to_adoption',
+        'deal_breakers', 'other_non_negotiables', 'health_constraints', 'skin_tone',
+        'marital_status', 'daily_routine', 'family_communication_frequency', 'relaxation_mode'
+    ]
+
+    # Check ALL expectation fields (both mandatory and optional)
+    for field_name in all_expectation_fields:
+        # Skip if already processed as mandatory
+        if field_name in mandatory_fields and mandatory_fields[field_name]:
+            continue
+            
+        # Map expectation field to profile field
+        profile_field_name = field_mapping.get(field_name, field_name)
+        expect_value = getattr(expect, field_name, None)
+        profile_value = getattr(profile, profile_field_name, None)
+        
+        # Special handling for location field
+        if field_name == 'pref_current_city' and not profile_value:
+            profile_value = profile.current_city
+        
+        # Only count if expectation has a value
+        if expect_value and str(expect_value).strip():
+            total_fields_checked += 1
+            # --- Console Log ---
+            print(
+                f"[COMPARE] Expectation -> {field_name}: '{expect_value}' "
+                f" | Profile -> {profile_field_name}: '{profile_value}'"
+            )
+
+            if profile_value and str(profile_value).strip():
+                if values_match(expect_value, profile_value, field_name):
+                    satisfied_fields += 1
+                    print(f"✅ COMPUTE_SCORE: Field match: {field_name}")
+                    field_details.append(f"✅ OPTIONAL: {field_name}: {expect_value} -> {profile_value}")
+                else:
+                    print(f"❌ COMPUTE_SCORE: Field mismatch: {field_name} - Expect: '{expect_value}', Profile: '{profile_value}'")
+                    field_details.append(f"❌ OPTIONAL: {field_name}: {expect_value} -> {profile_value}")
+            else:
+                print(f"❌ COMPUTE_SCORE: Field missing: {field_name} - Profile has no value")
+                field_details.append(f"⚠️ OPTIONAL: {field_name}: {expect_value} -> MISSING")
+        else:
+            field_details.append(f"➖ OPTIONAL NO PREF: {field_name}")
+
+    # 🚨 DEBUG: Print detailed field analysis
+    print(f"🔍 COMPUTE_SCORE: Field-by-field analysis:")
+    for detail in field_details:
+        print(f"   {detail}")
+    print(f"🔍 COMPUTE_SCORE: Total fields checked: {total_fields_checked}")
+    print(f"🔍 COMPUTE_SCORE: Satisfied fields: {satisfied_fields}")
+
+    # 🚨 Calculate percentage based on satisfied fields vs total fields checked
+    if total_fields_checked > 0:
+        percentage = (satisfied_fields / total_fields_checked) * 100
+        print(f"📊 COMPUTE_SCORE: Field Analysis: {satisfied_fields}/{total_fields_checked} fields satisfied = {percentage:.1f}%")
+        
+        # Special handling for expectation summary (bonus)
+        if hasattr(expect, 'expectation_summary') and expect.expectation_summary and profile.remark:
+            from difflib import SequenceMatcher
+            exp_summary = str(expect.expectation_summary).lower()
+            profile_remark = str(profile.remark).lower()
+            
+            sim = SequenceMatcher(None, exp_summary, profile_remark).ratio()
+            if sim > 0.3:
+                # Add bonus for summary similarity (up to 5%)
+                bonus = min(sim * 5, 5)
+                percentage = min(100, percentage + bonus)
+                print(f"✅ COMPUTE_SCORE: Summary similarity bonus: +{bonus:.1f}% (similarity: {sim:.2f})")
+        
+        final_percentage = round(percentage, 2)
+        print(f"🎯 COMPUTE_SCORE: Final expectation percentage: {final_percentage}%")
+        return final_percentage / 100  # Return as decimal for consistency
+    
+    print(f"⚠️ COMPUTE_SCORE: No expectation fields to check for profile {profile.user_id}")
+    return 0
+
+def match_expectation_with_profiles(user_id):
+    expectation = ExpectationResponse.query.filter_by(user_id=user_id).first()
+    if not expectation:
+        print(f"❌ No expectation data found for user {user_id}")
+        return []
+
+    # 🚨 CRITICAL FIX: Properly parse mandatory fields from database
+    mandatory_fields = {}
+    if hasattr(expectation, '_mandatory_fields') and expectation._mandatory_fields:
+        try:
+            if isinstance(expectation._mandatory_fields, str):
+                # Parse JSON string from database
+                mandatory_fields = json.loads(expectation._mandatory_fields)
+            else:
+                mandatory_fields = expectation._mandatory_fields
+        except Exception as e:
+            print(f"❌ Error parsing mandatory fields: {e}")
+            mandatory_fields = {}
+    else:
+        print("ℹ️ No mandatory fields found or empty")
+
+    print(f"🔍 DEBUG: Mandatory fields for user {user_id}: {mandatory_fields}")
+
+    # Get current user to know gender
+    current_user = Marriage.query.filter_by(user_id=user_id).first()
+    if not current_user:
+        print(f"❌ No marriage profile found for user {user_id}")
+        return []
+
+    user_gender = (current_user.gender or "").lower()
+    print(f"🔍 DEBUG: Current user gender: {user_gender}")
+
+    # Opposite gender profiles only
+    if user_gender.startswith('male'):
+        opposite_profiles = Marriage.query.filter(func.lower(func.trim(Marriage.gender)) == "female").all()
+    elif user_gender.startswith('female'):
+        opposite_profiles = Marriage.query.filter(func.lower(func.trim(Marriage.gender)) == "male").all()
+    else:
+        opposite_profiles = Marriage.query.filter(Marriage.gender != current_user.gender).all()
+
+    print(f"🔍 DEBUG: Found {len(opposite_profiles)} opposite gender profiles")
+
+    # 🚨 FIX: Initialize candidates list here
+    candidates = []
+    
+    # Evaluate all opposite gender profiles
+    for profile in opposite_profiles:
+        print(f"\n--- Evaluating Profile {profile.user_id} ---")
+        s = compute_expectation_score(expectation, profile, mandatory_fields)
+        if s > 0:
+            candidates.append({
+                "user_id": profile.user_id,
+                "name": profile.full_name,
+                "gender": profile.gender,
+                "location": profile.current_city,
+                "religion": profile.religion,
+                "remark": profile.remark,
+                "expectation_score": s,
+                "mandatory_matched": True
+            })
+            print(f"✅ Added candidate {profile.user_id} with score {s}")
+
+    print(f"📈 Total candidates after mandatory filtering: {len(candidates)}")
+
+    # 🚨 FIX: Get character compatibility for ALL candidates
+    all_ids = [c["user_id"] for c in candidates]
+    llm_data = LLMGeneratedQuestions.query.filter(LLMGeneratedQuestions.user_id.in_(all_ids)).all()
+    llm_map = {l.user_id: (l.blue, l.green, l.yellow, l.red) for l in llm_data}
+
+    # 🚨 FIX: Calculate character scores properly
+    for c in candidates:
+        if c["user_id"] in llm_map:
+            b, g, y, r = llm_map[c["user_id"]]
+            # Calculate character score as weighted sum of color percentages
+            total = b + g + y + r
+            if int(total) > 0:
+                # Normalize and calculate similarity to ideal distribution
+              
+                char_score = calculate_character_similarity(b, g, y, r)
+                c["character_score"] = round(char_score, 2)
+            else:
+                c["character_score"] = 0
+        else:
+            c["character_score"] = 0
+        
+        # Overall score combining both expectation and character
+        c["overall_score"] = round(0.7 * c["expectation_score"] + 0.3 * c["character_score"], 2)
+
+    # Return both sorted lists
+    expectation_sorted = sorted(candidates, key=lambda x: x["expectation_score"], reverse=True)
+    character_sorted = sorted(candidates, key=lambda x: x["character_score"], reverse=True)
+    overall_sorted = sorted(candidates, key=lambda x: x["overall_score"], reverse=True)
+    
+    print(f"🎯 Final ranked by expectation: {len(expectation_sorted)}")
+    print(f"🎯 Final ranked by character: {len(character_sorted)}")
+    
+    # 🚨 FIX: Return the appropriate list based on what the caller expects
+    return expectation_sorted
+
+def generate_expectation_explanation(expect_user: dict, profile_user: dict) -> list:
+    """
+    Compare user's expectations with another user's profile.
+    Gives a clean, correct, field-by-field explanation.
+    """
+
+    explanations = []
+    exact_matches = []
+    differences = []
+    missing_data = []
+
+    # -------------------------------------------
+    # 🔥 UNIVERSAL SAFE KEY LOOKUP
+    # -------------------------------------------
+    def get_profile_value(profile_dict, key_name):
+        """Case-insensitive and alias-safe key lookup."""
+        key_name = key_name.lower().strip()
+
+        # Special aliases for country
+        country_aliases = ["country", "location", "current_country",
+                           "residence_country", "live_country"]
+
+        for k, v in profile_dict.items():
+            k_clean = k.lower().strip()
+
+            # Match correct field
+            if k_clean == key_name:
+                return str(v).strip()
+
+            # Match ANY country-related field
+            if key_name == "country" and k_clean in country_aliases:
+                return str(v).strip()
+
+        # If not found
+        return ""
+
+    # -------------------------------------------
+    # 🔥 FIELD MAPPING (Same as compute_expectation_score)
+    # -------------------------------------------
+    field_mapping = {
+        'pref_age_range': 'date_of_birth',
+        'pref_height_range': 'height',
+        'pref_education_level': 'education_level',
+        'pref_employment_status': 'employment_status',
+
+        'pref_current_city': 'current_city',
+        'pref_countries': 'country',        # 👉 FIXED HERE
+        'pref_diet': 'food_preference',
+        'pref_fitness': 'fitness_level',
+        'pref_family_type': 'family_type',
+
+        'accept_smoking': 'smoking_habit',
+        'accept_alcohol': 'alcohol_habit',
+        'pref_languages': 'languages_spoken',
+        'religion_alignment': 'religion',
+        'pref_partner_relocation': 'relocation_willingness',
+
+        'pref_conflict_approach': 'conflict_approach',
+        'pref_financial_style': 'financial_style',
+        'pref_shared_hobbies': 'hobbies_interests',
+        'travel_pref': 'travel_preference',
+        'pet_pref': 'own_pets',
+        
+        'pref_income_range': 'income_range',
+        'live_with_inlaws': 'live_with_inlaws',
+        'pref_live_with_parents': 'live_with_parents',
+        'financial_support_to_parents': 'support_parents_financially',
+        'pref_career_aspirations': 'career_aspirations',
+
+        'children_timeline': 'children_timeline',
+        'open_to_adoption': 'open_to_adoption',
+        'deal_breakers': 'deal_breakers',
+        'other_non_negotiables': 'other_non_negotiables',
+        'health_constraints': 'health_constraints',
+
+        'skin_tone': 'skin_tone',
+        'marital_status': 'marital_status',
+        'daily_routine': 'daily_routine',
+        'family_communication_frequency': 'family_communication_frequency',
+        'relaxation_mode': 'relaxation_mode'
+    }
+
+    all_expectation_fields = list(field_mapping.keys())
+
+    # -------------------------------------------
+    # 🔥 FIELD COMPARISON LOGIC
+    # -------------------------------------------
+    for expect_key, profile_key in field_mapping.items():
+
+        label = expect_key.replace("pref_", "").replace("_", " ").title()
+        expect_value = str(expect_user.get(expect_key, "") or "").strip()
+
+        # If no preference → skip
+        if expect_value.lower() in ["", "no preference", "any", "all"]:
+            continue
+
+        # Correctly fetch profile value
+        profile_value = get_profile_value(profile_user, profile_key)
+
+        # Missing profile data (REAL missing only)
+        if profile_value == "":
+            missing_data.append((label, expect_value))
+            continue
+
+        # Perform match check
+        if values_match(expect_value, profile_value, expect_key):
+            exact_matches.append(f"• Profile matches your preference for {label.lower()} ({profile_value})")
+        else:
+            differences.append(
+                f"• Profile differs from your preference for {label.lower()} "
+                f"(you want: {expect_value}, they are: {profile_value})"
+            )
+
+    # -------------------------------------------
+    # 🔥 COMPUTE COMPATIBILITY (Same as compute_expectation_score)
+    # -------------------------------------------
+    total_pref_fields = 0
+    satisfied_count = 0
+
+    for field_name in all_expectation_fields:
+        expect_value = str(expect_user.get(field_name, "") or "").strip()
+        if expect_value.lower() in ["", "no preference", "any", "all"]:
+            continue
+
+        total_pref_fields += 1
+        profile_key = field_mapping[field_name]
+        profile_value = get_profile_value(profile_user, profile_key)
+
+        if profile_value and values_match(expect_value, profile_value, field_name):
+            satisfied_count += 1
+
+    if total_pref_fields > 0:
+        percent = round((satisfied_count / total_pref_fields) * 100, 2)
+    else:
+        percent = 0
+
+    # -------------------------------------------
+    # 🔥 BUILD EXPLANATION OUTPUT
+    # -------------------------------------------
+    explanations.append(f"📊 **Expectation Compatibility**: {percent}%")
+    explanations.append(f"• {satisfied_count} matches out of {total_pref_fields} preference fields")
+
+    if len(missing_data) > 0:
+        explanations.append(f"• ⚠️ {len(missing_data)} fields missing profile data")
+        explanations.append("")
+        explanations.append("**⚠️ Missing Profile Data:**")
+        for label, expect_val in missing_data:
+            explanations.append(f"• {label}: Profile missing (You want: {expect_val})")
+
+    if len(exact_matches) > 0:
+        explanations.append("")
+        explanations.append("**🔍 Detailed Field Analysis:**")
+        explanations.extend(exact_matches)
+        explanations.extend(differences)
+
+    return explanations

models.py

@@ -0,0 +1,125 @@
+# models.py
+from datetime import datetime
+from flask_sqlalchemy import SQLAlchemy
+
+db = SQLAlchemy()
+
+class Users(db.Model):
+    __tablename__ = "Users"
+    user_id = db.Column(db.Integer, primary_key=True)
+    name = db.Column(db.String(128), nullable=False)
+    email = db.Column(db.String(128))
+    password = db.Column(db.String(128))
+    created_at = db.Column(db.DateTime)
+
+class LLMGeneratedQuestions(db.Model):
+    __tablename__ = "LLMGeneratedQuestions"
+
+    llm_id     = db.Column(db.Integer, primary_key=True)
+    user_id    = db.Column(db.Integer, nullable=False, index=True)
+    role       = db.Column(db.String(64), nullable=True)
+    blue       = db.Column(db.Integer, nullable=False, default=0)
+    green      = db.Column(db.Integer, nullable=False, default=0)
+    yellow     = db.Column(db.Integer, nullable=False, default=0)
+    red        = db.Column(db.Integer, nullable=False, default=0)
+    created_at = db.Column(db.DateTime, default=datetime.utcnow)
+
+    def color_vec(self):
+        import numpy as np
+        v = np.array([self.blue, self.green, self.yellow, self.red], dtype=np.float32)
+        s = float(v.sum())
+        return v / s if s > 0 else v
+
+class Marriage(db.Model):
+    __tablename__ = "Marriage"
+    id = db.Column(db.Integer, primary_key=True)
+    user_id = db.Column(db.Integer, nullable=False)
+    full_name = db.Column(db.String(100))
+    date_of_birth = db.Column(db.String(50))
+    gender = db.Column(db.String(20))
+    current_city = db.Column(db.String(100))
+    marital_status = db.Column(db.String(50))
+    education_level = db.Column(db.String(100))
+    employment_status = db.Column(db.String(100))
+    number_of_siblings = db.Column(db.String(50))
+    family_type = db.Column(db.String(100))
+    hobbies_interests = db.Column(db.Text)
+    conflict_approach = db.Column(db.String(100))
+    financial_style = db.Column(db.String(100))
+    income_range = db.Column(db.String(100))
+    relocation_willingness = db.Column(db.String(100))
+    created_at = db.Column(db.DateTime, default=datetime.utcnow)
+    height = db.Column(db.String(100))  # Changed from height_weight
+    skin_tone = db.Column(db.String(50))
+    languages_spoken = db.Column(db.String(200))
+    country = db.Column(db.String(100))
+    blood_group = db.Column(db.String(10))
+    religion = db.Column(db.String(100))
+    dual_citizenship = db.Column(db.String(50))
+    siblings_position = db.Column(db.String(50))
+    parents_living_status = db.Column(db.String(100))
+    live_with_parents = db.Column(db.String(50))
+    support_parents_financially = db.Column(db.String(50))
+    family_communication_frequency = db.Column(db.String(100))
+    food_preference = db.Column(db.String(100))
+    smoking_habit = db.Column(db.String(50))
+    alcohol_habit = db.Column(db.String(50))
+    daily_routine = db.Column(db.String(200))
+    fitness_level = db.Column(db.String(100))
+    own_pets = db.Column(db.String(50))
+    travel_preference = db.Column(db.String(100))
+    relaxation_mode = db.Column(db.String(100))
+    job_role = db.Column(db.String(100))
+    work_experience_years = db.Column(db.String(50))
+    career_aspirations = db.Column(db.String(200))
+    field_of_study = db.Column(db.String(200))
+    remark = db.Column(db.Text)
+    # 🚨 NEW FIELDS
+    children_timeline = db.Column(db.String(100))
+    open_to_adoption = db.Column(db.String(50))
+    deal_breakers = db.Column(db.Text)
+    other_non_negotiables = db.Column(db.Text)
+    health_constraints = db.Column(db.String(200))
+    live_with_inlaws = db.Column(db.String(50))
+
+class ExpectationResponse(db.Model):
+    __tablename__ = "ExpectationResponse"
+
+    user_id = db.Column(db.Integer, primary_key=True)
+    pref_age_range = db.Column(db.String(100))
+    pref_height_range = db.Column(db.String(100))
+    pref_current_city = db.Column(db.String(100))
+    pref_countries = db.Column(db.String(100))
+    pref_languages = db.Column(db.String(100))
+    health_constraints = db.Column(db.String(200))
+    pref_diet = db.Column(db.String(100))
+    accept_smoking = db.Column(db.String(50))
+    accept_alcohol = db.Column(db.String(50))
+    pref_fitness = db.Column(db.String(100))
+    pref_family_type = db.Column(db.String(100))
+    live_with_inlaws = db.Column(db.String(50))  # 🚨 CHANGED: Remove 'pref_' prefix
+    children_timeline = db.Column(db.String(100))
+    open_to_adoption = db.Column(db.String(50))
+    pref_conflict_approach = db.Column(db.String(100))
+    pref_financial_style = db.Column(db.String(100))
+    religion_alignment = db.Column(db.String(50))
+    pref_shared_hobbies = db.Column(db.String(200))
+    travel_pref = db.Column(db.String(100))
+    pet_pref = db.Column(db.String(50))
+    pref_income_range = db.Column(db.String(100))
+    deal_breakers = db.Column(db.Text)
+    other_non_negotiables = db.Column(db.Text)
+    created_at = db.Column(db.DateTime, default=datetime.utcnow)
+    pref_education_level = db.Column(db.String(100))
+    pref_employment_status = db.Column(db.String(100))
+    expectation_summary = db.Column(db.Text)
+    _mandatory_fields = db.Column(db.Text)
+    skin_tone = db.Column(db.String(50))
+    marital_status = db.Column(db.String(50))
+    daily_routine = db.Column(db.String(200))
+    family_communication_frequency = db.Column(db.String(100))
+    relaxation_mode = db.Column(db.String(100))
+    pref_partner_relocation = db.Column(db.String(50))
+    financial_support_to_parents = db.Column(db.String(50))
+    pref_career_aspirations = db.Column(db.String(200))
+    pref_live_with_parents = db.Column(db.String(50))

routes/__init__.py

@@ -0,0 +1,8 @@
+# routes/__init__.py
+from .auth_routes import auth_bp
+from .profile_routes import profiles_bp
+from .expectation_routes import expectations_bp
+from .matching_routes import matching_bp
+from .llm_routes import llm_bp
+
+__all__ = ['auth_bp', 'profiles_bp', 'expectations_bp', 'matching_bp', 'llm_bp']

routes/auth_routes.py

@@ -0,0 +1,173 @@
+# auth_routes.py (with more debug logging)
+from flask import Blueprint, request, jsonify
+import pyodbc
+import traceback
+import os
+
+auth_bp = Blueprint('auth', __name__)
+print(f"✅ AUTH ROUTES: Blueprint '{auth_bp.name}' created")
+
+def get_db_connection():
+    # Use the same connection logic from your original server.py
+    SQL_DRIVER = os.getenv("PYMATCH_SQL_DRIVER", "ODBC Driver 17 for SQL Server")
+    SQL_SERVER = os.getenv("PYMATCH_SQL_SERVER", "localhost\sqlexpress")
+    SQL_DB = os.getenv("PYMATCH_SQL_DB", "Py_Match")
+    SQL_TRUSTED = os.getenv("PYMATCH_SQL_TRUSTED", "yes")
+    
+    print(f"🔗 AUTH ROUTES: Connecting to {SQL_SERVER}/{SQL_DB} with driver {SQL_DRIVER}")
+    
+    return pyodbc.connect(
+        f"DRIVER={SQL_DRIVER};"
+        f"SERVER={SQL_SERVER};"
+        f"DATABASE={SQL_DB};"
+        f"Trusted_Connection={SQL_TRUSTED};"
+    )
+
+@auth_bp.route('/signup', methods=['POST', 'OPTIONS'])
+def signup():
+    print(f"🎯 AUTH ROUTES: /signup endpoint called")
+    
+    if request.method == 'OPTIONS':
+        print(f"🔄 AUTH ROUTES: Handling OPTIONS preflight request")
+        response = jsonify({'success': True})
+        response.headers.add('Access-Control-Allow-Origin', '*')
+        response.headers.add('Access-Control-Allow-Headers', 'Content-Type, Authorization')
+        response.headers.add('Access-Control-Allow-Methods', 'POST, OPTIONS, GET')
+        return response
+    
+    try:
+        data = request.get_json(force=True) or {}
+        print(f"🟢 AUTH ROUTES: Received signup request with data: {data}")
+        
+        name = data.get("name")
+        email = data.get("email")
+        password = data.get("password")
+
+        if not name or not email or not password:
+            print(f"❌ AUTH ROUTES: Missing required fields")
+            return jsonify({"success": False, "message": "Name, email, and password are required."}), 400
+
+        try:
+            conn = get_db_connection()
+            cur = conn.cursor()
+            
+            print(f"🟢 AUTH ROUTES: Checking if email '{email}' already exists...")
+
+            # Check if email already exists
+            cur.execute("SELECT user_id FROM Users WHERE email = ?", (email,))
+            existing = cur.fetchone()
+            if existing:
+                print(f"❌ AUTH ROUTES: Email '{email}' already exists in database")
+                return jsonify({"success": False, "message": "User already exists. Please sign in."}), 409
+
+            # Use plain password (as per your original code)
+            plain_password = password
+
+            print(f"🟢 AUTH ROUTES: Inserting new user '{name}' with email '{email}'")
+
+            # Insert into Users table with plain password
+            cur.execute("""
+                INSERT INTO Users (name, email, password)
+                VALUES (?, ?, ?)
+            """, (name, email, plain_password))
+            conn.commit()
+
+            # Fetch the newly inserted user_id
+            cur.execute("SELECT @@IDENTITY AS user_id")
+            row = cur.fetchone()
+            user_id = row[0] if row else None
+            
+            print(f"✅ AUTH ROUTES: Successfully created user. User ID: {user_id}")
+
+            conn.close()
+            return jsonify({
+                "success": True,
+                "message": "Signup successful.",
+                "user_id": user_id,
+                "name": name,
+                "email": email
+            }), 201
+
+        except pyodbc.Error as e:
+            print(f"❌ AUTH ROUTES: Database Error: {e}")
+            print(f"❌ AUTH ROUTES: SQL State: {e.sqlstate if hasattr(e, 'sqlstate') else 'N/A'}")
+            print(f"❌ AUTH ROUTES: Error Code: {e.args[0] if e.args else 'N/A'}")
+            return jsonify({"success": False, "message": f"Database error: {str(e)}"}), 500
+
+        except Exception as e:
+            print(f"❌ AUTH ROUTES: Unexpected Error: {e}")
+            traceback.print_exc()
+            return jsonify({"success": False, "message": f"Unexpected error: {str(e)}"}), 500
+
+    except Exception as e:
+        print(f"❌ AUTH ROUTES: Outer exception: {e}")
+        traceback.print_exc()
+        return jsonify({"success": False, "message": f"Server error: {str(e)}"}), 500
+
+@auth_bp.route('/login', methods=['POST'])
+def login():
+    print(f"🎯 AUTH ROUTES: /login endpoint called")
+    try:
+        data = request.get_json(force=True) or {}
+        print(f"🟢 AUTH ROUTES: Received login request with email: {data.get('email', 'not provided')}")
+        
+        email = data.get("email")
+        password = data.get("password")
+
+        if not email or not password:
+            print(f"❌ AUTH ROUTES: Missing email or password")
+            return jsonify({"success": False, "message": "Email and password are required."}), 400
+
+        try:
+            conn = get_db_connection()
+            cur = conn.cursor()
+
+            print(f"🟢 AUTH ROUTES: Looking for user with email: {email}")
+            cur.execute("SELECT user_id, name, email, password FROM Users WHERE email = ?", (email,))
+            user = cur.fetchone()
+
+            if not user:
+                print(f"❌ AUTH ROUTES: User not found with email: {email}")
+                return jsonify({"success": False, "message": "User not found."}), 404
+
+            user_id, name, email, stored_password = user
+            print(f"🟢 AUTH ROUTES: Found user ID: {user_id}, Name: {name}")
+
+            # Use simple string comparison for plain text passwords
+            if stored_password != password:
+                print(f"❌ AUTH ROUTES: Password mismatch for user {user_id}")
+                return jsonify({"success": False, "message": "Invalid password."}), 401
+
+            print(f"✅ AUTH ROUTES: Successful login for user {user_id}")
+            conn.close()
+            return jsonify({
+                "success": True,
+                "message": "Login successful.",
+                "user_id": user_id,
+                "name": name,
+                "email": email
+            }), 200
+
+        except pyodbc.Error as e:
+            print(f"❌ AUTH ROUTES: Database Error: {e}")
+            return jsonify({"success": False, "message": f"Database error: {e}"}), 500
+
+        except Exception as e:
+            print(f"❌ AUTH ROUTES: Unexpected Error: {e}")
+            traceback.print_exc()
+            return jsonify({"success": False, "message": f"Unexpected error: {e}"}), 500
+
+    except Exception as e:
+        print(f"❌ AUTH ROUTES: Outer exception: {e}")
+        traceback.print_exc()
+        return jsonify({"success": False, "message": f"Server error: {str(e)}"}), 500
+    finally:
+        try:
+            conn.close()
+        except:
+            pass
+
+@auth_bp.route('/test', methods=['GET'])
+def test():
+    print("✅ AUTH ROUTES: /test endpoint hit!")
+    return jsonify({"message": "Auth routes are working!", "status": "ok", "blueprint": auth_bp.name}), 200

routes/expectation_routes.py

@@ -0,0 +1,415 @@
+# routes/expectation_routes.py
+from flask import Blueprint, request, jsonify
+import pyodbc
+import json
+from database import get_db_connection, row_to_dict
+
+expectations_bp = Blueprint('expectations', __name__)
+
+@expectations_bp.route('/api/existing-preferences/<int:user_id>', methods=['GET'])
+def get_existing_preferences(user_id: int):
+    """Get existing preferences data for a user"""
+    try:
+        conn = get_db_connection()
+        cur = conn.cursor()
+        
+        cur.execute("""
+            SELECT TOP 1 * FROM ExpectationResponse 
+            WHERE user_id = ? 
+            ORDER BY created_at DESC
+        """, (user_id,))
+        
+        row = cur.fetchone()
+        if row is None:
+            return jsonify({"error": "No preferences found"}), 404
+            
+        # Convert row to dict
+        preferences = row_to_dict(cur, row)
+        
+        # Process multi-select fields that are stored as comma-separated strings
+        # Get multi_select question keys from ExpectationQuestions
+        cur.execute("SELECT column_key FROM ExpectationQuestions WHERE input_type = 'multi_select'")
+        multi_select_keys = [row[0] for row in cur.fetchall()]
+        
+        for key in multi_select_keys:
+            if key in preferences and preferences[key]:
+                # Convert comma-separated string back to array
+                if isinstance(preferences[key], str):
+                    preferences[key] = [item.strip() for item in preferences[key].split(",") if item.strip()]
+        
+        return jsonify(preferences), 200
+        
+    except Exception as e:
+        print(f"Error fetching existing preferences: {e}")
+        return jsonify({"error": str(e)}), 500
+    finally:
+        try:
+            conn.close()
+        except:
+            pass
+
+@expectations_bp.route('/api/update-preferences/<int:user_id>', methods=['PUT'])
+def update_preferences(user_id: int):
+    """Update existing preferences"""
+    data = request.get_json(force=True) or {}
+    
+    if not user_id:
+        return jsonify({"error": "User ID is required."}), 400
+
+    try:
+        conn = get_db_connection()
+        cur = conn.cursor()
+
+        print("🟢 DEBUG UPDATE: Incoming data keys ->", list(data.keys()))
+        
+        # 🚨 CRITICAL FIX: Handle _mandatory_fields FIRST
+        mandatory_fields = data.get('_mandatory_fields', {})
+        print("🎯 DEBUG UPDATE: Mandatory fields received:", mandatory_fields)
+        
+        # Build SET clause for update
+        set_parts = []
+        values = []
+        
+        # Handle _mandatory_fields - convert to JSON string
+        if mandatory_fields:
+            set_parts.append('_mandatory_fields = ?')
+            if isinstance(mandatory_fields, dict):
+                mandatory_json = json.dumps(mandatory_fields, ensure_ascii=False)
+            else:
+                mandatory_json = str(mandatory_fields)
+            values.append(mandatory_json)
+            print("✅ DEBUG UPDATE: Adding _mandatory_fields to update:", mandatory_json)
+
+        # 🚨 CRITICAL: Field name mapping from frontend to database
+        field_mapping = {
+            'pref_live_with_inlaws': 'live_with_inlaws',
+            'accept_financial_support_to_parents': 'financial_support_to_parents'
+        }
+
+        # Define all valid ExpectationResponse fields
+        valid_fields = [
+            'pref_age_range', 'pref_height_range', 'pref_current_city', 'pref_countries',
+            'pref_languages', 'health_constraints', 'pref_diet', 'accept_smoking',
+            'accept_alcohol', 'pref_fitness', 'pref_family_type', 'live_with_inlaws',
+            'children_timeline', 'open_to_adoption', 'pref_conflict_approach',
+            'pref_financial_style', 'religion_alignment', 'pref_shared_hobbies',
+            'travel_pref', 'pet_pref', 'pref_income_range', 'deal_breakers',
+            'other_non_negotiables', 'pref_education_level', 'pref_employment_status',
+            'expectation_summary', 'skin_tone', 'marital_status', 'daily_routine',
+            'family_communication_frequency', 'relaxation_mode', 'pref_partner_relocation',
+            'financial_support_to_parents', 'pref_career_aspirations', 'pref_live_with_parents'
+        ]
+        
+        # Get input types from ExpectationQuestions
+        cur.execute("SELECT column_key, input_type FROM ExpectationQuestions")
+        field_types = {row[0]: row[1] for row in cur.fetchall()}
+        
+        # Process all fields
+        for key in valid_fields:
+            # 🚨 CRITICAL: Check if we need to map the field name
+            db_field_name = field_mapping.get(key, key)
+            
+            if key in data and key != 'user_id' and key != '_mandatory_fields':
+                value = data[key]
+                field_type = field_types.get(key, 'text')
+                print(f"🟡 Processing update field {key} -> {db_field_name} (type: {field_type}): {value}")
+                
+                if field_type == 'multi_select' and isinstance(value, list):
+                    clean_values = []
+                    for item in value:
+                        if isinstance(item, str) and item.strip():
+                            clean_item = item.strip()
+                            clean_item = clean_item.replace('[', '').replace(']', '').replace('"', '').strip()
+                            if clean_item and clean_item not in clean_values:
+                                clean_values.append(clean_item)
+                    
+                    if clean_values:
+                        final_value = ", ".join(clean_values)
+                        print(f"🟢 Converted multi_select array to string: '{final_value}'")
+                        set_parts.append(f"{db_field_name} = ?")  # 🚨 Use mapped field name
+                        values.append(final_value)
+                    else:
+                        set_parts.append(f"{db_field_name} = ?")  # 🚨 Use mapped field name
+                        values.append("")
+                        
+                elif field_type == 'multi_select' and isinstance(value, str):
+                    clean_value = value.strip()
+                    clean_value = clean_value.replace('[', '').replace(']', '').replace('"', '').strip()
+                    if clean_value.startswith(',') or clean_value.endswith(','):
+                        clean_value = clean_value.strip(',')
+                    
+                    print(f"🟢 Cleaning multi_select string: '{clean_value}'")
+                    set_parts.append(f"{db_field_name} = ?")  # 🚨 Use mapped field name
+                    values.append(clean_value)
+                    
+                elif value is not None:
+                    final_value = str(value).strip() if isinstance(value, str) else value
+                    print(f"🟢 Storing single value: '{final_value}'")
+                    set_parts.append(f"{db_field_name} = ?")  # 🚨 Use mapped field name
+                    values.append(final_value)
+                else:
+                    # Handle empty values
+                    set_parts.append(f"{db_field_name} = ?")  # 🚨 Use mapped field name
+                    values.append("")
+
+        if not set_parts and not mandatory_fields:
+            return jsonify({"error": "No valid fields to update"}), 400
+
+        # Add user_id for WHERE clause
+        values.append(user_id)
+        
+        set_clause = ", ".join(set_parts)
+        query = f"UPDATE ExpectationResponse SET {set_clause} WHERE user_id = ?"
+        
+        print(f"🟢 DEBUG UPDATE: Executing query: {query}")
+        print(f"🟢 DEBUG UPDATE: Values: {values}")
+
+        cur.execute(query, values)
+        conn.commit()
+
+        # Check if any row was updated
+        if cur.rowcount == 0:
+            print("⚠️ WARNING: No rows were updated - user might not exist")
+            return jsonify({"error": "No preferences found to update"}), 404
+
+        print(f"✅ SUCCESS: Updated {cur.rowcount} row(s) for user {user_id}")
+        return jsonify({"message": "Preferences updated successfully."}), 200
+
+    except Exception as e:
+        print(f"🔴 Error updating preferences: {e}")
+        import traceback
+        traceback.print_exc()
+        return jsonify({"error": str(e)}), 500
+    finally:
+        try:
+            conn.close()
+        except:
+            pass
+
+@expectations_bp.route('/api/check-mandatory-fields/<int:user_id>', methods=['GET'])
+def check_mandatory_fields(user_id: int):
+    """Check current mandatory fields in database"""
+    try:
+        conn = get_db_connection()
+        cur = conn.cursor()
+        
+        cur.execute("""
+            SELECT user_id, _mandatory_fields 
+            FROM ExpectationResponse 
+            WHERE user_id = ?
+        """, (user_id,))
+        
+        row = cur.fetchone()
+        if row is None:
+            return jsonify({"error": "No preferences found for user"}), 404
+            
+        result = {
+            "user_id": row[0],
+            "_mandatory_fields": row[1],
+            "_mandatory_fields_type": str(type(row[1])),
+            "exists_in_db": row[1] is not None
+        }
+        
+        print("🔍 CHECK MANDATORY FIELDS:", result)
+        
+        return jsonify(result), 200
+        
+    except Exception as e:
+        print(f"Error checking mandatory fields: {e}")
+        return jsonify({"error": str(e)}), 500
+    finally:
+        try:
+            conn.close()
+        except:
+            pass
+
+@expectations_bp.route('/api/expectation-questions', methods=['GET'])
+def get_expectation_questions():
+    try:
+        conn = get_db_connection()
+        cur = conn.cursor()
+        cur.execute("""
+            SELECT id, question, options, input_type, column_key, category
+            FROM ExpectationQuestions
+            ORDER BY id
+        """)
+        rows = cur.fetchall()
+
+        out = []
+        for r in rows:
+            out.append({
+                "id": r[0],
+                "question": r[1],
+                "options": (r[2].split(",") if r[2] else []),
+                "input_type": r[3],
+                "column_key": r[4],
+                "category": r[5]
+            })
+        return jsonify(out), 200
+    except Exception as e:
+        return jsonify({"error": str(e)}), 500
+    finally:
+        try: conn.close()
+        except: pass
+
+@expectations_bp.route('/api/expectation-response', methods=['POST'])
+def save_expectation_response():
+    data = request.get_json(force=True) or {}
+    user_id = data.get("user_id")
+
+    if not user_id:
+        return jsonify({"error": "user_id is required"}), 400
+
+    try:
+        conn = get_db_connection()
+        cur = conn.cursor()
+
+        # Get valid keys from ExpectationQuestions AND ExpectationResponse model
+        cur.execute("SELECT column_key, input_type FROM ExpectationQuestions")
+        valid_fields = {row[0]: row[1] for row in cur.fetchall()}
+        
+        # 🚨 CRITICAL FIX: Add all ExpectationResponse model fields
+        expectation_model_fields = [
+                    'pref_age_range', 'pref_height_range', 'pref_current_city', 'pref_countries',
+                    'pref_languages', 'health_constraints', 'pref_diet', 'accept_smoking',
+                    'accept_alcohol', 'pref_fitness', 'pref_family_type', 'live_with_inlaws',  # 🚨 CHANGED: Remove 'pref_' prefix
+                    'children_timeline', 'open_to_adoption', 'pref_conflict_approach',
+                    'pref_financial_style', 'religion_alignment', 'pref_shared_hobbies',
+                    'travel_pref', 'pet_pref', 'pref_income_range', 'deal_breakers',
+                    'other_non_negotiables', 'pref_education_level', 'pref_employment_status',
+                    'expectation_summary', '_mandatory_fields', 'skin_tone', 'marital_status',
+                    'daily_routine', 'family_communication_frequency', 'relaxation_mode',
+                    'pref_partner_relocation', 'financial_support_to_parents',  # 🚨 CHANGED: Remove 'accept_' prefix
+                    'pref_career_aspirations', 'pref_live_with_parents'
+                ]
+        
+        # Add model fields to valid_fields (default to 'text' input type if not in questions)
+        for field in expectation_model_fields:
+            if field not in valid_fields:
+                valid_fields[field] = 'text'  # default type
+
+        print("🟢 DEBUG: Valid fields ->", list(valid_fields.keys()))
+        print("🟢 DEBUG: Incoming data keys ->", list(data.keys()))
+
+        cols, vals = [], []
+        
+        # 🚨 CRITICAL: Handle _mandatory_fields FIRST
+        mandatory_fields = data.get('_mandatory_fields', {})
+        if mandatory_fields:
+            cols.append('_mandatory_fields')
+            if isinstance(mandatory_fields, dict):
+                mandatory_json = json.dumps(mandatory_fields, ensure_ascii=False)
+                vals.append(mandatory_json)
+            else:
+                vals.append(str(mandatory_fields))
+            print("✅ DEBUG: Added _mandatory_fields:", mandatory_fields)
+
+        # Process all other fields
+        for key, field_type in valid_fields.items():
+            if key in data and key != 'user_id' and key != '_mandatory_fields':
+                value = data[key]
+                print(f"🟡 Processing field {key} (type: {field_type}): {value}")
+                
+                if value is None or value == '':
+                    # Skip empty values
+                    continue
+                    
+                if field_type == 'multi_select' and isinstance(value, list):
+                    # Clean array data
+                    clean_values = []
+                    for item in value:
+                        if isinstance(item, str) and item.strip():
+                            clean_item = item.strip()
+                            clean_item = clean_item.replace('[', '').replace(']', '').replace('"', '').strip()
+                            if clean_item and clean_item not in clean_values:
+                                clean_values.append(clean_item)
+                    
+                    if clean_values:
+                        final_value = ", ".join(clean_values)
+                        print(f"🟢 Converted multi_select array to string: '{final_value}'")
+                        cols.append(key)
+                        vals.append(final_value)
+                    else:
+                        # Skip empty arrays
+                        continue
+                        
+                elif field_type == 'multi_select' and isinstance(value, str):
+                    # Clean string data
+                    clean_value = value.strip()
+                    clean_value = clean_value.replace('[', '').replace(']', '').replace('"', '').strip()
+                    if clean_value.startswith(',') or clean_value.endswith(','):
+                        clean_value = clean_value.strip(',')
+                    
+                    if clean_value:
+                        print(f"🟢 Cleaning multi_select string: '{clean_value}'")
+                        cols.append(key)
+                        vals.append(clean_value)
+                    else:
+                        continue
+                        
+                else:
+                    # For single values
+                    final_value = str(value).strip() if isinstance(value, str) else value
+                    if final_value:  # Only add non-empty values
+                        print(f"🟢 Storing single value: '{final_value}'")
+                        cols.append(key)
+                        vals.append(final_value)
+
+        if not cols:
+            return jsonify({"error": "No valid fields found in request"}), 400
+
+        # Build INSERT query
+        placeholders = ", ".join(["?"] * (len(cols) + 1))  # +1 for user_id
+        col_str = ", ".join([f"[{c}]" for c in cols])
+
+        query = f"""
+            INSERT INTO ExpectationResponse (user_id, {col_str})
+            VALUES ({placeholders})
+        """
+        print("🟢 DEBUG: Final query ->", query)
+        print("🟢 DEBUG: Values count ->", len([user_id] + vals))
+        print("🟢 DEBUG: Columns ->", cols)
+
+        cur.execute(query, [user_id] + vals)
+        conn.commit()
+
+        return jsonify({"message": "Preferences saved successfully"}), 201
+
+    except Exception as e:
+        import traceback
+        print("🔴 ERROR in save_expectation_response:")
+        traceback.print_exc()
+        return jsonify({"error": str(e)}), 500
+
+    finally:
+        try:
+            conn.close()
+        except:
+            pass
+
+@expectations_bp.route('/api/check-expectations/<int:user_id>', methods=['GET'])
+def check_expectations(user_id: int):
+    """Check if expectations exist for user"""
+    try:
+        conn = get_db_connection()
+        cur = conn.cursor()
+        
+        cur.execute("""
+            SELECT COUNT(*) as count 
+            FROM ExpectationResponse 
+            WHERE user_id = ?
+        """, (user_id,))
+        
+        row = cur.fetchone()
+        exists = row[0] > 0 if row else False
+        
+        return jsonify({"exists": exists}), 200
+        
+    except Exception as e:
+        print(f"Error checking expectations: {e}")
+        return jsonify({"error": str(e)}), 500
+    finally:
+        try:
+            conn.close()
+        except:
+            pass

routes/llm_routes.py

@@ -0,0 +1,181 @@
+# routes/llm_routes.py
+from flask import Blueprint, request, jsonify
+import uuid
+from llm_service import (
+    SessionState, SESSIONS, save_sessions, persist_final_progress,
+    choose_themes, generate_batch_questions
+)
+# Import fetch_profile_for_role from database instead of llm_service
+from database import fetch_profile_for_role
+
+llm_bp = Blueprint('llm', __name__)
+
+@llm_bp.route('/llm/start', methods=['POST'])
+def llm_start():
+    data = request.get_json(force=True) or {}
+    user_id = str(data.get("user_id") or "").strip()
+    role_in = (data.get("role") or "general").lower()
+    n_req = int(data.get("n_questions", 20))
+    b_req = int(data.get("batch_size", 10))
+
+    if not user_id:
+        return jsonify({"error": "user_id is required"}), 400
+    if role_in not in ["marriage", "interview", "partnership", "general"]:
+        return jsonify({"error": f"Invalid role. Allowed: marriage, interview, partnership, general"}), 400
+
+    # 🚨 CRITICAL: Check if user has already taken assessment
+    try:
+        from database import get_db_connection
+        conn = get_db_connection()
+        cur = conn.cursor()
+        cur.execute("""
+            SELECT COUNT(*) as count 
+            FROM LLMGeneratedQuestions 
+            WHERE user_id = ? AND (blue > 0 OR green > 0 OR yellow > 0 OR red > 0)
+        """, (user_id,))
+        row = cur.fetchone()
+        has_taken_assessment = row[0] > 0 if row else False
+        conn.close()
+        
+        if has_taken_assessment:
+            print(f"🚫 User {user_id} already took assessment, blocking new session")
+            return jsonify({
+                "error": "Assessment already completed",
+                "assessment_already_taken": True
+            }), 400
+    except Exception as e:
+        print(f"Warning: Could not check assessment status: {e}")
+
+    # Fetch profile from the correct table based on role
+    profile = fetch_profile_for_role(user_id, role_in)
+
+    # Create session
+    sid = str(uuid.uuid4())
+    sess = SessionState(
+        n_questions=n_req,
+        batch_size=b_req,
+        domain=role_in,
+        role=role_in,
+        profile=profile
+    )
+    SESSIONS[sid] = sess
+
+    # Generate first batch of questions from FAISS chunks
+    to_generate = min(sess.batch_size, sess.remaining())
+    themes = choose_themes(sess, to_generate)
+
+    context = ""
+    try:
+        from faiss_service import HAS_FAISS, FAISS_INDEX, TEXT_CHUNKS
+        if HAS_FAISS and FAISS_INDEX is not None and TEXT_CHUNKS:
+            import random
+            context = "\n".join(random.sample(TEXT_CHUNKS, min(3, len(TEXT_CHUNKS))))
+    except ImportError:
+        pass
+
+    # Generate questions
+    queue = generate_batch_questions(themes, sess.to_min_state(), context=context, previous_questions=sess.history_of_questions)
+
+    if not queue:
+        return jsonify({"error": "Question generation failed"}), 500
+
+    sess.queue = queue
+
+    # Serve first question
+    first = sess.queue.pop(0)
+    sess.asked += 1
+
+    # Track the asked question
+    sess.history_of_questions.append(first["question"])
+
+    save_sessions()
+
+    return jsonify({
+        "session_id": sid,
+        "index": 1,
+        "total": sess.n_questions,
+        "question": first["question"],
+        "options": first["options"],
+        "source": first.get("source", "unknown"),
+        "role": sess.role,
+        "profile_used": bool(profile),
+        "faiss_themes": themes,
+        "faiss_context": context
+    })
+
+@llm_bp.route('/llm/next', methods=['POST'])
+def llm_next():
+    data = request.get_json(force=True) or {}
+    sid = data.get("session_id")
+    color = str(data.get("selected_color") or "").lower()
+
+    if not sid or sid not in SESSIONS:
+        return jsonify({"error": "Invalid or missing session_id"}), 400
+    if color not in ["blue", "green", "red", "yellow"]:
+        return jsonify({"error": "selected_color must be blue|green|red|yellow"}), 400
+
+    sess = SESSIONS[sid]
+    if sess.finished:
+        return jsonify({"done": True, "message": "Session already finished."})
+
+    # record answer
+    sess.color_counts[color] += 1
+    sess.history.append({"selected_color": color})
+
+    # Initialize themes and context with default values
+    themes = []
+    context = ""
+
+    # finished?
+    if sess.asked >= sess.n_questions:
+        sess.finished = True
+        mix = sess.to_min_state()["mix"]
+        user_id = (sess.profile or {}).get("user_id")
+        db_ok = persist_final_progress(user_id=user_id, role=sess.role, mix=mix)
+        save_sessions()
+        return jsonify({
+            "done": True,
+            "message": "No more questions.",
+            "mix": mix,
+            "db_write": "ok" if db_ok else "failed"
+        })
+
+    # ensure queue; refill if needed
+    if not sess.queue:
+        to_generate = min(sess.batch_size, sess.remaining())
+        themes = choose_themes(sess, to_generate)
+
+        try:
+            from faiss_service import HAS_FAISS, FAISS_INDEX, TEXT_CHUNKS
+            if HAS_FAISS and FAISS_INDEX is not None and TEXT_CHUNKS:
+                import random
+                context = "\n".join(random.sample(TEXT_CHUNKS, min(3, len(TEXT_CHUNKS))))
+        except ImportError:
+            pass
+
+        # Generate questions
+        sess.queue = generate_batch_questions(themes, sess.to_min_state(), context=context, previous_questions=sess.history_of_questions)
+
+        if not sess.queue:
+            return jsonify({"error": "Question generation failed"}), 500
+
+    nxt = sess.queue.pop(0)
+    sess.asked += 1
+
+    # Track the asked question
+    sess.history_of_questions.append(nxt["question"])
+
+    save_sessions()
+
+    return jsonify({
+        "session_id": sid,
+        "index": sess.asked,
+        "total": sess.n_questions,
+        "question": nxt["question"],
+        "options": nxt["options"],
+        "progress": sess.to_min_state()["mix"],
+        "source": nxt.get("source", "unknown"),
+        "role": sess.role,
+        "faiss_themes": themes,
+        "faiss_context": context
+    })

routes/matching_routes.py

@@ -0,0 +1,325 @@
+# routes/matching_routes.py
+from flask import Blueprint, request, jsonify, current_app
+import numpy as np
+from sqlalchemy import func
+from models import LLMGeneratedQuestions, Marriage, Users, ExpectationResponse, db
+from matching_functions import match_expectation_with_profiles, generate_expectation_explanation
+from character_functions import cosine_sim, generate_character_llm_explanation, generate_character_fallback_explanation
+from database import fetch_expectation_data, fetch_marriage_profile_data
+from config import COLOR_KEYS
+
+matching_bp = Blueprint('matching', __name__)
+
+@matching_bp.route('/match')
+@matching_bp.route('/match/<int:user_id>')
+def unified_match(user_id=None):
+    """Unified match endpoint that handles all three modes"""
+    # Remove the incorrect db.app.app_context() and use current_app instead
+    # The app context is already provided by Flask for route handlers
+    
+    # Get user_id from either path parameter or query parameter
+    if user_id is None:
+        try:
+            user_id = int(request.args.get("user_id", ""))
+        except ValueError:
+            return jsonify({"error": "Missing or invalid user_id"}), 400
+
+    # Get parameters
+    role = request.args.get("role", None)
+    limit = int(request.args.get("limit", "10"))
+    exclude_self = request.args.get("exclude_self", "yes").lower() == "yes"
+    mode = request.args.get("mode", "expectation-only")  # Default to expectation-only
+    
+    print(f"🔍 DEBUG: Match request - user_id: {user_id}, mode: {mode}")
+
+    # 🚨 DECISION: Handle all three modes
+    if mode == "expectation-only":
+        print("🎯 Using PURE EXPECTATION matching")
+        # Pure expectation matching only (with mandatory filtering)
+        expectation_matches = match_expectation_with_profiles(user_id)
+        
+        if not expectation_matches:
+            return jsonify({"error": f"No matches found for user_id={user_id}"}), 404
+
+        # Convert to frontend format with expectation scores only
+        matches_by_range = {
+            "90-100": [],
+            "80-89": [], 
+            "70-79": [],
+            "60-69": [],
+            "below_60": []
+        }
+        
+        for match in expectation_matches:
+            # Use expectation score only (0-1 scale) to percentage (0-100)
+            score_percentage = match.get("expectation_score", 0) * 100
+            
+            # Determine which range this match belongs to
+            if score_percentage >= 90:
+                range_key = "90-100"
+            elif score_percentage >= 80:
+                range_key = "80-89"
+            elif score_percentage >= 70:
+                range_key = "70-79" 
+            elif score_percentage >= 60:
+                range_key = "60-69"
+            else:
+                range_key = "below_60"
+            
+            # Create match object with expectation score only
+            match_obj = {
+                "user_id": match["user_id"],
+                "name": match["name"],
+                "gender": match.get("gender", ""),
+                "city": match.get("location", ""),
+                "score_expect": match.get("expectation_score", 0),  # Raw score (0-1)
+                "score_color": match.get("character_score", 0),     # Still include but not used for sorting
+                "final_score": round(score_percentage, 2),          # Percentage for display
+                "blue": 0, "green": 0, "yellow": 0, "red": 0,
+                "explanations": [],
+                "explanation_source": "expectation"
+            }
+            
+            matches_by_range[range_key].append(match_obj)
+
+        # Get user data for input_user
+        user = Users.query.filter_by(user_id=user_id).first()
+        
+        input_user = {
+            "user_id": user_id,
+            "role": "marriage",
+            "name": user.name if user else "Unknown",
+            "blue": 0, "green": 0, "yellow": 0, "red": 0,  # Not used in this mode
+            "created_at": None,
+        }
+
+        print(f"✅ DEBUG: Returning {len(expectation_matches)} pure expectation matches")
+        
+        return jsonify({
+            "input_user": input_user,
+            "matches": matches_by_range,
+            "count": len(expectation_matches),
+            "mode": "expectation-only"
+        })
+        
+    elif mode == "character":
+        print("🎯 Using PURE CHARACTER matching - NO EXPECTATION FILTERING")
+        
+        # Get current user to know gender
+        current_user = Marriage.query.filter_by(user_id=user_id).first()
+        if not current_user:
+            return jsonify({"error": f"No marriage profile found for user_id={user_id}"}), 404
+
+        user_gender = (current_user.gender or "").lower()
+        print(f"🔍 DEBUG: Current user gender: {user_gender}")
+
+        # Opposite gender profiles only - NO MANDATORY FILTERING
+        if user_gender.startswith('male'):
+            opposite_profiles = Marriage.query.filter(func.lower(func.trim(Marriage.gender)) == "female").all()
+        elif user_gender.startswith('female'):
+            opposite_profiles = Marriage.query.filter(func.lower(func.trim(Marriage.gender)) == "male").all()
+        else:
+            opposite_profiles = Marriage.query.filter(Marriage.gender != current_user.gender).all()
+
+        print(f"🔍 DEBUG: Found {len(opposite_profiles)} opposite gender profiles (NO MANDATORY FILTERING)")
+
+        # Get base user's character data
+        base_llm = LLMGeneratedQuestions.query.filter_by(user_id=user_id).first()
+        if not base_llm:
+            return jsonify({"error": f"No character data found for user_id={user_id}"}), 404
+
+        u_vec = base_llm.color_vec()
+
+        # Calculate character scores for ALL opposite gender profiles
+        candidates = []
+        all_ids = [profile.user_id for profile in opposite_profiles]
+        
+        # Get LLM data for all candidates
+        llm_data = LLMGeneratedQuestions.query.filter(LLMGeneratedQuestions.user_id.in_(all_ids)).all()
+        llm_map = {l.user_id: l for l in llm_data}
+
+        for profile in opposite_profiles:
+            if profile.user_id in llm_map:
+                llm_other = llm_map[profile.user_id]
+                v_vec = llm_other.color_vec()
+                
+                # Compute character similarity
+                character_score = cosine_sim(u_vec, v_vec)
+                
+                # Convert to percentage for display
+                score_percentage = round(character_score * 100, 2)
+                
+                candidates.append({
+                    "user_id": profile.user_id,
+                    "name": profile.full_name,
+                    "gender": profile.gender,
+                    "location": profile.current_city,
+                    "score_color": character_score,  # Raw score (0-1)
+                    "score_expect": 0,  # Not used in this mode
+                    "final_score": score_percentage,  # Percentage for display
+                    "blue": llm_other.blue,
+                    "green": llm_other.green,
+                    "yellow": llm_other.yellow,
+                    "red": llm_other.red,
+                    "explanations": [],
+                    "explanation_source": "character"
+                })
+
+        # Sort by character score (highest first)
+        candidates.sort(key=lambda x: x["score_color"], reverse=True)
+        print(f"🔍 DEBUG: Pure character matching found {len(candidates)} candidates")
+
+        # 🚨 ADD: Detailed debug logging for score distribution
+        print("🔍 DEBUG: Candidate scores distribution:")
+        score_ranges = {"90+": 0, "80-89": 0, "70-79": 0, "60-69": 0, "below_60": 0}
+        for candidate in candidates:
+            score = candidate["final_score"]
+            if score >= 90:
+                score_ranges["90+"] += 1
+            elif score >= 80:
+                score_ranges["80-89"] += 1
+            elif score >= 70:
+                score_ranges["70-79"] += 1
+            elif score >= 60:
+                score_ranges["60-69"] += 1
+            else:
+                score_ranges["below_60"] += 1
+        
+        for range_name, count in score_ranges.items():
+            print(f"   {range_name}: {count} users")
+
+        # Show first 10 candidates with detailed scores
+        print("🔍 DEBUG: Top 10 candidate scores:")
+        for i, candidate in enumerate(candidates[:10]):
+            print(f"   {i+1}. {candidate['name']}: raw={candidate['score_color']:.3f}, percentage={candidate['final_score']}%")
+
+        # Group by score ranges
+        matches_by_range = {
+            "90-100": [],
+            "80-89": [], 
+            "70-79": [],
+            "60-69": [],
+            "below_60": []
+        }
+        
+        for candidate in candidates:
+            score_percentage = candidate["final_score"]
+            
+            # Determine which range this match belongs to
+            if score_percentage >= 90:
+                range_key = "90-100"
+            elif score_percentage >= 80:
+                range_key = "80-89"
+            elif score_percentage >= 70:
+                range_key = "70-79" 
+            elif score_percentage >= 60:
+                range_key = "60-69"
+            else:
+                range_key = "below_60"
+            
+            matches_by_range[range_key].append(candidate)
+
+        # 🚨 ADD: Debug logging to verify range assignment
+        print("🔍 DEBUG: Range distribution after grouping:")
+        for range_key, matches in matches_by_range.items():
+            if matches:
+                scores = [m["final_score"] for m in matches]
+                print(f"   {range_key}: {len(matches)} users, scores: {min(scores):.1f}% - {max(scores):.1f}%")
+            else:
+                print(f"   {range_key}: 0 users")
+
+        # Get user data for input_user
+        user = Users.query.filter_by(user_id=user_id).first()
+        
+        input_user = {
+            "user_id": user_id,
+            "role": "marriage",
+            "name": user.name if user else "Unknown",
+            "blue": base_llm.blue,
+            "green": base_llm.green,
+            "yellow": base_llm.yellow,
+            "red": base_llm.red,
+            "created_at": base_llm.created_at.isoformat() if base_llm.created_at else None,
+        }
+
+        print(f"✅ DEBUG: Returning {len(candidates)} pure character matches (NO EXPECTATION FILTERING)")
+        
+        return jsonify({
+            "input_user": input_user,
+            "matches": matches_by_range,
+            "count": len(candidates),
+            "mode": "character"
+        })
+    
+    else:  # expectation mode (default - expectation + character refinement)
+        print("🎯 Using EXPECTATION + CHARACTER matching")
+        # Use expectation-based matching with character refinement
+        expectation_matches = match_expectation_with_profiles(user_id)
+        
+        if not expectation_matches:
+            return jsonify({"error": f"No matches found for user_id={user_id}"}), 404
+
+        # Convert to the expected frontend format with combined scores
+        matches_by_range = {
+            "90-100": [],
+            "80-89": [], 
+            "70-79": [],
+            "60-69": [],
+            "below_60": []
+        }
+        
+        for match in expectation_matches:
+            # Convert overall_score (0-1 scale) to percentage (0-100)
+            score_percentage = match.get("overall_score", 0) * 100
+            
+            # Determine which range this match belongs to
+            if score_percentage >= 90:
+                range_key = "90-100"
+            elif score_percentage >= 80:
+                range_key = "80-89"
+            elif score_percentage >= 70:
+                range_key = "70-79" 
+            elif score_percentage >= 60:
+                range_key = "60-69"
+            else:
+                range_key = "below_60"
+            
+            # Create match object with combined scores
+            match_obj = {
+                "user_id": match["user_id"],
+                "name": match["name"],
+                "gender": match.get("gender", ""),
+                "city": match.get("location", ""),
+                "final_score": round(score_percentage, 2),
+                "score_expect": match.get("expectation_score", 0),
+                "score_color": match.get("character_score", 0),
+                "blue": 0, "green": 0, "yellow": 0, "red": 0,
+                "explanations": [],
+                "explanation_source": "expectation"
+            }
+            
+            matches_by_range[range_key].append(match_obj)
+
+        # Get user data for input_user
+        user = Users.query.filter_by(user_id=user_id).first()
+        llm_data = LLMGeneratedQuestions.query.filter_by(user_id=user_id).first()
+        
+        input_user = {
+            "user_id": user_id,
+            "role": "marriage",
+            "name": user.name if user else "Unknown",
+            "blue": llm_data.blue if llm_data else 0,
+            "green": llm_data.green if llm_data else 0, 
+            "yellow": llm_data.yellow if llm_data else 0,
+            "red": llm_data.red if llm_data else 0,
+            "created_at": llm_data.created_at.isoformat() if llm_data and llm_data.created_at else None,
+        }
+
+        print(f"✅ DEBUG: Returning {len(expectation_matches)} expectation + character matches")
+        
+        return jsonify({
+            "input_user": input_user,
+            "matches": matches_by_range,
+            "count": len(expectation_matches),
+            "mode": "expectation"
+        })

routes/profile_routes.py

@@ -0,0 +1,441 @@
+# routes/profile_routes.py
+from flask import Blueprint, request, jsonify
+import pyodbc
+import json
+from database import get_db_connection, row_to_dict
+
+profiles_bp = Blueprint('profiles', __name__)
+
+@profiles_bp.route('/api/questions/select-role', methods=['POST'])
+def select_role():
+    data = request.get_json(force=True) or {}
+    user_id = data.get("user_id")
+    role_name = data.get("role_name")
+    assigned_at = data.get("assigned_at")  # ISO or None
+
+    # Check if user_id and role_name are provided
+    if not user_id or not role_name:
+        return jsonify({"error": "User ID and role name are required."}), 400
+
+    try:
+        # Check if user_id exists in the Users table
+        conn = get_db_connection()
+        cur = conn.cursor()
+        cur.execute("SELECT COUNT(*) FROM Users WHERE user_id = ?", (user_id,))
+        user_exists = cur.fetchone()[0]
+
+        if user_exists == 0:
+            return jsonify({"error": "User ID does not exist in the Users table."}), 404
+
+        # Proceed with inserting into UserRoles
+        cur.execute("""
+            INSERT INTO UserRoles (user_id, role_name, assigned_at)
+            VALUES (?, ?, ?)
+        """, (user_id, role_name, assigned_at))
+        conn.commit()
+
+        return jsonify({"message": "Role assigned successfully."}), 201
+
+    except pyodbc.Error as e:
+        # Handle database error, including foreign key constraint violations
+        if "foreign key" in str(e).lower():
+            return jsonify({"error": "Foreign key violation: User ID not found."}), 400
+        return jsonify({"error": f"Database error: {str(e)}"}), 500
+    except Exception as e:
+        return jsonify({"error": f"Unexpected error: {str(e)}"}), 500
+    finally:
+        try:
+            conn.close()
+        except:
+            pass
+
+@profiles_bp.route('/api/questions/marriage', methods=['GET'])
+def get_questions():
+    try:
+        conn = get_db_connection()
+        cur = conn.cursor()
+        cur.execute("""
+            SELECT question, options, input_type, column_key, category
+            FROM RoleQuestions
+            WHERE role_name = 'marriage'
+            ORDER BY id
+        """)
+        rows = cur.fetchall()
+        out = []
+        for r in rows:
+            label = r[0]
+            options = (r[1].split(",") if r[1] else [])
+            input_type = r[2]
+            column_key = r[3]
+            category = r[4]
+            out.append({
+                "label": label,
+                "options": options,
+                "input_type": input_type,
+                "column_key": column_key,
+                "category": category
+            })
+        return jsonify(out), 200
+    except pyodbc.Error as e:
+        return jsonify({"error": str(e)}), 500
+    finally:
+        try: conn.close()
+        except: pass
+
+@profiles_bp.route('/api/questions/submit-answers/marriage', methods=['POST'])
+def submit_answers():
+    data = request.get_json(force=True) or {}
+    user_id = data.get("user_id")
+    if not user_id:
+        return jsonify({"error": "User ID is required."}), 400
+
+    role_fields = {
+            "marriage": [
+                "full_name", "date_of_birth", "gender", "current_city", "marital_status",
+                "education_level", "employment_status", "number_of_siblings", "family_type",
+                "hobbies_interests", "conflict_approach", "financial_style", "income_range",
+                "relocation_willingness", "height", "skin_tone", "languages_spoken", "country",
+                "blood_group", "religion", "dual_citizenship", "siblings_position",
+                "parents_living_status", "live_with_parents", "support_parents_financially",
+                "family_communication_frequency", "food_preference", "smoking_habit",
+                "alcohol_habit", "daily_routine", "fitness_level", "own_pets",
+                "travel_preference", "relaxation_mode", "job_role", "work_experience_years",
+                "career_aspirations", "field_of_study", "remark", "children_timeline",
+                "open_to_adoption", "deal_breakers", "other_non_negotiables",
+                "health_constraints", "live_with_inlaws"
+                # Note: "created_at" is excluded (auto-generated)
+            ]
+        }
+
+    # Validate all required fields are present
+    for f in role_fields["marriage"]:
+        if f not in data:
+            return jsonify({"error": f"{f} is required."}), 400
+
+    try:
+        conn = get_db_connection()
+        cur = conn.cursor()
+
+        table_name = "Marriage"
+        
+        # Build INSERT query without created_at
+        columns = ["user_id"] + role_fields["marriage"]
+        placeholders = ", ".join(["?"] * len(columns))
+        col_str = ", ".join([f"[{c}]" for c in columns])
+        
+        query = f"INSERT INTO {table_name} ({col_str}) VALUES ({placeholders})"
+
+        values = [user_id]
+        for f in role_fields["marriage"]:
+            val = data.get(f)
+                        # Handle radio button values (convert 1/0 to yes/no)
+            if f in ["dual_citizenship", "live_with_parents", "support_parents_financially", "own_pets"]:
+                if val == 1 or val == "1" or val is True:
+                    val = "Yes"
+                elif val == 0 or val == "0" or val is False:
+                    val = "No"
+                # If it's already "yes" or "no", leave it as is
+                elif val not in ["Yes", "No"]:
+                    val = "No"  # default to "no" if invalid value
+            # Handle list values (multiselect)
+            if isinstance(val, list):
+                val = ", ".join([str(v) for v in val])
+
+            # Convert to string or None
+            if val is None:
+                val = None
+            else:
+                val = str(val)
+
+            values.append(val)
+
+        print(f"DEBUG: Executing query: {query}")
+        print(f"DEBUG: Values: {values}")
+
+        cur.execute(query, values)
+        conn.commit()
+
+        return jsonify({"message": "Marriage record added successfully."}), 201
+
+    except pyodbc.Error as e:
+        print(f"Database Error: {e}")
+        return jsonify({"error": f"Database error: {str(e)}"}), 500
+    except Exception as e:
+        print(f"Unexpected Error: {e}")
+        return jsonify({"error": f"Unexpected error: {str(e)}"}), 500
+    finally:
+        try:
+            conn.close()
+        except:
+            pass
+
+@profiles_bp.route('/api/questions/existing-profile/<role>/<int:user_id>', methods=['GET'])
+def get_existing_profile(role: str, user_id: int):
+    """Get existing profile data for a user"""
+    try:
+        conn = get_db_connection()
+        cur = conn.cursor()
+        
+        # Determine table based on role
+        table_map = {
+            "marriage": "Marriage",
+            "interview": "Interview", 
+            "partnership": "Partnership"
+        }
+        
+        table_name = table_map.get(role.lower())
+        if not table_name:
+            return jsonify({"error": "Invalid role"}), 400
+        
+        cur.execute(f"""
+            SELECT TOP 1 * FROM {table_name} 
+            WHERE user_id = ? 
+            ORDER BY created_at DESC
+        """, (user_id,))
+        
+        row = cur.fetchone()
+        if row is None:
+            return jsonify({"error": "No profile found"}), 404
+            
+        # Convert row to dict
+        profile = row_to_dict(cur, row)
+        
+        # 🚨 CRITICAL: Clean up data for radio buttons
+        # Ensure radio button values are clean strings that match option values
+        for key, value in profile.items():
+            if value is not None:
+                # Convert to string and trim for consistency
+                if isinstance(value, bool):
+                    profile[key] = "Yes" if value else "No"
+                elif isinstance(value, (int, float)):
+                    profile[key] = str(value)
+                elif isinstance(value, str):
+                    profile[key] = value.strip()
+        
+        print(f"🟢 DEBUG: Returning cleaned profile data for user {user_id}")
+        return jsonify(profile), 200
+        
+    except Exception as e:
+        print(f"Error fetching existing profile: {e}")
+        return jsonify({"error": str(e)}), 500
+    finally:
+        try:
+            conn.close()
+        except:
+            pass
+
+@profiles_bp.route('/api/questions/update-answers/<role>', methods=['PUT'])
+def update_answers(role: str):
+    """Update existing profile answers"""
+    data = request.get_json(force=True) or {}
+    user_id = data.get("user_id")
+    
+    if not user_id:
+        return jsonify({"error": "User ID is required."}), 400
+
+    role_fields = {
+            "marriage": [
+                "full_name", "date_of_birth", "gender", "current_city", "marital_status",
+                "education_level", "employment_status", "number_of_siblings", "family_type",
+                "hobbies_interests", "conflict_approach", "financial_style", "income_range",
+                "relocation_willingness", "height", "skin_tone", "languages_spoken", "country",
+                "blood_group", "religion", "dual_citizenship", "siblings_position",
+                "parents_living_status", "live_with_parents", "support_parents_financially",
+                "family_communication_frequency", "food_preference", "smoking_habit",
+                "alcohol_habit", "daily_routine", "fitness_level", "own_pets",
+                "travel_preference", "relaxation_mode", "job_role", "work_experience_years",
+                "career_aspirations", "field_of_study", "remark", "children_timeline",
+                "open_to_adoption", "deal_breakers", "other_non_negotiables",
+                "health_constraints", "live_with_inlaws"
+                # Note: "created_at" is excluded (auto-generated)
+            ]
+        }
+
+    if role not in role_fields:
+        return jsonify({"error": f"Invalid role: {role}"}), 400
+
+    try:
+        conn = get_db_connection()
+        cur = conn.cursor()
+
+        table_name = "Marriage" if role == "marriage" else role.capitalize()
+        
+        # Build UPDATE query - only include fields that are present in data
+        set_parts = []
+        values = []
+        
+        for field in role_fields[role]:
+            if field in data:
+                set_parts.append(f"{field} = ?")
+                val = data.get(field)
+                # Handle radio button values (convert 1/0 to yes/no)
+                if field in ["dual_citizenship", "live_with_parents", "support_parents_financially", "own_pets"]:
+                    if val == 1 or val == "1" or val is True:
+                        val = "Yes"
+                    elif val == 0 or val == "0" or val is False:
+                        val = "No"
+                    # If it's already "Yes" or "no", leave it as is
+                    elif val not in ["Yes", "No"]:
+                        val = "No"  # default to "no" if invalid value
+                # Handle list values (multiselect)
+                if isinstance(val, list):
+                    val = ", ".join([str(v) for v in val])
+
+                # Convert to string or None
+                if val is None:
+                    val = None
+                else:
+                    val = str(val)
+
+                values.append(val)
+
+        if not set_parts:
+            return jsonify({"error": "No valid fields to update"}), 400
+
+        # Add user_id for WHERE clause
+        values.append(user_id)
+        
+        set_clause = ", ".join(set_parts)
+        query = f"UPDATE {table_name} SET {set_clause} WHERE user_id = ?"
+
+        print(f"DEBUG: Executing update query: {query}")
+        print(f"DEBUG: Values: {values}")
+
+        cur.execute(query, values)
+        conn.commit()
+
+        # Check if any row was updated
+        if cur.rowcount == 0:
+            return jsonify({"error": "No profile found to update"}), 404
+
+        return jsonify({"message": "Profile updated successfully."}), 200
+
+    except pyodbc.Error as e:
+        print(f"Database Error: {e}")
+        return jsonify({"error": f"Database error: {str(e)}"}), 500
+    except Exception as e:
+        print(f"Unexpected Error: {e}")
+        return jsonify({"error": f"Unexpected error: {str(e)}"}), 500
+    finally:
+        try:
+            conn.close()
+        except:
+            pass
+
+@profiles_bp.route('/api/marriage-profile/<int:user_id>', methods=['GET'])
+def get_marriage_profile(user_id: int):
+    """Get marriage profile by user_id"""
+    try:
+        conn = get_db_connection()
+        cur = conn.cursor()
+
+        cur.execute("""
+            SELECT * FROM Marriage
+            WHERE user_id = ?
+            ORDER BY created_at DESC
+        """, (user_id,))
+
+        row = cur.fetchone()
+        if row is None:
+            return jsonify({"error": "Marriage profile not found"}), 404
+
+        # Convert row to dict
+        profile = row_to_dict(cur, row)
+
+        return jsonify(profile), 200
+
+    except Exception as e:
+        print(f"Error fetching marriage profile: {e}")
+        return jsonify({"error": str(e)}), 500
+    finally:
+        try:
+            conn.close()
+        except:
+            pass
+
+@profiles_bp.route('/api/check-marriage-profile/<int:user_id>', methods=['GET'])
+def check_marriage_profile(user_id: int):
+    """Check if marriage profile exists for user"""
+    try:
+        conn = get_db_connection()
+        cur = conn.cursor()
+        
+        cur.execute("""
+            SELECT COUNT(*) as count 
+            FROM Marriage 
+            WHERE user_id = ?
+        """, (user_id,))
+        
+        row = cur.fetchone()
+        exists = row[0] > 0 if row else False
+        
+        return jsonify({"exists": exists}), 200
+        
+    except Exception as e:
+        print(f"Error checking marriage profile: {e}")
+        return jsonify({"error": str(e)}), 500
+    finally:
+        try:
+            conn.close()
+        except:
+            pass
+
+@profiles_bp.route('/api/check-assessment/<int:user_id>', methods=['GET'])
+def check_assessment(user_id: int):
+    """Check if assessment is completed for user"""
+    try:
+        conn = get_db_connection()
+        cur = conn.cursor()
+        
+        cur.execute("""
+            SELECT COUNT(*) as count 
+            FROM LLMGeneratedQuestions 
+            WHERE user_id = ?
+        """, (user_id,))
+        
+        row = cur.fetchone()
+        exists = row[0] > 0 if row else False
+        
+        return jsonify({"exists": exists}), 200
+        
+    except Exception as e:
+        print(f"Error checking assessment: {e}")
+        return jsonify({"error": str(e)}), 500
+    finally:
+        try:
+            conn.close()
+        except:
+            pass
+
+@profiles_bp.route('/api/check-assessment-completion/<int:user_id>', methods=['GET'])
+def check_assessment_completion(user_id: int):
+    """Check if user has already completed the assessment"""
+    try:
+        conn = get_db_connection()
+        cur = conn.cursor()
+        
+        # 🚨 CRITICAL FIX: Check if user exists in LLMGeneratedQuestions table with valid data
+        cur.execute("""
+            SELECT COUNT(*) as count 
+            FROM LLMGeneratedQuestions 
+            WHERE user_id = ? AND (blue > 0 OR green > 0 OR yellow > 0 OR red > 0)
+        """, (user_id,))
+        
+        row = cur.fetchone()
+        has_taken_assessment = row[0] > 0 if row else False
+        
+        print(f"🔍 Assessment check for user {user_id}: {has_taken_assessment} (count: {row[0] if row else 0})")
+        
+        return jsonify({
+            "has_taken_assessment": has_taken_assessment,
+            "message": "User has already taken assessment" if has_taken_assessment else "User can take assessment"
+        }), 200
+        
+    except Exception as e:
+        print(f"Error checking assessment completion: {e}")
+        return jsonify({"error": str(e)}), 500
+    finally:
+        try:
+            conn.close()
+        except:
+            pass

server.py

@@ -1,2039 +0,0 @@
-# server.py
-# -----------------------------------------------------------------------------
-# Unified Py-Match Service (Flask)
-# - Local Windows: Trusted_Connection to SQL Server (e.g., localhost\SQLEXPRESS)
-# - Cloud (AWS RDS / Hugging Face): SQL auth via UID/PWD + Encrypt
-# - ODBC driver name defaults to {ODBC Driver 17 for SQL Server}
-# -----------------------------------------------------------------------------
-
-# ADD this line at the top with your imports
-
-import os, uuid, json, random, threading, hashlib, pickle
-from typing import Dict, List, Optional, Literal, Tuple
-from datetime import datetime
-from sqlalchemy import func
-# --- load .env so OPENAI_API_KEY (and others) are available ---
-from dotenv import load_dotenv
-BASE_DIR = os.path.dirname(os.path.abspath(__file__))
-
-# Load environment variables - priority: Hugging Face secrets > .env file
-IS_HUGGING_FACE = os.environ.get('HUGGINGFACE_SPACES') == 'true' or os.environ.get('SPACE_ID') is not None
-if not IS_HUGGING_FACE:
-    # Only load from .env file when running locally
-    load_dotenv(os.path.join(BASE_DIR, ".env"))
-    load_dotenv(os.path.join(BASE_DIR, ".env"))
-else:
-    # On Hugging Face, secrets are automatically available as environment variables
-    print("Running on Hugging Face Spaces - using secrets from environment variables")
-
-from flask import Flask, request, jsonify
-from flask_cors import CORS
-import pyodbc
-from flask_sqlalchemy import SQLAlchemy
-import urllib.parse
-import numpy as np
-# ----------------------------------------------------------------------
-# Expectation-Based Matching (Strict + Soft Comparison)
-# ----------------------------------------------------------------------
-def match_expectation_with_profiles(user_id):
-    expectation = ExpectationResponse.query.filter_by(user_id=user_id).first()
-    if not expectation:
-        return []
-
-    # Get current user to know gender
-    current_user = Marriage.query.filter_by(user_id=user_id).first()
-    if not current_user:
-        return []
-
-    # Opposite gender profiles only
-    opposite_profiles = Marriage.query.filter(Marriage.gender != current_user.gender).all()
-
-    def compute_expectation_score(expect, profile):
-        """Compute a weighted expectation match between expectation and profile"""
-        score, total = 0, 0
-
-        # ----- Strict rules -----
-        # Religion: strict match unless 'different faith acceptable'
-        if expect.religion_alignment and "different" not in expect.religion_alignment.lower():
-            if expect.religion_alignment.lower() not in str(profile.religion or "").lower():
-                return 0  # strict fail
-
-        # Smoking strict rule
-        if expect.accept_smoking and expect.accept_smoking.lower() == "never":
-            if str(profile.smoking_habit or "").lower() != "never":
-                return 0
-
-        # Alcohol strict rule
-        if expect.accept_alcohol and expect.accept_alcohol.lower() == "never":
-            if str(profile.alcohol_habit or "").lower() != "never":
-                return 0
-
-        # ----- Flexible rules -----
-        # Location
-        if expect.pref_location and str(expect.pref_location).lower() in str(profile.current_city or "").lower():
-            score += 1
-        total += 1
-
-        # Country
-        if expect.pref_countries and str(expect.pref_countries).lower() in str(profile.country or "").lower():
-            score += 1
-        total += 1
-
-        # Language
-        if expect.pref_languages and any(lang.lower() in str(profile.languages_spoken or "").lower() for lang in eval(expect.pref_languages)):
-            score += 1
-        total += 1
-
-        # Diet
-        if expect.pref_diet and str(expect.pref_diet).lower() in str(profile.food_preference or "").lower():
-            score += 1
-        total += 1
-
-        # Family type
-        if expect.pref_family_type and str(expect.pref_family_type).lower() in str(profile.family_type or "").lower():
-            score += 1
-        total += 1
-
-        # Fitness
-        if expect.pref_fitness and str(expect.pref_fitness).lower() in str(profile.fitness_level or "").lower():
-            score += 1
-        total += 1
-
-        # Expectation summary ↔ remark text match
-        if expect.expectation_summary and profile.remark:
-            from difflib import SequenceMatcher
-            sim = SequenceMatcher(None, expect.expectation_summary.lower(), str(profile.remark).lower()).ratio()
-            if sim > 0.3:
-                score += sim * 2  # higher weight for similar mindset
-            total += 2
-
-        return round(score / total, 2) if total > 0 else 0
-
-    # Evaluate all opposite gender profiles
-    candidates = []
-    for profile in opposite_profiles:
-        s = compute_expectation_score(expectation, profile)
-        if s > 0:
-            candidates.append({
-                "user_id": profile.user_id,
-                "name": profile.full_name,
-                "gender": profile.gender,
-                "location": profile.current_city,
-                "religion": profile.religion,
-                "remark": profile.remark,
-                "expectation_score": s
-            })
-
-    # Sort by expectation score descending
-    candidates.sort(key=lambda x: x["expectation_score"], reverse=True)
-
-    # Once sorted, get character compatibility for top matches
-    top_ids = [c["user_id"] for c in candidates[:10]]
-    llm_data = LLMGeneratedQuestions.query.filter(LLMGeneratedQuestions.user_id.in_(top_ids)).all()
-    llm_map = {l.user_id: (l.blue, l.green, l.yellow, l.red) for l in llm_data}
-
-    # Combine both results
-    for c in candidates:
-        if c["user_id"] in llm_map:
-            b, g, y, r = llm_map[c["user_id"]]
-            c["character_score"] = round((b + g + y + r) / 400, 2)
-            c["overall_score"] = round(0.7 * c["expectation_score"] + 0.3 * c["character_score"], 2)
-        else:
-            c["character_score"] = 0
-            c["overall_score"] = c["expectation_score"]
-
-    candidates.sort(key=lambda x: x["overall_score"], reverse=True)
-    return candidates
-
-
-
-# Try importing faiss (may be "faiss" or "faiss-cpu" depending on installation)
-try:
-    import faiss
-    HAS_FAISS = True
-except Exception as e:
-    print("faiss import failed:", e)
-    HAS_FAISS = False
-
-# Try importing sentence-transformers
-try:
-    from sentence_transformers import SentenceTransformer  # type: ignore
-    HAS_EMBEDDER = True
-except Exception:
-    SentenceTransformer = None
-    HAS_EMBEDDER = False
-
-# ---------- Optional LLM deps (fallback if missing) ----------
-try:
-    from pydantic import BaseModel, Field
-    from langchain_core.prompts import ChatPromptTemplate
-    from langchain_core.output_parsers import PydanticOutputParser, StrOutputParser
-    from langchain_openai import ChatOpenAI
-    HAS_LLM_STACK = True
-    HAS_LLM = True
-except Exception:
-    HAS_LLM_STACK = False
-    HAS_LLM = False
-
-# ==============================
-# Configuration / DB Connection
-# ==============================
-
-# ==============================
-# Configuration / DB Connection
-# ==============================
-
-#IS_HUGGING_FACE = os.environ.get('HUGGINGFACE_SPACES') == 'true' or os.environ.get('SPACE_ID') is not None
-
-if IS_HUGGING_FACE:
-    # Hugging Face Spaces configuration
-    DEFAULT_SQL_SERVER = "pykara-sqlserver.c5aosm6ie5j3.eu-north-1.rds.amazonaws.com,1433"
-    DEFAULT_SQL_DB = "PyMatch"
-    DEFAULT_SQL_TRUSTED = "yes"  # Use SQL authentication on Hugging Face
-else:
-    # Local development configuration
-    DEFAULT_SQL_SERVER = "localhost\\SQLEXPRESS"
-    DEFAULT_SQL_DB = "Py_Match"
-    DEFAULT_SQL_TRUSTED = "yes"  # Use Windows authentication locally
-
-SQL_DRIVER   = os.getenv("PYMATCH_SQL_DRIVER", "ODBC Driver 17 for SQL Server")
-SQL_SERVER   = os.getenv("PYMATCH_SQL_SERVER", DEFAULT_SQL_SERVER)
-SQL_DB       = os.getenv("PYMATCH_SQL_DB", DEFAULT_SQL_DB)
-SQL_TRUSTED  = os.getenv("PYMATCH_SQL_TRUSTED", DEFAULT_SQL_TRUSTED)  # yes/no
-SQL_USER      = os.getenv("PYMATCH_SQL_USER", "")
-SQL_PASSWORD  = os.getenv("PYMATCH_SQL_PASSWORD", "")
-SQL_PORT      = os.getenv("PYMATCH_SQL_PORT", "")
-SQL_ENCRYPT   = os.getenv("PYMATCH_SQL_ENCRYPT", "no").lower().strip()
-SQL_TRUSTCERT = os.getenv("PYMATCH_SQL_TRUST_CERT", "yes").lower().strip()
-
-PROGRESS_TBL = os.getenv("PYMATCH_PROGRESS_TABLE", "LLMGeneratedQuestions")
-DEFAULT_N_QUESTIONS = int(os.getenv("PYMATCH_DEFAULT_N_QUESTIONS", "20"))
-DEFAULT_BATCH_SIZE = int(os.getenv("PYMATCH_DEFAULT_BATCH_SIZE", "10"))
-MAX_QUESTIONS = int(os.getenv("PYMATCH_MAX_QUESTIONS", "50"))
-
-# Some constants used across the app
-COLOR_KEYS = ["blue", "green", "red", "yellow"]
-DOMAINS = ["marriage", "interview", "partnership", "general"]
-
-# Faiss index / chunks defaults - user should update FAISS_INDEX_PATH or provide companion chunks file
-FAISS_INDEX_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "faiss_index_file.index")
-# try companion files: same base name + (.chunks.json | _chunks.json | .chunks.pkl)
-
-
-def get_db_connection():
-    return pyodbc.connect(
-        f"DRIVER={SQL_DRIVER};"
-        f"SERVER={SQL_SERVER};"
-        f"DATABASE={SQL_DB};"
-        f"Trusted_Connection={SQL_TRUSTED};"
-    )
-
-# ==========
-# Flask App
-# ==========
-app = Flask(__name__)
-CORS(app, resources={r"/*": {"origins": "*"}})
-
-# =================================================================
-# SQLAlchemy & Database Models (from matching.py)
-# =================================================================
-_server = SQL_SERVER
-if SQL_PORT:
-    _server = f"{SQL_SERVER},{SQL_PORT}"
-
-if SQL_TRUSTED == "yes":
-    raw = (
-        f"DRIVER={{{SQL_DRIVER}}};"
-        f"SERVER={_server};"
-        f"DATABASE={SQL_DB};"
-        f"Trusted_Connection=yes;"
-    )
-else:
-    raw = (
-        f"DRIVER={{{SQL_DRIVER}}};"
-        f"SERVER={_server};"
-        f"DATABASE={SQL_DB};"
-        f"UID={SQL_USER};PWD={SQL_PASSWORD};"
-    )
-
-if SQL_ENCRYPT == "yes":
-    raw += "Encrypt=yes;"
-if SQL_TRUSTCERT == "yes":
-    raw += "TrustServerCertificate=yes;"
-
-params = urllib.parse.quote_plus(raw)
-SQLALCHEMY_DATABASE_URI = f"mssql+pyodbc:///?odbc_connect={params}"
-
-app.config["SQLALCHEMY_DATABASE_URI"] = SQLALCHEMY_DATABASE_URI
-app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
-
-db = SQLAlchemy(app)
-
-class Users(db.Model):
-    __tablename__ = "Users"
-    user_id = db.Column(db.Integer, primary_key=True)
-    name = db.Column(db.String(128), nullable=False)
-    email = db.Column(db.String(128))
-    # phone = db.Column(db.String(32))
-    # gender = db.Column(db.String(16))
-    password = db.Column(db.String(128))
-    created_at = db.Column(db.DateTime)
-
-class LLMGeneratedQuestions(db.Model):
-    __tablename__ = "LLMGeneratedQuestions"
-
-    llm_id     = db.Column(db.Integer, primary_key=True)
-    user_id    = db.Column(db.Integer, nullable=False, index=True)
-    role       = db.Column(db.String(64), nullable=True)
-    blue       = db.Column(db.Integer, nullable=False, default=0)
-    green      = db.Column(db.Integer, nullable=False, default=0)
-    yellow     = db.Column(db.Integer, nullable=False, default=0)
-    red        = db.Column(db.Integer, nullable=False, default=0)
-    created_at = db.Column(db.DateTime, default=datetime.utcnow)
-
-    def color_vec(self) -> np.ndarray:
-        v = np.array([self.blue, self.green, self.yellow, self.red], dtype=np.float32)
-        s = float(v.sum())
-        return v / s if s > 0 else v
-
-class Marriage(db.Model):
-    __tablename__ = "Marriage"
-    id = db.Column(db.Integer, primary_key=True)
-    user_id = db.Column(db.Integer, nullable=False)
-    full_name = db.Column(db.String(100))
-    date_of_birth = db.Column(db.String(50))
-    gender = db.Column(db.String(20))
-    current_city = db.Column(db.String(100))
-    marital_status = db.Column(db.String(50))
-    education_level = db.Column(db.String(100))
-    employment_status = db.Column(db.String(100))
-    number_of_siblings = db.Column(db.String(50))
-    family_type = db.Column(db.String(100))
-    hobbies_interests = db.Column(db.Text)
-    conflict_approach = db.Column(db.String(100))
-    financial_style = db.Column(db.String(100))
-    income_range = db.Column(db.String(100))
-    relocation_willingness = db.Column(db.String(100))
-    created_at = db.Column(db.DateTime, default=datetime.utcnow)
-      # Newly added columns
-    height_weight = db.Column(db.String(100))
-    eye_colour = db.Column(db.String(50))
-    skin_tone = db.Column(db.String(50))
-    languages_spoken = db.Column(db.String(200))
-    country = db.Column(db.String(100))
-    blood_group = db.Column(db.String(10))
-    religion = db.Column(db.String(100))
-    dual_citizenship = db.Column(db.String(50))
-    siblings_position = db.Column(db.String(50))
-    parents_living_status = db.Column(db.String(100))
-    live_with_parents = db.Column(db.String(50))
-    support_parents_financially = db.Column(db.String(50))
-    family_communication_frequency = db.Column(db.String(100))
-    food_preference = db.Column(db.String(100))
-    smoking_habit = db.Column(db.String(50))
-    alcohol_habit = db.Column(db.String(50))
-    daily_routine = db.Column(db.String(200))
-    fitness_level = db.Column(db.String(100))
-    own_pets = db.Column(db.String(50))
-    travel_preference = db.Column(db.String(100))
-    relaxation_mode = db.Column(db.String(100))
-    job_role = db.Column(db.String(100))
-    work_experience_years = db.Column(db.String(50))
-    career_aspirations = db.Column(db.String(200))
-    field_of_study = db.Column(db.String(200))
-    work_preference = db.Column(db.String(100))
-    remark = db.Column(db.Text)
-
-class ExpectationResponse(db.Model):
-    __tablename__ = "ExpectationResponse"
-
-    user_id = db.Column(db.Integer, primary_key=True)
-    religion_alignment = db.Column(db.String(50))
-    accept_smoking = db.Column(db.String(50))
-    accept_alcohol = db.Column(db.String(50))
-    pref_location = db.Column(db.String(100))
-    pref_countries = db.Column(db.String(100))
-    pref_languages = db.Column(db.String(100))
-    pref_diet = db.Column(db.String(100))
-    pref_family_type = db.Column(db.String(100))
-    pref_fitness = db.Column(db.String(100))
-    expectation_summary = db.Column(db.Text)
-
-# =================================================================
-# Knowledge source (FAISS) and LLM setup (from matching.py)
-# =================================================================
-
-
-BASE_DIR = os.path.dirname(os.path.abspath(__file__))
-
-BOOKS = [
-    {
-        "index": os.path.join(BASE_DIR, "psychology2e.index"),
-        "meta":  os.path.join(BASE_DIR, "psychology2e_meta.pkl"),
-        "name":  "Psychology 2e",
-    },
-    {
-        "index": os.path.join(BASE_DIR, "surrounded_by_idiots.index"),
-        "meta":  os.path.join(BASE_DIR, "surrounded_by_idiots_meta.pkl"),
-        "name":  "Surrounded by Idiots",
-    },
-]
-
-
-
-class KnowledgeSource:
-    def __init__(self):
-        self.indices: List = []
-        self.metas: List[List[Dict]] = []
-        self.embedder = None
-        if not HAS_FAISS:  # Use the global flag instead of checking faiss directly
-            return
-        try:
-            if SentenceTransformer:
-                self.embedder = SentenceTransformer("all-MiniLM-L6-v2")
-        except Exception:
-            self.embedder = None
-        for b in BOOKS:
-            try:
-                idx_path = b["index"]
-                meta_path = b["meta"]
-                if os.path.exists(idx_path) and os.path.exists(meta_path):
-                    index = faiss.read_index(idx_path)
-                    with open(meta_path, "rb") as f:
-                        meta = pickle.load(f)
-                    self.indices.append(index)
-                    self.metas.append(meta)
-            except Exception:
-                continue
-
-    def get_relevant_context(self, query: str, topk: int = 3) -> List[str]:
-        """Get relevant context from knowledge base for relationship matching"""
-        if not self.indices or not self.embedder:
-            return []
-        try:
-            vec = self.embedder.encode([query]).astype("float32")
-            results: List[Tuple[float, str]] = []
-            for index, meta in zip(self.indices, self.metas):
-                D, I = index.search(vec, topk)
-                for d, i in zip(D[0], I[0]):
-                    if 0 <= i < len(meta):
-                        txt = meta[i].get("text", "")[:500]  # Increased length for better context
-                        results.append((float(d), txt))
-            results.sort(key=lambda x: x[0])
-            return [t for _, t in results[:topk]]
-        except Exception:
-            return []
-
-# Initialize knowledge base only if FAISS is available
-knowledge = None
-if HAS_FAISS:
-    knowledge = KnowledgeSource()
-else:
-    print("FAISS not available - KnowledgeSource disabled")
-    # Create a dummy knowledge object with empty methods
-    class DummyKnowledge:
-        def get_relevant_context(self, *args, **kwargs):
-            return []
-    knowledge = DummyKnowledge()
-
-# ---------------------------------------------------------------------------
-# LLM Setup for Dynamic Explanations (from matching.py)
-# ---------------------------------------------------------------------------
-
-def create_llm_chain():
-    """Create LLM chain for dynamic match explanations"""
-    if not HAS_LLM or not os.getenv("OPENAI_API_KEY"):
-        return None
-
-    try:
-        llm = ChatOpenAI(
-            model="gpt-4o-mini",
-            temperature=0.7,
-            max_retries=2,
-            timeout=30
-        )
-
-        prompt_template = ChatPromptTemplate.from_messages([
-            ("system", """You are a relationship compatibility expert specializing in personality color analysis (Blue, Green, Yellow, Red).
-            Based on the personality profiles, relevant psychological context, and user profiles, provide a detailed compatibility analysis.
-
-            CRITICAL GUIDELINES:
-            1. Focus on practical relationship dynamics, not just theoretical compatibility
-            2. Use insights from the provided psychological context when relevant
-            3. Be specific about strengths and potential challenges
-            4. Provide actionable advice for the couple
-            5. Keep explanations natural and conversational, not robotic
-            6. Reference specific personality traits and how they interact
-            7. Consider cultural and personal background when relevant
-            8. Balance positivity with realistic expectations
-
-            Structure your response with:
-            - Compatibility overview (1-2 sentences)
-            - Key strengths of this pairing
-            - Potential challenges to be aware of
-            - Practical advice for success
-            - Daily life compatibility"""),
-            ("human", """Personality Profiles:
-User 1 ({user1_name}, {user1_gender}): Blue {user1_blue}%, Green {user1_green}%, Yellow {user1_yellow}%, Red {user1_red}%
-User 2 ({user2_name}, {user2_gender}): Blue {user2_blue}%, Green {user2_green}%, Yellow {user2_yellow}%, Red {user2_red}%
-
-User 1 Background: {user1_background}
-User 2 Background: {user2_background}
-
-Relevant Psychological Context:
-{context}
-
-Please provide a comprehensive compatibility analysis:""")
-        ])
-
-        return prompt_template | llm | StrOutputParser()
-    except Exception as e:
-        print(f"Failed to create LLM chain: {e}")
-        return None
-
-llm_chain = create_llm_chain() if HAS_LLM else None
-
-# =================================================================
-# Enhanced Helpers with LLM Integration (from matching.py)
-# =================================================================
-
-def cosine_sim(a: np.ndarray, b: np.ndarray) -> float:
-    na, nb = np.linalg.norm(a), np.linalg.norm(b)
-    if na == 0.0 or nb == 0.0: return 0.0
-    return float(np.dot(a, b) / (na * nb))
-
-def get_user_background(user_id: int) -> Dict:
-    """Get comprehensive user background for LLM analysis"""
-    background = {}
-
-    # Get basic user info
-    user = Users.query.filter_by(user_id=user_id).first()
-    if user:
-        background.update({
-            "name": user.name or "Unknown",
-            # "gender": user.gender or "Not specified",
-            "email": user.email or "",
-            # "phone": user.phone or ""
-        })
-
-    # Get marriage profile if exists
-    marriage_profile = Marriage.query.filter_by(user_id=user_id).first()
-    if marriage_profile:
-        background.update({
-            "current_location": marriage_profile.current_city or "",
-            "education": marriage_profile.education_level or "",
-            "employment": marriage_profile.employment_status or "",
-            "hobbies": marriage_profile.hobbies_interests or "",
-            "conflict_style": marriage_profile.conflict_approach or "",
-            "financial_style": marriage_profile.financial_style or "",
-            "family_type": marriage_profile.family_type or "",
-
-        })
-
-    return background
-
-def generate_dynamic_explanation(user1_id: int, user2_id: int, user1_vec: np.ndarray, user2_vec: np.ndarray) -> List[str]:
-    """Generate dynamic explanation using LLM and knowledge base"""
-
-    # Get user backgrounds
-    user1_bg = get_user_background(user1_id)
-    user2_bg = get_user_background(user2_id)
-
-    # Create query for knowledge base
-    query = f"compatibility between personality types: {user1_bg.get('conflict_style', '')} and {user2_bg.get('conflict_style', '')}"
-    context_chunks = knowledge.get_relevant_context(query, topk=2)
-    context = "\n".join(context_chunks) if context_chunks else "No specific psychological context available."
-
-    # Prepare data for LLM
-    llm_data = {
-        "user1_name": user1_bg.get("name", "User 1"),
-        "user1_gender": user1_bg.get("gender", "Not specified"),
-        "user1_blue": round(user1_vec[0] * 100, 1),
-        "user1_green": round(user1_vec[1] * 100, 1),
-        "user1_yellow": round(user1_vec[2] * 100, 1),
-        "user1_red": round(user1_vec[3] * 100, 1),
-        "user2_name": user2_bg.get("name", "User 2"),
-        "user2_gender": user2_bg.get("gender", "Not specified"),
-        "user2_blue": round(user2_vec[0] * 100, 1),
-        "user2_green": round(user2_vec[1] * 100, 1),
-        "user2_yellow": round(user2_vec[2] * 100, 1),
-        "user2_red": round(user2_vec[3] * 100, 1),
-        "user1_background": json.dumps(user1_bg, indent=2),
-        "user2_background": json.dumps(user2_bg, indent=2),
-        "context": context
-    }
-
-    # Try LLM first
-    if llm_chain is not None:
-        try:
-            response = llm_chain.invoke(llm_data)
-            # Parse LLM response into structured points
-            points = []
-            lines = response.split('\n')
-            for line in lines:
-                line = line.strip()
-                if line and not line.startswith(('- Compatibility', '- Key', '- Potential', '- Practical', '- Daily')):
-                    if line.startswith('•') or line.startswith('-'):
-                        points.append(line[1:].strip())
-                    elif len(line) > 20:  # Substantive lines
-                        points.append(line)
-
-            if points:
-                return points[:5]  # Return top 5 most relevant points
-        except Exception as e:
-            print(f"LLM explanation failed: {e}")
-
-    # Fallback to rule-based explanations
-    return generate_rule_based_explanation(user1_vec, user2_vec, user1_bg, user2_bg)
-
-def generate_rule_based_explanation(user1_vec: np.ndarray, user2_vec: np.ndarray, user1_bg: Dict, user2_bg: Dict) -> List[str]:
-    """Rule-based fallback explanation"""
-    labels = ["Blue", "Green", "Yellow", "Red"]
-    user1_dom = labels[int(np.argmax(user1_vec))]
-    user2_dom = labels[int(np.argmax(user2_vec))]
-
-    explanations = []
-
-    # Dominant trait analysis
-    if user1_dom == user2_dom:
-        explanations.append(f"Both share {user1_dom} dominance: Strong alignment in core approach and values.")
-    else:
-        explanations.append(f"{user1_dom}-{user2_dom} pairing: Complementary strengths create balanced dynamics.")
-
-    # Difference analysis
-    diffs = user2_vec - user1_vec
-    for idx, diff in enumerate(diffs):
-        color = labels[idx]
-        if abs(diff) > 0.15:
-            if diff > 0:
-                explanations.append(f"Higher {color} influence brings {get_color_strength(color)} to the relationship.")
-            else:
-                explanations.append(f"Lower {color} presence allows for more {get_color_balance(color)} in dynamics.")
-
-    # Background considerations
-    if user1_bg.get("hobbies") and user2_bg.get("hobbies"):
-        explanations.append("Shared interests and hobbies create strong bonding opportunities.")
-
-    if user1_bg.get("conflict_style") and user2_bg.get("conflict_style"):
-        explanations.append("Complementary conflict styles can lead to effective problem-solving.")
-
-    return explanations[:4]  # Limit to 4 points
-
-def get_color_strength(color: str) -> str:
-    strengths = {
-        "Blue": "analytical precision and structured thinking",
-        "Green": "emotional stability and patient understanding",
-        "Yellow": "creative energy and social connection",
-        "Red": "decisive action and goal orientation"
-    }
-    return strengths.get(color, "unique strengths")
-
-def get_color_balance(color: str) -> str:
-    balances = {
-        "Blue": "flexibility and spontaneity",
-        "Green": "directness and assertiveness",
-        "Yellow": "focus and routine",
-        "Red": "collaboration and patience"
-    }
-    return balances.get(color, "balanced approaches")
-
-def detailed_explanation(user1_id: int, user2_id: int, u_vec: np.ndarray, v_vec: np.ndarray) -> List[str]:
-    """Main function to generate detailed explanations"""
-    return generate_dynamic_explanation(user1_id, user2_id, u_vec, v_vec)
-
-def expectation_similarity(exp1: dict, exp2: dict) -> float:
-    """Compare expectation data fields and return score between 0–1."""
-    if not exp1 or not exp2:
-        return 0.0
-
-    keys = ["pref_conflict_approach", "pref_financial_style", "pref_core_values",
-            "pref_family_type", "work_life_pref", "ambition_pref",
-            "social_pref", "move_for_career", "deal_breakers"]
-
-    matches = 0
-    total = 0
-
-    for k in keys:
-        if k in exp1 and k in exp2:
-            total += 1
-            v1 = str(exp1[k]).strip().lower()
-            v2 = str(exp2[k]).strip().lower()
-            if v1 == v2:
-                matches += 1
-            elif v1 in v2 or v2 in v1:
-                matches += 0.5  # partial match
-
-    return matches / total if total else 0.0
-
-def generate_expectation_explanation(expect1: dict, expect2: dict) -> list:
-    """
-    Compare two users' expectations and produce text explanations
-    describing why they match or differ, based only on expectation fields.
-    """
-
-    explanations = []
-
-    def compare_field(key, label):
-        v1 = str(expect1.get(key, "") or "").strip().lower()
-        v2 = str(expect2.get(key, "") or "").strip().lower()
-        if not v1 or not v2:
-            return
-
-        if v1 == v2:
-            explanations.append(f"Both share similar preferences in {label} ({v1.capitalize()}).")
-        elif v1 in v2 or v2 in v1:
-            explanations.append(f"They have partly aligned {label} preferences ({v1} vs {v2}).")
-        else:
-            explanations.append(f"Their {label} expectations differ ({v1} vs {v2}).")
-
-    # Key expectation areas
-    compare_field("religion_alignment", "religion or faith")
-    compare_field("accept_smoking", "smoking preference")
-    compare_field("accept_alcohol", "alcohol preference")
-    compare_field("pref_diet", "dietary habits")
-    compare_field("pref_family_type", "family type")
-    compare_field("pref_fitness", "fitness lifestyle")
-    compare_field("pref_location", "preferred location")
-    compare_field("pref_countries", "preferred country")
-    compare_field("pref_languages", "spoken languages")
-    compare_field("pref_conflict_approach", "conflict approach")
-    compare_field("pref_financial_style", "financial management style")
-    compare_field("work_life_pref", "work–life balance")
-    compare_field("pref_core_values", "core values")
-    compare_field("social_pref", "social interaction style")
-    compare_field("ambition_pref", "ambition level")
-    compare_field("pref_income_range", "income expectation")
-    compare_field("move_for_career", "career relocation preference")
-    compare_field("deal_breakers", "deal breakers")
-    compare_field("other_non_negotiables", "non-negotiable expectations")
-
-    # If both have expectation_summary text
-    if expect1.get("expectation_summary") and expect2.get("expectation_summary"):
-        from difflib import SequenceMatcher
-        sim = SequenceMatcher(None, expect1["expectation_summary"].lower(), expect2["expectation_summary"].lower()).ratio()
-        if sim > 0.6:
-            explanations.append("Their overall expectation summaries express similar relationship outlooks.")
-        else:
-            explanations.append("They express different overall expectations about relationship goals.")
-
-    # Clean up and limit
-    if not explanations:
-        explanations.append("No strong similarities or differences could be identified in expectations.")
-
-    return explanations[:8]  # Limit to 8 concise points
-
-# ----------------------------------------------------------------------
-# Step 2: Character-Based Filtering for Expectation Matches
-# ----------------------------------------------------------------------
-def refine_with_character_match(expectation_matches: list, base_user_id: int):
-    """Given a list of expectation-matched profiles, rank them further by character similarity."""
-
-    # Fetch base user's color data
-    base_llm = LLMGeneratedQuestions.query.filter_by(user_id=base_user_id).first()
-    if not base_llm:
-        return expectation_matches  # No character data → return as-is
-
-    u_vec = base_llm.color_vec()  # [blue, green, yellow, red]
-
-    refined = []
-    for match in expectation_matches:
-        other_id = match["user_id"]
-        llm_other = LLMGeneratedQuestions.query.filter_by(user_id=other_id).first()
-        if llm_other:
-            v_vec = llm_other.color_vec()
-            # Compute cosine similarity for personality color match
-            score_color = cosine_sim(u_vec, v_vec)
-            match["character_match_score"] = round(score_color, 3)
-            # Blend both expectation + character score (e.g., 70–30 weighting)
-            match["final_combined_score"] = round(0.7 * match["expectation_score"] + 0.3 * score_color, 3)
-        else:
-            match["character_match_score"] = 0
-            match["final_combined_score"] = match["expectation_score"]
-
-        refined.append(match)
-
-    # Sort by final combined score descending
-    refined.sort(key=lambda x: x["final_combined_score"], reverse=True)
-    return refined
-
-def _compute_matches(user_id: int, role: Optional[str], limit: int, exclude_self: bool):
-    """Compute matches – initial (expectation only), characterwise sorting triggered separately."""
-    base_user_query = db.session.query(LLMGeneratedQuestions, Users, Marriage)\
-        .join(Users, Users.user_id == LLMGeneratedQuestions.user_id)\
-        .join(Marriage, Marriage.user_id == Users.user_id)\
-        .filter(LLMGeneratedQuestions.user_id == user_id)
-
-    if role:
-        base_user_query = base_user_query.filter(LLMGeneratedQuestions.role == role)
-
-    base_user = base_user_query.first()
-    if not base_user:
-        return None, []
-
-    llm_src, user_src, marriage_src = base_user
-    u_vec = llm_src.color_vec()
-    user_gender = (marriage_src.gender or "").lower()
-
-    cq = db.session.query(LLMGeneratedQuestions, Users, Marriage)\
-        .join(Users, Users.user_id == LLMGeneratedQuestions.user_id)\
-        .join(Marriage, Marriage.user_id == Users.user_id)
-
-    if role:
-        cq = cq.filter(LLMGeneratedQuestions.role == role)
-    if exclude_self:
-        cq = cq.filter(LLMGeneratedQuestions.user_id != user_id)
-
-    if role == "marriage":
-        if user_gender.startswith("male"):
-            cq = cq.filter(func.lower(func.trim(Marriage.gender)) == "female")
-        elif user_gender.startswith("female"):
-            cq = cq.filter(func.lower(func.trim(Marriage.gender)) == "male")
-
-    candidates = cq.all()
-    results = []
-
-    exp_user = fetch_expectation_data(user_id)
-    weight_color = float(request.args.get("weight_color", 0))
-    weight_expect = float(request.args.get("weight_expect", 1))
-
-    show_characterwise = request.args.get("characterwise", "false").lower() == "true"
-
-    for llm, u, m in candidates:
-        v_vec = llm.color_vec()
-        score_color = cosine_sim(u_vec, v_vec)
-        exp_candidate = fetch_expectation_data(llm.user_id)
-        score_expect = expectation_similarity(exp_user, exp_candidate)
-        final_score = (weight_color * score_color) + (weight_expect * score_expect)
-        percentage = round(final_score * 100, 2)
-
-        if not show_characterwise:
-            # First load: Expectation-based explanation only
-            explanations = generate_expectation_explanation(exp_user, exp_candidate)
-            source_type = "expectation"
-        else:
-            # After clicking Characterwise button: LLM + psychology book explanation
-            explanations = detailed_explanation(user_id, llm.user_id, u_vec, v_vec)
-            source_type = "character"
-
-        results.append({
-            "user_id": llm.user_id,
-            "name": u.name,
-            "gender": m.gender,
-            "blue": llm.blue,
-            "green": llm.green,
-            "yellow": llm.yellow,
-            "red": llm.red,
-            "score_color": round(score_color, 4),
-            "score_expect": round(score_expect, 4),
-            "final_score": round(final_score, 4),
-            "percentage": percentage,
-            "created_at": llm.created_at.isoformat() if llm.created_at else None,
-            "explanations": explanations,
-            "explanation_source": source_type
-        })
-
-    results.sort(key=lambda x: x["final_score"], reverse=True)
-
-    grouped = {
-        "90-100": [r for r in results if r["percentage"] >= 90],
-        "80-89": [r for r in results if 80 <= r["percentage"] < 90],
-        "70-79": [r for r in results if 70 <= r["percentage"] < 80],
-        "60-69": [r for r in results if 60 <= r["percentage"] < 70],
-        "below_60": [r for r in results if r["percentage"] < 60],
-    }
-
-    return llm_src, grouped
-
-
-
-
-# ==========
-# Utilities
-# ==========
-def hash_password(password: str) -> str:
-    return hashlib.sha256(password.encode("utf-8")).hexdigest()
-
-
-def row_to_dict(cursor, row) -> Dict:
-    if row is None:
-        return {}
-    cols = [col[0] for col in cursor.description]
-    return {cols[i]: row[i] for i in range(len(cols))}
-
-# --------------------
-# FAISS helpers
-# --------------------
-FAISS_INDEX = None
-TEXT_CHUNKS: List[str] = []
-
-
-def try_load_chunks_from_disk(index_path: str) -> List[str]:
-    """Try several companion filenames for the chunk/text mapping."""
-    # base = os.path.splitext(index_path)[0]
-    # base = r"C:\Malini AI\Py-Match back end\New folder\faiss_index_file"
-    base = os.path.splitext(index_path)[0]
-    candidates = [base + ".chunks.json", base + "_chunks.json", base + ".chunks.pkl", base + "_chunks.pkl"]
-
-    for c in candidates:
-        if os.path.exists(c):
-            try:
-                if c.endswith(".json"):
-                    with open(c, "r", encoding="utf-8") as f:
-                        data = json.load(f)
-                        # expecting list of strings
-                        if isinstance(data, list):
-                            return data
-                        # sometimes stored as {"chunks": [...]}
-                        if isinstance(data, dict) and "chunks" in data:
-                            return data["chunks"]
-                else:
-                    with open(c, "rb") as f:
-                        data = pickle.load(f)
-                        if isinstance(data, list):
-                            return data
-            except Exception as e:
-                print(f"Failed to load chunks from {c}:", e)
-    return []
-
-
-def load_faiss_index(index_path: str):
-    global FAISS_INDEX, TEXT_CHUNKS
-    if not HAS_FAISS:
-        print("FAISS not installed. Skipping index load.")
-        return
-    if not os.path.exists(index_path):
-        print("Faiss index path does not exist:", index_path)
-        return
-    try:
-        FAISS_INDEX = faiss.read_index(index_path)
-        # try to load chunks from companion files
-        TEXT_CHUNKS = try_load_chunks_from_disk(index_path)
-        if not TEXT_CHUNKS:
-            print("Warning: Faiss index loaded but no companion text chunks found.")
-            print("Provide a companion .chunks.json or .chunks.pkl file with a list of text chunks.")
-    except Exception as e:
-        print("Failed to load faiss index:", e)
-        FAISS_INDEX = None
-
-
-def get_nearest_context(query_emb: List[float] = None, k: int = 5, query_vector: Optional[List[float]] = None):
-    """Return concatenated top-k chunks for a query."""
-    if FAISS_INDEX is None or not HAS_FAISS:
-        return ""
-    try:
-        # ... rest of the existing code remains the same
-        if query_vector is None:
-            # no embedding generation in this script: rely on an external embedding or LLM to provide a context id
-            return ""
-        import numpy as np
-        vec = np.array([query_vector], dtype='float32')
-        D, I = FAISS_INDEX.search(vec, k)
-        idxs = I[0].tolist()
-        texts = []
-        for idx in idxs:
-            if 0 <= idx < len(TEXT_CHUNKS):
-                texts.append(TEXT_CHUNKS[idx])
-        return "\n\n".join(texts)
-    except Exception as e:
-        print("Faiss search failed:", e)
-        return ""
-
-# attempt to load faiss index at startup if path given
-if FAISS_INDEX is None and HAS_FAISS and FAISS_INDEX is None and FAISS_INDEX != False:
-    load_faiss_index(FAISS_INDEX_PATH)
-
-# =======================
-# 1) AUTH / SIGNUP (auth)
-# =======================
-@app.post("/api/signup")
-def signup():
-    data = request.get_json(force=True) or {}
-    name = data.get("name")
-    email = data.get("email")
-    password = data.get("password")
-
-    if not name or not email or not password:
-        return jsonify({"error": "Name, email, and password are required."}), 400
-
-    password_hash = hash_password(password)
-
-    try:
-        conn = get_db_connection()
-        cur = conn.cursor()
-        cur.execute("""
-            INSERT INTO Users (name, email, password)
-            VALUES (?, ?, ?)
-        """, (name, email, password_hash))
-        conn.commit()
-
-        # Get the last inserted user_id
-        cur.execute("SELECT @@IDENTITY AS user_id")
-        user_id_row = cur.fetchone()
-        user_id = user_id_row[0] if user_id_row else None
-
-        if user_id:
-            return jsonify({"message": "User created successfully.", "user_id": user_id}), 201
-        else:
-            return jsonify({"error": "Failed to retrieve user ID."}), 500
-    except pyodbc.Error as e:
-        print(f"Database Error: {e}")  # Add this line to log the specific error
-        return jsonify({"error": f"DB error: {e}"}), 500
-    except Exception as e:
-        print(f"Unexpected Error: {e}")  # Log unexpected errors
-        return jsonify({"error": f"Unexpected error: {e}"}), 500
-    finally:
-        try: conn.close()
-        except: pass
-
-
-
-# ==================================================
-# 2) ROLE SELECTION + STATIC QUESTION FETCH + SAVE
-# ==================================================
-@app.post("/api/questions/select-role")
-def select_role():
-    data = request.get_json(force=True) or {}
-    user_id = data.get("user_id")
-    role_name = data.get("role_name")
-    assigned_at = data.get("assigned_at")  # ISO or None
-
-    # Check if user_id and role_name are provided
-    if not user_id or not role_name:
-        return jsonify({"error": "User ID and role name are required."}), 400
-
-    try:
-        # Check if user_id exists in the Users table
-        conn = get_db_connection()
-        cur = conn.cursor()
-        cur.execute("SELECT COUNT(*) FROM Users WHERE user_id = ?", (user_id,))
-        user_exists = cur.fetchone()[0]
-
-        if user_exists == 0:
-            return jsonify({"error": "User ID does not exist in the Users table."}), 404
-
-        # Proceed with inserting into UserRoles
-        cur.execute("""
-            INSERT INTO UserRoles (user_id, role_name, assigned_at)
-            VALUES (?, ?, ?)
-        """, (user_id, role_name, assigned_at))
-        conn.commit()
-
-        return jsonify({"message": "Role assigned successfully."}), 201
-
-    except pyodbc.Error as e:
-        # Handle database error, including foreign key constraint violations
-        if "foreign key" in str(e).lower():
-            return jsonify({"error": "Foreign key violation: User ID not found."}), 400
-        return jsonify({"error": f"Database error: {str(e)}"}), 500
-    except Exception as e:
-        return jsonify({"error": f"Unexpected error: {str(e)}"}), 500
-    finally:
-        try:
-            conn.close()
-        except:
-            pass
-
-@app.get("/api/questions/marriage")
-def get_questions():
-    try:
-        conn = get_db_connection()
-        cur = conn.cursor()
-        cur.execute("""
-            SELECT question, options, input_type, column_key, category
-            FROM RoleQuestions
-            WHERE role_name = 'marriage'
-            ORDER BY id
-        """)
-        rows = cur.fetchall()
-        out = []
-        for r in rows:
-            label = r[0]
-            options = (r[1].split(",") if r[1] else [])
-            input_type = r[2]
-            column_key = r[3]
-            category = r[4]
-            out.append({
-                "label": label,
-                "options": options,
-                "input_type": input_type,
-                "column_key": column_key,
-                "category": category
-            })
-        return jsonify(out), 200
-    except pyodbc.Error as e:
-        return jsonify({"error": str(e)}), 500
-    finally:
-        try: conn.close()
-        except: pass
-
-@app.post("/api/questions/submit-answers/marriage")
-def submit_answers():
-    data = request.get_json(force=True) or {}
-    user_id = data.get("user_id")
-    if not user_id:
-        return jsonify({"error": "User ID is required."}), 400
-
-    role_fields = {
-        "marriage": [
-            "full_name", "date_of_birth", "gender", "current_city", "marital_status",
-            "education_level", "employment_status", "number_of_siblings", "family_type", "hobbies_interests",
-            "conflict_approach", "financial_style", "income_range", "relocation_willingness", "height_weight",
-            "eye_colour", "skin_tone", "languages_spoken", "country", "blood_group", "religion", "dual_citizenship",
-            "siblings_position", "parents_living_status", "live_with_parents", "support_parents_financially",
-            "family_communication_frequency", "food_preference", "smoking_habit", "alcohol_habit", "daily_routine",
-            "fitness_level", "own_pets", "travel_preference", "relaxation_mode", "job_role", "work_experience_years",
-            "career_aspirations", "field_of_study", "work_preference", "remark", "created_at"
-        ]
-    }
-
-    # Validate all required fields are present
-    for f in role_fields["marriage"]:
-        if f not in data:
-            return jsonify({"error": f"{f} is required."}), 400
-
-    try:
-        conn = get_db_connection()
-        cur = conn.cursor()
-
-        table_name = "Marriage"
-        placeholders = ", ".join(["?"] * (len(role_fields["marriage"]) + 1))
-        query = f"INSERT INTO {table_name} (user_id, {', '.join(role_fields['marriage'])}) VALUES ({placeholders})"
-
-        values = [user_id]
-        for f in role_fields["marriage"]:
-            val = data.get(f)
-
-            # Handle list values (multiselect)
-            if isinstance(val, list):
-                val = ", ".join([str(v) for v in val])
-
-            # Convert to string or None
-            if val is None:
-                val = None
-            else:
-                val = str(val)
-
-            values.append(val)
-
-        print(f"DEBUG: Executing query: {query}")  # Debug log
-        print(f"DEBUG: Values: {values}")  # Debug log
-
-        cur.execute(query, values)
-        conn.commit()
-
-        return jsonify({"message": "Marriage record added successfully."}), 201
-
-    except pyodbc.Error as e:
-        print(f"Database Error: {e}")  # Detailed error logging
-        return jsonify({"error": f"Database error: {str(e)}"}), 500
-    except Exception as e:
-        print(f"Unexpected Error: {e}")  # Detailed error logging
-        return jsonify({"error": f"Unexpected error: {str(e)}"}), 500
-    finally:
-        try:
-            conn.close()
-        except:
-            pass
-
-@app.get("/api/expectation-questions")
-def get_expectation_questions():
-    try:
-        conn = get_db_connection()
-        cur = conn.cursor()
-        cur.execute("""
-            SELECT id, question, options, input_type, column_key, category
-            FROM ExpectationQuestions
-            ORDER BY id
-        """)
-        rows = cur.fetchall()
-
-        out = []
-        for r in rows:
-            out.append({
-                "id": r[0],
-                "question": r[1],
-                "options": (r[2].split(",") if r[2] else []),
-                "input_type": r[3],
-                "column_key": r[4],
-                "category": r[5]
-            })
-        return jsonify(out), 200
-    except Exception as e:
-        return jsonify({"error": str(e)}), 500
-    finally:
-        try: conn.close()
-        except: pass
-
-
-@app.post("/api/expectation-response")
-def save_expectation_response():
-    data = request.get_json(force=True) or {}
-    user_id = data.get("user_id")
-
-    if not user_id:
-        return jsonify({"error": "user_id is required"}), 400
-
-    try:
-        conn = get_db_connection()
-        cur = conn.cursor()
-
-        # Get valid keys from ExpectationQuestions
-        cur.execute("SELECT column_key FROM ExpectationQuestions")
-        valid_keys = [row[0] for row in cur.fetchall()]
-        print("DEBUG: Valid keys from DB ->", valid_keys)
-        print("DEBUG: Incoming data keys ->", list(data.keys()))
-
-        cols, vals = [], []
-        for k, v in data.items():
-            if k in valid_keys:
-                cols.append(k)
-                if isinstance(v, (list, dict)):
-                    vals.append(json.dumps(v))
-                else:
-                    vals.append(v)
-
-        if not cols:
-            return jsonify({"error": "No valid fields found in request"}), 400
-
-        # Build placeholders safely
-        placeholders = ", ".join(["?"] * (len(cols) + 1))  # +1 for user_id
-        col_str = ", ".join([f"[{c}]" for c in cols])      # safe column quoting
-
-        query = f"""
-            INSERT INTO ExpectationResponse (user_id, {col_str})
-            VALUES ({placeholders})
-        """
-        print("DEBUG: Final query ->", query)
-        print("DEBUG: Values ->", [user_id] + vals)
-
-        cur.execute(query, [user_id] + vals)
-        conn.commit()
-
-        return jsonify({"message": "Preferences saved successfully"}), 201
-
-    except Exception as e:
-        import traceback
-        traceback.print_exc()  # full error in server logs
-        return jsonify({"error": str(e)}), 500
-
-    finally:
-        try:
-            conn.close()
-        except:
-            pass
-
-
-# ==========================================
-# 3) LLM BATCH Q-GEN + COLOR % PERSIST (LLM)
-# ==========================================
-# Marriage-specific topics
-# TOPIC_BANK_BY_DOMAIN = {
-#     "marriage": [
-#         "household budget plan", "holiday travel decision", "child's school choice", "conflict about chores",
-#         "time with in-laws", "health and fitness routine", "weekend family schedule", "saving vs spending debate",
-#         "home renovation plan", "vacation destination", "career prioritization", "parenting disagreements",
-#         "personal space and boundaries", "cultural value clashes", "major financial decision", "health emergency reaction",
-#         "supporting partner’s dream", "anniversary planning", "family crisis handling", "friend group conflict"
-#     ],
-#     "general": [
-#         "time management", "work-life balance", "goal setting", "communication challenge",
-#         "team collaboration", "conflict resolution", "planning an event", "change management"
-#     ]
-# }
-
-# COLOR_PHRASES_BY_DOMAIN = {
-#     "marriage": {
-#         "blue": "evidence-based family decision",
-#         "green": "routine and planning at home",
-#         "red": "direct discussion and action",
-#         "yellow": "creative family options"
-#     },
-#     "general": {
-#         "blue": "data-driven choice",
-#         "green": "process and planning",
-#         "red": "people and action",
-#         "yellow": "creative possibility"
-#     }
-# }
-
-# Expand parser/chain initialization to include context slot in prompt
-PARSER_BATCH = None
-CHAIN_BATCH = None
-if HAS_LLM_STACK and os.getenv("OPENAI_API_KEY"):
-    class Option(BaseModel):
-        text: str
-        color: Literal["blue", "green", "red", "yellow"]
-
-    class QAItem(BaseModel):
-        question: str
-        options: List[Option] = Field(min_items=4, max_items=4)
-
-    class BatchQA(BaseModel):
-        items: List[QAItem] = Field(..., min_items=1)
-
-    SYSTEM_PROMPT = (
-        "You write marriage compatibility assessment questions that reveal four personality colors through forced choices:\n"
-        "- blue=analytical, fact-based (positive: thorough, precise | negative: overly critical, data-obsessed)\n"
-        "- green=organized, process-oriented (positive: systematic, reliable | negative: rigid, bureaucratic)\n"
-        "- red=decisive, action-oriented (positive: direct, results-driven | negative: impulsive, controlling)\n"
-        "- yellow=creative, big-picture (positive: innovative, visionary | negative: unrealistic, scattered)\n"
-        "\n"
-        "CRITICAL RULES:\n"
-        "1) Each option MUST clearly represent one color's typical behavior (include both positive and negative aspects)\n"
-        "2) Questions should force a choice that reveals CORE PERSONALITY TRAITS, not future plans\n"
-        "3) Use simple, everyday language but maintain situational specificity\n"
-        "4) Options should be practical actions someone would actually take (show both strengths and weaknesses)\n"
-        "5) Focus on CURRENT CHARACTER ANALYSIS, not future hypotheticals\n"
-        "6) Ensure each color option is distinctly different from others\n"
-        "7) Each option MUST be maximum 15 words - keep them concise and clear\n"
-        "8) For 20 total questions, distribute as:\n"
-        "   - 25% (5 questions): Profile-based (use user's background, education, job, family background)\n"
-        "   - 25% (5 questions): Expectation-based (use relationship expectations, values, lifestyle preferences)\n"
-        "   - 50% (10 questions): Character-based scenarios (current behavior in various situations)\n"
-        "9) AVOID future-focused questions about children, family planning, or long-term hypotheticals\n"
-        "10) Include some options that reveal potential negative/shadow aspects of each color\n"
-        "\n"
-        "EXPECTATION-BASED QUESTION SOURCES:\n"
-        "- Conflict Approach: How user prefers to handle disagreements\n"
-        "- Financial Style: Money management and financial values\n"
-        "- Work-Life Balance: Career vs personal life priorities\n"
-        "- Social Preferences: Introversion/extroversion levels\n"
-        "- Core Values: Fundamental beliefs and principles\n"
-        "- Family Values: Views on family structure and relationships\n"
-        "- Ambition Level: Career and personal growth aspirations\n"
-        "- Deal Breakers: Non-negotiable relationship requirements\n"
-        "\n"
-        "PROFILE-BASED QUESTION SOURCES:\n"
-        "- Education Level: Academic background\n"
-        "- Employment Status: Current profession/work situation\n"
-        "- Hobbies/Interests: Leisure activities and passions\n"
-        "- Family Background: Siblings, family type, upbringing\n"
-        "- Current Location: Living situation and environment\n"
-        "\n"
-        "CHARACTER-BASED QUESTIONS:\n"
-        "- CURRENT behavior scenarios that reveal core personality colors\n"
-        "- Decision-making styles in everyday situations\n"
-        "- Problem-solving approaches\n"
-        "- Communication patterns\n"
-        "- Stress response behaviors\n"
-    )
-
-    USER_PROMPT_BATCH = (
-        "Context (from Surrounded by Idiots or other corpus):\n{context}\n\n"
-        "User Profile (Current Background):\n"
-        "- Education: {education}\n"
-        "- Employment: {employment}\n"
-        "- Hobbies: {hobbies}\n"
-        "- Family Background: {family_type}\n"
-        "- Current Lifestyle: {current_lifestyle}\n"
-        "\n"
-        "User Relationship Expectations:\n"
-        "- Conflict Style: {conflict_style}\n"
-        "- Financial Style: {financial_style}\n"
-        "- Income Expectations: {income_range}\n"
-        "- Career Mobility: {relocation_willingness}\n"
-        "- Family Values: {family_values}\n"
-        "- Core Values: {core_values}\n"
-        "- Work-Life Balance: {lifestyle_pref}\n"
-        "- Social Preference: {social_pref}\n"
-        "- Ambition Level: {ambition_pref}\n"
-        "- Deal Breakers: {deal_breakers}\n"
-        "\n"
-        "Themes (array of short strings): {themes_json}\n"
-        "Previously asked questions: {previous_questions}\n\n"
-        "{format_instructions}\n\n"
-        "Generate CHARACTER ANALYSIS questions with this distribution:\n"
-        "- 25% PROFILE-BASED (5 questions): Use the user's CURRENT background above\n"
-        "- 25% EXPECTATION-BASED (5 questions): Use relationship expectations and values above\n"
-        "- 50% CHARACTER-BASED (10 questions): Current behavior in various life situations\n\n"
-        "FOCUS ON CURRENT TRAITS, NOT FUTURE PLANS.\n"
-        "AVOID questions about marriage plans, children, or long-term future.\n"
-        "EACH OPTION MUST REVEAL CORE PERSONALITY COLORS (MAX 15 WORDS).\n"
-        "INCLUDE SHADOW ASPECTS THAT SHOW POTENTIAL CHALLENGES.\n"
-        "ALL QUESTIONS SHOULD HELP ANALYZE CURRENT CHARACTER FOR COMPATIBILITY MATCHING.\n"
-    )
-
-
-
-    PARSER_BATCH = PydanticOutputParser(pydantic_object=BatchQA)
-
-    def build_batch_chain():
-        llm = ChatOpenAI(
-            model="gpt-4o-mini",
-            temperature=0.7,
-            max_retries=2,
-            timeout=30,
-            model_kwargs={"response_format": {"type": "json_object"}},
-        )
-        prompt = ChatPromptTemplate.from_messages([
-            ("system", SYSTEM_PROMPT),
-            ("user", USER_PROMPT_BATCH),
-        ])
-        return prompt | llm | PARSER_BATCH
-
-    try:
-        CHAIN_BATCH = build_batch_chain()
-    except Exception as e:
-        print("Failed to build CHAIN_BATCH:", e)
-        CHAIN_BATCH = None
-
-
-
-def ensure_valid_colors(options: List[Dict]) -> List[Dict]:
-    seen, fixed = set(), []
-    defaults = {
-        "blue": "Verify facts and numbers",
-        "green": "Outline a clear process",
-        "red": "Coordinate people and act",
-        "yellow": "Propose a fresh idea",
-    }
-    for o in options:
-        c = str(o.get("color", "")).lower()
-        t = str(o.get("text", "")).strip()
-        if c in COLOR_KEYS and c not in seen and t:
-            seen.add(c); fixed.append({"text": t[:80], "color": c})
-    for c in COLOR_KEYS:
-        if c not in seen:
-            fixed.append({"text": defaults[c], "color": c})
-    return fixed[:4]
-
-# Expanded: include more profile fields so LLM can use them safely
-def summarize_profile(profile: Dict) -> Dict:
-    out: Dict = {}
-
-    # name (optional) — if you prefer first name only, derive it here
-    if isinstance(profile.get("full_name"), str) and profile["full_name"].strip():
-        out["full_name"] = profile["full_name"].strip()
-
-    # safe, non-PII signals to expose
-    keys_allowed = [
-        "gender","current_city","marital_status","family_type","number_of_siblings",
-        "education_level","employment_status","hobbies_interests",
-        "conflict_approach","financial_style","income_range","relocation_willingness",
-        "user_id"
-    ]
-    for k in keys_allowed:
-        v = profile.get(k)
-        if v not in (None, "", []):
-            out[k] = v
-    return out
-
-# Fallback uses name + scenario bank and lightly weaves profile signals
-# Fallback uses name + scenario bank and lightly weaves profile signals
-def offline_generate_batch(themes: List[str], state: Dict, context: str = "") -> List[Dict]:
-    prof = state.get("profile", {}) or {}
-    name = prof.get("full_name") or "Partner"
-    conflict = (prof.get("conflict_approach") or "").lower()
-    money = (prof.get("financial_style") or "").lower()
-    hobby = None
-    if isinstance(prof.get("hobbies_interests"), list) and prof["hobbies_interests"]:
-        hobby = random.choice(prof["hobbies_interests"])
-
-    def flavor():
-        bits = []
-        if conflict:
-            bits.append(f"{conflict} style")
-        if money:
-            bits.append(f"{money} finances")
-        if hobby:
-            bits.append(f"likes {hobby}")
-        return ", ".join(bits)
-
-    items = []
-    for theme in themes:
-        short = theme.split(" around ")[-1].strip()
-
-        # Simple fallback question generation without PROMPT_SCENARIOS_BY_THEME
-        tail = (", " + flavor()) if flavor() else ""
-        q = f"{name}, what would you do about {short}?".strip()
-
-        # incorporate small bit from context if available (first 120 chars)
-        if context:
-            ctx_snip = context.replace('\n', ' ')[:120]
-            q = f"{q} (Note: {ctx_snip})"
-
-        # Keep concise
-        if len(q.split()) > 20:
-            q = " ".join(q.split()[:20])
-
-        opts = [
-            {"text": "Check data and facts", "color": "blue"},
-            {"text": "Draft a step-by-step plan", "color": "green"},
-            {"text": "Align people and act", "color": "red"},
-            {"text": "Brainstorm bold ideas", "color": "yellow"},
-        ]
-        random.shuffle(opts)
-        items.append({"question": q, "options": opts, "source": "fallback"})
-    return items
-
-
-def generate_batch_questions(themes: List[str], state: Dict, context: str = "", previous_questions: List[str] = None) -> List[Dict]:
-    # Extract PROFILE-BASED data from Marriage table
-    profile = state.get("profile", {})
-    user_id = profile.get("user_id")
-
-    # Fetch EXPECTATION data from ExpectationResponse table
-    expectation_data = fetch_expectation_data(user_id) if user_id else {}
-
-    # PROFILE-BASED COLUMNS (from Marriage table)
-    education = profile.get("education_level", "Not specified")
-    employment = profile.get("employment_status", "Not specified")
-    hobbies = profile.get("hobbies_interests", "Not specified")
-    family_type = profile.get("family_type", "Not specified")
-    current_city = profile.get("current_city", "Not specified")
-    marital_status = profile.get("marital_status", "Not specified")
-    number_of_siblings = profile.get("number_of_siblings", "Not specified")
-
-    # EXPECTATION-BASED COLUMNS (from ExpectationResponse table)
-    conflict_style = expectation_data.get("pref_conflict_approach", "Not specified")
-    financial_style = expectation_data.get("pref_financial_style", "Not specified")
-    income_range = expectation_data.get("pref_income_range", "Not specified")
-    relocation_willingness = expectation_data.get("move_for_career", "Not specified")
-    family_values = expectation_data.get("pref_family_type", "Not specified")
-    core_values = expectation_data.get("pref_core_values", "Not specified")
-    lifestyle_pref = expectation_data.get("work_life_pref", "Not specified")
-    social_pref = expectation_data.get("social_pref", "Not specified")
-    ambition_pref = expectation_data.get("ambition_pref", "Not specified")
-    deal_breakers = expectation_data.get("deal_breakers", "Not specified")
-
-    # Calculate current lifestyle from PROFILE data
-    current_lifestyle = f"Education: {education}, Employment: {employment}, Location: {current_city}, Hobbies: {hobbies}"
-
-    # Prepare expectation summary
-    expectation_summary = f"""
-    Relationship Expectations:
-    - Conflict Style: {conflict_style}
-    - Financial Style: {financial_style}
-    - Income Range: {income_range}
-    - Career Mobility: {relocation_willingness}
-    - Family Values: {family_values}
-    - Core Values: {core_values}
-    - Work-Life Balance: {lifestyle_pref}
-    - Social Preference: {social_pref}
-    - Ambition Level: {ambition_pref}
-    - Deal Breakers: {deal_breakers}
-    """
-
-    if CHAIN_BATCH is not None and PARSER_BATCH is not None:
-        try:
-            payload = {
-                "state": json.dumps(state, ensure_ascii=False),
-                "themes_json": json.dumps(themes, ensure_ascii=False),
-                "previous_questions": json.dumps(previous_questions or [], ensure_ascii=False),
-                "format_instructions": PARSER_BATCH.get_format_instructions(),
-                "context": context[:4000],
-
-                # PROFILE-BASED parameters (25% questions)
-                "education": education,
-                "employment": employment,
-                "hobbies": str(hobbies),
-                "family_type": family_type,
-                "current_lifestyle": current_lifestyle,
-                "marital_status": marital_status,
-                "number_of_siblings": number_of_siblings,
-
-                # EXPECTATION-BASED parameters (25% questions)
-                "conflict_style": conflict_style,
-                "financial_style": financial_style,
-                "income_range": income_range,
-                "relocation_willingness": relocation_willingness,
-                "family_values": family_values,
-                "core_values": core_values,
-                "lifestyle_pref": lifestyle_pref,
-                "social_pref": social_pref,
-                "ambition_pref": ambition_pref,
-                "deal_breakers": deal_breakers,
-                "expectation_summary": expectation_summary
-            }
-
-            result = CHAIN_BATCH.invoke(payload)
-
-            if hasattr(result, "items"):
-                items_raw = result.items
-            elif isinstance(result, dict) and "items" in result:
-                items_raw = result["items"]
-            else:
-                items_raw = []
-
-            items: List[Dict] = []
-            for qa in items_raw:
-                out = qa.dict() if hasattr(qa, "dict") else dict(qa)
-                out["options"] = ensure_valid_colors(out.get("options", []))
-                out["source"] = "llm"
-                random.shuffle(out["options"])  # reduce position bias
-                items.append(out)
-
-            if items:
-                return items
-        except Exception as e:
-            print("LLM batch generation failed:", e)
-
-    # Fallback generator (always returns items if themes not empty)
-    return offline_generate_batch(themes, state, context=context)
-
-
-class SessionState:
-    def __init__(self, n_questions: int, batch_size: int, domain: str = "general", role: Optional[str] = None, profile: Optional[Dict] = None):
-        domain = (domain or role or "general").lower()
-        self.domain = domain if domain in DOMAINS else "general"
-        self.role = (role or self.domain)
-        self.profile = profile or {}
-        self.n_questions = max(1, min(n_questions, MAX_QUESTIONS))
-        self.batch_size = max(1, batch_size)
-        self.asked = 0
-        self.color_counts = {c: 0 for c in COLOR_KEYS}
-        self.history: List[Dict] = []
-        self.queue: List[Dict] = []
-        self.finished = False
-        self.used_topics: List[str] = []
-        self.history_of_questions: List[str] = []  # Add this line to track question texts
-
-    def to_min_state(self) -> Dict:
-        total = sum(self.color_counts.values()) or 1
-        mix_percentages = {k: round((v / total) * 100, 2) for k, v in self.color_counts.items()}
-        dominant = max(self.color_counts, key=self.color_counts.get) if total else None
-        return {
-            "asked": self.asked,
-            "dominant": dominant,
-            "mix": mix_percentages,
-            "domain": self.domain,
-            "role": self.role,
-            "profile": summarize_profile(self.profile),
-        }
-
-    def remaining(self) -> int:
-        return self.n_questions - self.asked
-
-SESSIONS_FILE = os.getenv("PYMATCH_SESSIONS_FILE", "sessions.json")
-_sessions_lock = threading.Lock()
-SESSIONS: Dict[str, SessionState] = {}
-
-def save_sessions():
-    try:
-        with _sessions_lock:
-            serializable = {sid: s.__dict__ for sid, s in SESSIONS.items()}
-            tmp = SESSIONS_FILE + ".tmp"
-            with open(tmp, "w", encoding="utf-8") as f:
-                json.dump(serializable, f, ensure_ascii=False, indent=2, default=str)
-            os.replace(tmp, SESSIONS_FILE)
-    except Exception as e:
-        print("Failed to save sessions:", e)
-
-
-def persist_final_progress(user_id: Optional[str], role: str, mix: Dict[str, float]) -> bool:
-    llm_id = str(uuid.uuid4())
-    blue   = float(mix.get("blue", 0.0))
-    green  = float(mix.get("green", 0.0))
-    yellow = float(mix.get("yellow", 0.0))
-    red    = float(mix.get("red", 0.0))
-    try:
-        conn = get_db_connection()
-        cur = conn.cursor()
-        # Try with llm_id; if identity error, retry without it
-        try:
-            cur.execute(f"""
-                INSERT INTO [dbo].[{PROGRESS_TBL}]
-                  ([llm_id],[user_id],[role],[blue],[green],[yellow],[red],[created_at])
-                VALUES (?,?,?,?,?,?,?,SYSUTCDATETIME())
-            """, (llm_id, str(user_id) if user_id is not None else None, role, blue, green, yellow, red))
-            conn.commit()
-            return True
-        except pyodbc.Error as e:
-            if "IDENTITY_INSERT" in str(e) or "(544)" in str(e):
-                cur.execute(f"""
-                    INSERT INTO [dbo].[{PROGRESS_TBL}]
-                      ([user_id],[role],[blue],[green],[yellow],[red],[created_at])
-                    VALUES (?,?,?,?,?,?,SYSUTCDATETIME())
-                """, (str(user_id) if user_id is not None else None, role, blue, green, yellow, red))
-                conn.commit()
-                return True
-            else:
-                print("Persist failed:", e)
-                return False
-    except Exception as ex:
-        print("Persist final progress failed:", ex)
-        return False
-    finally:
-        try: conn.close()
-        except: pass
-
-# -------------------------
-# Profile fetch by role/id
-# -------------------------
-def fetch_profile_for_role(user_id: str, role: str) -> Dict:
-    table = {
-        "marriage": "Marriage",
-        "interview": "Interview",
-        "partnership": "Partnership"
-    }.get(role.lower())
-
-    if not table:
-        return {}
-
-    try:
-        conn = get_db_connection()
-        cur = conn.cursor()
-        cur.execute(f"""
-            SELECT TOP 1 *
-            FROM {table}
-            WHERE user_id = ?
-            ORDER BY created_at DESC
-        """, (user_id,))
-        row = cur.fetchone()
-        if row is None:
-            return {}
-        prof = row_to_dict(cur, row)
-        # Normalize hobbies_interests if it exists
-        if "hobbies_interests" in prof and isinstance(prof["hobbies_interests"], str):
-            if prof["hobbies_interests"].strip().startswith("["):
-                try:
-                    prof["hobbies_interests"] = json.loads(prof["hobbies_interests"])
-                except Exception:
-                    prof["hobbies_interests"] = [s.strip() for s in prof["hobbies_interests"].split(",") if s.strip()]
-            else:
-                prof["hobbies_interests"] = [s.strip() for s in prof["hobbies_interests"].split(",") if s.strip()]
-        prof["user_id"] = str(user_id)
-        return prof
-    except pyodbc.Error as e:
-        print("Profile fetch error:", e)
-        return {}
-    finally:
-        try: conn.close()
-        except: pass
-
-def fetch_expectation_data(user_id: str) -> Dict:
-    """Fetch expectation data from ExpectationResponse table"""
-    try:
-        conn = get_db_connection()
-        cur = conn.cursor()
-        cur.execute("""
-            SELECT * FROM ExpectationResponse
-            WHERE user_id = ?
-            ORDER BY created_at DESC
-        """, (user_id,))
-        row = cur.fetchone()
-        if row is None:
-            return {}
-        return row_to_dict(cur, row)
-    except Exception as e:
-        print(f"Error fetching expectation data: {e}")
-        return {}
-    finally:
-        try: conn.close()
-        except: pass
-# -------------------
-# Theme chooser
-# -------------------
-def choose_themes(sess: SessionState, k: int) -> List[str]:
-    """
-    Instead of generic topic banks, use FAISS to retrieve text chunks from the document.
-    """
-    if HAS_FAISS and FAISS_INDEX is not None and TEXT_CHUNKS:
-        # Just grab k random chunks from the indexed document
-        selected = random.sample(TEXT_CHUNKS, min(k, len(TEXT_CHUNKS)))
-        # Wrap them as "themes" but really they're just context
-        return selected
-    else:
-        # fallback: use generic themes
-        fallback_themes = [
-            "communication style", "conflict resolution", "decision making",
-            "problem solving", "team collaboration", "personal values",
-            "work habits", "social interaction", "stress management",
-            "goal setting", "time management", "relationship dynamics"
-        ]
-        return random.sample(fallback_themes, min(k, len(fallback_themes)))
-
-
-
-# ---------------
-# Health / Home
-# ---------------
-@app.get("/health")
-def health():
-    return {
-        "status": "ok",
-        "llm": ("openai" if CHAIN_BATCH is not None else "offline-fallback"),
-        "has_openai_key": bool(os.getenv("OPENAI_API_KEY")),
-        "db": {"server": SQL_SERVER, "database": SQL_DB, "table": PROGRESS_TBL},
-        "faiss_loaded": bool(FAISS_INDEX is not None and HAS_FAISS),  # Added HAS_FAISS check
-        "faiss_chunks": len(TEXT_CHUNKS),
-        "matching_llm_available": llm_chain is not None,
-        "knowledge_base_loaded": knowledge is not None and hasattr(knowledge, 'indices') and len(knowledge.indices) > 0,
-        "faiss_available": HAS_FAISS  # Use the global flag
-    }
-
-
-
-@app.get("/")
-def home():
-    return {
-        "message": "Unified Py-Match Service (FAISS-enabled)",
-        "try": [
-            "POST /api/signup",
-            "POST /api/questions/select-role",
-            "GET  /api/questions/<role>",
-            "POST /api/questions/submit-answers/<role>",
-            "POST /llm/start   (body: { user_id, role, n_questions, batch_size })",
-            "POST /llm/next    (body: { session_id, selected_color })",
-            "GET  /api/match/<user_id> (query: ?role=<role>&limit=<num>)",
-        ]
-    }
-
-# -------------------------
-# LLM Session: start / next
-# ----------------------
-
-
-@app.post("/llm/start")
-def llm_start():
-    data = request.get_json(force=True) or {}
-    user_id = str(data.get("user_id") or "").strip()
-    role_in = (data.get("role") or "general").lower()
-    n_req = int(data.get("n_questions", DEFAULT_N_QUESTIONS))
-    b_req = int(data.get("batch_size", DEFAULT_BATCH_SIZE))
-
-    if not user_id:
-        return jsonify({"error": "user_id is required"}), 400
-    if role_in not in DOMAINS:
-        return jsonify({"error": f"Invalid role. Allowed: {', '.join(DOMAINS)}"}), 400
-
-    # Fetch profile from the correct table based on role
-    profile = fetch_profile_for_role(user_id, role_in)
-
-    # Create session
-    sid = str(uuid.uuid4())
-    sess = SessionState(
-        n_questions=n_req,
-        batch_size=b_req,
-        domain=role_in,
-        role=role_in,
-        profile=profile
-    )
-    SESSIONS[sid] = sess
-
-    # Generate first batch of questions from FAISS chunks
-    to_generate = min(sess.batch_size, sess.remaining())
-    themes = choose_themes(sess, to_generate)
-
-    context = ""
-    if HAS_FAISS and FAISS_INDEX is not None and TEXT_CHUNKS:
-        context = "\n".join(random.sample(TEXT_CHUNKS, min(3, len(TEXT_CHUNKS))))
-
-    # INSERT THIS LINE (replace the existing queue generation line):
-    queue = generate_batch_questions(themes, sess.to_min_state(), context=context, previous_questions=sess.history_of_questions)
-
-    if not queue:
-        return jsonify({"error": "Question generation failed"}), 500
-
-    sess.queue = queue
-
-    # Serve first question
-    first = sess.queue.pop(0)
-    sess.asked += 1
-
-    # INSERT THIS LINE (to track the asked question):
-    sess.history_of_questions.append(first["question"])
-
-    save_sessions()
-
-    return jsonify({
-        "session_id": sid,
-        "index": 1,
-        "total": sess.n_questions,
-        "question": first["question"],
-        "options": first["options"],
-        "source": first.get("source", "unknown"),
-        "role": sess.role,
-        "profile_used": bool(profile),
-        "faiss_themes": themes,
-        "faiss_context": context
-    })
-
-
-
-@app.post("/llm/next")
-def llm_next():
-    data = request.get_json(force=True) or {}
-    sid = data.get("session_id")
-    color = str(data.get("selected_color") or "").lower()
-
-    if not sid or sid not in SESSIONS:
-        return jsonify({"error": "Invalid or missing session_id"}), 400
-    if color not in COLOR_KEYS:
-        return jsonify({"error": "selected_color must be blue|green|red|yellow"}), 400
-
-    sess = SESSIONS[sid]
-    if sess.finished:
-        return jsonify({"done": True, "message": "Session already finished."})
-
-    # record answer
-    sess.color_counts[color] += 1
-    sess.history.append({"selected_color": color})
-
-    # Initialize themes and context with default values
-    themes = []
-    context = ""
-
-    # finished?
-    if sess.asked >= sess.n_questions:
-        sess.finished = True
-        mix = sess.to_min_state()["mix"]
-        user_id = (sess.profile or {}).get("user_id")
-        db_ok = persist_final_progress(user_id=user_id, role=sess.role, mix=mix)
-        save_sessions()
-        return jsonify({
-            "done": True,
-            "message": "No more questions.",
-            "mix": mix,
-            "db_write": "ok" if db_ok else "failed"
-        })
-
-    # ensure queue; refill if needed
-    if not sess.queue:
-        to_generate = min(sess.batch_size, sess.remaining())
-        themes = choose_themes(sess, to_generate)
-
-        if HAS_FAISS and FAISS_INDEX is not None and TEXT_CHUNKS:
-            context = "\n".join(random.sample(TEXT_CHUNKS, min(3, len(TEXT_CHUNKS))))
-
-        # INSERT THIS LINE (replace the existing queue generation line):
-        sess.queue = generate_batch_questions(themes, sess.to_min_state(), context=context, previous_questions=sess.history_of_questions)
-
-        if not sess.queue:
-            return jsonify({"error": "Question generation failed"}), 500
-
-    nxt = sess.queue.pop(0)
-    sess.asked += 1
-
-    # INSERT THIS LINE (to track the asked question):
-    sess.history_of_questions.append(nxt["question"])
-
-    save_sessions()
-
-    return jsonify({
-        "session_id": sid,
-        "index": sess.asked,
-        "total": sess.n_questions,
-        "question": nxt["question"],
-        "options": nxt["options"],
-        "progress": sess.to_min_state()["mix"],
-        "source": nxt.get("source", "unknown"),
-        "role": sess.role,
-        "faiss_themes": themes,
-        "faiss_context": context
-    })
-
-
-# =================================================================
-# Routes (from matching.py)
-# =================================================================
-@app.get("/api/marriage-profile/<int:user_id>")
-def get_marriage_profile(user_id: int):
-    """Get marriage profile by user_id"""
-    try:
-        conn = get_db_connection()
-        cur = conn.cursor()
-
-        cur.execute("""
-            SELECT * FROM Marriage
-            WHERE user_id = ?
-            ORDER BY created_at DESC
-        """, (user_id,))
-
-        row = cur.fetchone()
-        if row is None:
-            return jsonify({"error": "Marriage profile not found"}), 404
-
-        # Convert row to dict
-        profile = row_to_dict(cur, row)
-
-        return jsonify(profile), 200
-
-    except Exception as e:
-        print(f"Error fetching marriage profile: {e}")
-        return jsonify({"error": str(e)}), 500
-    finally:
-        try:
-            conn.close()
-        except:
-            pass
-
-
-@app.get("/api/match/<int:user_id>")
-def api_match(user_id: int):
-    with app.app_context():
-        role = request.args.get("role", None)
-        limit = int(request.args.get("limit", "10"))
-        exclude_self = request.args.get("exclude_self", "yes").lower() == "yes"
-
-        src, top = _compute_matches(user_id, role, limit, exclude_self)
-        if src is None:
-            return jsonify({"error": f"No data found for user_id={user_id}"}), 404
-
-        # Ensure the name field is included in the input_user object
-        user = Users.query.filter_by(user_id=user_id).first()
-        input_user = {
-            "user_id": src.user_id,
-            "role": src.role,
-            "blue": src.blue,
-            "green": src.green,
-            "yellow": src.yellow,
-            "red": src.red,
-            "created_at": src.created_at.isoformat() if src.created_at else None,
-            "name": user.name if user else "Unknown"  # Add name field
-        }
-
-        return jsonify({
-            "input_user": input_user,
-            "matches": top,
-            "count": len(top),
-        })
-
-@app.get("/api/match")
-def compat_match():
-    with app.app_context():
-        try:
-            user_id = int(request.args.get("user_id", ""))
-        except ValueError:
-            return jsonify({"error": "Missing or invalid user_id"}), 400
-
-        role = request.args.get("role", None)
-        limit = int(request.args.get("limit", "10"))
-        exclude_self = request.args.get("exclude_self", "yes").lower() == "yes"
-
-        src, top = _compute_matches(user_id, role, limit, exclude_self)
-        if src is None:
-            return jsonify({"error": f"No data found for user_id={user_id}"}), 404
-
-        return jsonify({
-            "input_user": {
-                "user_id": src.user_id,
-                "role": src.role,
-                "blue": src.blue,
-                "green": src.green,
-                "yellow": src.yellow,
-                "red": src.red,
-                "created_at": src.created_at.isoformat() if src.created_at else None,
-            },
-            "matches": top,
-            "count": len(top),
-        })
-
-
-# =========
-# Run app
-# =========
-if __name__ == "__main__":
-    # If you want to force-load faiss on startup, set FAISS_INDEX_PATH env var
-    if HAS_FAISS and FAISS_INDEX is None and os.path.exists(FAISS_INDEX_PATH):
-        load_faiss_index(FAISS_INDEX_PATH)
-
-    # Initialize the knowledge base for matching.py functionality
-    knowledge = KnowledgeSource()
-
-    with app.app_context():
-        print(f"LLM Chain Available: {llm_chain is not None}")
-        print(f"Knowledge Base Loaded: {len(knowledge.indices)} indices")
-        print(f"FAISS Available: {faiss is not None}")
-
-    app.run(host="0.0.0.0", port=5000, debug=True)