Add validation to prevent always 'keep' recommendations - override lazy OpenAI responses with intelligent analysis

app/smart_recommendation.py

@@ -69,6 +69,25 @@ class SmartBudgetRecommender:
                 recommended_budget = ai_result.get("recommended_budget")
                 reason = ai_result.get("reason", f"AI recommendation for {category_name}")
                 action = ai_result.get("action")
+                
+                # Validate OpenAI recommendation (same logic as in get_recommendation_for_category)
+                if recommended_budget == avg_expense and action == "keep":
+                    std_dev = data.get("std_dev", 0.0)
+                    monthly_values = data.get("monthly_values", [])
+                    has_trend = len(monthly_values) > 1 and (monthly_values[-1] != monthly_values[0])
+                    
+                    if has_trend or std_dev > avg_expense * 0.05:
+                        # Override with intelligent recommendation
+                        if has_trend and monthly_values[-1] > monthly_values[0]:
+                            recommended_budget = avg_expense * 1.15
+                            action = "increase"
+                        elif std_dev > avg_expense * 0.05:
+                            recommended_budget = avg_expense * 1.20
+                            action = "increase"
+                        else:
+                            recommended_budget = avg_expense * 1.05
+                            action = "increase"
+                
                 print(f"✅ OpenAI recommendation for {category_name}: {recommended_budget} (action: {action})")
             else:
                 # Fallback to rule-based recommendation if OpenAI fails
@@ -234,8 +253,16 @@ class SmartBudgetRecommender:
                 ai_result = self._get_ai_recommendation(category_name, data, avg_expense)
                 if ai_result and ai_result.get("recommended_budget"):
                     recommended_budget = ai_result.get("recommended_budget")
-                    reason = ai_result.get("reason", f"AI recommendation for {category_name} based on your budget of Rs.{budget_amount:,.2f}")
+                    reason = ai_result.get("reason", f"AI recommendation for {category_name} based on your budget of {budget_amount:,.2f}")
                     action = ai_result.get("action")
+                    
+                    # Validate OpenAI recommendation
+                    if recommended_budget == avg_expense and action == "keep":
+                        # For budget_amount only, always add buffer
+                        recommended_budget = avg_expense * 1.10
+                        action = "increase"
+                        reason = f"Based on your budget amount, I recommend increasing by 10% to {recommended_budget:,.0f} to account for variability and inflation."
+                    
                     print(f"✅ OpenAI recommendation for {category_name} (budget: {budget_amount}): {recommended_budget} (action: {action})")
                 else:
                     # Fallback to rule-based recommendation if OpenAI fails
@@ -404,7 +431,49 @@ class SmartBudgetRecommender:
             recommended_budget = ai_result.get("recommended_budget")
             reason = ai_result.get("reason", f"AI recommendation for {category_name}")
             action = ai_result.get("action")
-            print(f"✅ OpenAI recommendation for {category_name}: {recommended_budget} (action: {action})")
+            
+            # VALIDATION: Check if OpenAI returned a lazy "keep" recommendation
+            # If recommended_budget equals avg_expense and action is "keep", validate if it's justified
+            monthly_values = data.get("monthly_values", [])
+            std_dev = data.get("std_dev", 0.0)
+            
+            if recommended_budget == avg_expense and action == "keep":
+                # Check if this is justified
+                has_trend = False
+                if len(monthly_values) > 1:
+                    # Check for upward trend
+                    if monthly_values[-1] > monthly_values[0]:
+                        has_trend = True
+                    # Check for downward trend
+                    elif monthly_values[-1] < monthly_values[0]:
+                        has_trend = True
+                
+                # If there's a trend or high variation, force a better recommendation
+                if has_trend or std_dev > avg_expense * 0.05:
+                    print(f"⚠️ OpenAI returned 'keep' but data shows trend/variation - overriding with intelligent recommendation")
+                    # Force increase with buffer
+                    if has_trend and monthly_values[-1] > monthly_values[0]:
+                        # Upward trend - increase by 15%
+                        recommended_budget = avg_expense * 1.15
+                        action = "increase"
+                        reason = f"Your spending shows an upward trend. I recommend increasing your budget by 15% to {recommended_budget:,.0f} to accommodate this growth pattern and provide a buffer for continued increases."
+                    elif has_trend and monthly_values[-1] < monthly_values[0]:
+                        # Downward trend - decrease by 10%
+                        recommended_budget = avg_expense * 0.90
+                        action = "decrease"
+                        reason = f"Your spending shows a downward trend. I recommend decreasing your budget by 10% to {recommended_budget:,.0f} to reflect this reduction pattern."
+                    elif std_dev > avg_expense * 0.05:
+                        # High variation - increase by 20%
+                        recommended_budget = avg_expense * 1.20
+                        action = "increase"
+                        reason = f"Your spending shows high variability (std_dev: {std_dev:,.0f}). I recommend increasing your budget by 20% to {recommended_budget:,.0f} to create a safety buffer for unpredictable expenses."
+                else:
+                    # Even for stable spending, add inflation buffer
+                    recommended_budget = avg_expense * 1.05
+                    action = "increase"
+                    reason = f"While your spending is stable, I recommend adding a 5% buffer ({recommended_budget:,.0f}) to account for inflation and unexpected expenses."
+            
+            print(f"✅ OpenAI recommendation for {category_name}: {recommended_budget:,.0f} (action: {action}, avg: {avg_expense:,.0f})")
         else:
             # Fallback to rule-based recommendation if OpenAI fails
             recommended_budget = self._calculate_recommended_budget(avg_expense, data)
@@ -998,13 +1067,37 @@ class SmartBudgetRecommender:
             "   - Accounts for category-specific factors\n"
             "   - Includes appropriate buffer for variability and inflation\n"
             "   - Is justified by your analysis and knowledge\n\n"
-            "CRITICAL RULES:\n"
-            "- Do NOT always recommend 'keep' - use your intelligence to determine the best action\n"
-            "- If there's an upward trend, recommend INCREASE (don't ignore the trend)\n"
-            "- If there's high variation, recommend INCREASE to create buffer\n"
-            "- If spending is stable AND category is predictable, you can recommend KEEP with small buffer (5-10%)\n"
-            "- The recommended_budget should be SMART: Specific, Measurable, Achievable, Relevant, Time-bound\n"
-            "- Consider inflation: even stable spending should account for 2-5% annual inflation\n\n"
+            "CRITICAL RULES - READ CAREFULLY:\n"
+            "⚠️ DO NOT ALWAYS RECOMMEND 'KEEP' - This is a common mistake. Analyze the data first!\n\n"
+            "MANDATORY ANALYSIS STEPS:\n"
+            "1. Look at the monthly_values array - is there a trend?\n"
+            "   - If values increase over time → MUST recommend INCREASE\n"
+            "   - If values decrease over time → MUST recommend DECREASE\n"
+            "   - Only if values are truly flat (all same) AND std_dev is very low → can recommend KEEP\n\n"
+            "2. Check the std_dev (standard deviation):\n"
+            "   - If std_dev > 10% of average → MUST recommend INCREASE (high variability needs buffer)\n"
+            "   - If std_dev is moderate (5-10%) → Recommend INCREASE with 10-15% buffer\n"
+            "   - Only if std_dev < 3% AND no trend → can recommend KEEP with 5% buffer\n\n"
+            "3. Consider inflation and best practices:\n"
+            "   - Even if spending is stable, inflation (2-5% annually) means you should INCREASE by at least 5-10%\n"
+            "   - Always add a buffer for unexpected expenses (5-15% depending on category)\n\n"
+            "4. For single data point or new budgets:\n"
+            "   - MUST recommend INCREASE by 10-20% to account for variability\n"
+            "   - Never recommend KEEP for new/limited data\n\n"
+            "DECISION TREE:\n"
+            "- Upward trend? → INCREASE (10-25%)\n"
+            "- Downward trend? → DECREASE (5-15%)\n"
+            "- High variation (std_dev > 15%)? → INCREASE (20-30%)\n"
+            "- Moderate variation (std_dev 5-15%)? → INCREASE (10-20%)\n"
+            "- Stable with low variation (std_dev < 3%) AND no trend? → KEEP with 5-10% buffer\n"
+            "- Single data point? → INCREASE (10-20%)\n\n"
+            "⚠️ IMPORTANT: The recommended_budget MUST be different from average_expense in most cases.\n"
+            "Only recommend the same amount if ALL of these are true:\n"
+            "1. Spending is perfectly stable (all monthly values identical)\n"
+            "2. Std_dev is very low (< 3% of average)\n"
+            "3. No upward or downward trend\n"
+            "4. Category is highly predictable\n"
+            "Even then, add at least 5% buffer for inflation!\n\n"
             "Respond strictly as JSON with the following keys:\n"
             '{ "recommended_budget": number, "action": "increase|decrease|keep", "reason": "string" }.\n\n'
             "The 'reason' field is CRITICAL - it must:\n"
@@ -1032,7 +1125,7 @@ class SmartBudgetRecommender:
                     "messages": [
                         {"role": "user", "content": prompt}
                     ],
-                    "temperature": 0.7,  # Increased for more varied and creative recommendations
+                    "temperature": 0.9,  # Higher temperature for more varied and creative recommendations
                     "response_format": {"type": "json_object"},
                 },
                 timeout=30,