executable_code/app.py

import os
import re
from dotenv import load_dotenv

# Load secret keys from .env file
load_dotenv()
import csv
import json
import warnings
import requests
import joblib
import nltk
import spacy
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import xgboost as xgb
from datetime import datetime, timedelta
from flask import Flask, request, jsonify, session, render_template
from flask_session import Session
from pyngrok import ngrok
from prophet import Prophet
from transformers import pipeline
from sklearn.preprocessing import LabelEncoder, OneHotEncoder
from sklearn.metrics import mean_absolute_error, r2_score, classification_report, precision_score, recall_score, f1_score, accuracy_score, confusion_matrix
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestRegressor
from sklearn import tree

warnings.filterwarnings('ignore')

# Download required NLTK data
try:
    nltk.data.find('tokenizers/punkt')
except LookupError:
    nltk.download('punkt')

# Load Spacy model (Ensure it's installed or fallback)
try:
    nlp = spacy.load('en_core_web_sm')
except:
    import subprocess
    import sys
    subprocess.run([sys.executable, "-m", "spacy", "download", "en_core_web_sm"])
    nlp = spacy.load('en_core_web_sm')

# Load Pipelines
print("Loading NLP pipelines... this might take a moment.")
intent_classifier = pipeline("text-classification", model="distilbert-base-uncased")
sentiment_analyzer = pipeline("sentiment-analysis")
response_generator = pipeline("text-generation", model="gpt2")

# Load Models
# Assuming the script runs from 'executable_code', models are in '../models/'
MODEL_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), "../models/"))
print(f"Loading models from: {MODEL_PATH}")

try:
    crop_model = joblib.load(os.path.join(MODEL_PATH, 'adaboost_model_soil.pkl'))
    intent_model = joblib.load(os.path.join(MODEL_PATH, 'intent_model.pkl'))
    vectorizer = joblib.load(os.path.join(MODEL_PATH, 'vectorizer_intent.pkl'))
    soil_model = joblib.load(os.path.join(MODEL_PATH, 'soilpred.pkl'))
    forecast_model = joblib.load(os.path.join(MODEL_PATH, 'prophet.pkl'))
    print("Models loaded successfully.")
except Exception as e:
    print(f"Error loading models: {e}")
    print("Please ensure models exist in the 'models' directory.")

app = Flask(__name__)
app.secret_key = 'your_secret_key'

# Initialize session management
app.config['SESSION_TYPE'] = 'filesystem'
Session(app)

# Questions for crop recommendation
questions = [
    'Please provide the Nitrogen (N) value:',
    'Please provide the Phosphorous (P) value:',
    'Please provide the Potassium (K) value:',
    'Please provide the pH value:'
]

crop_durations = {
    "bajra": 90, "barley": 120, "turmeric": 250, "tur": 180, "sugarcane": 365,
    "soybeans": 100, "ragi": 120, "potato": 90, "onion": 120, "maize": 100,
    "ladyfinger": 60, "jute": 150, "jowar": 120, "green gram": 70, "cotton": 180,
    "coffee": 1200, "chickpea": 110, "cabbage": 90, "wheat": 150, "rice": 40,
    "Paddy(Dhan)(Common)": 120
}

def fetch_weather_data(location):
    api_key = os.environ.get('WEATHER_API_KEY')
    base_url = f'http://api.openweathermap.org/data/2.5/weather?q={location}&appid={api_key}&units=metric'
    try:
        response = requests.get(base_url, timeout=5)
        if response.status_code == 200:
            data = response.json()
            temperature = data['main']['temp']
            humidity = data['main']['humidity']
            rainfall = data.get('rain', {}).get('1h', 0)
            return temperature, humidity, rainfall
    except:
        pass
    return None, None, None

def detect_intent(user_message):
    result = intent_classifier(user_message)
    intent = result[0]['label']
    confidence = result[0]['score']
    return intent, confidence

def analyze_sentiment(user_message):
    result = sentiment_analyzer(user_message)
    sentiment = result[0]['label']
    return sentiment

def classify_intent(user_message):
    user_message_vector = vectorizer.transform([user_message])
    predicted_intent = intent_model.predict(user_message_vector)
    return predicted_intent[0]

@app.route('/')
def index():
    return render_template('index.html')

@app.route('/chatbot', methods=['POST'])
def chatbot_route():
    greet = ['hi', 'hello', 'wassup', 'heya', 'hey']
    user_message = request.json['message'].lower()
    
    # Handle Greetings (and clear session)
    if user_message in greet:
        session.clear()
        return jsonify({'reply': "Hello! I'm <b>Cropiee</b>, your smart farming assistant. 😊<br>How can I help you today?<br>🌿 Try asking for a <b>crop recommendation</b> or <b>soil advice</b>!"})

    # Identity and Help check
    if "who are you" in user_message or "your name" in user_message:
        return jsonify({'reply': "I am <b>Cropiee</b>! 🌾 I'm here to help you make informed decisions about your crops and soil."})

    if "help" in user_message or "what can you do" in user_message:
        return jsonify({'reply': "I can help you with two main things:<br>1️⃣ <b>Crop Recommendation</b>: Tell me your location and soil data, and I'll suggest the best crop.<br>2️⃣ <b>Soil Advice</b>: Ask about a specific crop and location (e.g., 'soil for rice in Trichy')."})

    # PRIORITIZE SOIL: If user mentions 'soil', always try the soil behavior first
    if 'soil' in user_message:
        return handle_soil_conditions(user_message)

    # Check for active session context
    intent = session.get('intent')
    if intent == 'recommend_crop':
        return recommend_crop(user_message)
    elif intent == 'soil_advice':
        return handle_soil_conditions(user_message)

    # Determine Intent via ML if no clear keyword found
    intent_label = classify_intent(user_message)
    if intent_label == 'Crop Recommend':
        return recommend_crop(user_message)
    elif intent_label == 'Soil Character':
        return handle_soil_conditions(user_message)
    
    # POLISHED FALLBACK: For anything else outside its knowledge
    fallback_responses = [
        "I'm sorry, that's a bit outside my field of knowledge! 🌾 I'm specifically trained to help with <b>crop recommendations</b> and <b>soil requirements</b>. Could we try one of those?",
        "I didn't quite catch that. My expertise is rooted in <b>agriculture</b>! 🌿 I can advise on which crops to grow or what soil conditions they need. What would you like to explore?",
        "That sounds interesting, but I'm not sure how to help with it yet! 🧐 I'm best at suggesting <b>crops</b> or analyzing <b>soil</b>. Feel free to ask me about those!"
    ]
    import random
    return jsonify({'reply': random.choice(fallback_responses)})

def recommend_crop(user_message):
    recommend_synonyms = ['recommend', 'suggest', 'advise', 'what crops can i grow', 'what should i plant', 'crop recommendation']
    positive_feedback = ['thanks', 'great', 'good', 'helpful', 'useful', 'love', 'like', 'happy', 'nic']
    negative_feedback = ['bad', 'not good', 'useless', 'hate', 'dislike', 'terrible', 'worst']

    # Check if user explicitly wants to restart / start a new recommendation
    if any(word in user_message.lower() for word in recommend_synonyms):
        session['intent'] = 'recommend_crop'
        session['step'] = 0
        session['data'] = []
        session['recommendations'] = []
        session['feedback_stage'] = False
        session['state'] = None
        return jsonify({'reply': '🏠 Please provide your <b>district</b> name:'})

    # Feedback handling
    if session.get('feedback_stage', False):
        sentiment = analyze_sentiment(user_message)
        if any(word in user_message.lower() for word in negative_feedback) or sentiment == 'NEGATIVE':
            session['feedback_stage'] = False
            session.pop('intent', None)
            return jsonify({'reply': 'I understand. Feel free to ask if you need other recommendations later!'})
        elif any(word in user_message.lower() for word in positive_feedback) or sentiment == 'POSITIVE':
            session['feedback_stage'] = False
            return jsonify({'reply': 'I\'m glad to hear that! 😊 Would you like to get another recommendation?'})
        else:
            return jsonify({'reply': "Thank you for your feedback! If you have more to ask, I'm here."})

    # Yes/No responses
    if user_message.lower() in ['yes', 'ya'] and session.get('intent') == 'recommend_crop' and session.get('feedback_stage') == False:
        session['step'] = 0
        session['data'] = []
        session['recommendations'] = []
        return jsonify({'reply': 'Great! Please provide your <b>district</b> name:'})
    elif user_message.lower() in ['no', 'nah'] and session.get('intent') == 'recommend_crop':
        session.pop('intent', None)
        return jsonify({'reply': 'Thanks for your time! Have a great day.'})

    # Step 0: Get location and fetch weather
    if session.get('intent') == 'recommend_crop' and session['step'] == 0:
        session['location'] = user_message
        temperature, humidity, rainfall = fetch_weather_data(session['location'])

        if temperature is None:
            return jsonify({'reply': 'Could not fetch weather data for that location. Please try a valid <b>district</b> name (e.g., Delhi, Mumbai, Trichy).'})

        session['weather'] = {'temperature': temperature, 'humidity': humidity, 'rainfall': rainfall}
        session['step'] += 1
        return jsonify({'reply': f'🏡 Got it! Which <b>state</b> does {user_message.title()} belong to?'})

    # Step 1: Get state
    if session.get('intent') == 'recommend_crop' and session['step'] == 1:
        session['state'] = user_message.title()
        session['step'] += 1
        return jsonify({'reply': f"Got it. Now for some soil data. <br>{questions[0]}"})

    # Step 2+: Collect input data
    if session.get('intent') == 'recommend_crop':
        try:
            val = float(user_message)
            if len(session['data']) >= len(questions):
                session['data'][-1] = val  # replace last value if already full
            else:
                session['data'].append(val)
        except ValueError:
            return jsonify({'reply': "Please enter a valid numeric value for the soil reading."})

        step_idx = session['step'] - 2
        if step_idx < len(questions) - 1:
            session['step'] += 1
            return jsonify({'reply': questions[session['step'] - 2]})
        else:
            try:
                # Prepare features for crop model
                features = session['data'] + [
                    session['weather']['temperature'],
                    session['weather']['humidity'],
                    session['weather']['rainfall']
                ]
                features_arr = np.array(features, dtype=float).reshape(1, -1)
                predicted_crops = crop_model.predict(features_arr)
                session['recommendations'] = predicted_crops

                if len(predicted_crops) == 0:
                    session.pop('intent', None)
                    return jsonify({'reply': 'No suitable crops found based on your input. Try different soil values.'})

                crop = str(predicted_crops[0]).title()
                state = str(session['state']).title()
                district = str(session['location']).title()

                response_text = f"🍃 Based on the provided data, I recommend: <b>{crop}</b>.<br><br>"
                
                # Market Data & Forecasting
                market_msg = ""
                try:
                    url = "https://api.data.gov.in/resource/35985678-0d79-46b4-9ed6-6f13308a1d24"
                    api_key = os.environ.get('MARKET_API_KEY')
                    
                    # Fetch historical data (higher limit for better forecast)
                    params = {
                        "api-key": api_key, "format": "json", "limit": "200", 
                        "filters[Commodity]": crop, "filters[District]": district, "filters[State]": state
                    }
                    res = requests.get(url, params=params, timeout=10)
                    records = res.json().get("records", [])
                    
                    if records:
                        df = pd.DataFrame(records)
                        # Clean and prepare data for Prophet
                        df['ds'] = pd.to_datetime(df['Arrival_Date'], format='%d/%m/%Y', errors='coerce')
                        df['y'] = pd.to_numeric(df['Modal_Price'], errors='coerce')
                        df.dropna(subset=['ds', 'y'], inplace=True)
                        
                        if len(df) > 5:  # Need at least a few points to forecast
                            # Initialize Prophet
                            model = Prophet(
                                changepoint_prior_scale=0.05,
                                seasonality_mode='additive',
                                yearly_seasonality=True,
                                weekly_seasonality=False,
                                daily_seasonality=False
                            )
                            model.fit(df)
                            
                            # Predict based on crop duration
                            duration = crop_durations.get(crop.lower(), 120)
                            future = model.make_future_dataframe(periods=duration)
                            forecast = model.predict(future)
                            
                            today = datetime.today()
                            harvest_date = today + timedelta(days=duration)
                            
                            # Filter Forecast for profit analysis
                            current_val = forecast[forecast['ds'].dt.month == today.month]['yhat'].mean()
                            harvest_val = forecast[forecast['ds'].dt.month == harvest_date.month]['yhat'].mean()
                            
                            if not np.isnan(current_val) and not np.isnan(harvest_val):
                                diff = harvest_val - current_val
                                result = "Profit" if diff > 0 else "Loss"
                                percentage = round(abs(diff / current_val) * 100, 2)
                                market_msg = f"📊 <b>Market Analysis:</b> Growing {crop} in {district} is predicted to yield a <b>{result} of {percentage}%</b> by harvest time.🪴<br>"
                            else:
                                market_msg = f"📊 Market Insight: Average price observed is <b>₹{round(df['y'].mean(), 2)}</b> (Insufficient data for trend analysis).<br>"
                        else:
                            market_msg = f"📊 Market Insight: Current modal price is <b>₹{round(df['y'].mean(), 2)}</b>.<br>"
                    else:
                        market_msg = f"📊 Market Insight: No recent market data found for {crop} in {district}.<br>"
                except Exception as forecast_err:
                    print(f"Forecasting error: {forecast_err}")
                    market_msg = "📊 Market Insight: Market prediction service is currently unavailable.<br>"

                response_text += market_msg
                response_text += "<br><b>Analysis Chart Generated Below:</b>"
                
                # Chart Data: N, P, K, pH
                chart_data = {
                    "labels": ["Nitrogen", "Phosphorus", "Potassium", "pH Value"],
                    "values": [session['data'][0], session['data'][1], session['data'][2], session['data'][3]]
                }

                session['feedback_stage'] = True
                return jsonify({
                    'reply': response_text,
                    'chart': chart_data,
                    'chart_type': 'input_analysis'
                })

            except Exception as e:
                print(f"Prediction error: {e}")
                return jsonify({'reply': 'An error occurred during prediction. Please try again or check model configuration.'})

def handle_soil_conditions(user_message):
    try:
        session['intent'] = 'soil_advice'
        crop_list = ["bajra", "barley", "turmeric", "tur", "sugarcane", "soybeans",
                    "ragi", "potato", "onion", "maize", "ladyfinger", "jute",
                    "jowar", "green gram", "cotton", "coffee", "chickpea",
                    "cabbage", "wheat", "rice"]

        user_message_lower = user_message.lower()
        
        # 1. Attempt to find a crop in the current message or retrieve from session
        crop_found = session.get('soil_crop')
        for c in crop_list:
            if c in user_message_lower:
                crop_found = c
                session['soil_crop'] = c
                break

        if not crop_found:
            return jsonify({'reply': "🌿 Which crop would you like soil advice for? (e.g., 'soil for rice')", 'status': 'need_crop'})

        # 2. Attempt to find location
        location = None
        # Check if the user used the 'in [location]' format
        if ' in ' in user_message_lower:
            parts = user_message_lower.split(' in ')
            location = parts[1].strip()
        # If no 'in', and we ALREADY had the crop, treat the whole message as the location
        elif session.get('soil_crop') and user_message_lower not in ['soil', 'soil analysis', session.get('soil_crop')]:
            location = user_message.strip()
        
        if not location:
            return jsonify({'reply': f"📍 I've got <b>{crop_found.title()}</b>. Now, which <b>location</b> (district) are you planting in?", 'status': 'need_location'})

        temperature, humidity, rainfall = fetch_weather_data(location)
        
        if temperature is None:
            return jsonify({'reply': f"❌ Could not fetch weather for '<b>{location.title()}</b>'. Please try a valid district name (e.g., Trichy, Delhi)."})

        # OneHot encode crop for Soil Model
        encoder = OneHotEncoder(sparse_output=False, handle_unknown='ignore')
        encoder.fit(np.array(crop_list).reshape(-1, 1))
        encoded_crop = encoder.transform([[crop_found]])[0]
        
        full_features = np.concatenate([encoded_crop, [temperature, humidity, rainfall]])
        prediction = soil_model.predict(pd.DataFrame([full_features]))

        # N, P, K, pH prediction
        vals = [round(float(prediction[0][1]), 2), round(float(prediction[0][2]), 2), round(float(prediction[0][3]), 2), round(float(prediction[0][0]), 2)]

        reply_message = (
            f"🌱 Ideal soil conditions for <b>{crop_found.title()}</b> in {location.title()}:<br>"
            f"<b>- pH:</b> {vals[3]}<br>"
            f"<b>- Nitrogen (N):</b> {vals[0]}<br>"
            f"<b>- Phosphorus (P):</b> {vals[1]}<br>"
            f"<b>- Potassium (K):</b> {vals[2]}<br>"
            f"<br>📊 <b>Soil Balance Chart:</b>"
        )

        # Clear intent after success
        session.pop('intent', None)
        session.pop('soil_crop', None)

        return jsonify({
            'reply': reply_message, 
            'status': 'success',
            'chart': {
                'labels': ["Nitrogen", "Phosphorus", "Potassium", "pH Value"],
                'values': vals
            },
            'chart_type': 'input_analysis'
        })

    except Exception as e:
        return jsonify({'reply': f"Sorry, I couldn't process that request. Error: {str(e)}", 'status': 'error'})

if __name__ == '__main__':
    # Initialize Ngrok for exposure (Optional)
    # Get your token from: https://dashboard.ngrok.com/get-started/your-authtoken
    token = os.environ.get('NGROK_TOKEN')
    if token:
        try:
            ngrok.set_auth_token(token)
            public_url = ngrok.connect(5000)
            print(f"\n * Ngrok Tunnel Active: {public_url}")
            print(f" * Access your chatbot here: {public_url}\n")
        except Exception as e:
            print(f" * Ngrok failed to start: {e}. Running locally only.")

    print("--- Starting Flask Application ---")
    app.run(port=5000)