Initial commit - Project Samarth Intelligent Q&A System

.gitignore

@@ -0,0 +1,6 @@
+venv/
+__pycache__/
+*.csv
+*.ipynb_checkpoints
+.DS_Store
+.env

README.md

@@ -0,0 +1,141 @@
+# 🌾 Project Samarth — Intelligent Q&A System
+**Bridging Agriculture & Climate Insights using Live Government Data**
+
+---
+
+### 🧠 Overview
+
+**Project Samarth** is an intelligent **Q&A system** built to analyze and answer complex, data-driven questions about **India’s agricultural economy** and its relationship with **climate patterns** — powered entirely by **live datasets from [data.gov.in](https://data.gov.in/)**.
+
+This system fetches real-time data from the:
+- 🏛️ **Ministry of Agriculture & Farmers Welfare**
+- 🌦️ **India Meteorological Department (IMD)**
+
+It integrates both datasets and allows users to query them in **natural language** through a clean **Streamlit-based interface**.
+
+---
+
+### 🎯 Problem Statement
+
+Government portals like **data.gov.in** contain thousands of valuable datasets — but they exist in diverse formats across ministries, making it difficult to extract cross-domain insights.
+
+**Your Mission:**  
+To design and build a **functional end-to-end prototype** that:
+1. Fetches live government data using APIs.
+2. Integrates multiple datasets (Agriculture + IMD Rainfall).
+3. Enables users to ask **natural language questions**.
+4. Returns accurate, traceable, and data-backed insights with proper citations.
+
+---
+
+### 🚀 Features
+
+✅ **Real-Time API Integration**
+- Fetches data directly from `data.gov.in` via official API keys and resource IDs.  
+- Agriculture: Crop production data (1997–2014)  
+- IMD: Sub-divisional rainfall data (1901–2017)
+
+✅ **Data Integration Layer**
+- Automatically merges climate and crop production datasets using cleaned and normalized state names.
+
+✅ **Intelligent Q&A Engine**
+- Understands queries like:
+  - “Compare rainfall and rice production in Bihar and Jharkhand for the last 5 years.”
+  - “Analyze crop trends in Andhra Pradesh.”
+
+✅ **Streamlit Chat Interface**
+- Simple user input box.
+- Clean, markdown-based formatted answers.
+- Auto-citation of data sources.
+
+✅ **Accuracy & Traceability**
+- Every answer is directly backed by the live dataset and cited source.
+
+---
+
+### 🧩 System Architecture
+
+User (Streamlit UI)
+│
+▼
+Natural Language Parser (LLM / Keyword Extractor)
+│
+▼
+Query Engine (Pandas Logic)
+│
+▼
+Data Layer (APIs + Local CSV Integration)
+│
+▼
+Answer Generator (Formatter + Citation)
+
+
+---
+
+### 🧰 Tech Stack
+
+| Layer | Tools / Libraries |
+|-------|--------------------|
+| Data Fetching | `requests`, `pandas`, `json` |
+| Data Integration | `pandas`, `numpy` |
+| NLP Parsing | Custom keyword parser / rule-based |
+| Visualization | `matplotlib`, `seaborn`, `plotly` |
+| Frontend | `streamlit`, `style.css` |
+| Backend Logic | Python 3.10+ |
+| Source | [data.gov.in](https://data.gov.in) APIs |
+
+---
+
+### ⚙️ Setup Instructions
+
+1️⃣ **Clone the Repository**
+```bash
+git clone https://github.com/<your-username>/Project_Samarth.git
+cd Project_Samarth
+
+2️⃣ Create a Virtual Environment
+
+python -m venv venv
+source venv/bin/activate   # (or venv\Scripts\activate on Windows)
+
+
+3️⃣ Install Dependencies
+
+pip install -r requirements.txt
+
+
+4️⃣ Fetch & Integrate Data
+
+python main.py
+
+
+5️⃣ Run the Streamlit Q&A Interface
+
+streamlit run ui/app_streamlit.py
+
+🧠 Example Query
+
+Input:
+
+Compare rainfall and rice production in Andaman and Nicobar Islands for the last 5 years
+
+
+Output:
+
+📊 Analysis for Andaman and Nicobar Islands — Crop: Rice
+
+🌧️ Average Rainfall (mm):
+ • Andaman and Nicobar Islands: 1142.46
+
+🌾 Total Production (tonnes):
+ • Andaman and Nicobar Islands: 45,451
+
+📚 Data Source: Ministry of Agriculture & Farmers Welfare and India Meteorological Department (IMD), data.gov.in
+
+🧩 Key Dataset References
+Dataset	Ministry	API Resource ID
+District-wise Crop Production Statistics (1997–2014)	Ministry of Agriculture & Farmers Welfare	xxxxx
+Sub-divisional Rainfall Data (1901–2017)	India Meteorological Department (IMD)	xxxxxx
+
+👨‍💻 Developed By
+Satyam Kumar

answer_generator/citation_manager.py

@@ -0,0 +1,22 @@
+def get_source(query_type: str):
+    """
+    Returns the accurate data source(s) based on query type.
+    Ensures correct citation for all Q&A responses.
+    """
+
+    if query_type in ["compare_rainfall", "compare_rainfall_production"]:
+        return "Ministry of Agriculture & Farmers Welfare and India Meteorological Department (IMD), data.gov.in"
+
+    elif query_type in ["highest_production", "crop_trend"]:
+        return "Ministry of Agriculture & Farmers Welfare, data.gov.in"
+
+    elif query_type == "climate_correlation":
+        return "India Meteorological Department (IMD), data.gov.in"
+
+    # Default fallback
+    return "Government Open Data Portal (data.gov.in)"
+
+
+# 🧪 Test
+if __name__ == "__main__":
+    print(get_source("compare_rainfall_production"))

answer_generator/formatter.py

@@ -0,0 +1,29 @@
+def format_response(result: dict, source: str):
+    """Format the Q&A result for Streamlit display."""
+
+    if not result or "error" in result:
+        return f"❌ {result.get('error', 'No valid data found.')}"
+
+    states = [s.title() for s in result.get("states", [])]
+    crop = result.get("crop", "N/A").title()
+    text = f"📊 Analysis for {', '.join(states)} — Crop: {crop}\n\n"
+
+    # 🌧️ Rainfall Summary
+    rainfall = result.get("rainfall_summary", {})
+    if rainfall:
+        text += "🌧️ Average Rainfall (mm):\n"
+        for state, value in rainfall.items():
+            text += f" • {state.title()}: {round(value, 2)}\n"
+        text += "\n"
+
+    # 🌾 Production Summary
+    production = result.get("production_summary", {})
+    if production:
+        text += "🌾 Total Production (tonnes):\n"
+        for state, value in production.items():
+            text += f" • {state.title()}: {int(value):,}\n"
+        text += "\n"
+
+    # 📚 Citation
+    text += f"📚 Data Source: {source}"
+    return text

data_layer/config.py

@@ -0,0 +1,4 @@
+BASE_URL = "https://api.data.gov.in/resource/"
+API_KEY = "579b464db66ec23bdd000001375fd3eede8e49af7458c4f371a43d02"
+AGRI_RESOURCE_ID = "35be999b-0208-4354-b557-f6ca9a5355de"
+IMD_RESOURCE_ID  = "8e0bd482-4aba-4d99-9cb9-ff124f6f1c2f"

data_layer/fetch_agriculture_api.py

@@ -0,0 +1,69 @@
+import requests
+import pandas as pd
+import os
+import time
+from data_layer.config import BASE_URL, API_KEY, AGRI_RESOURCE_ID
+
+def fetch_agriculture_data(limit=500, retries=3, max_records=2000):
+    """
+    Fetch agriculture data from data.gov.in API in chunks and save as CSV.
+    Handles rate limits and saves automatically into hybrid_dataset folder.
+    """
+
+    os.makedirs("hybrid_dataset", exist_ok=True)
+    csv_path = "hybrid_dataset/agriculture_data.csv"
+    all_data = []
+
+    print("🌾 Starting Agriculture data fetch...")
+
+    offset = 0
+    total_fetched = 0
+
+    while total_fetched < max_records:
+        url = f"{BASE_URL}{AGRI_RESOURCE_ID}?api-key={API_KEY}&format=json&limit={limit}&offset={offset}"
+
+        for attempt in range(retries):
+            try:
+                response = requests.get(url, timeout=20)
+                response.raise_for_status()
+
+                data = response.json().get("records", [])
+                if not data:
+                    print("✅ No more records found.")
+                    break
+
+                df_chunk = pd.DataFrame(data)
+                all_data.append(df_chunk)
+
+                total_fetched += len(df_chunk)
+                offset += limit
+
+                print(f"✅ Chunk fetched: {len(df_chunk)} rows (Total: {total_fetched})")
+
+                # small delay to avoid rate limit
+                time.sleep(2)
+                break
+
+            except requests.exceptions.HTTPError as e:
+                if "429" in str(e):
+                    print("⚠️ Too Many Requests — waiting 20 seconds...")
+                    time.sleep(20)
+                elif "403" in str(e):
+                    print("🚫 Forbidden: Check your API key or URL in config.py")
+                    return pd.DataFrame()
+                else:
+                    print(f"⚠️ Attempt {attempt+1} failed: {e}")
+                    time.sleep(3)
+
+        else:
+            print("❌ Max retries reached, skipping this chunk.")
+            break
+
+    if all_data:
+        final_df = pd.concat(all_data, ignore_index=True)
+        final_df.to_csv(csv_path, index=False)
+        print(f"✅ Agriculture data fetched & saved → {csv_path} ({len(final_df)} rows total)")
+        return final_df
+    else:
+        print("❌ No data fetched.")
+        return pd.DataFrame()

data_layer/fetch_imd_api.py

@@ -0,0 +1,66 @@
+import requests
+import pandas as pd
+import os
+import time
+from data_layer.config import BASE_URL, API_KEY, IMD_RESOURCE_ID
+
+def fetch_rainfall_data(limit=500, retries=3, max_records=2000):
+    """
+    Fetch IMD rainfall data from data.gov.in API in chunks and save as CSV.
+    Automatically handles rate limits and saves into hybrid_dataset folder.
+    """
+    os.makedirs("hybrid_dataset", exist_ok=True)
+    csv_path = "hybrid_dataset/imd_rainfall_data.csv"
+    all_data = []
+
+    print("🌦️  Starting IMD Rainfall data fetch...")
+
+    offset = 0
+    total_fetched = 0
+
+    while total_fetched < max_records:
+        url = f"{BASE_URL}{IMD_RESOURCE_ID}?api-key={API_KEY}&format=json&limit={limit}&offset={offset}"
+
+        for attempt in range(retries):
+            try:
+                response = requests.get(url, timeout=20)
+                response.raise_for_status()
+
+                data = response.json().get("records", [])
+                if not data:
+                    print("✅ No more records found.")
+                    break
+
+                df_chunk = pd.DataFrame(data)
+                all_data.append(df_chunk)
+
+                total_fetched += len(df_chunk)
+                offset += limit
+
+                print(f"✅ Chunk fetched: {len(df_chunk)} rows (Total: {total_fetched})")
+
+                time.sleep(2)  # avoid rate-limit
+                break
+
+            except requests.exceptions.HTTPError as e:
+                if "429" in str(e):
+                    print("⚠️ Too Many Requests — waiting 20 seconds...")
+                    time.sleep(20)
+                elif "403" in str(e):
+                    print("🚫 Forbidden: check API key or IMD resource ID in config.py")
+                    return pd.DataFrame()
+                else:
+                    print(f"⚠️ Attempt {attempt+1} failed: {e}")
+                    time.sleep(3)
+        else:
+            print("❌ Max retries reached, skipping this chunk.")
+            break
+
+    if all_data:
+        final_df = pd.concat(all_data, ignore_index=True)
+        final_df.to_csv(csv_path, index=False)
+        print(f"✅ Rainfall data fetched & saved → {csv_path} ({len(final_df)} rows total)")
+        return final_df
+    else:
+        print("❌ No rainfall data fetched.")
+        return pd.DataFrame()

data_layer/integrate_data.py

@@ -0,0 +1,88 @@
+import pandas as pd
+import os
+import re
+
+def clean_state_name(name: str):
+    """Cleans and standardizes state/subdivision names."""
+    if not isinstance(name, str):
+        return ""
+    name = name.lower().strip()
+    name = re.sub(r"&", "and", name)
+    name = re.sub(r"\s+", " ", name)
+    name = re.sub(r"[^a-z\s]", "", name)  # remove special chars
+    return name
+
+def integrate_data(agri_df: pd.DataFrame, rain_df: pd.DataFrame):
+    """
+    🔗 Final Integration Logic — Clean, Normalize, and Merge
+    Works even if & or trailing spaces exist.
+    """
+
+    os.makedirs("hybrid_dataset", exist_ok=True)
+
+    print(f"🧾 Agriculture unique states: {agri_df['state_name'].nunique()}")
+    print(f"☁️ Rainfall unique subdivisions: {rain_df['subdivision'].nunique()}")
+
+    # Clean columns
+    agri_df.columns = agri_df.columns.str.lower().str.strip()
+    rain_df.columns = rain_df.columns.str.lower().str.strip()
+
+    # Clean text values
+    agri_df["state_name"] = agri_df["state_name"].apply(clean_state_name)
+    rain_df["subdivision"] = rain_df["subdivision"].apply(clean_state_name)
+
+    # Create mapping
+    mapping = {
+        "andaman and nicobar islands": "andaman and nicobar islands",
+        "orissa": "odisha",
+        "sub himalayan west bengal and sikkim": "west bengal",
+        "gangetic west bengal": "west bengal",
+        "east uttar pradesh": "uttar pradesh",
+        "west uttar pradesh": "uttar pradesh",
+        "east rajasthan": "rajasthan",
+        "west rajasthan": "rajasthan",
+        "haryana delhi and chandigarh": "haryana",
+        "assam and meghalaya": "assam",
+        "naga mani mizo tripura": "tripura",
+    }
+
+    # Apply mapping to rainfall data
+    rain_df["state_name"] = rain_df["subdivision"].replace(mapping)
+
+    # Ensure year columns match type
+    agri_df["crop_year"] = pd.to_numeric(agri_df["crop_year"], errors="coerce").astype("Int64")
+    rain_df["year"] = pd.to_numeric(rain_df["year"], errors="coerce").astype("Int64")
+    rain_df.rename(columns={"year": "crop_year"}, inplace=True)
+
+    # Show what’s common after full cleaning
+    common_states = sorted(set(agri_df["state_name"].unique()) & set(rain_df["state_name"].unique()))
+    print(f"✅ Common states found: {common_states}")
+
+    if not common_states:
+        print("⚠️ No matching states even after cleaning — check character mismatches manually!")
+        print("🔍 Example Agri states:", agri_df['state_name'].unique().tolist())
+        print("🔍 Example Rainfall states:", rain_df['state_name'].unique().tolist())
+        return pd.DataFrame()
+
+    # Filter only matching states
+    agri_df = agri_df[agri_df["state_name"].isin(common_states)]
+    rain_df = rain_df[rain_df["state_name"].isin(common_states)]
+
+    # Merge datasets
+    merged = pd.merge(agri_df, rain_df, on=["state_name", "crop_year"], how="inner")
+
+    # Save output
+    output_path = "hybrid_dataset/merged_agri_rainfall.csv"
+    merged.to_csv(output_path, index=False)
+
+    print(f"✅ Data integrated and saved → {output_path} ({len(merged)} rows, {len(merged.columns)} columns)")
+    print("🏛️ Unique merged states:", merged["state_name"].unique().tolist())
+
+    return merged
+
+
+# 🧪 Quick standalone test
+if __name__ == "__main__":
+    ag = pd.read_csv("hybrid_dataset/agriculture_data.csv")
+    rd = pd.read_csv("hybrid_dataset/imd_rainfall_data.csv")
+    integrate_data(ag, rd)

main.py

@@ -0,0 +1,16 @@
+from data_layer.fetch_agriculture_api import fetch_agriculture_data   # 🌾 fetches Agriculture data from API
+from data_layer.fetch_imd_api import fetch_rainfall_data               # 🌧️ fetches IMD rainfall data from API
+from data_layer.integrate_data import integrate_data                   # 🔗 merges both datasets
+
+if __name__ == "__main__":
+    print("🌾 Fetching Agriculture Data ...")
+    agri_df = fetch_agriculture_data()
+
+    print("🌧️ Fetching IMD Rainfall Data ...")
+    rain_df = fetch_rainfall_data()
+
+    print("🔗 Integrating Datasets ...")
+    integrate_data(agri_df, rain_df)
+
+    print("\n✅ Phase 1 Completed Successfully!")
+    print("Now run: streamlit run ui/app_streamlit.py")

notebooks/01_data_discovery.ipynb

@@ -0,0 +1,1619 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "ec455bd3",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Active code page: 1252\n",
+      "Requirement already satisfied: requests in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (2.32.5)\n",
+      "Requirement already satisfied: pandas in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (2.3.2)\n",
+      "Requirement already satisfied: numpy in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (2.3.2)\n",
+      "Collecting matplotlib\n",
+      "  Using cached matplotlib-3.10.7-cp313-cp313-win_amd64.whl.metadata (11 kB)\n",
+      "Collecting plotly\n",
+      "  Downloading plotly-6.3.1-py3-none-any.whl.metadata (8.5 kB)\n",
+      "Requirement already satisfied: streamlit in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (1.49.1)\n",
+      "Requirement already satisfied: langchain in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (0.3.27)\n",
+      "Requirement already satisfied: transformers in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (4.56.1)\n",
+      "Requirement already satisfied: charset_normalizer<4,>=2 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from requests) (3.4.3)\n",
+      "Requirement already satisfied: idna<4,>=2.5 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from requests) (3.10)\n",
+      "Requirement already satisfied: urllib3<3,>=1.21.1 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from requests) (2.5.0)\n",
+      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from requests) (2025.8.3)\n",
+      "Requirement already satisfied: python-dateutil>=2.8.2 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from pandas) (2.9.0.post0)\n",
+      "Requirement already satisfied: pytz>=2020.1 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from pandas) (2025.2)\n",
+      "Requirement already satisfied: tzdata>=2022.7 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from pandas) (2025.2)\n",
+      "Collecting contourpy>=1.0.1 (from matplotlib)\n",
+      "  Using cached contourpy-1.3.3-cp313-cp313-win_amd64.whl.metadata (5.5 kB)\n",
+      "Collecting cycler>=0.10 (from matplotlib)\n",
+      "  Using cached cycler-0.12.1-py3-none-any.whl.metadata (3.8 kB)\n",
+      "Collecting fonttools>=4.22.0 (from matplotlib)\n",
+      "  Using cached fonttools-4.60.1-cp313-cp313-win_amd64.whl.metadata (114 kB)\n",
+      "Collecting kiwisolver>=1.3.1 (from matplotlib)\n",
+      "  Using cached kiwisolver-1.4.9-cp313-cp313-win_amd64.whl.metadata (6.4 kB)\n",
+      "Requirement already satisfied: packaging>=20.0 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from matplotlib) (23.2)\n",
+      "Requirement already satisfied: pillow>=8 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from matplotlib) (11.3.0)\n",
+      "Collecting pyparsing>=3 (from matplotlib)\n",
+      "  Using cached pyparsing-3.2.5-py3-none-any.whl.metadata (5.0 kB)\n",
+      "Requirement already satisfied: narwhals>=1.15.1 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from plotly) (2.4.0)\n",
+      "Requirement already satisfied: altair!=5.4.0,!=5.4.1,<6,>=4.0 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from streamlit) (5.5.0)\n",
+      "Requirement already satisfied: blinker<2,>=1.5.0 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from streamlit) (1.9.0)\n",
+      "Requirement already satisfied: cachetools<7,>=4.0 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from streamlit) (6.2.0)\n",
+      "Requirement already satisfied: click<9,>=7.0 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from streamlit) (8.2.1)\n",
+      "Requirement already satisfied: protobuf<7,>=3.20 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from streamlit) (6.32.0)\n",
+      "Requirement already satisfied: pyarrow>=7.0 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from streamlit) (21.0.0)\n",
+      "Requirement already satisfied: tenacity<10,>=8.1.0 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from streamlit) (8.5.0)\n",
+      "Requirement already satisfied: toml<2,>=0.10.1 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from streamlit) (0.10.2)\n",
+      "Requirement already satisfied: typing-extensions<5,>=4.4.0 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from streamlit) (4.15.0)\n",
+      "Requirement already satisfied: watchdog<7,>=2.1.5 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from streamlit) (6.0.0)\n",
+      "Requirement already satisfied: gitpython!=3.1.19,<4,>=3.0.7 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from streamlit) (3.1.45)\n",
+      "Requirement already satisfied: pydeck<1,>=0.8.0b4 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from streamlit) (0.9.1)\n",
+      "Requirement already satisfied: tornado!=6.5.0,<7,>=6.0.3 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from streamlit) (6.5.2)\n",
+      "Requirement already satisfied: langchain-core<1.0.0,>=0.3.72 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from langchain) (0.3.76)\n",
+      "Requirement already satisfied: langchain-text-splitters<1.0.0,>=0.3.9 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from langchain) (0.3.11)\n",
+      "Requirement already satisfied: langsmith>=0.1.17 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from langchain) (0.4.27)\n",
+      "Requirement already satisfied: pydantic<3.0.0,>=2.7.4 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from langchain) (2.11.7)\n",
+      "Requirement already satisfied: SQLAlchemy<3,>=1.4 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from langchain) (2.0.43)\n",
+      "Requirement already satisfied: PyYAML>=5.3 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from langchain) (6.0.2)\n",
+      "Requirement already satisfied: filelock in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from transformers) (3.19.1)\n",
+      "Requirement already satisfied: huggingface-hub<1.0,>=0.34.0 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from transformers) (0.34.4)\n",
+      "Requirement already satisfied: regex!=2019.12.17 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from transformers) (2025.9.1)\n",
+      "Requirement already satisfied: tokenizers<=0.23.0,>=0.22.0 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from transformers) (0.22.0)\n",
+      "Requirement already satisfied: safetensors>=0.4.3 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from transformers) (0.6.2)\n",
+      "Requirement already satisfied: tqdm>=4.27 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from transformers) (4.67.1)\n",
+      "Requirement already satisfied: jinja2 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from altair!=5.4.0,!=5.4.1,<6,>=4.0->streamlit) (3.1.6)\n",
+      "Requirement already satisfied: jsonschema>=3.0 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from altair!=5.4.0,!=5.4.1,<6,>=4.0->streamlit) (4.25.1)\n",
+      "Requirement already satisfied: colorama in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from click<9,>=7.0->streamlit) (0.4.6)\n",
+      "Requirement already satisfied: gitdb<5,>=4.0.1 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from gitpython!=3.1.19,<4,>=3.0.7->streamlit) (4.0.12)\n",
+      "Requirement already satisfied: fsspec>=2023.5.0 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from huggingface-hub<1.0,>=0.34.0->transformers) (2024.2.0)\n",
+      "Requirement already satisfied: jsonpatch<2.0,>=1.33 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from langchain-core<1.0.0,>=0.3.72->langchain) (1.33)\n",
+      "Requirement already satisfied: httpx<1,>=0.23.0 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from langsmith>=0.1.17->langchain) (0.28.1)\n",
+      "Requirement already satisfied: orjson>=3.9.14 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from langsmith>=0.1.17->langchain) (3.11.3)\n",
+      "Requirement already satisfied: requests-toolbelt>=1.0.0 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from langsmith>=0.1.17->langchain) (1.0.0)\n",
+      "Requirement already satisfied: zstandard>=0.23.0 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from langsmith>=0.1.17->langchain) (0.24.0)\n",
+      "Requirement already satisfied: annotated-types>=0.6.0 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from pydantic<3.0.0,>=2.7.4->langchain) (0.7.0)\n",
+      "Requirement already satisfied: pydantic-core==2.33.2 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from pydantic<3.0.0,>=2.7.4->langchain) (2.33.2)\n",
+      "Requirement already satisfied: typing-inspection>=0.4.0 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from pydantic<3.0.0,>=2.7.4->langchain) (0.4.1)\n",
+      "Requirement already satisfied: six>=1.5 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from python-dateutil>=2.8.2->pandas) (1.17.0)\n",
+      "Requirement already satisfied: greenlet>=1 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from SQLAlchemy<3,>=1.4->langchain) (3.2.4)\n",
+      "Requirement already satisfied: smmap<6,>=3.0.1 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from gitdb<5,>=4.0.1->gitpython!=3.1.19,<4,>=3.0.7->streamlit) (5.0.2)\n",
+      "Requirement already satisfied: anyio in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from httpx<1,>=0.23.0->langsmith>=0.1.17->langchain) (4.10.0)\n",
+      "Requirement already satisfied: httpcore==1.* in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from httpx<1,>=0.23.0->langsmith>=0.1.17->langchain) (1.0.9)\n",
+      "Requirement already satisfied: h11>=0.16 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from httpcore==1.*->httpx<1,>=0.23.0->langsmith>=0.1.17->langchain) (0.16.0)\n",
+      "Requirement already satisfied: MarkupSafe>=2.0 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from jinja2->altair!=5.4.0,!=5.4.1,<6,>=4.0->streamlit) (3.0.2)\n",
+      "Requirement already satisfied: jsonpointer>=1.9 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from jsonpatch<2.0,>=1.33->langchain-core<1.0.0,>=0.3.72->langchain) (3.0.0)\n",
+      "Requirement already satisfied: attrs>=22.2.0 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from jsonschema>=3.0->altair!=5.4.0,!=5.4.1,<6,>=4.0->streamlit) (25.3.0)\n",
+      "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from jsonschema>=3.0->altair!=5.4.0,!=5.4.1,<6,>=4.0->streamlit) (2025.9.1)\n",
+      "Requirement already satisfied: referencing>=0.28.4 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from jsonschema>=3.0->altair!=5.4.0,!=5.4.1,<6,>=4.0->streamlit) (0.36.2)\n",
+      "Requirement already satisfied: rpds-py>=0.7.1 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from jsonschema>=3.0->altair!=5.4.0,!=5.4.1,<6,>=4.0->streamlit) (0.27.1)\n",
+      "Requirement already satisfied: sniffio>=1.1 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from anyio->httpx<1,>=0.23.0->langsmith>=0.1.17->langchain) (1.3.1)\n",
+      "Using cached matplotlib-3.10.7-cp313-cp313-win_amd64.whl (8.1 MB)\n",
+      "Downloading plotly-6.3.1-py3-none-any.whl (9.8 MB)\n",
+      "   ---------------------------------------- 0.0/9.8 MB ? eta -:--:--\n",
+      "   ---- ----------------------------------- 1.0/9.8 MB 7.0 MB/s eta 0:00:02\n",
+      "   ---- ----------------------------------- 1.0/9.8 MB 7.0 MB/s eta 0:00:02\n",
+      "   ---------- ----------------------------- 2.6/9.8 MB 4.5 MB/s eta 0:00:02\n",
+      "   ------------- -------------------------- 3.4/9.8 MB 4.4 MB/s eta 0:00:02\n",
+      "   ----------------- ---------------------- 4.2/9.8 MB 4.3 MB/s eta 0:00:02\n",
+      "   -------------------- ------------------- 5.0/9.8 MB 4.2 MB/s eta 0:00:02\n",
+      "   ----------------------- ---------------- 5.8/9.8 MB 4.1 MB/s eta 0:00:01\n",
+      "   --------------------------- ------------ 6.8/9.8 MB 4.1 MB/s eta 0:00:01\n",
+      "   ------------------------------ --------- 7.6/9.8 MB 4.1 MB/s eta 0:00:01\n",
+      "   ---------------------------------- ----- 8.4/9.8 MB 4.1 MB/s eta 0:00:01\n",
+      "   ------------------------------------- -- 9.2/9.8 MB 4.0 MB/s eta 0:00:01\n",
+      "   ---------------------------------------  9.7/9.8 MB 4.0 MB/s eta 0:00:01\n",
+      "   ---------------------------------------- 9.8/9.8 MB 3.9 MB/s eta 0:00:00\n",
+      "Using cached contourpy-1.3.3-cp313-cp313-win_amd64.whl (226 kB)\n",
+      "Using cached cycler-0.12.1-py3-none-any.whl (8.3 kB)\n",
+      "Using cached fonttools-4.60.1-cp313-cp313-win_amd64.whl (2.3 MB)\n",
+      "Using cached kiwisolver-1.4.9-cp313-cp313-win_amd64.whl (73 kB)\n",
+      "Using cached pyparsing-3.2.5-py3-none-any.whl (113 kB)\n",
+      "Installing collected packages: pyparsing, plotly, kiwisolver, fonttools, cycler, contourpy, matplotlib\n",
+      "Successfully installed contourpy-1.3.3 cycler-0.12.1 fonttools-4.60.1 kiwisolver-1.4.9 matplotlib-3.10.7 plotly-6.3.1 pyparsing-3.2.5\n",
+      "Note: you may need to restart the kernel to use updated packages.\n"
+     ]
+    }
+   ],
+   "source": [
+    "pip install requests pandas numpy matplotlib plotly streamlit langchain transformers\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "42263596",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "🔍 Dataset Loaded Successfully!\n",
+      "Total Rows: 203\n",
+      "Total Columns: 25\n",
+      "\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.microsoft.datawrangler.viewer.v0+json": {
+       "columns": [
+        {
+         "name": "index",
+         "rawType": "int64",
+         "type": "integer"
+        },
+        {
+         "name": "state_name",
+         "rawType": "object",
+         "type": "string"
+        },
+        {
+         "name": "district_name",
+         "rawType": "object",
+         "type": "string"
+        },
+        {
+         "name": "crop_year",
+         "rawType": "int64",
+         "type": "integer"
+        },
+        {
+         "name": "season",
+         "rawType": "object",
+         "type": "string"
+        },
+        {
+         "name": "crop",
+         "rawType": "object",
+         "type": "string"
+        },
+        {
+         "name": "area_",
+         "rawType": "float64",
+         "type": "float"
+        },
+        {
+         "name": "production_",
+         "rawType": "float64",
+         "type": "float"
+        },
+        {
+         "name": "subdivision",
+         "rawType": "object",
+         "type": "string"
+        },
+        {
+         "name": "jan",
+         "rawType": "float64",
+         "type": "float"
+        },
+        {
+         "name": "feb",
+         "rawType": "float64",
+         "type": "float"
+        },
+        {
+         "name": "mar",
+         "rawType": "float64",
+         "type": "float"
+        },
+        {
+         "name": "apr",
+         "rawType": "float64",
+         "type": "float"
+        },
+        {
+         "name": "may",
+         "rawType": "float64",
+         "type": "float"
+        },
+        {
+         "name": "jun",
+         "rawType": "float64",
+         "type": "float"
+        },
+        {
+         "name": "jul",
+         "rawType": "float64",
+         "type": "float"
+        },
+        {
+         "name": "aug",
+         "rawType": "float64",
+         "type": "float"
+        },
+        {
+         "name": "sep",
+         "rawType": "float64",
+         "type": "float"
+        },
+        {
+         "name": "oct",
+         "rawType": "float64",
+         "type": "float"
+        },
+        {
+         "name": "nov",
+         "rawType": "float64",
+         "type": "float"
+        },
+        {
+         "name": "dec",
+         "rawType": "float64",
+         "type": "float"
+        },
+        {
+         "name": "annual",
+         "rawType": "float64",
+         "type": "float"
+        },
+        {
+         "name": "jf",
+         "rawType": "float64",
+         "type": "float"
+        },
+        {
+         "name": "mam",
+         "rawType": "float64",
+         "type": "float"
+        },
+        {
+         "name": "jjas",
+         "rawType": "float64",
+         "type": "float"
+        },
+        {
+         "name": "ond",
+         "rawType": "float64",
+         "type": "float"
+        }
+       ],
+       "ref": "633f3f12-0965-479f-8a47-3a1c5a2b8105",
+       "rows": [
+        [
+         "0",
+         "andaman and nicobar islands",
+         "NICOBARS",
+         "2000",
+         "Kharif",
+         "Arecanut",
+         "1254.0",
+         "2000.0",
+         "andaman & nicobar islands",
+         "53.0",
+         "59.0",
+         "171.3",
+         "218.1",
+         "422.8",
+         "357.0",
+         "176.3",
+         "460.8",
+         "250.1",
+         "321.2",
+         "158.3",
+         "115.2",
+         "2763.2",
+         "112.0",
+         "812.2",
+         "1244.2",
+         "594.7"
+        ],
+        [
+         "1",
+         "andaman and nicobar islands",
+         "NICOBARS",
+         "2000",
+         "Kharif",
+         "Other Kharif pulses",
+         "2.0",
+         "1.0",
+         "andaman & nicobar islands",
+         "53.0",
+         "59.0",
+         "171.3",
+         "218.1",
+         "422.8",
+         "357.0",
+         "176.3",
+         "460.8",
+         "250.1",
+         "321.2",
+         "158.3",
+         "115.2",
+         "2763.2",
+         "112.0",
+         "812.2",
+         "1244.2",
+         "594.7"
+        ],
+        [
+         "2",
+         "andaman and nicobar islands",
+         "NICOBARS",
+         "2000",
+         "Kharif",
+         "Rice",
+         "102.0",
+         "321.0",
+         "andaman & nicobar islands",
+         "53.0",
+         "59.0",
+         "171.3",
+         "218.1",
+         "422.8",
+         "357.0",
+         "176.3",
+         "460.8",
+         "250.1",
+         "321.2",
+         "158.3",
+         "115.2",
+         "2763.2",
+         "112.0",
+         "812.2",
+         "1244.2",
+         "594.7"
+        ],
+        [
+         "3",
+         "andaman and nicobar islands",
+         "NICOBARS",
+         "2000",
+         "Whole Year",
+         "Banana",
+         "176.0",
+         "641.0",
+         "andaman & nicobar islands",
+         "53.0",
+         "59.0",
+         "171.3",
+         "218.1",
+         "422.8",
+         "357.0",
+         "176.3",
+         "460.8",
+         "250.1",
+         "321.2",
+         "158.3",
+         "115.2",
+         "2763.2",
+         "112.0",
+         "812.2",
+         "1244.2",
+         "594.7"
+        ],
+        [
+         "4",
+         "andaman and nicobar islands",
+         "NICOBARS",
+         "2000",
+         "Whole Year",
+         "Cashewnut",
+         "720.0",
+         "165.0",
+         "andaman & nicobar islands",
+         "53.0",
+         "59.0",
+         "171.3",
+         "218.1",
+         "422.8",
+         "357.0",
+         "176.3",
+         "460.8",
+         "250.1",
+         "321.2",
+         "158.3",
+         "115.2",
+         "2763.2",
+         "112.0",
+         "812.2",
+         "1244.2",
+         "594.7"
+        ]
+       ],
+       "shape": {
+        "columns": 25,
+        "rows": 5
+       }
+      },
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>state_name</th>\n",
+       "      <th>district_name</th>\n",
+       "      <th>crop_year</th>\n",
+       "      <th>season</th>\n",
+       "      <th>crop</th>\n",
+       "      <th>area_</th>\n",
+       "      <th>production_</th>\n",
+       "      <th>subdivision</th>\n",
+       "      <th>jan</th>\n",
+       "      <th>feb</th>\n",
+       "      <th>...</th>\n",
+       "      <th>aug</th>\n",
+       "      <th>sep</th>\n",
+       "      <th>oct</th>\n",
+       "      <th>nov</th>\n",
+       "      <th>dec</th>\n",
+       "      <th>annual</th>\n",
+       "      <th>jf</th>\n",
+       "      <th>mam</th>\n",
+       "      <th>jjas</th>\n",
+       "      <th>ond</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>andaman and nicobar islands</td>\n",
+       "      <td>NICOBARS</td>\n",
+       "      <td>2000</td>\n",
+       "      <td>Kharif</td>\n",
+       "      <td>Arecanut</td>\n",
+       "      <td>1254.0</td>\n",
+       "      <td>2000.0</td>\n",
+       "      <td>andaman &amp; nicobar islands</td>\n",
+       "      <td>53.0</td>\n",
+       "      <td>59.0</td>\n",
+       "      <td>...</td>\n",
+       "      <td>460.8</td>\n",
+       "      <td>250.1</td>\n",
+       "      <td>321.2</td>\n",
+       "      <td>158.3</td>\n",
+       "      <td>115.2</td>\n",
+       "      <td>2763.2</td>\n",
+       "      <td>112.0</td>\n",
+       "      <td>812.2</td>\n",
+       "      <td>1244.2</td>\n",
+       "      <td>594.7</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>andaman and nicobar islands</td>\n",
+       "      <td>NICOBARS</td>\n",
+       "      <td>2000</td>\n",
+       "      <td>Kharif</td>\n",
+       "      <td>Other Kharif pulses</td>\n",
+       "      <td>2.0</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>andaman &amp; nicobar islands</td>\n",
+       "      <td>53.0</td>\n",
+       "      <td>59.0</td>\n",
+       "      <td>...</td>\n",
+       "      <td>460.8</td>\n",
+       "      <td>250.1</td>\n",
+       "      <td>321.2</td>\n",
+       "      <td>158.3</td>\n",
+       "      <td>115.2</td>\n",
+       "      <td>2763.2</td>\n",
+       "      <td>112.0</td>\n",
+       "      <td>812.2</td>\n",
+       "      <td>1244.2</td>\n",
+       "      <td>594.7</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>andaman and nicobar islands</td>\n",
+       "      <td>NICOBARS</td>\n",
+       "      <td>2000</td>\n",
+       "      <td>Kharif</td>\n",
+       "      <td>Rice</td>\n",
+       "      <td>102.0</td>\n",
+       "      <td>321.0</td>\n",
+       "      <td>andaman &amp; nicobar islands</td>\n",
+       "      <td>53.0</td>\n",
+       "      <td>59.0</td>\n",
+       "      <td>...</td>\n",
+       "      <td>460.8</td>\n",
+       "      <td>250.1</td>\n",
+       "      <td>321.2</td>\n",
+       "      <td>158.3</td>\n",
+       "      <td>115.2</td>\n",
+       "      <td>2763.2</td>\n",
+       "      <td>112.0</td>\n",
+       "      <td>812.2</td>\n",
+       "      <td>1244.2</td>\n",
+       "      <td>594.7</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>andaman and nicobar islands</td>\n",
+       "      <td>NICOBARS</td>\n",
+       "      <td>2000</td>\n",
+       "      <td>Whole Year</td>\n",
+       "      <td>Banana</td>\n",
+       "      <td>176.0</td>\n",
+       "      <td>641.0</td>\n",
+       "      <td>andaman &amp; nicobar islands</td>\n",
+       "      <td>53.0</td>\n",
+       "      <td>59.0</td>\n",
+       "      <td>...</td>\n",
+       "      <td>460.8</td>\n",
+       "      <td>250.1</td>\n",
+       "      <td>321.2</td>\n",
+       "      <td>158.3</td>\n",
+       "      <td>115.2</td>\n",
+       "      <td>2763.2</td>\n",
+       "      <td>112.0</td>\n",
+       "      <td>812.2</td>\n",
+       "      <td>1244.2</td>\n",
+       "      <td>594.7</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>andaman and nicobar islands</td>\n",
+       "      <td>NICOBARS</td>\n",
+       "      <td>2000</td>\n",
+       "      <td>Whole Year</td>\n",
+       "      <td>Cashewnut</td>\n",
+       "      <td>720.0</td>\n",
+       "      <td>165.0</td>\n",
+       "      <td>andaman &amp; nicobar islands</td>\n",
+       "      <td>53.0</td>\n",
+       "      <td>59.0</td>\n",
+       "      <td>...</td>\n",
+       "      <td>460.8</td>\n",
+       "      <td>250.1</td>\n",
+       "      <td>321.2</td>\n",
+       "      <td>158.3</td>\n",
+       "      <td>115.2</td>\n",
+       "      <td>2763.2</td>\n",
+       "      <td>112.0</td>\n",
+       "      <td>812.2</td>\n",
+       "      <td>1244.2</td>\n",
+       "      <td>594.7</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>5 rows × 25 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                    state_name district_name  crop_year      season  \\\n",
+       "0  andaman and nicobar islands      NICOBARS       2000      Kharif   \n",
+       "1  andaman and nicobar islands      NICOBARS       2000      Kharif   \n",
+       "2  andaman and nicobar islands      NICOBARS       2000      Kharif   \n",
+       "3  andaman and nicobar islands      NICOBARS       2000  Whole Year   \n",
+       "4  andaman and nicobar islands      NICOBARS       2000  Whole Year   \n",
+       "\n",
+       "                  crop   area_  production_                subdivision   jan  \\\n",
+       "0             Arecanut  1254.0       2000.0  andaman & nicobar islands  53.0   \n",
+       "1  Other Kharif pulses     2.0          1.0  andaman & nicobar islands  53.0   \n",
+       "2                 Rice   102.0        321.0  andaman & nicobar islands  53.0   \n",
+       "3               Banana   176.0        641.0  andaman & nicobar islands  53.0   \n",
+       "4            Cashewnut   720.0        165.0  andaman & nicobar islands  53.0   \n",
+       "\n",
+       "    feb  ...    aug    sep    oct    nov    dec  annual     jf    mam    jjas  \\\n",
+       "0  59.0  ...  460.8  250.1  321.2  158.3  115.2  2763.2  112.0  812.2  1244.2   \n",
+       "1  59.0  ...  460.8  250.1  321.2  158.3  115.2  2763.2  112.0  812.2  1244.2   \n",
+       "2  59.0  ...  460.8  250.1  321.2  158.3  115.2  2763.2  112.0  812.2  1244.2   \n",
+       "3  59.0  ...  460.8  250.1  321.2  158.3  115.2  2763.2  112.0  812.2  1244.2   \n",
+       "4  59.0  ...  460.8  250.1  321.2  158.3  115.2  2763.2  112.0  812.2  1244.2   \n",
+       "\n",
+       "     ond  \n",
+       "0  594.7  \n",
+       "1  594.7  \n",
+       "2  594.7  \n",
+       "3  594.7  \n",
+       "4  594.7  \n",
+       "\n",
+       "[5 rows x 25 columns]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "📊 Columns in dataset:\n",
+      "['state_name', 'district_name', 'crop_year', 'season', 'crop', 'area_', 'production_', 'subdivision', 'jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec', 'annual', 'jf', 'mam', 'jjas', 'ond']\n",
+      "\n",
+      "🏛️ Unique States in Dataset:\n",
+      "- andaman and nicobar islands\n",
+      "\n",
+      "🌾 Unique Crops in Dataset:\n",
+      "- Arecanut\n",
+      "- Arhar/Tur\n",
+      "- Banana\n",
+      "- Black pepper\n",
+      "- Cashewnut\n",
+      "- Coconut\n",
+      "- Dry chillies\n",
+      "- Dry ginger\n",
+      "- Groundnut\n",
+      "- Maize\n",
+      "- Moong(Green Gram)\n",
+      "- Other Kharif pulses\n",
+      "- Rice\n",
+      "- Sugarcane\n",
+      "- Sunflower\n",
+      "- Sweet potato\n",
+      "- Tapioca\n",
+      "- Turmeric\n",
+      "- Urad\n",
+      "- other oilseeds\n",
+      "... (Total 20 unique crops)\n",
+      "\n",
+      "📅 Crop Year Range: 2000 - 2010\n",
+      "\n",
+      "📈 Number of unique crops per state:\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.microsoft.datawrangler.viewer.v0+json": {
+       "columns": [
+        {
+         "name": "index",
+         "rawType": "int64",
+         "type": "integer"
+        },
+        {
+         "name": "state_name",
+         "rawType": "object",
+         "type": "string"
+        },
+        {
+         "name": "unique_crops",
+         "rawType": "int64",
+         "type": "integer"
+        }
+       ],
+       "ref": "16808a21-275e-4e3e-a212-22ed467e3c22",
+       "rows": [
+        [
+         "0",
+         "andaman and nicobar islands",
+         "20"
+        ]
+       ],
+       "shape": {
+        "columns": 2,
+        "rows": 1
+       }
+      },
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>state_name</th>\n",
+       "      <th>unique_crops</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>andaman and nicobar islands</td>\n",
+       "      <td>20</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                    state_name  unique_crops\n",
+       "0  andaman and nicobar islands            20"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "# -----------------------------------------------\n",
+    "# 📘 Project Samarth - Phase 1: Data Discovery\n",
+    "# -----------------------------------------------\n",
+    "\n",
+    "import pandas as pd\n",
+    "\n",
+    "# ✅ Load merged dataset\n",
+    "file_path = \"../hybrid_dataset/merged_agri_rainfall.csv\"  # adjust if needed\n",
+    "df = pd.read_csv(file_path)\n",
+    "\n",
+    "# ✅ Basic info\n",
+    "print(\"🔍 Dataset Loaded Successfully!\")\n",
+    "print(f\"Total Rows: {len(df)}\")\n",
+    "print(f\"Total Columns: {len(df.columns)}\\n\")\n",
+    "\n",
+    "# ✅ Display first few rows\n",
+    "display(df.head())\n",
+    "\n",
+    "# ✅ Show all available columns\n",
+    "print(\"📊 Columns in dataset:\")\n",
+    "print(df.columns.tolist())\n",
+    "\n",
+    "# ✅ Check unique states\n",
+    "if \"state_name\" in df.columns:\n",
+    "    states = sorted(df[\"state_name\"].dropna().unique().tolist())\n",
+    "    print(\"\\n🏛️ Unique States in Dataset:\")\n",
+    "    for s in states:\n",
+    "        print(\"-\", s)\n",
+    "\n",
+    "# ✅ Check unique crops\n",
+    "if \"crop\" in df.columns:\n",
+    "    crops = sorted(df[\"crop\"].dropna().unique().tolist())\n",
+    "    print(\"\\n🌾 Unique Crops in Dataset:\")\n",
+    "    for c in crops[:20]:  # limit to first 20\n",
+    "        print(\"-\", c)\n",
+    "    print(f\"... (Total {len(crops)} unique crops)\")\n",
+    "\n",
+    "# ✅ Check year range\n",
+    "if \"crop_year\" in df.columns:\n",
+    "    min_year, max_year = int(df[\"crop_year\"].min()), int(df[\"crop_year\"].max())\n",
+    "    print(f\"\\n📅 Crop Year Range: {min_year} - {max_year}\")\n",
+    "\n",
+    "# ✅ Quick count by state and crop\n",
+    "if {\"state_name\", \"crop\"} <= set(df.columns):\n",
+    "    summary = (\n",
+    "        df.groupby(\"state_name\")[\"crop\"]\n",
+    "        .nunique()\n",
+    "        .sort_values(ascending=False)\n",
+    "        .reset_index()\n",
+    "        .rename(columns={\"crop\": \"unique_crops\"})\n",
+    "    )\n",
+    "    print(\"\\n📈 Number of unique crops per state:\")\n",
+    "    display(summary)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "49e9d168",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Columns available in this file:\n",
+      "['subdivision', 'year', 'jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec', 'annual', 'jf', 'mam', 'jjas', 'ond']\n"
+     ]
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "\n",
+    "df = pd.read_csv(r\"C:\\Users\\satya\\Downloads\\Project_Samarth\\task\\hybrid_dataset\\imd_rainfall_data.csv\")\n",
+    "\n",
+    "print(\"Columns available in this file:\")\n",
+    "print(df.columns.tolist())\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "1f616bd6",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "✅ Datasets Loaded Successfully!\n",
+      "\n",
+      "Agriculture Data Shape: (5000, 7)\n",
+      "IMD Rainfall Data Shape: (2000, 19)\n"
+     ]
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "\n",
+    "# Paths to your saved files (use raw string format to avoid escape issues)\n",
+    "agri_path = r\"C:\\Users\\satya\\Downloads\\Project_Samarth\\task\\hybrid_dataset\\agriculture_data.csv\"\n",
+    "imd_path = r\"C:\\Users\\satya\\Downloads\\Project_Samarth\\task\\hybrid_dataset\\imd_rainfall_data.csv\"\n",
+    "\n",
+    "# Load both datasets\n",
+    "agri_df = pd.read_csv(agri_path)\n",
+    "imd_df = pd.read_csv(imd_path)\n",
+    "\n",
+    "print(\"✅ Datasets Loaded Successfully!\\n\")\n",
+    "print(f\"Agriculture Data Shape: {agri_df.shape}\")\n",
+    "print(f\"IMD Rainfall Data Shape: {imd_df.shape}\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "485e1cd8",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "🌾 Agriculture Data Columns:\n",
+      "['state_name', 'district_name', 'crop_year', 'season', 'crop', 'area_', 'production_']\n",
+      "\n",
+      "☁️ IMD Rainfall Data Columns:\n",
+      "['subdivision', 'year', 'jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec', 'annual', 'jf', 'mam', 'jjas', 'ond']\n",
+      "\n",
+      "�� Agriculture Data Sample:\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.microsoft.datawrangler.viewer.v0+json": {
+       "columns": [
+        {
+         "name": "index",
+         "rawType": "int64",
+         "type": "integer"
+        },
+        {
+         "name": "state_name",
+         "rawType": "object",
+         "type": "string"
+        },
+        {
+         "name": "district_name",
+         "rawType": "object",
+         "type": "string"
+        },
+        {
+         "name": "crop_year",
+         "rawType": "int64",
+         "type": "integer"
+        },
+        {
+         "name": "season",
+         "rawType": "object",
+         "type": "string"
+        },
+        {
+         "name": "crop",
+         "rawType": "object",
+         "type": "string"
+        },
+        {
+         "name": "area_",
+         "rawType": "float64",
+         "type": "float"
+        },
+        {
+         "name": "production_",
+         "rawType": "float64",
+         "type": "float"
+        }
+       ],
+       "ref": "0d1e680d-db1a-4e1b-a014-078e92fcf760",
+       "rows": [
+        [
+         "0",
+         "Andaman and Nicobar Islands",
+         "NICOBARS",
+         "2000",
+         "Kharif",
+         "Arecanut",
+         "1254.0",
+         "2000.0"
+        ],
+        [
+         "1",
+         "Andaman and Nicobar Islands",
+         "NICOBARS",
+         "2000",
+         "Kharif",
+         "Other Kharif pulses",
+         "2.0",
+         "1.0"
+        ],
+        [
+         "2",
+         "Andaman and Nicobar Islands",
+         "NICOBARS",
+         "2000",
+         "Kharif",
+         "Rice",
+         "102.0",
+         "321.0"
+        ],
+        [
+         "3",
+         "Andaman and Nicobar Islands",
+         "NICOBARS",
+         "2000",
+         "Whole Year",
+         "Banana",
+         "176.0",
+         "641.0"
+        ],
+        [
+         "4",
+         "Andaman and Nicobar Islands",
+         "NICOBARS",
+         "2000",
+         "Whole Year",
+         "Cashewnut",
+         "720.0",
+         "165.0"
+        ]
+       ],
+       "shape": {
+        "columns": 7,
+        "rows": 5
+       }
+      },
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>state_name</th>\n",
+       "      <th>district_name</th>\n",
+       "      <th>crop_year</th>\n",
+       "      <th>season</th>\n",
+       "      <th>crop</th>\n",
+       "      <th>area_</th>\n",
+       "      <th>production_</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>Andaman and Nicobar Islands</td>\n",
+       "      <td>NICOBARS</td>\n",
+       "      <td>2000</td>\n",
+       "      <td>Kharif</td>\n",
+       "      <td>Arecanut</td>\n",
+       "      <td>1254.0</td>\n",
+       "      <td>2000.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>Andaman and Nicobar Islands</td>\n",
+       "      <td>NICOBARS</td>\n",
+       "      <td>2000</td>\n",
+       "      <td>Kharif</td>\n",
+       "      <td>Other Kharif pulses</td>\n",
+       "      <td>2.0</td>\n",
+       "      <td>1.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>Andaman and Nicobar Islands</td>\n",
+       "      <td>NICOBARS</td>\n",
+       "      <td>2000</td>\n",
+       "      <td>Kharif</td>\n",
+       "      <td>Rice</td>\n",
+       "      <td>102.0</td>\n",
+       "      <td>321.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>Andaman and Nicobar Islands</td>\n",
+       "      <td>NICOBARS</td>\n",
+       "      <td>2000</td>\n",
+       "      <td>Whole Year</td>\n",
+       "      <td>Banana</td>\n",
+       "      <td>176.0</td>\n",
+       "      <td>641.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>Andaman and Nicobar Islands</td>\n",
+       "      <td>NICOBARS</td>\n",
+       "      <td>2000</td>\n",
+       "      <td>Whole Year</td>\n",
+       "      <td>Cashewnut</td>\n",
+       "      <td>720.0</td>\n",
+       "      <td>165.0</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                    state_name district_name  crop_year      season  \\\n",
+       "0  Andaman and Nicobar Islands      NICOBARS       2000      Kharif   \n",
+       "1  Andaman and Nicobar Islands      NICOBARS       2000      Kharif   \n",
+       "2  Andaman and Nicobar Islands      NICOBARS       2000      Kharif   \n",
+       "3  Andaman and Nicobar Islands      NICOBARS       2000  Whole Year   \n",
+       "4  Andaman and Nicobar Islands      NICOBARS       2000  Whole Year   \n",
+       "\n",
+       "                  crop   area_  production_  \n",
+       "0             Arecanut  1254.0       2000.0  \n",
+       "1  Other Kharif pulses     2.0          1.0  \n",
+       "2                 Rice   102.0        321.0  \n",
+       "3               Banana   176.0        641.0  \n",
+       "4            Cashewnut   720.0        165.0  "
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "🌦️ IMD Rainfall Data Sample:\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.microsoft.datawrangler.viewer.v0+json": {
+       "columns": [
+        {
+         "name": "index",
+         "rawType": "int64",
+         "type": "integer"
+        },
+        {
+         "name": "subdivision",
+         "rawType": "object",
+         "type": "string"
+        },
+        {
+         "name": "year",
+         "rawType": "int64",
+         "type": "integer"
+        },
+        {
+         "name": "jan",
+         "rawType": "float64",
+         "type": "float"
+        },
+        {
+         "name": "feb",
+         "rawType": "float64",
+         "type": "float"
+        },
+        {
+         "name": "mar",
+         "rawType": "float64",
+         "type": "float"
+        },
+        {
+         "name": "apr",
+         "rawType": "float64",
+         "type": "float"
+        },
+        {
+         "name": "may",
+         "rawType": "float64",
+         "type": "float"
+        },
+        {
+         "name": "jun",
+         "rawType": "float64",
+         "type": "float"
+        },
+        {
+         "name": "jul",
+         "rawType": "float64",
+         "type": "float"
+        },
+        {
+         "name": "aug",
+         "rawType": "float64",
+         "type": "float"
+        },
+        {
+         "name": "sep",
+         "rawType": "float64",
+         "type": "float"
+        },
+        {
+         "name": "oct",
+         "rawType": "float64",
+         "type": "float"
+        },
+        {
+         "name": "nov",
+         "rawType": "float64",
+         "type": "float"
+        },
+        {
+         "name": "dec",
+         "rawType": "float64",
+         "type": "float"
+        },
+        {
+         "name": "annual",
+         "rawType": "float64",
+         "type": "float"
+        },
+        {
+         "name": "jf",
+         "rawType": "float64",
+         "type": "float"
+        },
+        {
+         "name": "mam",
+         "rawType": "float64",
+         "type": "float"
+        },
+        {
+         "name": "jjas",
+         "rawType": "float64",
+         "type": "float"
+        },
+        {
+         "name": "ond",
+         "rawType": "float64",
+         "type": "float"
+        }
+       ],
+       "ref": "032f16ab-7b47-4a43-bcbc-1c17f86ef3bb",
+       "rows": [
+        [
+         "0",
+         "Andaman & Nicobar Islands",
+         "1901",
+         "49.2",
+         "87.1",
+         "29.2",
+         "2.3",
+         "528.8",
+         "517.5",
+         "365.1",
+         "481.1",
+         "332.6",
+         "388.5",
+         "558.2",
+         "33.6",
+         "3373.2",
+         "136.3",
+         "560.3",
+         "1696.3",
+         "980.3"
+        ],
+        [
+         "1",
+         "Andaman & Nicobar Islands",
+         "1902",
+         "0.0",
+         "159.8",
+         "12.2",
+         "0.0",
+         "446.1",
+         "537.1",
+         "228.9",
+         "753.7",
+         "666.2",
+         "197.2",
+         "359.0",
+         "160.5",
+         "3520.7",
+         "159.8",
+         "458.3",
+         "2185.9",
+         "716.7"
+        ],
+        [
+         "2",
+         "Andaman & Nicobar Islands",
+         "1903",
+         "12.7",
+         "144.0",
+         "0.0",
+         "1.0",
+         "235.1",
+         "479.9",
+         "728.4",
+         "326.7",
+         "339.0",
+         "181.2",
+         "284.4",
+         "225.0",
+         "2957.4",
+         "156.7",
+         "236.1",
+         "1874.0",
+         "690.6"
+        ],
+        [
+         "3",
+         "Andaman & Nicobar Islands",
+         "1904",
+         "9.4",
+         "14.7",
+         "0.0",
+         "202.4",
+         "304.5",
+         "495.1",
+         "502.0",
+         "160.1",
+         "820.4",
+         "222.2",
+         "308.7",
+         "40.1",
+         "3079.6",
+         "24.1",
+         "506.9",
+         "1977.6",
+         "571.0"
+        ],
+        [
+         "4",
+         "Andaman & Nicobar Islands",
+         "1905",
+         "1.3",
+         "0.0",
+         "3.3",
+         "26.9",
+         "279.5",
+         "628.7",
+         "368.7",
+         "330.5",
+         "297.0",
+         "260.7",
+         "25.4",
+         "344.7",
+         "2566.7",
+         "1.3",
+         "309.7",
+         "1624.9",
+         "630.8"
+        ]
+       ],
+       "shape": {
+        "columns": 19,
+        "rows": 5
+       }
+      },
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>subdivision</th>\n",
+       "      <th>year</th>\n",
+       "      <th>jan</th>\n",
+       "      <th>feb</th>\n",
+       "      <th>mar</th>\n",
+       "      <th>apr</th>\n",
+       "      <th>may</th>\n",
+       "      <th>jun</th>\n",
+       "      <th>jul</th>\n",
+       "      <th>aug</th>\n",
+       "      <th>sep</th>\n",
+       "      <th>oct</th>\n",
+       "      <th>nov</th>\n",
+       "      <th>dec</th>\n",
+       "      <th>annual</th>\n",
+       "      <th>jf</th>\n",
+       "      <th>mam</th>\n",
+       "      <th>jjas</th>\n",
+       "      <th>ond</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>Andaman &amp; Nicobar Islands</td>\n",
+       "      <td>1901</td>\n",
+       "      <td>49.2</td>\n",
+       "      <td>87.1</td>\n",
+       "      <td>29.2</td>\n",
+       "      <td>2.3</td>\n",
+       "      <td>528.8</td>\n",
+       "      <td>517.5</td>\n",
+       "      <td>365.1</td>\n",
+       "      <td>481.1</td>\n",
+       "      <td>332.6</td>\n",
+       "      <td>388.5</td>\n",
+       "      <td>558.2</td>\n",
+       "      <td>33.6</td>\n",
+       "      <td>3373.2</td>\n",
+       "      <td>136.3</td>\n",
+       "      <td>560.3</td>\n",
+       "      <td>1696.3</td>\n",
+       "      <td>980.3</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>Andaman &amp; Nicobar Islands</td>\n",
+       "      <td>1902</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>159.8</td>\n",
+       "      <td>12.2</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>446.1</td>\n",
+       "      <td>537.1</td>\n",
+       "      <td>228.9</td>\n",
+       "      <td>753.7</td>\n",
+       "      <td>666.2</td>\n",
+       "      <td>197.2</td>\n",
+       "      <td>359.0</td>\n",
+       "      <td>160.5</td>\n",
+       "      <td>3520.7</td>\n",
+       "      <td>159.8</td>\n",
+       "      <td>458.3</td>\n",
+       "      <td>2185.9</td>\n",
+       "      <td>716.7</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>Andaman &amp; Nicobar Islands</td>\n",
+       "      <td>1903</td>\n",
+       "      <td>12.7</td>\n",
+       "      <td>144.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>235.1</td>\n",
+       "      <td>479.9</td>\n",
+       "      <td>728.4</td>\n",
+       "      <td>326.7</td>\n",
+       "      <td>339.0</td>\n",
+       "      <td>181.2</td>\n",
+       "      <td>284.4</td>\n",
+       "      <td>225.0</td>\n",
+       "      <td>2957.4</td>\n",
+       "      <td>156.7</td>\n",
+       "      <td>236.1</td>\n",
+       "      <td>1874.0</td>\n",
+       "      <td>690.6</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>Andaman &amp; Nicobar Islands</td>\n",
+       "      <td>1904</td>\n",
+       "      <td>9.4</td>\n",
+       "      <td>14.7</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>202.4</td>\n",
+       "      <td>304.5</td>\n",
+       "      <td>495.1</td>\n",
+       "      <td>502.0</td>\n",
+       "      <td>160.1</td>\n",
+       "      <td>820.4</td>\n",
+       "      <td>222.2</td>\n",
+       "      <td>308.7</td>\n",
+       "      <td>40.1</td>\n",
+       "      <td>3079.6</td>\n",
+       "      <td>24.1</td>\n",
+       "      <td>506.9</td>\n",
+       "      <td>1977.6</td>\n",
+       "      <td>571.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>Andaman &amp; Nicobar Islands</td>\n",
+       "      <td>1905</td>\n",
+       "      <td>1.3</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>3.3</td>\n",
+       "      <td>26.9</td>\n",
+       "      <td>279.5</td>\n",
+       "      <td>628.7</td>\n",
+       "      <td>368.7</td>\n",
+       "      <td>330.5</td>\n",
+       "      <td>297.0</td>\n",
+       "      <td>260.7</td>\n",
+       "      <td>25.4</td>\n",
+       "      <td>344.7</td>\n",
+       "      <td>2566.7</td>\n",
+       "      <td>1.3</td>\n",
+       "      <td>309.7</td>\n",
+       "      <td>1624.9</td>\n",
+       "      <td>630.8</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                 subdivision  year   jan    feb   mar    apr    may    jun  \\\n",
+       "0  Andaman & Nicobar Islands  1901  49.2   87.1  29.2    2.3  528.8  517.5   \n",
+       "1  Andaman & Nicobar Islands  1902   0.0  159.8  12.2    0.0  446.1  537.1   \n",
+       "2  Andaman & Nicobar Islands  1903  12.7  144.0   0.0    1.0  235.1  479.9   \n",
+       "3  Andaman & Nicobar Islands  1904   9.4   14.7   0.0  202.4  304.5  495.1   \n",
+       "4  Andaman & Nicobar Islands  1905   1.3    0.0   3.3   26.9  279.5  628.7   \n",
+       "\n",
+       "     jul    aug    sep    oct    nov    dec  annual     jf    mam    jjas  \\\n",
+       "0  365.1  481.1  332.6  388.5  558.2   33.6  3373.2  136.3  560.3  1696.3   \n",
+       "1  228.9  753.7  666.2  197.2  359.0  160.5  3520.7  159.8  458.3  2185.9   \n",
+       "2  728.4  326.7  339.0  181.2  284.4  225.0  2957.4  156.7  236.1  1874.0   \n",
+       "3  502.0  160.1  820.4  222.2  308.7   40.1  3079.6   24.1  506.9  1977.6   \n",
+       "4  368.7  330.5  297.0  260.7   25.4  344.7  2566.7    1.3  309.7  1624.9   \n",
+       "\n",
+       "     ond  \n",
+       "0  980.3  \n",
+       "1  716.7  \n",
+       "2  690.6  \n",
+       "3  571.0  \n",
+       "4  630.8  "
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "print(\"\\n🌾 Agriculture Data Columns:\")\n",
+    "print(agri_df.columns.tolist())\n",
+    "\n",
+    "print(\"\\n☁️ IMD Rainfall Data Columns:\")\n",
+    "print(imd_df.columns.tolist())\n",
+    "\n",
+    "print(\"\\n📊 Agriculture Data Sample:\")\n",
+    "display(agri_df.head(5))\n",
+    "\n",
+    "print(\"\\n🌦️ IMD Rainfall Data Sample:\")\n",
+    "display(imd_df.head(5))\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "4d48c1f1",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "🏛️ Unique States in Agriculture Data:\n",
+      "['Andaman and Nicobar Islands', 'Andhra Pradesh']\n",
+      "\n",
+      "📅 Year Range in Agriculture Data:\n",
+      "1997 → 2014\n",
+      "\n",
+      "🌾 Top 10 Crops:\n",
+      "['Arecanut', 'Other Kharif pulses', 'Rice', 'Banana', 'Cashewnut', 'Coconut', 'Dry ginger', 'Sugarcane', 'Sweet potato', 'Tapioca']\n",
+      "\n",
+      "🏛️ Unique Subdivisions in IMD Rainfall Data:\n",
+      "['Andaman & Nicobar Islands', 'Arunachal Pradesh', 'Assam & Meghalaya', 'Naga Mani Mizo Tripura', 'Sub Himalayan West Bengal & Sikkim', 'Gangetic West Bengal', 'Orissa', 'Jharkhand', 'Bihar', 'East Uttar Pradesh', 'West Uttar Pradesh', 'Uttarakhand', 'Haryana Delhi & Chandigarh', 'Punjab', 'Himachal Pradesh', 'Jammu & Kashmir', 'West Rajasthan', 'East Rajasthan']\n",
+      "\n",
+      "📅 Year Range in IMD Rainfall Data:\n",
+      "1901 → 2017\n"
+     ]
+    }
+   ],
+   "source": [
+    "# ---- AGRICULTURE ----\n",
+    "print(\"\\n🏛️ Unique States in Agriculture Data:\")\n",
+    "print(agri_df['state_name'].unique().tolist())\n",
+    "\n",
+    "print(\"\\n📅 Year Range in Agriculture Data:\")\n",
+    "if 'crop_year' in agri_df.columns:\n",
+    "    print(int(agri_df['crop_year'].min()), \"→\", int(agri_df['crop_year'].max()))\n",
+    "\n",
+    "print(\"\\n🌾 Top 10 Crops:\")\n",
+    "print(agri_df['crop'].unique().tolist()[:10])\n",
+    "\n",
+    "# ---- IMD RAINFALL ----\n",
+    "print(\"\\n🏛️ Unique Subdivisions in IMD Rainfall Data:\")\n",
+    "print(imd_df['subdivision'].unique().tolist())\n",
+    "\n",
+    "print(\"\\n📅 Year Range in IMD Rainfall Data:\")\n",
+    "if 'year' in imd_df.columns:\n",
+    "    print(int(imd_df['year'].min()), \"→\", int(imd_df['year'].max()))\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "8cfe6ee8",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "✅ Common Names Found Between Agriculture & IMD Data (0):\n",
+      "[]\n",
+      "\n",
+      "⚠️ States in Agriculture but not in IMD (2):\n",
+      "['andaman and nicobar islands', 'andhra pradesh']\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Lowercase and trim for consistency\n",
+    "agri_states = set(agri_df['state_name'].str.lower().str.strip().unique())\n",
+    "imd_subdiv = set(imd_df['subdivision'].str.lower().str.strip().unique())\n",
+    "\n",
+    "common = sorted(agri_states.intersection(imd_subdiv))\n",
+    "\n",
+    "print(f\"\\n✅ Common Names Found Between Agriculture & IMD Data ({len(common)}):\")\n",
+    "print(common[:10])\n",
+    "\n",
+    "missing_from_imd = sorted(list(agri_states - imd_subdiv))\n",
+    "print(f\"\\n⚠️ States in Agriculture but not in IMD ({len(missing_from_imd)}):\")\n",
+    "print(missing_from_imd[:10])\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "21653121",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "myenv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.13.0"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebooks/03_qna_demo.ipynb

@@ -0,0 +1,322 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "f92a389b",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Active code page: 1252\n",
+      "Requirement already satisfied: pandas in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (2.3.2)\n",
+      "Requirement already satisfied: numpy in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (2.3.2)\n",
+      "Requirement already satisfied: matplotlib in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (3.10.7)\n",
+      "Requirement already satisfied: seaborn in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (0.13.2)\n",
+      "Requirement already satisfied: nltk in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (3.9.1)\n",
+      "Requirement already satisfied: python-dateutil>=2.8.2 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from pandas) (2.9.0.post0)\n",
+      "Requirement already satisfied: pytz>=2020.1 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from pandas) (2025.2)\n",
+      "Requirement already satisfied: tzdata>=2022.7 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from pandas) (2025.2)\n",
+      "Requirement already satisfied: contourpy>=1.0.1 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from matplotlib) (1.3.3)\n",
+      "Requirement already satisfied: cycler>=0.10 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from matplotlib) (0.12.1)\n",
+      "Requirement already satisfied: fonttools>=4.22.0 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from matplotlib) (4.60.1)\n",
+      "Requirement already satisfied: kiwisolver>=1.3.1 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from matplotlib) (1.4.9)\n",
+      "Requirement already satisfied: packaging>=20.0 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from matplotlib) (23.2)\n",
+      "Requirement already satisfied: pillow>=8 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from matplotlib) (11.3.0)\n",
+      "Requirement already satisfied: pyparsing>=3 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from matplotlib) (3.2.5)\n",
+      "Requirement already satisfied: click in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from nltk) (8.2.1)\n",
+      "Requirement already satisfied: joblib in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from nltk) (1.5.2)\n",
+      "Requirement already satisfied: regex>=2021.8.3 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from nltk) (2025.9.1)\n",
+      "Requirement already satisfied: tqdm in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from nltk) (4.67.1)\n",
+      "Requirement already satisfied: six>=1.5 in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from python-dateutil>=2.8.2->pandas) (1.17.0)\n",
+      "Requirement already satisfied: colorama in c:\\users\\satya\\anaconda3\\envs\\myenv\\lib\\site-packages (from click->nltk) (0.4.6)\n",
+      "Note: you may need to restart the kernel to use updated packages.\n"
+     ]
+    }
+   ],
+   "source": [
+    "pip install pandas numpy matplotlib seaborn nltk\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "6bf0d886",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "[nltk_data] Downloading package stopwords to\n",
+      "[nltk_data]     C:\\Users\\satya\\AppData\\Roaming\\nltk_data...\n",
+      "[nltk_data]   Package stopwords is already up-to-date!\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "True"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import nltk\n",
+    "nltk.download('stopwords')\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "6c16aeb7",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "✅ Dataset Loaded Successfully!\n",
+      "Rows: 203, Columns: 25\n",
+      "\n",
+      "📊 Columns: ['state_name', 'district_name', 'crop_year', 'season', 'crop', 'area_', 'production_', 'subdivision', 'jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec', 'annual', 'jf', 'mam', 'jjas', 'ond']\n",
+      "\n",
+      "🏛️ States: ['andaman and nicobar islands']\n",
+      "🔍 Parsed Query: {'states': ['andaman', 'nicobar islands'], 'crop': 'rice', 'years': 5, 'metrics': ['rainfall', 'production']}\n",
+      "\n",
+      "📊 Q&A Result Summary:\n",
+      " {'message': 'No matching records found for your query.'}\n",
+      "ℹ️ No matching records found for your query.\n",
+      "\n",
+      "✅ Notebook Execution Completed Successfully!\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "[nltk_data] Downloading package stopwords to\n",
+      "[nltk_data]     C:\\Users\\satya\\AppData\\Roaming\\nltk_data...\n",
+      "[nltk_data]   Package stopwords is already up-to-date!\n"
+     ]
+    }
+   ],
+   "source": [
+    "# ===============================================\n",
+    "# 🌾 Project Samarth - Notebook 03\n",
+    "# Phase 2: Intelligent Q&A System (Final Fixed)\n",
+    "# ===============================================\n",
+    "\n",
+    "# ✅ Step 1: Import Libraries\n",
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "import re\n",
+    "import matplotlib.pyplot as plt\n",
+    "import seaborn as sns\n",
+    "import nltk\n",
+    "from nltk.corpus import stopwords\n",
+    "nltk.download('stopwords')\n",
+    "\n",
+    "# ✅ Step 2: Load Integrated Dataset\n",
+    "data_path = \"../hybrid_dataset/merged_agri_rainfall.csv\"\n",
+    "df = pd.read_csv(data_path)\n",
+    "\n",
+    "df.columns = df.columns.str.lower().str.strip()\n",
+    "df[\"crop_year\"] = pd.to_numeric(df[\"crop_year\"], errors=\"coerce\")\n",
+    "\n",
+    "print(\"✅ Dataset Loaded Successfully!\")\n",
+    "print(f\"Rows: {len(df)}, Columns: {len(df.columns)}\")\n",
+    "print(\"\\n📊 Columns:\", df.columns.tolist())\n",
+    "print(\"\\n🏛️ States:\", df['state_name'].dropna().unique().tolist())\n",
+    "\n",
+    "# ===============================================\n",
+    "# 🧠 Step 3: Improved NLP Query Parser\n",
+    "# ===============================================\n",
+    "\n",
+    "def parse_query(query: str):\n",
+    "    \"\"\"\n",
+    "    Improved NLP parser that cleanly extracts states, crop, years, and metrics.\n",
+    "    Example:\n",
+    "      'Compare rainfall and rice production in Andaman and Nicobar Islands and Andhra Pradesh for the last 5 years'\n",
+    "    \"\"\"\n",
+    "    query = query.lower().strip()\n",
+    "\n",
+    "    # Extract crop\n",
+    "    crop_match = re.findall(r\"\\b(rice|wheat|maize|sugarcane|banana|cotton)\\b\", query)\n",
+    "\n",
+    "    # Extract metrics\n",
+    "    metrics = []\n",
+    "    if \"rainfall\" in query: metrics.append(\"rainfall\")\n",
+    "    if \"production\" in query: metrics.append(\"production\")\n",
+    "\n",
+    "    # Extract year info\n",
+    "    year_match = re.search(r\"last (\\d+)\", query)\n",
+    "    years = int(year_match.group(1)) if year_match else 5\n",
+    "\n",
+    "    # Extract state names (clean multiple cases)\n",
+    "    state_part = re.search(r\"in (.*)\", query)\n",
+    "    states = []\n",
+    "    if state_part:\n",
+    "        # Break by 'and', ',', or 'with'\n",
+    "        parts = re.split(r\"\\band\\b|,|with\", state_part.group(1))\n",
+    "        for p in parts:\n",
+    "            p = p.strip()\n",
+    "            # Stop reading after phrases like \"for the last\"\n",
+    "            if \"for the last\" in p:\n",
+    "                break\n",
+    "            if p:\n",
+    "                states.append(p.strip())\n",
+    "\n",
+    "    return {\n",
+    "        \"states\": states,\n",
+    "        \"crop\": crop_match[0] if crop_match else None,\n",
+    "        \"years\": years,\n",
+    "        \"metrics\": metrics\n",
+    "    }\n",
+    "\n",
+    "# ===============================================\n",
+    "# ⚙️ Step 4: Query Execution Logic\n",
+    "# ===============================================\n",
+    "\n",
+    "def run_query(parsed_query: dict):\n",
+    "    \"\"\"Perform analysis using parsed query information.\"\"\"\n",
+    "    if df.empty:\n",
+    "        return {\"error\": \"Dataset not found or empty.\"}\n",
+    "\n",
+    "    states = [s.lower() for s in parsed_query.get(\"states\", [])]\n",
+    "    crop = parsed_query.get(\"crop\")\n",
+    "    years = parsed_query.get(\"years\", 5)\n",
+    "    metrics = parsed_query.get(\"metrics\", [])\n",
+    "\n",
+    "    filtered = df.copy()\n",
+    "\n",
+    "    if states:\n",
+    "        filtered = filtered[filtered[\"state_name\"].str.lower().isin(states)]\n",
+    "    if crop:\n",
+    "        filtered = filtered[filtered[\"crop\"].str.lower() == crop]\n",
+    "\n",
+    "    # Handle missing crop_year safely\n",
+    "    if \"crop_year\" in filtered.columns and not filtered.empty:\n",
+    "        latest_year = filtered[\"crop_year\"].max()\n",
+    "        if pd.notna(latest_year):\n",
+    "            latest_year = int(latest_year)\n",
+    "            start_year = latest_year - years + 1\n",
+    "            filtered = filtered[(filtered[\"crop_year\"] >= start_year) & (filtered[\"crop_year\"] <= latest_year)]\n",
+    "\n",
+    "    if filtered.empty:\n",
+    "        return {\"message\": \"No matching records found for your query.\"}\n",
+    "\n",
+    "    result = {\"states\": states, \"crop\": crop, \"years\": years}\n",
+    "\n",
+    "    # Rainfall Analysis\n",
+    "    if \"rainfall\" in metrics:\n",
+    "        rain_cols = [c for c in [\"annual\", \"jjas\", \"jf\", \"mam\", \"ond\"] if c in filtered.columns]\n",
+    "        if rain_cols:\n",
+    "            filtered[\"avg_rainfall\"] = filtered[rain_cols].apply(pd.to_numeric, errors=\"coerce\").mean(axis=1)\n",
+    "            rainfall_summary = (\n",
+    "                filtered.groupby(\"state_name\")[\"avg_rainfall\"].mean().round(2).to_dict()\n",
+    "            )\n",
+    "            result[\"rainfall_summary\"] = rainfall_summary\n",
+    "\n",
+    "    # Production Analysis\n",
+    "    if \"production\" in metrics and \"production_\" in filtered.columns:\n",
+    "        prod_summary = (\n",
+    "            filtered.groupby(\"state_name\")[\"production_\"].sum().round(2).to_dict()\n",
+    "        )\n",
+    "        result[\"production_summary\"] = prod_summary\n",
+    "\n",
+    "    return result\n",
+    "\n",
+    "# ===============================================\n",
+    "# 🗣️ Step 5: Test with a Query\n",
+    "# ===============================================\n",
+    "\n",
+    "example_query = \"Compare rainfall and rice production in Andaman and Nicobar Islands and Andhra Pradesh for the last 5 years\"\n",
+    "\n",
+    "parsed = parse_query(example_query)\n",
+    "print(\"🔍 Parsed Query:\", parsed)\n",
+    "\n",
+    "result = run_query(parsed)\n",
+    "print(\"\\n📊 Q&A Result Summary:\\n\", result)\n",
+    "\n",
+    "# ===============================================\n",
+    "# ✅ Step 6: Display Final Answer\n",
+    "# ===============================================\n",
+    "\n",
+    "if \"message\" in result:\n",
+    "    print(\"ℹ️\", result[\"message\"])\n",
+    "elif \"rainfall_summary\" in result or \"production_summary\" in result:\n",
+    "    print(f\"\\n📊 Analysis for {', '.join(parsed['states'])} — Crop: {parsed['crop'].title() if parsed['crop'] else 'All'}\")\n",
+    "\n",
+    "    if \"rainfall_summary\" in result:\n",
+    "        print(\"\\n🌧️ Average Rainfall (mm):\")\n",
+    "        for s, v in result[\"rainfall_summary\"].items():\n",
+    "            print(f\" • {s.title()}: {v} mm\")\n",
+    "\n",
+    "    if \"production_summary\" in result:\n",
+    "        print(\"\\n🌾 Total Production (tonnes):\")\n",
+    "        for s, v in result[\"production_summary\"].items():\n",
+    "            print(f\" • {s.title()}: {int(v)} tonnes\")\n",
+    "\n",
+    "    print(\"\\n📚 Data Source: Government Open Data Portal (data.gov.in)\")\n",
+    "    print(\"Developed for Project Samarth — Integrating Agriculture & Climate Data 🌦️🌾\")\n",
+    "\n",
+    "print(\"\\n✅ Notebook Execution Completed Successfully!\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "798fe45c",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Available States in merged dataset: ['andaman and nicobar islands']\n",
+      "Available Crops: ['Arecanut', 'Other Kharif pulses', 'Rice', 'Banana', 'Cashewnut', 'Coconut', 'Dry ginger', 'Sugarcane', 'Sweet potato', 'Tapioca']\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(\"Available States in merged dataset:\", df['state_name'].unique().tolist())\n",
+    "print(\"Available Crops:\", df['crop'].unique().tolist()[:10])\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8c59d29d",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "myenv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.13.0"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

query_engine/logic_engine.py

@@ -0,0 +1,103 @@
+# -----------------------------------------------------------
+# 🌾 Project Samarth — Logic Engine (Final Polished Version)
+# -----------------------------------------------------------
+
+import pandas as pd
+import numpy as np
+
+DATA_PATH = "hybrid_dataset/merged_agri_rainfall.csv"
+
+try:
+    df = pd.read_csv(DATA_PATH)
+    df.columns = df.columns.str.lower().str.strip()
+    df["crop_year"] = pd.to_numeric(df.get("crop_year", pd.Series()), errors="coerce")
+    df["state_name"] = df["state_name"].fillna("").astype(str)
+    df["crop"] = df["crop"].fillna("").astype(str)
+    print(f"✅ Dataset loaded successfully → {DATA_PATH} ({len(df)} rows)")
+except Exception as e:
+    print(f"⚠️ Error loading dataset: {e}")
+    df = pd.DataFrame()
+
+
+def run_query(parsed_query: dict):
+    """Executes logic for a given parsed query using integrated dataset."""
+
+    if df.empty:
+        return {"error": "Dataset not found or empty."}
+
+    if not parsed_query or not isinstance(parsed_query, dict):
+        return {"error": "Invalid query format."}
+
+    states = [s.lower().strip() for s in parsed_query.get("states", [])]
+    crop = parsed_query.get("crop", "").lower().strip()
+    years = parsed_query.get("years", 5)
+    metrics = parsed_query.get("metrics", [])
+    result = {"states": states, "crop": crop, "years": years}
+
+    filtered = df.copy()
+
+    # ✅ Safely filter by state
+    if "state_name" in filtered.columns and states:
+        filtered = filtered[filtered["state_name"].str.lower().isin(states)]
+
+    # ✅ Safely filter by crop
+    if "crop" in filtered.columns and crop:
+        filtered = filtered[filtered["crop"].str.lower() == crop]
+
+    # ✅ Filter by year range
+    if "crop_year" in filtered.columns and not filtered["crop_year"].isna().all():
+        latest_year = int(filtered["crop_year"].max())
+        start_year = latest_year - years + 1
+        filtered = filtered[
+            (filtered["crop_year"] >= start_year)
+            & (filtered["crop_year"] <= latest_year)
+        ]
+
+    if filtered.empty:
+        return {"message": "No matching records found for your query."}
+
+    # 🌧️ Compute Average Rainfall
+    if "rainfall" in metrics:
+        rain_cols = [c for c in ["annual", "jjas", "jf", "mam", "ond"] if c in filtered.columns]
+        if rain_cols:
+            filtered["avg_rainfall"] = filtered[rain_cols].apply(
+                pd.to_numeric, errors="coerce"
+            ).mean(axis=1)
+            rainfall_summary = (
+                filtered.groupby("state_name")["avg_rainfall"]
+                .mean()
+                .round(2)
+                .to_dict()
+            )
+            result["rainfall_summary"] = rainfall_summary
+
+    # 🌾 Compute Total Crop Production
+    if "production" in metrics and "production_" in filtered.columns:
+        prod_summary = (
+            filtered.groupby("state_name")["production_"]
+            .sum()
+            .round(2)
+            .to_dict()
+        )
+        result["production_summary"] = prod_summary
+
+    # 📊 If no data found
+    if "rainfall_summary" not in result and "production_summary" not in result:
+        result["message"] = "No metrics found in dataset for the given query."
+
+    # ✅ Format clean output
+    result["states"] = sorted([s.title() for s in result.get("states", [])])
+
+    return result
+
+
+# 🧪 Quick test
+if __name__ == "__main__":
+    test_query = {
+        "states": ["andaman and nicobar islands", "andhra pradesh"],
+        "crop": "rice",
+        "years": 5,
+        "metrics": ["rainfall", "production"],
+    }
+    print("\n🧠 Running test query...\n")
+    print(run_query(test_query))

query_engine/parser.py

@@ -0,0 +1,79 @@
+import re
+
+def parse_query(user_input: str):
+    """
+    🌾 Project Samarth — Query Parser (Final Version)
+    --------------------------------
+    Converts user natural language question into structured query.
+    """
+
+    query = (user_input or "").lower().strip()
+    result = {
+        "states": [],
+        "crop": None,
+        "years": 5,  # Default
+        "metrics": [],
+        "query_type": "general"
+    }
+
+    # 1️⃣ Extract number of years
+    match = re.search(r"last (\d+) years?", query)
+    if match:
+        result["years"] = int(match.group(1))
+
+    # 2️⃣ Extract states — only ones that exist in your merged dataset
+    state_list = [
+        "andaman and nicobar islands", "andhra pradesh", "bihar", "jharkhand",
+        "odisha", "tamil nadu", "rajasthan", "uttar pradesh", "west bengal",
+        "kerala", "karnataka", "maharashtra"
+    ]
+    found_states = [s for s in state_list if s in query]
+    if found_states:
+        result["states"] = found_states
+
+    # 3️⃣ Extract crop
+    crop_list = [
+        "rice", "maize", "wheat", "sugarcane", "turmeric", "banana", "groundnut",
+        "arecanut", "sunflower", "moong", "urad", "black pepper", "cashewnut"
+    ]
+    for crop in crop_list:
+        if crop in query:
+            result["crop"] = crop
+            break
+
+    # 4️⃣ Extract metrics
+    if "rainfall" in query:
+        result["metrics"].append("rainfall")
+    if "production" in query:
+        result["metrics"].append("production")
+
+    # Default metrics
+    if not result["metrics"]:
+        result["metrics"] = ["rainfall", "production"]
+
+    # 5️⃣ Determine query type
+    if "compare" in query:
+        result["query_type"] = "compare_rainfall_production"
+    elif "trend" in query:
+        result["query_type"] = "crop_trend"
+    elif "highest" in query:
+        result["query_type"] = "highest_production"
+    elif "policy" in query or "promote" in query:
+        result["query_type"] = "policy_support"
+    else:
+        result["query_type"] = "general"
+
+    return result
+
+
+# 🧪 Quick test
+if __name__ == "__main__":
+    queries = [
+        "Compare rainfall and rice production in Andaman and Nicobar Islands for the last 5 years",
+        "Show rainfall trend for Rice in Andhra Pradesh for the last 10 years",
+        "Which district had highest rice production in Andhra Pradesh?",
+        "Suggest policy to promote drought-resistant crops in Odisha"
+    ]
+    for q in queries:
+        print(f"\n🔍 Query: {q}")
+        print("Parsed Output:", parse_query(q))

requirements.txt

@@ -0,0 +1,8 @@
+requests
+pandas
+numpy
+matplotlib
+plotly
+streamlit
+langchain
+transformers

ui/app_streamlit.py

@@ -0,0 +1,91 @@
+# ---------------------------------------------------
+# 🌾 Project Samarth — Intelligent Q&A System
+# ---------------------------------------------------
+
+import sys, os
+import streamlit as st
+
+# ✅ Ensure Python finds your project modules
+sys.path.append(os.path.dirname(os.path.abspath(__file__)) + "/..")
+
+from query_engine.parser import parse_query
+from query_engine.logic_engine import run_query
+from answer_generator.citation_manager import get_source
+
+# ---------------------------------------------------
+# ⚙️ Streamlit Page Setup
+# ---------------------------------------------------
+st.set_page_config(page_title="🌾 Project Samarth — Intelligent Q&A", layout="centered")
+
+# ✅ Load custom CSS
+try:
+    st.markdown("<style>" + open("ui/style.css").read() + "</style>", unsafe_allow_html=True)
+except Exception as e:
+    st.warning("⚠️ Could not load CSS file. Using default Streamlit styling.")
+
+# ---------------------------------------------------
+# 🧠 Title and Info
+# ---------------------------------------------------
+st.title("🌾 Project Samarth — Intelligent Q&A System")
+st.caption("Ask intelligent, data-driven questions about agriculture and climate using live datasets from data.gov.in.")
+
+# ---------------------------------------------------
+# ✍️ Input Section
+# ---------------------------------------------------
+query = st.text_area(
+    "🧠 Ask your question:",
+    height=100,
+    placeholder="Example: Compare rainfall and rice production in Andaman and Nicobar Islands and Andhra Pradesh for the last 5 years"
+)
+
+# ---------------------------------------------------
+# 🔍 Analyze Button Logic
+# ---------------------------------------------------
+if st.button("🔍 Analyze"):
+    if not query.strip():
+        st.warning("Please enter a valid question.")
+    else:
+        with st.spinner("Analyzing your question..."):
+            try:
+                # 1️⃣ Parse user query
+                parsed_query = parse_query(query)
+
+                # 2️⃣ Run analysis logic
+                result = run_query(parsed_query)
+
+                # 3️⃣ Get citation source
+                source = get_source(parsed_query.get("query_type"))
+
+                # 4️⃣ Display structured results
+                st.markdown("---")
+                st.markdown("### 📊 Result Summary")
+
+                if "error" in result:
+                    st.error(result["error"])
+
+                elif "message" in result:
+                    st.info(result["message"])
+
+                else:
+                    states = parsed_query.get("states", [])
+                    crop = parsed_query.get("crop", "")
+                    st.markdown(f"### 📊 Analysis for {', '.join(states)} — Crop: {crop.title()}")
+
+                    # 🌧️ Rainfall Summary
+                    if "rainfall_summary" in result:
+                        st.markdown("#### 🌧️ Average Rainfall (mm):")
+                        for state, value in result["rainfall_summary"].items():
+                            st.markdown(f"- **{state.title()}**: `{value}` mm")
+
+                    # 🌾 Production Summary
+                    if "production_summary" in result:
+                        st.markdown("#### 🌾 Total Production (tonnes):")
+                        for state, value in result["production_summary"].items():
+                            st.markdown(f"- **{state.title()}**: `{int(value)}` tonnes")
+
+                    st.markdown("---")
+                    st.markdown(f"📚 **Data Source:** {source}")
+                    st.caption("Developed for Project Samarth — Integrating Agriculture & Climate Data 🌦️🌾")
+
+            except Exception as e:
+                st.error(f"❌ Something went wrong: {e}")

ui/style.css

@@ -0,0 +1,56 @@
+/* 🌾 Project Samarth – Smart Minimal UI Styling */
+
+body {
+  font-family: 'Poppins', 'Segoe UI', sans-serif;
+  background-color: #fafafa;
+  color: #333;
+}
+
+h1, h2, h3 {
+  color: #2e7d32;
+  text-align: center;
+  font-weight: 600;
+}
+
+textarea, input {
+  border-radius: 8px;
+  border: 1px solid #070707;
+  padding: 10px;
+  width: 100%;
+  background-color: #fff; /* changed to white */
+  color: #000; /* text color black */
+}
+
+button, .stButton>button {
+  background-color: #4caf50 !important;
+  color: #fff !important;
+  border: none !important;
+  padding: 8px 16px !important;
+  border-radius: 8px !important;
+  cursor: pointer !important;
+  transition: background-color 0.3s ease;
+}
+
+button:hover, .stButton>button:hover {
+  background-color: #2e7d32 !important;
+}
+
+hr {
+  border: none;
+  border-top: 1px solid #050505;
+  margin: 20px 0;
+}
+
+.stTextArea textarea {
+  border: 1px solid #ccc !important;
+  border-radius: 8px !important;
+  background-color: #fff !important;
+  color: #000 !important; /* black text color inside text area */
+}
+
+footer {
+  text-align: center;
+  color: #777;
+  font-size: 13px;
+  margin-top: 20px;
+}

utils/helper.py

@@ -0,0 +1,19 @@
+# utils/helper.py
+# Basic helper functions used across the project
+
+import pandas as pd
+
+def load_csv(path):
+    """Safely loads a CSV file."""
+    try:
+        df = pd.read_csv(path)
+        print(f"✅ Loaded file: {path} ({len(df)} rows)")
+        return df
+    except FileNotFoundError:
+        print(f"⚠️ File not found: {path}")
+        return pd.DataFrame()
+
+def save_csv(df, path):
+    """Saves a DataFrame as CSV."""
+    df.to_csv(path, index=False)
+    print(f"💾 Data saved to {path}")

utils/visualizer.py

@@ -0,0 +1,10 @@
+import matplotlib.pyplot as plt
+
+def plot_trend(df, x_col, y_col, title):
+    plt.figure(figsize=(8,5))
+    plt.plot(df[x_col], df[y_col], marker='o')
+    plt.title(title)
+    plt.xlabel(x_col)
+    plt.ylabel(y_col)
+    plt.grid(True)
+    plt.show()