scripts/analyze_combined_data.py

#!/usr/bin/env python3
"""
Analysis script for the combined crop dataset.
This script provides insights into the combined data.
"""

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

def load_data():
    """Load the combined dataset."""
    return pd.read_csv('combined_crop_data.csv')

def basic_analysis(df):
    """Perform basic analysis of the dataset."""
    print("="*60)
    print("BASIC DATASET ANALYSIS")
    print("="*60)
    
    print(f"Dataset shape: {df.shape}")
    print(f"Memory usage: {df.memory_usage().sum() / 1024**2:.2f} MB")
    
    print("\nColumn information:")
    print(df.info())
    
    print("\nFirst 5 rows:")
    print(df.head())
    
    print("\nBasic statistics for numeric columns:")
    print(df.describe())

def data_distribution_analysis(df):
    """Analyze data distribution across different dimensions."""
    print("\n" + "="*60)
    print("DATA DISTRIBUTION ANALYSIS")
    print("="*60)
    
    # Year distribution
    print("\n1. Data distribution by year:")
    year_dist = df['Crop_Year'].value_counts().sort_index()
    print(year_dist)
    
    # State distribution
    print("\n2. Top 15 states by record count:")
    state_dist = df['State'].value_counts().head(15)
    print(state_dist)
    
    # Crop distribution
    print("\n3. Top 15 crops by record count:")
    crop_dist = df['Crop'].value_counts().head(15)
    print(crop_dist)
    
    # Season distribution
    print("\n4. Season distribution:")
    season_dist = df['Season'].value_counts()
    print(season_dist)

def data_quality_analysis(df):
    """Analyze data quality and completeness."""
    print("\n" + "="*60)
    print("DATA QUALITY ANALYSIS")
    print("="*60)
    
    # Missing values
    print("\n1. Missing values analysis:")
    missing_data = df.isnull().sum()
    missing_percent = (missing_data / len(df)) * 100
    
    quality_df = pd.DataFrame({
        'Column': df.columns,
        'Missing_Count': missing_data,
        'Missing_Percent': missing_percent,
        'Data_Type': df.dtypes
    })
    print(quality_df)
    
    # Duplicate records
    print(f"\n2. Duplicate records: {df.duplicated().sum()}")
    
    # Zero values in production and area
    print(f"\n3. Zero values in Area: {(df['Area'] == 0).sum()}")
    print(f"4. Zero values in Production: {(df['Production'] == 0).sum()}")
    print(f"5. Zero values in Yield: {(df['Yield'] == 0).sum()}")

def yield_analysis(df):
    """Analyze yield patterns."""
    print("\n" + "="*60)
    print("YIELD ANALYSIS")
    print("="*60)
    
    # Remove zero yields for meaningful analysis
    df_yield = df[df['Yield'] > 0].copy()
    
    print(f"\nYield statistics (excluding zero yields):")
    print(f"Records with valid yield: {len(df_yield):,}")
    print(f"Mean yield: {df_yield['Yield'].mean():.2f}")
    print(f"Median yield: {df_yield['Yield'].median():.2f}")
    print(f"Standard deviation: {df_yield['Yield'].std():.2f}")
    
    # Top crops by average yield
    print(f"\nTop 10 crops by average yield:")
    crop_yield = df_yield.groupby('Crop')['Yield'].agg(['mean', 'count']).reset_index()
    crop_yield = crop_yield[crop_yield['count'] >= 50]  # At least 50 records
    crop_yield = crop_yield.sort_values('mean', ascending=False).head(10)
    print(crop_yield)
    
    # State-wise average yield
    print(f"\nTop 10 states by average yield:")
    state_yield = df_yield.groupby('State')['Yield'].agg(['mean', 'count']).reset_index()
    state_yield = state_yield[state_yield['count'] >= 100]  # At least 100 records
    state_yield = state_yield.sort_values('mean', ascending=False).head(10)
    print(state_yield)

def temporal_analysis(df):
    """Analyze temporal patterns."""
    print("\n" + "="*60)
    print("TEMPORAL ANALYSIS")
    print("="*60)
    
    # Data availability by year
    year_coverage = df.groupby('Crop_Year').agg({
        'State': 'nunique',
        'Crop': 'nunique', 
        'District': 'nunique'
    }).reset_index()
    
    print("\nData coverage by year:")
    print(year_coverage.tail(10))
    
    # Historical vs Recent data
    historical_data = df[df['Crop_Year'] < 2020]
    recent_data = df[df['Crop_Year'] >= 2020]
    
    print(f"\nHistorical data (before 2020): {len(historical_data):,} records")
    print(f"Recent data (2020 onwards): {len(recent_data):,} records")

def create_summary_report(df):
    """Create a summary report."""
    print("\n" + "="*60)
    print("SUMMARY REPORT")
    print("="*60)
    
    # Key insights
    insights = []
    
    # Dataset size
    insights.append(f"📊 Combined dataset contains {len(df):,} records")
    insights.append(f"📅 Data spans from {df['Crop_Year'].min()} to {df['Crop_Year'].max()}")
    insights.append(f"🌍 Covers {df['State'].nunique()} states/UTs")
    insights.append(f"🏘️  Includes {df['District'].nunique()} districts")
    insights.append(f"🌾 Contains data for {df['Crop'].nunique()} different crops")
    
    # Data completeness
    complete_records = df.dropna(subset=['Area', 'Production', 'Yield']).shape[0]
    completeness_pct = (complete_records / len(df)) * 100
    insights.append(f"✅ {completeness_pct:.1f}% records have complete Area/Production/Yield data")
    
    # Recent vs Historical
    recent_pct = (len(df[df['Crop_Year'] >= 2020]) / len(df)) * 100
    insights.append(f"🕐 {recent_pct:.1f}% of data is from 2020 onwards")
    
    print("\nKey Insights:")
    for i, insight in enumerate(insights, 1):
        print(f"{i}. {insight}")
    
    print("\nRecommendations for further analysis:")
    print("• Focus on crops with sufficient historical data for trend analysis")
    print("• Consider handling missing rainfall/fertilizer data through imputation")
    print("• Analyze seasonal patterns in crop yield")
    print("• Investigate state-wise agricultural productivity")
    print("• Use district-level data for more granular insights")

def main():
    """Main function to run the analysis."""
    print("Loading combined crop dataset...")
    df = load_data()
    
    # Run all analyses
    basic_analysis(df)
    data_distribution_analysis(df)
    data_quality_analysis(df)
    yield_analysis(df)
    temporal_analysis(df)
    create_summary_report(df)
    
    print(f"\n✨ Analysis complete! Combined dataset is ready for use.")

if __name__ == "__main__":
    main()