Spaces:
Sleeping
Sleeping
File size: 6,403 Bytes
bbd5f9c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 | #!/usr/bin/env python3
"""
Analysis script for the combined crop dataset.
This script provides insights into the combined data.
"""
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
def load_data():
"""Load the combined dataset."""
return pd.read_csv('combined_crop_data.csv')
def basic_analysis(df):
"""Perform basic analysis of the dataset."""
print("="*60)
print("BASIC DATASET ANALYSIS")
print("="*60)
print(f"Dataset shape: {df.shape}")
print(f"Memory usage: {df.memory_usage().sum() / 1024**2:.2f} MB")
print("\nColumn information:")
print(df.info())
print("\nFirst 5 rows:")
print(df.head())
print("\nBasic statistics for numeric columns:")
print(df.describe())
def data_distribution_analysis(df):
"""Analyze data distribution across different dimensions."""
print("\n" + "="*60)
print("DATA DISTRIBUTION ANALYSIS")
print("="*60)
# Year distribution
print("\n1. Data distribution by year:")
year_dist = df['Crop_Year'].value_counts().sort_index()
print(year_dist)
# State distribution
print("\n2. Top 15 states by record count:")
state_dist = df['State'].value_counts().head(15)
print(state_dist)
# Crop distribution
print("\n3. Top 15 crops by record count:")
crop_dist = df['Crop'].value_counts().head(15)
print(crop_dist)
# Season distribution
print("\n4. Season distribution:")
season_dist = df['Season'].value_counts()
print(season_dist)
def data_quality_analysis(df):
"""Analyze data quality and completeness."""
print("\n" + "="*60)
print("DATA QUALITY ANALYSIS")
print("="*60)
# Missing values
print("\n1. Missing values analysis:")
missing_data = df.isnull().sum()
missing_percent = (missing_data / len(df)) * 100
quality_df = pd.DataFrame({
'Column': df.columns,
'Missing_Count': missing_data,
'Missing_Percent': missing_percent,
'Data_Type': df.dtypes
})
print(quality_df)
# Duplicate records
print(f"\n2. Duplicate records: {df.duplicated().sum()}")
# Zero values in production and area
print(f"\n3. Zero values in Area: {(df['Area'] == 0).sum()}")
print(f"4. Zero values in Production: {(df['Production'] == 0).sum()}")
print(f"5. Zero values in Yield: {(df['Yield'] == 0).sum()}")
def yield_analysis(df):
"""Analyze yield patterns."""
print("\n" + "="*60)
print("YIELD ANALYSIS")
print("="*60)
# Remove zero yields for meaningful analysis
df_yield = df[df['Yield'] > 0].copy()
print(f"\nYield statistics (excluding zero yields):")
print(f"Records with valid yield: {len(df_yield):,}")
print(f"Mean yield: {df_yield['Yield'].mean():.2f}")
print(f"Median yield: {df_yield['Yield'].median():.2f}")
print(f"Standard deviation: {df_yield['Yield'].std():.2f}")
# Top crops by average yield
print(f"\nTop 10 crops by average yield:")
crop_yield = df_yield.groupby('Crop')['Yield'].agg(['mean', 'count']).reset_index()
crop_yield = crop_yield[crop_yield['count'] >= 50] # At least 50 records
crop_yield = crop_yield.sort_values('mean', ascending=False).head(10)
print(crop_yield)
# State-wise average yield
print(f"\nTop 10 states by average yield:")
state_yield = df_yield.groupby('State')['Yield'].agg(['mean', 'count']).reset_index()
state_yield = state_yield[state_yield['count'] >= 100] # At least 100 records
state_yield = state_yield.sort_values('mean', ascending=False).head(10)
print(state_yield)
def temporal_analysis(df):
"""Analyze temporal patterns."""
print("\n" + "="*60)
print("TEMPORAL ANALYSIS")
print("="*60)
# Data availability by year
year_coverage = df.groupby('Crop_Year').agg({
'State': 'nunique',
'Crop': 'nunique',
'District': 'nunique'
}).reset_index()
print("\nData coverage by year:")
print(year_coverage.tail(10))
# Historical vs Recent data
historical_data = df[df['Crop_Year'] < 2020]
recent_data = df[df['Crop_Year'] >= 2020]
print(f"\nHistorical data (before 2020): {len(historical_data):,} records")
print(f"Recent data (2020 onwards): {len(recent_data):,} records")
def create_summary_report(df):
"""Create a summary report."""
print("\n" + "="*60)
print("SUMMARY REPORT")
print("="*60)
# Key insights
insights = []
# Dataset size
insights.append(f"📊 Combined dataset contains {len(df):,} records")
insights.append(f"📅 Data spans from {df['Crop_Year'].min()} to {df['Crop_Year'].max()}")
insights.append(f"🌍 Covers {df['State'].nunique()} states/UTs")
insights.append(f"🏘️ Includes {df['District'].nunique()} districts")
insights.append(f"🌾 Contains data for {df['Crop'].nunique()} different crops")
# Data completeness
complete_records = df.dropna(subset=['Area', 'Production', 'Yield']).shape[0]
completeness_pct = (complete_records / len(df)) * 100
insights.append(f"✅ {completeness_pct:.1f}% records have complete Area/Production/Yield data")
# Recent vs Historical
recent_pct = (len(df[df['Crop_Year'] >= 2020]) / len(df)) * 100
insights.append(f"🕐 {recent_pct:.1f}% of data is from 2020 onwards")
print("\nKey Insights:")
for i, insight in enumerate(insights, 1):
print(f"{i}. {insight}")
print("\nRecommendations for further analysis:")
print("• Focus on crops with sufficient historical data for trend analysis")
print("• Consider handling missing rainfall/fertilizer data through imputation")
print("• Analyze seasonal patterns in crop yield")
print("• Investigate state-wise agricultural productivity")
print("• Use district-level data for more granular insights")
def main():
"""Main function to run the analysis."""
print("Loading combined crop dataset...")
df = load_data()
# Run all analyses
basic_analysis(df)
data_distribution_analysis(df)
data_quality_analysis(df)
yield_analysis(df)
temporal_analysis(df)
create_summary_report(df)
print(f"\n✨ Analysis complete! Combined dataset is ready for use.")
if __name__ == "__main__":
main()
|