Spaces:
Sleeping
Sleeping
File size: 6,181 Bytes
bbd5f9c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 | #!/usr/bin/env python3
"""
Quick comparison between original and cleaned dataset performance
"""
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestRegressor
from sklearn.preprocessing import LabelEncoder
from sklearn.metrics import r2_score, mean_squared_error
import warnings
warnings.filterwarnings('ignore')
def test_dataset_performance(df, dataset_name="Dataset"):
"""Test model performance on a given dataset"""
print(f"\n{'='*50}")
print(f"TESTING {dataset_name.upper()}")
print(f"{'='*50}")
# Basic stats
print(f"Shape: {df.shape}")
print(f"Missing values: {df.isnull().sum().sum():,}")
# Prepare data - find yield column
yield_cols = [col for col in df.columns if 'yield' in col.lower()]
if not yield_cols:
print("❌ No yield column found!")
return None
target_col = yield_cols[0]
print(f"Target column: {target_col}")
# Remove zero yields for modeling
df_model = df[df[target_col] > 0].copy()
print(f"Non-zero yield records: {len(df_model):,}")
if len(df_model) < 100:
print("❌ Insufficient non-zero yield data!")
return None
# Select features
numerical_cols = df_model.select_dtypes(include=[np.number]).columns.tolist()
numerical_cols = [col for col in numerical_cols if col not in [target_col, 'Crop_Year']]
# Add categorical features with encoding
categorical_cols = ['Crop', 'Season', 'State']
feature_cols = []
# Add numerical features
for col in numerical_cols:
if col in df_model.columns and df_model[col].notna().sum() > len(df_model) * 0.5:
feature_cols.append(col)
X = df_model[feature_cols].copy()
# Handle missing values by filling with median
for col in X.columns:
X[col] = X[col].fillna(X[col].median())
# Add encoded categorical features
le_dict = {}
for col in categorical_cols:
if col in df_model.columns:
le = LabelEncoder()
X[col + '_encoded'] = le.fit_transform(df_model[col].fillna('Unknown'))
le_dict[col] = le
y = df_model[target_col]
print(f"Features used: {len(X.columns)}")
print(f"Target range: {y.min():.2f} to {y.max():.2f}")
# Train model
try:
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
model = RandomForestRegressor(n_estimators=50, random_state=42) # Smaller for quick test
model.fit(X_train, y_train)
y_pred = model.predict(X_test)
r2 = r2_score(y_test, y_pred)
rmse = np.sqrt(mean_squared_error(y_test, y_pred))
print(f"✅ R² Score: {r2:.4f}")
print(f"✅ RMSE: {rmse:.2f}")
return {
'dataset': dataset_name,
'records_used': len(df_model),
'features': len(X.columns),
'r2_score': r2,
'rmse': rmse,
'missing_values': df.isnull().sum().sum()
}
except Exception as e:
print(f"❌ Error: {str(e)}")
return None
def main():
print("🔍 COMPARING ORIGINAL VS CLEANED DATASET PERFORMANCE")
print("="*80)
# Load datasets
try:
original_df = pd.read_csv("/home/aiavid/Yeild_pred_SIH/data/combined_crop_data.csv")
cleaned_df = pd.read_csv("/home/aiavid/Yeild_pred_SIH/data/combined_crop_data_cleaned.csv")
except Exception as e:
print(f"❌ Error loading datasets: {str(e)}")
return
# Test both datasets
original_results = test_dataset_performance(original_df, "ORIGINAL DATASET")
cleaned_results = test_dataset_performance(cleaned_df, "CLEANED DATASET")
# Compare results
if original_results and cleaned_results:
print(f"\n{'='*80}")
print("COMPARISON RESULTS")
print(f"{'='*80}")
print("\n| Metric | Original | Cleaned | Improvement |")
print("|--------|----------|---------|-------------|")
# Missing values
orig_missing = original_results['missing_values']
clean_missing = cleaned_results['missing_values']
missing_improvement = ((orig_missing - clean_missing) / orig_missing * 100) if orig_missing > 0 else 0
print(f"| Missing Values | {orig_missing:,} | {clean_missing:,} | {missing_improvement:.1f}% reduction |")
# Records used
orig_records = original_results['records_used']
clean_records = cleaned_results['records_used']
record_improvement = ((clean_records - orig_records) / orig_records * 100) if orig_records > 0 else 0
print(f"| Usable Records | {orig_records:,} | {clean_records:,} | {record_improvement:+.1f}% |")
# R² Score
orig_r2 = original_results['r2_score']
clean_r2 = cleaned_results['r2_score']
r2_improvement = clean_r2 - orig_r2
print(f"| R² Score | {orig_r2:.4f} | {clean_r2:.4f} | {r2_improvement:+.4f} |")
# RMSE
orig_rmse = original_results['rmse']
clean_rmse = cleaned_results['rmse']
rmse_improvement = ((orig_rmse - clean_rmse) / orig_rmse * 100) if orig_rmse > 0 else 0
print(f"| RMSE | {orig_rmse:.2f} | {clean_rmse:.2f} | {rmse_improvement:.1f}% better |")
print(f"\n🎯 **OVERALL ASSESSMENT:**")
print(f" 📈 Model Accuracy Improved by {r2_improvement:+.4f} R² points")
print(f" 📉 Prediction Error Reduced by {rmse_improvement:.1f}%")
print(f" 🧹 Data Quality Improved: {missing_improvement:.1f}% fewer missing values")
if clean_r2 > 0.9:
print(f" 🏆 **EXCELLENT** model performance achieved!")
elif clean_r2 > 0.8:
print(f" 🥇 **VERY GOOD** model performance achieved!")
elif clean_r2 > 0.7:
print(f" 🥈 **GOOD** model performance achieved!")
else:
print(f" 📊 Model performance: **ACCEPTABLE**")
if __name__ == "__main__":
main()
|