# ```python from fastapi import FastAPI, HTTPException from pydantic import BaseModel import pandas as pd import numpy as np from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor from sklearn.preprocessing import LabelEncoder, StandardScaler from sklearn.metrics import accuracy_score import matplotlib.pyplot as plt import json import base64 from io import BytesIO import warnings import logging # Set up logging logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) warnings.filterwarnings("ignore") app = FastAPI() class WorkerIdRequest(BaseModel): worker_id: int @app.post("/predict_worker_earnings/") async def predict_worker_earnings(request: WorkerIdRequest): try: worker_id = request.worker_id logger.info(f"Processing request for worker_id: {worker_id}") # Initialize result dictionary results = { 'worker_id': worker_id, 'classification_metrics': {}, 'worker_profile': {}, 'plot': '' } # Load data try: df = pd.read_csv('/app/extended_worker_dataset.csv') logger.info(f"CSV loaded successfully. Columns: {list(df.columns)}") except FileNotFoundError: logger.error("CSV file not found at /app/extended_worker_dataset.csv") raise HTTPException(status_code=500, detail="CSV file not found at /app/data/extended_worker_dataset.csv") except Exception as e: logger.error(f"Error reading CSV file: {str(e)}") raise HTTPException(status_code=500, detail=f"Error reading CSV file: {str(e)}") # Verify expected columns expected_columns = ['worker_id', 'state', 'labour_category', 'contracted_wage', 'age', 'gender', 'migration_status', 'years_of_experience', 'feedback_score', 'job_type', 'timestamp'] if not all(col in df.columns for col in expected_columns): missing_cols = [col for col in expected_columns if col not in df.columns] logger.error(f"Missing columns in CSV: {missing_cols}") raise HTTPException(status_code=500, detail=f"Missing columns in CSV: {missing_cols}") # Filter for one worker_id df = df[df['worker_id'] == worker_id].copy() if df.empty: logger.warning(f"No data found for worker_id {worker_id}") raise HTTPException(status_code=404, detail=f"No data found for worker_id {worker_id}") logger.info(f"Filtered data for worker_id {worker_id}: {len(df)} rows") # Data preprocessing try: df['timestamp'] = pd.to_datetime(df['timestamp']) except Exception as e: logger.error(f"Error converting timestamp: {str(e)}") raise HTTPException(status_code=500, detail=f"Error converting timestamp: {str(e)}") df['has_job'] = (df['job_type'] != "No Job").astype(int) try: wage_cap = df[df['contracted_wage'] > 0]['contracted_wage'].quantile(0.90) if np.isnan(wage_cap) or wage_cap <= 500: logger.error("Invalid wage cap calculated") raise ValueError("Invalid wage cap calculated") df['contracted_wage'] = df['contracted_wage'].clip(lower=500, upper=wage_cap) except Exception as e: logger.error(f"Error processing wage data: {str(e)}") raise HTTPException(status_code=500, detail=f"Error processing wage data: {str(e)}") # Encode job_type le = LabelEncoder() try: df['job_type_encoded'] = le.fit_transform(df['job_type']) except Exception as e: logger.error(f"Error encoding job_type: {str(e)}") raise HTTPException(status_code=500, detail=f"Error encoding job_type: {str(e)}") # Split data if len(df) < 2: logger.warning("Insufficient data points for training and testing") raise HTTPException(status_code=400, detail="Insufficient data points for training and testing") split_point = int(len(df) * 0.8) train_df = df.iloc[:split_point].copy() test_df = df.iloc[split_point:].copy() # Scale features scaler = StandardScaler() try: train_df[['job_type_scaled', 'years_exp_scaled']] = scaler.fit_transform( train_df[['job_type_encoded', 'years_of_experience']] ) train_df['job_exp_interaction'] = train_df['job_type_scaled'] * train_df['years_exp_scaled'] except Exception as e: logger.error(f"Error scaling features: {str(e)}") raise HTTPException(status_code=500, detail=f"Error scaling features: {str(e)}") for subset in [train_df, test_df]: subset['dayofweek'] = subset['timestamp'].dt.dayofweek subset['month'] = subset['timestamp'].dt.month subset['year'] = subset['timestamp'].dt.year subset['dayofyear'] = subset['timestamp'].dt.dayofyear subset['is_weekend'] = subset['dayofweek'].isin([5, 6]).astype(int) # Train classifier X_train_class = train_df[['dayofweek', 'month', 'year', 'dayofyear', 'is_weekend', 'job_type_encoded', 'feedback_score', 'years_of_experience']] y_train_class = train_df['has_job'] try: classifier = RandomForestClassifier( n_estimators=500, max_depth=12, min_samples_split=5, random_state=42 ) classifier.fit(X_train_class, y_train_class) except Exception as e: logger.error(f"Error training classifier: {str(e)}") raise HTTPException(status_code=500, detail=f"Error training classifier: {str(e)}") # Train regressor train_df_reg = train_df[train_df['has_job'] == 1].copy() if train_df_reg.empty: logger.warning("No data available for regression (all has_job == 0)") raise HTTPException(status_code=404, detail="No data available for regression (all has_job == 0)") X_train_reg = train_df_reg[['dayofweek', 'month', 'year', 'dayofyear', 'is_weekend', 'job_type_scaled', 'feedback_score', 'years_exp_scaled', 'job_exp_interaction']] y_train_reg = train_df_reg['contracted_wage'] try: regressor = RandomForestRegressor( n_estimators=300, max_depth=10, min_samples_split=4, random_state=42 ) regressor.fit(X_train_reg, y_train_reg) except Exception as e: logger.error(f"Error training regressor: {str(e)}") raise HTTPException(status_code=500, detail=f"Error training regressor: {str(e)}") # Prepare future dataframe future_df = test_df[['timestamp', 'job_type', 'job_type_encoded', 'feedback_score', 'years_of_experience']].rename(columns={'timestamp': 'ds'}) future_df['dayofweek'] = future_df['ds'].dt.dayofweek future_df['month'] = future_df['ds'].dt.month future_df['year'] = future_df['ds'].dt.year future_df['dayofyear'] = future_df['ds'].dt.dayofyear future_df['is_weekend'] = future_df['dayofweek'].isin([5, 6]).astype(int) try: future_df[['job_type_scaled', 'years_exp_scaled']] = scaler.transform( future_df[['job_type_encoded', 'years_of_experience']] ) future_df['job_exp_interaction'] = future_df['job_type_scaled'] * future_df['years_exp_scaled'] except Exception as e: logger.error(f"Error transforming future dataframe: {str(e)}") raise HTTPException(status_code=500, detail=f"Error transforming future dataframe: {str(e)}") # Predict job/no-job try: future_df['has_job_predicted'] = classifier.predict( future_df[['dayofweek', 'month', 'year', 'dayofyear', 'is_weekend', 'job_type_encoded', 'feedback_score', 'years_of_experience']] ) except Exception as e: logger.error(f"Error predicting has_job: {str(e)}") raise HTTPException(status_code=500, detail=f"Error predicting has_job: {str(e)}") # Evaluate classifier accuracy test_df['has_job'] = (test_df['job_type'] != "No Job").astype(int) try: acc = accuracy_score(test_df['has_job'], future_df['has_job_predicted']) results['classification_metrics']['accuracy'] = round(acc * 100, 2) except Exception as e: logger.error(f"Error calculating accuracy: {str(e)}") raise HTTPException(status_code=500, detail=f"Error calculating accuracy: {str(e)}") # Predict wages try: future_df['yhat'] = regressor.predict( future_df[['dayofweek', 'month', 'year', 'dayofyear', 'is_weekend', 'job_type_scaled', 'feedback_score', 'years_exp_scaled', 'job_exp_interaction']] ) except Exception as e: logger.error(f"Error predicting wages: {str(e)}") raise HTTPException(status_code=500, detail=f"Error predicting wages: {str(e)}") # Apply job prediction mask final_forecast_df = future_df.copy() final_forecast_df['yhat'] = np.where(final_forecast_df['has_job_predicted'] == 0, 0, final_forecast_df['yhat']) final_forecast_df['yhat'] = np.minimum(final_forecast_df['yhat'], wage_cap) # Uncertainty intervals try: predictions = regressor.predict(X_train_reg) std_dev = np.std([tree.predict(X_train_reg) for tree in regressor.estimators_], axis=0) future_df['yhat_lower'] = np.maximum(final_forecast_df['yhat'] - 1.96 * std_dev.mean(), 0) future_df['yhat_upper'] = final_forecast_df['yhat'] + 1.96 * std_dev.mean() final_forecast_df['yhat_lower'] = np.where(final_forecast_df['has_job_predicted'] == 0, 0, future_df['yhat_lower']) final_forecast_df['yhat_upper'] = np.where(final_forecast_df['has_job_predicted'] == 0, 0, future_df['yhat_upper']) except Exception as e: logger.error(f"Error calculating uncertainty intervals: {str(e)}") raise HTTPException(status_code=500, detail=f"Error calculating uncertainty intervals: {str(e)}") # Evaluation try: comparison_df = pd.merge( test_df[['timestamp', 'contracted_wage']].rename(columns={'timestamp': 'ds', 'contracted_wage': 'y'}), final_forecast_df[['ds', 'yhat', 'yhat_lower', 'yhat_upper']], on='ds' ) valid_comparison_df = comparison_df[comparison_df['y'] > 0] if not valid_comparison_df.empty: weights = valid_comparison_df['y'] / valid_comparison_df['y'].mean() mae = np.average([abs(a - p) for a, p in zip(valid_comparison_df['y'], valid_comparison_df['yhat'])], weights=weights) mape = np.average([abs((a - p) / a) * 100 for a, p in zip(valid_comparison_df['y'], valid_comparison_df['yhat'])], weights=weights) else: mae = np.nan mape = np.nan results['classification_metrics']['mae'] = round(mae, 2) if not np.isnan(mae) else None results['classification_metrics']['mape'] = round(mape, 2) if not np.isnan(mape) else None except Exception as e: logger.error(f"Error evaluating predictions: {str(e)}") raise HTTPException(status_code=500, detail=f"Error evaluating predictions: {str(e)}") # Plot results try: plt.figure(figsize=(12, 6)) plt.plot(comparison_df['ds'], comparison_df['y'], 'o-', label='Actual Values', markersize=4) plt.plot(comparison_df['ds'], comparison_df['yhat'], '-', label='Forecasted Values') plt.fill_between(comparison_df['ds'], comparison_df['yhat_lower'], comparison_df['yhat_upper'], color='gray', alpha=0.2, label='Uncertainty Interval') plt.title('Actual vs. Forecasted Daily Earnings (Last 20% of Dataset)') plt.xlabel('Date') plt.ylabel('Contracted Wage') plt.legend() plt.grid(True) plt.xticks(rotation=45) plt.tight_layout() buffer = BytesIO() plt.savefig(buffer, format='png') buffer.seek(0) plot_base64 = base64.b64encode(buffer.getvalue()).decode('utf-8') results['plot'] = f'data:image/png;base64,{plot_base64}' plt.close() except Exception as e: logger.error(f"Error generating plot: {str(e)}") raise HTTPException(status_code=500, detail=f"Error generating plot: {str(e)}") # Worker Profile for Microfinance try: worker_data = df.copy() avg_daily_earning = worker_data[worker_data['contracted_wage'] > 0]['contracted_wage'].mean() avg_monthly_earning = avg_daily_earning * 30 if not np.isnan(avg_daily_earning) else 0 job_distribution = worker_data['job_type'].value_counts(normalize=True) * 100 avg_feedback = worker_data['feedback_score'].mean() workholic_index = job_distribution.drop(labels=['No Job'], errors='ignore').sum() / 100 if avg_daily_earning > 0: earning_stability = worker_data[worker_data['contracted_wage'] > 0]['contracted_wage'].std() / avg_daily_earning else: earning_stability = np.nan results['worker_profile'] = { 'average_daily_earning': round(avg_daily_earning, 2) if not np.isnan(avg_daily_earning) else None, 'estimated_monthly_earning': round(avg_monthly_earning, 2) if not np.isnan(avg_monthly_earning) else None, 'job_distribution': job_distribution.round(2).to_dict(), 'average_feedback_score': round(avg_feedback, 2) if not np.isnan(avg_feedback) else None, 'workholic_index': round(workholic_index, 2) if not np.isnan(workholic_index) else None, 'earning_stability': round(earning_stability, 2) if not np.isnan(earning_stability) else None } except Exception as e: logger.error(f"Error generating worker profile: {str(e)}") raise HTTPException(status_code=500, detail=f"Error generating worker profile: {str(e)}") def convert_to_serializable(obj): if isinstance(obj, np.floating): return float(obj) if isinstance(obj, np.integer): return int(obj) if isinstance(obj, np.ndarray): return obj.tolist() return obj logger.info("Request processed successfully") return json.loads(json.dumps(results, default=convert_to_serializable)) except Exception as e: logger.error(f"Error processing request: {str(e)}") raise HTTPException(status_code=500, detail=f"Error processing request: {str(e)}") if __name__ == "__main__": import uvicorn uvicorn.run(app, host="0.0.0.0", port=8000)