updated

app.py

@@ -3,15 +3,17 @@ import numpy as np
 import matplotlib.pyplot as plt
 import seaborn as sns
 import warnings
-import gradio as gr
+import io
+import base64
+import os
+import tempfile
 from sklearn.model_selection import train_test_split
 from sklearn.ensemble import RandomForestClassifier
 from sklearn.metrics import classification_report
 from sklearn.preprocessing import StandardScaler
 from sklearn.cluster import KMeans
 from sklearn.decomposition import PCA
-import io
-from PIL import Image
+import gradio as gr
 
 # Suppress specific FutureWarnings
 warnings.filterwarnings("ignore", category=FutureWarning)
@@ -39,6 +41,19 @@ def clean_data(df):
 
     df = df.drop(columns=['nsn'], errors='ignore')
 
+    category_cols = [
+        'gender', 'ethnicity', 'year_level', 'contributing_primary_school',
+        'year_11_english_teacher', 'year_11_maths_teacher', 'year_12_english_teacher',
+        'year_12_maths_teacher', 'form_teacher', 'leaving_date', 'primary_language',
+        'first_language', 'secondary_language', 'term_1_intervention',
+        'term_2_intervention', 'term_3_intervention', 'term_4_intervention',
+        'major_life_event', 'learning_difficulty', 'pastoral_care_incident',
+        'pastoral_care_action_taken', 'pastoral_care_follow_up'
+    ]
+    for col in category_cols:
+        if col in df.columns:
+            df[col] = df[col].astype('category')
+
     if 'ncea_results' in df.columns:
         ncea_results = []
         for idx, row in df.iterrows():
@@ -55,11 +70,22 @@ def clean_data(df):
     else:
         df['Total Credits'] = 0
 
+    if 'pastoral_care_follow_up' in df.columns:
+        df['action_effective'] = df['pastoral_care_follow_up'].apply(
+            lambda x: 'Effective' if 'resolved' in str(x).lower() else 'Not Effective'
+        )
+
     df['credit_threshold'] = df['year_level'].apply(lambda x: 80 if x == 'Year 11' else 60)
     df['credit_achievement_rate'] = df['Total Credits'] / df['credit_threshold']
 
     return df
 
+def plt_to_file():
+    with tempfile.NamedTemporaryFile(delete=False, suffix='.png') as tmpfile:
+        plt.savefig(tmpfile.name)
+        plt.close()
+    return tmpfile.name
+
 def identify_at_risk_students(df):
     def prepare_data_for_modeling(df):
         df_model = df.drop(columns=[
@@ -88,43 +114,43 @@ def identify_at_risk_students(df):
     model.fit(X_train, y_train)
     y_pred = model.predict(X_test)
 
-    classification_rep = classification_report(y_test, y_pred)
+    report = classification_report(y_test, y_pred)
 
     importances = model.feature_importances_
     feature_names = features.columns
     feature_importance_df = pd.DataFrame({'Feature': feature_names, 'Importance': importances})
     feature_importance_df = feature_importance_df.sort_values(by='Importance', ascending=False)
 
-    return classification_rep, feature_importance_df
-
-def plot_credit_achievement_rate(df):
-    plt.figure(figsize=(10, 6))
-    sns.histplot(df['credit_achievement_rate'], kde=True, bins=30)
-    plt.title('Distribution of Credit Achievement Rate')
-    plt.xlabel('Credit Achievement Rate')
-    plt.ylabel('Frequency')
-    buf = io.BytesIO()
-    plt.savefig(buf, format='png')
-    buf.seek(0)
-    plt.close()
-    return Image.open(buf)
-
-def plot_feature_importance(feature_importance_df):
-    plt.figure(figsize=(12, 6))
-    sns.barplot(data=feature_importance_df.head(10), x='Importance', y='Feature', palette='viridis')
-    plt.title('Top 10 Important Features for Predicting At-Risk Students', fontsize=14)
-    plt.xlabel('Importance', fontsize=12)
-    plt.ylabel('Feature', fontsize=12)
-    plt.tight_layout()
-    buf = io.BytesIO()
-    plt.savefig(buf, format='png')
-    buf.seek(0)
-    plt.close()
-    return Image.open(buf)
+    graphs = []
+    tables = {}
+
+    tables['classification_report'] = report
+    tables['feature_importance'] = feature_importance_df.head(10).to_string()
+
+    if feature_importance_df['Importance'].sum() > 0:
+        plt.figure(figsize=(12, 6))
+        sns.barplot(data=feature_importance_df.head(10), x='Importance', y='Feature', palette='viridis')
+        plt.title('Top 10 Important Features for Predicting At-Risk Students', fontsize=14)
+        plt.xlabel('Importance', fontsize=12)
+        plt.ylabel('Feature', fontsize=12)
+        plt.tight_layout()
+        graphs.append(plt_to_file())
+
+    return graphs, tables
+
+def process_extra_curricular(df):
+    df['extra_curricular_activities'] = df['extra_curricular_activities'].apply(
+        lambda x: eval(x) if isinstance(x, str) else []
+    )
+    activities = df['extra_curricular_activities'].explode().unique()
+    activities = [activity for activity in activities if activity is not None]
+    for activity in activities:
+        df[activity] = df['extra_curricular_activities'].apply(lambda x: int(activity in x))
+    return df
 
 def analyze_extra_curricular_impact(df):
+    graphs = []
     activity_cols = [col for col in df.columns if col in ['Cricket', 'Debating', 'Football', 'Art Club', 'Drama Club', 'Rugby']]
-    images = []
     for activity in activity_cols:
         if activity in df.columns:
             data = df.copy()
@@ -136,19 +162,86 @@ def analyze_extra_curricular_impact(df):
             plt.xlabel('Participation Status', fontsize=12)
             plt.ylabel('Average Credit Achievement Rate', fontsize=12)
             plt.tight_layout()
-            buf = io.BytesIO()
-            plt.savefig(buf, format='png')
-            buf.seek(0)
-            plt.close()
-            images.append(Image.open(buf))
-    return images
+            graphs.append(plt_to_file())
+    return graphs
+
+def analyze_teacher_performance(df):
+    graphs = []
+    tables = {}
+    teacher_year_levels = {
+        'year_11_english_teacher': 'Year 11',
+        'year_11_maths_teacher': 'Year 11',
+        'year_12_english_teacher': 'Year 12',
+        'year_12_maths_teacher': 'Year 12'
+    }
+    for col, year_level in teacher_year_levels.items():
+        data = df[(df[col] != 'Unknown') & (df['year_level'] == year_level)]
+        if not data.empty:
+            group = data.groupby(col)['credit_achievement_rate'].mean().reset_index()
+            plt.figure(figsize=(10, 6))
+            sns.barplot(data=group, x=col, y='credit_achievement_rate', palette='Set3', edgecolor='w', errorbar=None)
+            plt.title(f'Average Credit Achievement Rate by {col.replace("_", " ").title()} ({year_level})', fontsize=14)
+            plt.xlabel('Teacher', fontsize=12)
+            plt.ylabel('Average Credit Achievement Rate', fontsize=12)
+            plt.xticks(rotation=45)
+            plt.tight_layout()
+            graphs.append(plt_to_file())
+        else:
+            tables[f"{col}_{year_level}"] = f"No data available for {col} in {year_level}."
+    return graphs, tables
+
+def analyze_language_impact(df):
+    graphs = []
+    tables = {}
+    data = df[df['primary_language'] != 'Unknown']
+    if not data.empty:
+        group = data.groupby('primary_language')['credit_achievement_rate'].mean().reset_index()
+        plt.figure(figsize=(10, 6))
+        sns.barplot(data=group, x='primary_language', y='credit_achievement_rate', palette='Pastel1', edgecolor='w', errorbar=None)
+        plt.title('Average Credit Achievement Rate by Primary Language', fontsize=14)
+        plt.xlabel('Primary Language', fontsize=12)
+        plt.ylabel('Average Credit Achievement Rate', fontsize=12)
+        plt.xticks(rotation=45)
+        plt.tight_layout()
+        graphs.append(plt_to_file())
+    else:
+        tables['language_impact'] = "No data available for primary languages."
+    return graphs, tables
+
+def perform_clustering(df):
+    graphs = []
+    tables = {}
+    attendance_cols = [col for col in df.columns if 'attendance' in col]
+    features = df[['credit_achievement_rate', 'age'] + attendance_cols]
+    features = features.fillna(0)
+    scaler = StandardScaler()
+    scaled_features = scaler.fit_transform(features)
+    pca = PCA(n_components=2)
+    principal_components = pca.fit_transform(scaled_features)
+    kmeans = KMeans(n_clusters=3, random_state=42)
+    clusters = kmeans.fit_predict(principal_components)
+    df['Cluster'] = clusters
+    cluster_analysis = df.groupby('Cluster')[['credit_achievement_rate', 'age'] + attendance_cols].mean()
+    tables['cluster_analysis'] = cluster_analysis.to_string()
+    plt.figure(figsize=(8, 6))
+    sns.scatterplot(x=principal_components[:,0], y=principal_components[:,1], hue=clusters, palette='Set1', s=100, alpha=0.7)
+    plt.title('Student Clusters', fontsize=14)
+    plt.xlabel('Principal Component 1', fontsize=12)
+    plt.ylabel('Principal Component 2', fontsize=12)
+    plt.legend(title='Cluster')
+    plt.tight_layout()
+    graphs.append(plt_to_file())
+    return graphs, tables
 
 def plot_correlation_with_credit_achievement(df):
+    graphs = []
+    tables = {}
     numeric_cols = df.select_dtypes(include=['float64', 'int64']).columns
     corr_matrix = df[numeric_cols].corr()
 
     if 'credit_achievement_rate' not in corr_matrix.columns:
-        return None
+        tables['correlation_error'] = "Error: 'credit_achievement_rate' column not found in the dataset."
+        return graphs, tables
 
     corr_matrix = corr_matrix.drop(['credit_threshold', 'Total Credits'], axis=1, errors='ignore')
     corr_matrix = corr_matrix.drop(['credit_threshold', 'Total Credits'], axis=0, errors='ignore')
@@ -160,52 +253,135 @@ def plot_correlation_with_credit_achievement(df):
     plt.title('Correlation with Credit Achievement Rate', fontsize=16)
     plt.xticks(rotation=45, ha='right', fontsize=10)
     plt.tight_layout()
-    buf = io.BytesIO()
-    plt.savefig(buf, format='png')
-    buf.seek(0)
-    plt.close()
-    return Image.open(buf)
-
-def analyze_data(file):
-    try:
-        df = pd.read_csv(file)
-        df = clean_data(df)
-        # Generate classification report and feature importance
-        classification_rep, feature_importance_df = identify_at_risk_students(df)
-        # Generate plots
-        plot_img = plot_credit_achievement_rate(df)
-        feature_importance_img = plot_feature_importance(feature_importance_df)
-        extra_curricular_imgs = analyze_extra_curricular_impact(df)
-        correlation_img = plot_correlation_with_credit_achievement(df)
-        result_text = f"Data analysis complete.\n\nClassification Report:\n{classification_rep}"
-        return result_text, plot_img, feature_importance_img, extra_curricular_imgs, correlation_img
-    except Exception as e:
-        return f"An error occurred: {str(e)}", None, None, [], None
-
-# Gradio Interface
-def analyze_uploaded_file(file):
-    text, plot, feature_importance_plot, extra_curricular_plots, correlation_plot = analyze_data(file)
-    outputs = [text, plot if plot else None, feature_importance_plot if feature_importance_plot else None]
-    outputs.extend(extra_curricular_plots if extra_curricular_plots else [None] * 6)
-    outputs.append(correlation_plot if correlation_plot else None)
-    return outputs
-
-with gr.Blocks() as demo:
-    gr.Markdown("""
-        # Student Data Analysis Tool
-        Upload your CSV file to analyze student data and generate insights.
-    """)
+    graphs.append(plt_to_file())
+
+    tables['correlation_with_credit'] = correlation_with_credit.to_string()
+
+    corr_matrix_clean = corr_matrix.replace([np.inf, -np.inf], np.nan).fillna(0)
+
+    plt.figure(figsize=(12, 12))
+    sns.clustermap(corr_matrix_clean, annot=False, cmap='coolwarm', figsize=(12, 12), method='average')
+    plt.title('Cluster Map of Feature Correlations (excluding credit_threshold, Total Credits)', fontsize=16)
+    graphs.append(plt_to_file())
+
+    return graphs, tables
+
+def plot_top_features_vs_credit(df):
+    graphs = []
+    tables = {}
+    numeric_cols = df.select_dtypes(include=['float64', 'int64']).columns
+    corr_matrix = df[numeric_cols].corr()
+
+    corr_matrix = corr_matrix.drop(['credit_threshold', 'Total Credits'], axis=0, errors='ignore')
+    corr_matrix = corr_matrix.drop(['credit_threshold', 'Total Credits'], axis=1, errors='ignore')
+
+    top_corr_features = corr_matrix['credit_achievement_rate'].abs().sort_values(ascending=False).index[1:6]
     
-    with gr.Row():
-        file_input = gr.File(label="Upload CSV File")
-        text_output = gr.Textbox(label="Analysis Summary")
+    tables['top_corr_features'] = f"Top features most correlated with Credit Achievement Rate:\n{', '.join(top_corr_features)}"
+
+    for feature in top_corr_features:
+        if pd.api.types.is_numeric_dtype(df[feature]):
+            df_sorted = df[[feature, 'credit_achievement_rate']].sort_values(by=feature)
+
+            plt.figure(figsize=(10, 6))
+            sns.lineplot(x=df_sorted[feature], y=df_sorted['credit_achievement_rate'], marker='o')
+            plt.title(f'Line Graph: {feature} vs Credit Achievement Rate', fontsize=14)
+            plt.xlabel(feature.replace('_', ' ').title(), fontsize=12)
+            plt.ylabel('Credit Achievement Rate', fontsize=12)
+            plt.tight_layout()
+            graphs.append(plt_to_file())
+        elif pd.api.types.is_categorical_dtype(df[feature]) or pd.api.types.is_object_dtype(df[feature]):
+            group = df.groupby(feature)['credit_achievement_rate'].mean().reset_index()
+
+            plt.figure(figsize=(10, 6))
+            sns.barplot(x=group[feature], y=group['credit_achievement_rate'], palette='Set2')
+            plt.title(f'Bar Plot: {feature} vs Credit Achievement Rate', fontsize=14)
+            plt.xlabel(feature.replace('_', ' ').title(), fontsize=12)
+            plt.ylabel('Average Credit Achievement Rate', fontsize=12)
+            plt.xticks(rotation=45)
+            plt.tight_layout()
+            graphs.append(plt_to_file())
+
+    return graphs, tables
+
+def perform_comprehensive_analysis(df):
+    all_graphs = []
+    all_tables = {}
+
+    # 1. Identifying At-Risk Students
+    graphs, tables = identify_at_risk_students(df)
+    all_graphs.extend(graphs)
+    all_tables.update(tables)
+
+    # 2. Analyzing Impact of Extra-Curricular Activities
+    df = process_extra_curricular(df)
+    graphs = analyze_extra_curricular_impact(df)
+    all_graphs.extend(graphs)
+
+    # 3. Analyzing Teacher Performance
+    graphs, tables = analyze_teacher_performance(df)
+    all_graphs.extend(graphs)
+    all_tables.update(tables)
+
+    # 4. Analyzing Language Proficiency Impact
+    graphs, tables = analyze_language_impact(df)
+    all_graphs.extend(graphs)
+    all_tables.update(tables)
+
+    # 5. Performing Cluster Analysis
+    graphs, tables = perform_clustering(df)
+    all_graphs.extend(graphs)
+    all_tables.update(tables)
+
+    # 6. Correlation Analysis for Credit Achievement Rate
+    graphs, tables = plot_correlation_with_credit_achievement(df)
+    all_graphs.extend(graphs)
+    all_tables.update(tables)
+
+    # 7. Plotting Top Features vs Credit Achievement Rate
+    graphs, tables = plot_top_features_vs_credit(df)
+    all_graphs.extend(graphs)
+    all_tables.update(tables)
+
+    return all_graphs, all_tables
+
+def gradio_wrapper(file):
+    df = pd.read_csv(file.name)
+    df = clean_data(df)
+    graphs, tables = perform_comprehensive_analysis(df)
     
-    with gr.Row():
-        image_output = gr.Image(label="Generated Plot", type="pil")
-        feature_importance_output = gr.Image(label="Feature Importance Plot", type="pil")
-        extra_curricular_outputs = [gr.Image(label=f"Extra Curricular Impact Plot {i+1}", type="pil") for i in range(6)]
-        correlation_output = gr.Image(label="Correlation with Credit Achievement Rate", type="pil")
+    # Convert tables to a list of strings for easier display
+    table_outputs = [f"### {k}\n```\n{v}\n```" for k, v in tables.items()]
     
-    file_input.change(analyze_uploaded_file, inputs=file_input, outputs=[text_output, image_output, feature_importance_output] + extra_curricular_outputs + [correlation_output])
-
-demo.launch()
+    return [graphs] + table_outputs
+
+# Create Gradio interface
+iface = gr.Interface(
+    fn=gradio_wrapper,
+    inputs=gr.File(label="Upload CSV"),
+    outputs=[
+        gr.Gallery(label="Graphs", columns=2, rows=3, height="auto"),
+        gr.Markdown(label="Classification Report"),
+        gr.Markdown(label="Feature Importance"),
+        gr.Markdown(label="Teacher Performance"),
+        gr.Markdown(label="Language Impact"),
+        gr.Markdown(label="Cluster Analysis"),
+        gr.Markdown(label="Correlation with Credit Achievement Rate"),
+        gr.Markdown(label="Top Correlated Features")
+    ],
+    title="Comprehensive Student Data Analysis",
+    description="Upload a CSV file to analyze student data. The analysis includes identifying at-risk students, impact of extra-curricular activities, teacher performance, language proficiency impact, cluster analysis, and correlation analysis."
+)
+
+# Launch the interface
+iface.launch()
+
+# Clean up temporary files
+def cleanup_temp_files():
+    for filename in os.listdir(tempfile.gettempdir()):
+        if filename.endswith(".png"):
+            os.remove(os.path.join(tempfile.gettempdir(), filename))
+
+# Register the cleanup function to be called when the script exits
+import atexit
+atexit.register(cleanup_temp_files)