updated

app.py

@@ -3,17 +3,21 @@ import numpy as np
 import matplotlib.pyplot as plt
 import seaborn as sns
 import warnings
-import io
-import base64
-import os
 import tempfile
+import os
+import dash
+import dash_core_components as dcc
+import dash_html_components as html
+import dash_table
 from sklearn.model_selection import train_test_split
 from sklearn.ensemble import RandomForestClassifier
-from sklearn.metrics import classification_report
-from sklearn.preprocessing import StandardScaler
-from sklearn.cluster import KMeans
+from sklearn.linear_model import LogisticRegression
+from sklearn.preprocessing import StandardScaler, LabelEncoder
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.cluster import KMeans, DBSCAN
+from sklearn.metrics import classification_report, accuracy_score, silhouette_score
 from sklearn.decomposition import PCA
-import gradio as gr
+from sklearn.manifold import TSNE
 
 # Suppress specific FutureWarnings
 warnings.filterwarnings("ignore", category=FutureWarning)
@@ -21,373 +25,165 @@ warnings.filterwarnings("ignore", category=FutureWarning)
 # Set seaborn style for better aesthetics
 sns.set(style="whitegrid")
 
-def clean_data(df):
-    df = df.drop_duplicates()
-    df.columns = df.columns.str.strip().str.lower().str.replace(' ', '_').str.replace('-', '_')
-
+def enhanced_preprocessing(df):
+    # Handling missing values
+    df = df.fillna('Unknown')
+    
+    # Encoding categorical features
     categorical_cols = df.select_dtypes(include=['object']).columns.tolist()
-    numerical_cols = df.select_dtypes(include=['float64', 'int64']).columns.tolist()
-
-    df[categorical_cols] = df[categorical_cols].fillna('Unknown')
-    df[numerical_cols] = df[numerical_cols].fillna(df[numerical_cols].median())
-
-    attendance_cols = [col for col in df.columns if 'attendance' in col]
-    for col in attendance_cols:
-        df[col] = df[col].astype(str).str.rstrip('%').replace('Unknown', '0').astype(float) / 100.0
-
-    df['date_of_birth'] = pd.to_datetime(df['date_of_birth'], format='%d/%m/%Y', errors='coerce')
-    df = df.dropna(subset=['date_of_birth'])
-    df['age'] = (pd.Timestamp('today') - df['date_of_birth']).dt.days // 365
-
-    df = df.drop(columns=['nsn'], errors='ignore')
-
-    category_cols = [
-        'gender', 'ethnicity', 'year_level', 'contributing_primary_school',
-        'year_11_english_teacher', 'year_11_maths_teacher', 'year_12_english_teacher',
-        'year_12_maths_teacher', 'form_teacher', 'leaving_date', 'primary_language',
-        'first_language', 'secondary_language', 'term_1_intervention',
-        'term_2_intervention', 'term_3_intervention', 'term_4_intervention',
-        'major_life_event', 'learning_difficulty', 'pastoral_care_incident',
-        'pastoral_care_action_taken', 'pastoral_care_follow_up'
-    ]
-    for col in category_cols:
-        if col in df.columns:
-            df[col] = df[col].astype('category')
-
-    if 'ncea_results' in df.columns:
-        ncea_results = []
-        for idx, row in df.iterrows():
-            try:
-                ncea_data = eval(row['ncea_results'])
-                total_credits = sum([result.get('Credits', 0) for result in ncea_data])
-                ncea_results.append({'Index': idx, 'Total Credits': total_credits})
-            except:
-                ncea_results.append({'Index': idx, 'Total Credits': 0})
-
-        ncea_df = pd.DataFrame(ncea_results)
-        df = df.merge(ncea_df, left_index=True, right_on='Index', how='left')
-        df = df.drop(columns=['Index', 'ncea_results'], errors='ignore')
-    else:
-        df['Total Credits'] = 0
-
-    if 'pastoral_care_follow_up' in df.columns:
-        df['action_effective'] = df['pastoral_care_follow_up'].apply(
-            lambda x: 'Effective' if 'resolved' in str(x).lower() else 'Not Effective'
-        )
-
-    df['credit_threshold'] = df['year_level'].apply(lambda x: 80 if x == 'Year 11' else 60)
-    df['credit_achievement_rate'] = df['Total Credits'] / df['credit_threshold']
-
-    return df
-
-def plt_to_file():
-    with tempfile.NamedTemporaryFile(delete=False, suffix='.png') as tmpfile:
-        plt.savefig(tmpfile.name)
-        plt.close()
-    return tmpfile.name
-
-def identify_at_risk_students(df):
-    def prepare_data_for_modeling(df):
-        df_model = df.drop(columns=[
-            'first_name', 'last_name', 'date_of_birth', 'form_teacher',
-            'leaving_date', 'pastoral_care', 'pastoral_care_follow_up',
-            'pastoral_care_action_taken', 'pastoral_care_incident',
-            'extra_curricular_activities', 'contributing_primary_school',
-            'year_11_english_teacher', 'year_11_maths_teacher',
-            'year_12_english_teacher', 'year_12_maths_teacher', 'primary_language',
-            'first_language', 'secondary_language', 'action_effective'
-        ], errors='ignore')
-
-        categorical_cols = df_model.select_dtypes(include=['object', 'category']).columns
-        df_encoded = pd.get_dummies(df_model, columns=categorical_cols, drop_first=True)
-        df_encoded = df_encoded.fillna(0)
-
-        features = df_encoded.drop(['Total Credits', 'credit_threshold', 'credit_achievement_rate'], axis=1, errors='ignore')
-        target = (df_encoded['credit_achievement_rate'] < 1).astype(int)
-
-        return features, target
-
-    features, target = prepare_data_for_modeling(df)
-    X_train, X_test, y_train, y_test = train_test_split(features, target, test_size=0.2, random_state=42)
-
-    model = RandomForestClassifier(n_estimators=100, random_state=42)
-    model.fit(X_train, y_train)
-    y_pred = model.predict(X_test)
-
-    report = classification_report(y_test, y_pred)
-
-    importances = model.feature_importances_
-    feature_names = features.columns
-    feature_importance_df = pd.DataFrame({'Feature': feature_names, 'Importance': importances})
-    feature_importance_df = feature_importance_df.sort_values(by='Importance', ascending=False)
-
-    graphs = []
-    tables = {}
-
-    tables['classification_report'] = report
-    tables['feature_importance'] = feature_importance_df.head(10).to_string()
-
-    if feature_importance_df['Importance'].sum() > 0:
-        plt.figure(figsize=(12, 6))
-        sns.barplot(data=feature_importance_df.head(10), x='Importance', y='Feature', palette='viridis')
-        plt.title('Top 10 Important Features for Predicting At-Risk Students', fontsize=14)
-        plt.xlabel('Importance', fontsize=12)
-        plt.ylabel('Feature', fontsize=12)
-        plt.tight_layout()
-        graphs.append(plt_to_file())
-
-    return graphs, tables
-
-def process_extra_curricular(df):
-    df['extra_curricular_activities'] = df['extra_curricular_activities'].apply(
-        lambda x: eval(x) if isinstance(x, str) else []
-    )
-    activities = df['extra_curricular_activities'].explode().unique()
-    activities = [activity for activity in activities if activity is not None]
-    for activity in activities:
-        df[activity] = df['extra_curricular_activities'].apply(lambda x: int(activity in x))
+    for col in categorical_cols:
+        if len(df[col].unique()) < 20:  # Label Encoding for columns with low cardinality
+            label_encoder = LabelEncoder()
+            df[col] = label_encoder.fit_transform(df[col])
+        else:  # One-Hot Encoding for high-cardinality features
+            one_hot = pd.get_dummies(df[col], prefix=col)
+            df = pd.concat([df, one_hot], axis=1).drop(col, axis=1)
+            
+    # Vectorizing free-text columns (example: interventions column)
+    if 'interventions' in df.columns:
+        tfidf = TfidfVectorizer()
+        tfidf_matrix = tfidf.fit_transform(df['interventions'])
+        tfidf_df = pd.DataFrame(tfidf_matrix.toarray(), columns=tfidf.get_feature_names_out())
+        df = pd.concat([df, tfidf_df], axis=1).drop('interventions', axis=1)
+    
     return df
 
-def analyze_extra_curricular_impact(df):
-    graphs = []
-    activity_cols = [col for col in df.columns if col in ['Cricket', 'Debating', 'Football', 'Art Club', 'Drama Club', 'Rugby']]
-    for activity in activity_cols:
-        if activity in df.columns:
-            data = df.copy()
-            group = data.groupby(activity)['credit_achievement_rate'].mean().reset_index()
-            group[activity] = group[activity].map({0: 'Not Involved', 1: 'Involved'})
-            plt.figure(figsize=(6, 4))
-            sns.barplot(data=group, x=activity, y='credit_achievement_rate', palette='Set2', edgecolor='w', errorbar=None)
-            plt.title(f'Impact of {activity} on Credit Achievement Rate', fontsize=14)
-            plt.xlabel('Participation Status', fontsize=12)
-            plt.ylabel('Average Credit Achievement Rate', fontsize=12)
-            plt.tight_layout()
-            graphs.append(plt_to_file())
-    return graphs
-
-def analyze_teacher_performance(df):
-    graphs = []
-    tables = {}
-    teacher_year_levels = {
-        'year_11_english_teacher': 'Year 11',
-        'year_11_maths_teacher': 'Year 11',
-        'year_12_english_teacher': 'Year 12',
-        'year_12_maths_teacher': 'Year 12'
-    }
-    for col, year_level in teacher_year_levels.items():
-        data = df[(df[col] != 'Unknown') & (df['year_level'] == year_level)]
-        if not data.empty:
-            group = data.groupby(col)['credit_achievement_rate'].mean().reset_index()
-            plt.figure(figsize=(10, 6))
-            sns.barplot(data=group, x=col, y='credit_achievement_rate', palette='Set3', edgecolor='w', errorbar=None)
-            plt.title(f'Average Credit Achievement Rate by {col.replace("_", " ").title()} ({year_level})', fontsize=14)
-            plt.xlabel('Teacher', fontsize=12)
-            plt.ylabel('Average Credit Achievement Rate', fontsize=12)
-            plt.xticks(rotation=45)
-            plt.tight_layout()
-            graphs.append(plt_to_file())
-        else:
-            tables[f"{col}_{year_level}"] = f"No data available for {col} in {year_level}."
-    return graphs, tables
-
-def analyze_language_impact(df):
-    graphs = []
-    tables = {}
-    data = df[df['primary_language'] != 'Unknown']
-    if not data.empty:
-        group = data.groupby('primary_language')['credit_achievement_rate'].mean().reset_index()
-        plt.figure(figsize=(10, 6))
-        sns.barplot(data=group, x='primary_language', y='credit_achievement_rate', palette='Pastel1', edgecolor='w', errorbar=None)
-        plt.title('Average Credit Achievement Rate by Primary Language', fontsize=14)
-        plt.xlabel('Primary Language', fontsize=12)
-        plt.ylabel('Average Credit Achievement Rate', fontsize=12)
-        plt.xticks(rotation=45)
-        plt.tight_layout()
-        graphs.append(plt_to_file())
-    else:
-        tables['language_impact'] = "No data available for primary languages."
-    return graphs, tables
+def calculate_correlations(df, threshold=0.3):
+    correlations = df.corr()
+    significant_corr = correlations[abs(correlations) > threshold].stack().reset_index()
+    significant_corr = significant_corr[significant_corr['level_0'] != significant_corr['level_1']]
+    significant_corr.columns = ['Feature 1', 'Feature 2', 'Correlation']
+    
+    return significant_corr
 
 def perform_clustering(df):
-    graphs = []
-    tables = {}
-    attendance_cols = [col for col in df.columns if 'attendance' in col]
-    features = df[['credit_achievement_rate', 'age'] + attendance_cols]
-    features = features.fillna(0)
+    # Normalize the data for clustering
     scaler = StandardScaler()
-    scaled_features = scaler.fit_transform(features)
-    pca = PCA(n_components=2)
-    principal_components = pca.fit_transform(scaled_features)
-    kmeans = KMeans(n_clusters=3, random_state=42)
-    clusters = kmeans.fit_predict(principal_components)
-    df['Cluster'] = clusters
-    cluster_analysis = df.groupby('Cluster')[['credit_achievement_rate', 'age'] + attendance_cols].mean()
-    tables['cluster_analysis'] = cluster_analysis.to_string()
-    plt.figure(figsize=(8, 6))
-    sns.scatterplot(x=principal_components[:,0], y=principal_components[:,1], hue=clusters, palette='Set1', s=100, alpha=0.7)
-    plt.title('Student Clusters', fontsize=14)
-    plt.xlabel('Principal Component 1', fontsize=12)
-    plt.ylabel('Principal Component 2', fontsize=12)
-    plt.legend(title='Cluster')
-    plt.tight_layout()
-    graphs.append(plt_to_file())
-    return graphs, tables
-
-def plot_correlation_with_credit_achievement(df):
-    graphs = []
-    tables = {}
-    numeric_cols = df.select_dtypes(include=['float64', 'int64']).columns
-    corr_matrix = df[numeric_cols].corr()
-
-    if 'credit_achievement_rate' not in corr_matrix.columns:
-        tables['correlation_error'] = "Error: 'credit_achievement_rate' column not found in the dataset."
-        return graphs, tables
-
-    corr_matrix = corr_matrix.drop(['credit_threshold', 'Total Credits'], axis=1, errors='ignore')
-    corr_matrix = corr_matrix.drop(['credit_threshold', 'Total Credits'], axis=0, errors='ignore')
-
-    correlation_with_credit = corr_matrix[['credit_achievement_rate']].sort_values(by='credit_achievement_rate', ascending=False)
-
-    plt.figure(figsize=(8, 10))
-    sns.heatmap(correlation_with_credit, annot=True, cmap='coolwarm', fmt='.2f', annot_kws={"size": 10}, cbar=True)
-    plt.title('Correlation with Credit Achievement Rate', fontsize=16)
-    plt.xticks(rotation=45, ha='right', fontsize=10)
-    plt.tight_layout()
-    graphs.append(plt_to_file())
-
-    tables['correlation_with_credit'] = correlation_with_credit.to_string()
-
-    corr_matrix_clean = corr_matrix.replace([np.inf, -np.inf], np.nan).fillna(0)
-
-    plt.figure(figsize=(12, 12))
-    sns.clustermap(corr_matrix_clean, annot=False, cmap='coolwarm', figsize=(12, 12), method='average')
-    plt.title('Cluster Map of Feature Correlations (excluding credit_threshold, Total Credits)', fontsize=16)
-    graphs.append(plt_to_file())
+    df_scaled = scaler.fit_transform(df)
 
-    return graphs, tables
+    # Determine best clustering method based on dataset characteristics
+    kmeans = KMeans(n_clusters=4, random_state=42)
+    dbscan = DBSCAN(eps=0.5, min_samples=5)
 
-def plot_top_features_vs_credit(df):
-    graphs = []
-    tables = {}
-    numeric_cols = df.select_dtypes(include=['float64', 'int64']).columns
-    corr_matrix = df[numeric_cols].corr()
+    kmeans_labels = kmeans.fit_predict(df_scaled)
+    dbscan_labels = dbscan.fit_predict(df_scaled)
 
-    corr_matrix = corr_matrix.drop(['credit_threshold', 'Total Credits'], axis=0, errors='ignore')
-    corr_matrix = corr_matrix.drop(['credit_threshold', 'Total Credits'], axis=1, errors='ignore')
+    kmeans_score = silhouette_score(df_scaled, kmeans_labels)
+    dbscan_score = silhouette_score(df_scaled, dbscan_labels) if len(set(dbscan_labels)) > 1 else -1
 
-    top_corr_features = corr_matrix['credit_achievement_rate'].abs().sort_values(ascending=False).index[1:6]
-    
-    tables['top_corr_features'] = f"Top features most correlated with Credit Achievement Rate:\n{', '.join(top_corr_features)}"
-
-    for feature in top_corr_features:
-        if pd.api.types.is_numeric_dtype(df[feature]):
-            df_sorted = df[[feature, 'credit_achievement_rate']].sort_values(by=feature)
-
-            plt.figure(figsize=(10, 6))
-            sns.lineplot(x=df_sorted[feature], y=df_sorted['credit_achievement_rate'], marker='o')
-            plt.title(f'Line Graph: {feature} vs Credit Achievement Rate', fontsize=14)
-            plt.xlabel(feature.replace('_', ' ').title(), fontsize=12)
-            plt.ylabel('Credit Achievement Rate', fontsize=12)
-            plt.tight_layout()
-            graphs.append(plt_to_file())
-        elif pd.api.types.is_categorical_dtype(df[feature]) or pd.api.types.is_object_dtype(df[feature]):
-            group = df.groupby(feature)['credit_achievement_rate'].mean().reset_index()
-
-            plt.figure(figsize=(10, 6))
-            sns.barplot(x=group[feature], y=group['credit_achievement_rate'], palette='Set2')
-            plt.title(f'Bar Plot: {feature} vs Credit Achievement Rate', fontsize=14)
-            plt.xlabel(feature.replace('_', ' ').title(), fontsize=12)
-            plt.ylabel('Average Credit Achievement Rate', fontsize=12)
-            plt.xticks(rotation=45)
-            plt.tight_layout()
-            graphs.append(plt_to_file())
-
-    return graphs, tables
+    if kmeans_score > dbscan_score:
+        df['Cluster'] = kmeans_labels
+        best_model = 'K-Means'
+    else:
+        df['Cluster'] = dbscan_labels
+        best_model = 'DBSCAN'
 
-def perform_comprehensive_analysis(df):
-    all_graphs = []
-    all_tables = {}
+    # Use PCA for visualization
+    pca = PCA(n_components=2)
+    pca_components = pca.fit_transform(df_scaled)
+    df['PCA1'] = pca_components[:, 0]
+    df['PCA2'] = pca_components[:, 1]
 
-    # 1. Identifying At-Risk Students
-    graphs, tables = identify_at_risk_students(df)
-    all_graphs.extend(graphs)
-    all_tables.update(tables)
+    return df, best_model
 
-    # 2. Analyzing Impact of Extra-Curricular Activities
-    df = process_extra_curricular(df)
-    graphs = analyze_extra_curricular_impact(df)
-    all_graphs.extend(graphs)
+def perform_predictions(df):
+    results = []
+    target_cols = [col for col in df.columns if col in ['skip_class', 'final_grade']]
 
-    # 3. Analyzing Teacher Performance
-    graphs, tables = analyze_teacher_performance(df)
-    all_graphs.extend(graphs)
-    all_tables.update(tables)
+    for target in target_cols:
+        X = df.drop(target, axis=1)
+        y = df[target]
 
-    # 4. Analyzing Language Proficiency Impact
-    graphs, tables = analyze_language_impact(df)
-    all_graphs.extend(graphs)
-    all_tables.update(tables)
+        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
 
-    # 5. Performing Cluster Analysis
-    graphs, tables = perform_clustering(df)
-    all_graphs.extend(graphs)
-    all_tables.update(tables)
+        # Model 1: Random Forest
+        rf_model = RandomForestClassifier(random_state=42)
+        rf_model.fit(X_train, y_train)
+        rf_pred = rf_model.predict(X_test)
+        rf_accuracy = accuracy_score(y_test, rf_pred)
 
-    # 6. Correlation Analysis for Credit Achievement Rate
-    graphs, tables = plot_correlation_with_credit_achievement(df)
-    all_graphs.extend(graphs)
-    all_tables.update(tables)
+        # Model 2: Logistic Regression
+        lr_model = LogisticRegression(max_iter=1000)
+        lr_model.fit(X_train, y_train)
+        lr_pred = lr_model.predict(X_test)
+        lr_accuracy = accuracy_score(y_test, lr_pred)
 
-    # 7. Plotting Top Features vs Credit Achievement Rate
-    graphs, tables = plot_top_features_vs_credit(df)
-    all_graphs.extend(graphs)
-    all_tables.update(tables)
+        if rf_accuracy > lr_accuracy:
+            results.append({'Target': target, 'Model': 'Random Forest', 'Accuracy': rf_accuracy})
+        else:
+            results.append({'Target': target, 'Model': 'Logistic Regression', 'Accuracy': lr_accuracy})
 
-    return all_graphs, all_tables
+    return results
 
-def gradio_wrapper(file):
-    df = pd.read_csv(file.name)
-    df = clean_data(df)
-    graphs, tables = perform_comprehensive_analysis(df)
-    
-    # Convert tables to a list of strings for easier display
-    table_outputs = [
-        f"### {k}\n```\n{v}\n```" for k, v in tables.items()
-    ]
+def create_dashboard(df, correlation_data, clustering_data, prediction_results):
+    app = dash.Dash(__name__)
     
-    # Ensure we have exactly 7 table outputs (plus the gallery of graphs)
-    while len(table_outputs) < 7:
-        table_outputs.append("No data available for this section.")
-    
-    return [graphs] + table_outputs
-
-# Create Gradio interface
-iface = gr.Interface(
-    fn=gradio_wrapper,
-    inputs=gr.File(label="Upload CSV"),
-    outputs=[
-        gr.Gallery(label="Graphs", columns=2, rows=3, height="auto"),
-        gr.Markdown(label="Classification Report"),
-        gr.Markdown(label="Feature Importance"),
-        gr.Markdown(label="Teacher Performance"),
-        gr.Markdown(label="Language Impact"),
-        gr.Markdown(label="Cluster Analysis"),
-        gr.Markdown(label="Correlation with Credit Achievement Rate"),
-        gr.Markdown(label="Top Correlated Features")
-    ],
-    title="Comprehensive Student Data Analysis",
-    description="Upload a CSV file to analyze student data. The analysis includes identifying at-risk students, impact of extra-curricular activities, teacher performance, language proficiency impact, cluster analysis, and correlation analysis."
-)
-
-# Launch the interface
-iface.launch()
-
-# Clean up temporary files
-def cleanup_temp_files():
-    for filename in os.listdir(tempfile.gettempdir()):
-        if filename.endswith(".png"):
-            os.remove(os.path.join(tempfile.gettempdir(), filename))
-
-# Register the cleanup function to be called when the script exits
-import atexit
-atexit.register(cleanup_temp_files)
+    app.layout = html.Div([
+        html.H1('Comprehensive Student Data Analysis'),
+
+        html.Div([
+            html.H2('Correlation Analysis'),
+            dash_table.DataTable(
+                id='correlation_table',
+                columns=[{'name': i, 'id': i} for i in correlation_data.columns],
+                data=correlation_data.to_dict('records')
+            )
+        ]),
+
+        html.Div([
+            html.H2('Clustering Analysis'),
+            html.P(f'Best Clustering Algorithm: {clustering_data["best_model"]}'),
+            dcc.Graph(
+                id='clustering_scatter',
+                figure={
+                    'data': [
+                        {
+                            'x': df['PCA1'],
+                            'y': df['PCA2'],
+                            'mode': 'markers',
+                            'marker': {'color': df['Cluster'], 'colorscale': 'Viridis', 'size': 10},
+                            'text': df['Cluster'],
+                            'type': 'scatter'
+                        }
+                    ],
+                    'layout': {
+                        'title': 'Cluster Visualization using PCA',
+                        'xaxis': {'title': 'PCA Component 1'},
+                        'yaxis': {'title': 'PCA Component 2'}
+                    }
+                }
+            )
+        ]),
+
+        html.Div([
+            html.H2('Prediction Models'),
+            dash_table.DataTable(
+                id='prediction_table',
+                columns=[{'name': i, 'id': i} for i in prediction_results.columns],
+                data=prediction_results.to_dict('records')
+            )
+        ])
+    ])
+
+    app.run_server(debug=True)
+
+# Main execution
+if __name__ == "__main__":
+    # Load dataset
+    df = pd.read_csv('student_data.csv')  # Replace with your CSV file
+
+    # Preprocess the data
+    df = enhanced_preprocessing(df)
+
+    # Perform correlation analysis
+    correlation_data = calculate_correlations(df)
+
+    # Perform clustering analysis
+    df, best_model = perform_clustering(df)
+    clustering_data = {'best_model': best_model}
+
+    # Perform prediction analysis
+    prediction_results = pd.DataFrame(perform_predictions(df))
+
+    # Create and launch the dashboard
+    create_dashboard(df, correlation_data, clustering_data, prediction_results)

requirements.txt

@@ -4,3 +4,4 @@ matplotlib
 seaborn
 scikit-learn
 gradio
+dash