Create app.py

app.py

@@ -0,0 +1,143 @@
+import gradio as gr
+import lime
+from lime.lime_text import LimeTextExplainer
+import numpy as np
+from datasets import load_dataset
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.model_selection import train_test_split
+from sklearn.linear_model import LogisticRegression
+from sklearn.pipeline import make_pipeline
+import shap
+import matplotlib.pyplot as plt
+import io
+from PIL import Image
+import pandas as pd
+
+
+# Load the IMDB dataset using Hugging Face datasets
+dataset = load_dataset('imdb')
+
+# Extract the training and test sets
+text_train = [review['text'] for review in dataset['train']]
+y_train = [review['label'] for review in dataset['train']]
+text_test = [review['text'] for review in dataset['test']]
+y_test = [review['label'] for review in dataset['test']]
+
+# Convert the text data into a TF-IDF representation
+vectorizer = TfidfVectorizer(stop_words='english', max_features=5000)
+X_train = vectorizer.fit_transform(text_train)
+X_test = vectorizer.transform(text_test)
+
+# Split the training data into train and validation sets
+X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.2, random_state=42)
+
+# Train a logistic regression model
+model = LogisticRegression(max_iter=1000)
+model.fit(X_train, y_train)
+
+# Initialize LIME explainer
+lime_explainer = LimeTextExplainer(class_names=['Negative', 'Positive'])
+
+# Create a SHAP explainer object
+shap_explainer = shap.LinearExplainer(model, X_train)
+
+def explain_text(input_text):
+    # Predict label
+    input_vector = vectorizer.transform([input_text])
+    predicted_label = model.predict(input_vector)[0]
+    label_name = 'Positive' if predicted_label == 1 else 'Negative'
+
+    # LIME explanation
+    def predict_proba_for_lime(texts):
+        return model.predict_proba(vectorizer.transform(texts))
+
+    lime_exp = lime_explainer.explain_instance(input_text, predict_proba_for_lime, num_features=10)
+    lime_fig = lime_exp.as_pyplot_figure()
+    lime_img = fig_to_nparray(lime_fig)
+
+    # Get the complete HTML for LIME explanation
+    lime_html = lime_exp.as_html()
+
+    # SHAP explanation
+    shap_values = shap_explainer.shap_values(input_vector)[0]
+    feature_names = vectorizer.get_feature_names_out()
+
+    # Create a SHAP explanation object for the selected instance
+    shap_explanation = shap.Explanation(
+        values=shap_values,
+        base_values=shap_explainer.expected_value,
+        feature_names=feature_names,
+        data=input_vector.toarray()[0]
+    )
+
+    # Function to highlight text based on SHAP values
+    def highlight_text_shap(text, word_importances, feature_names, max_num_features):
+        words = text.split()
+        word_to_importance = {}
+        for idx, word in enumerate(feature_names):
+            if word in text.lower():
+                word_to_importance[word] = word_importances[idx]
+
+        sorted_word_importance = sorted(word_to_importance.items(), key=lambda x: abs(x[1]), reverse=True)[:max_num_features]
+        top_words = {word: importance for word, importance in sorted_word_importance}
+
+        highlighted_text = []
+        for word in words:
+            cleaned_word = ''.join(filter(str.isalnum, word)).lower()
+            if cleaned_word in top_words:
+                importance = top_words[cleaned_word]
+                color = 'red' if importance > 0 else 'blue'
+                highlighted_text.append(f'<span style="color:{color}">{word}</span>')
+            else:
+                highlighted_text.append(word)
+
+        return ' '.join(highlighted_text)
+
+    # Set the maximum number of features to display
+    max_num_features = 10
+
+    # Create a DataFrame for SHAP values
+    shap_df = pd.DataFrame({
+        'Feature': shap_explanation.feature_names,
+        'SHAP Value': shap_explanation.values
+    }).sort_values(by='SHAP Value', ascending=False).head(max_num_features)
+
+    # Plot the SHAP values
+    plt.figure(figsize=(10, 6))
+    plt.barh(shap_df['Feature'], shap_df['SHAP Value'], color=['red' if val > 0 else 'blue' for val in shap_df['SHAP Value']])
+    plt.xlabel('SHAP Value')
+    plt.title('Top 10 Feature Importance')
+    plt.tight_layout()
+    shap_fig = fig_to_nparray(plt.gcf())
+
+    # Highlight the text based on SHAP values
+    shap_highlighted_text = highlight_text_shap(input_text, shap_values, feature_names, max_num_features)
+
+    return label_name, lime_img, shap_fig, lime_html, shap_highlighted_text
+
+def fig_to_nparray(fig):
+    """Convert a matplotlib figure to a NumPy array."""
+    buf = io.BytesIO()
+    fig.savefig(buf, format='png')
+    buf.seek(0)
+    img = Image.open(buf)
+    return np.array(img)
+
+
+# Create Gradio interface
+iface = gr.Interface(
+    fn=explain_text,
+    inputs=gr.Textbox(lines=2, placeholder="Enter your text here..."),
+    outputs=[
+        gr.Label(label="Predicted Label"),
+        gr.Image(type="numpy", label="LIME Explanation"),
+        gr.Image(type="numpy", label="SHAP Explanation"),
+        gr.HTML(label="LIME Highlighted Text Explanation"),
+        gr.HTML(label="SHAP Highlighted Text Explanation"),
+    ],
+    title="LIME and SHAP Explanations",
+    description="Enter a text sample to see its prediction and explanations using LIME and SHAP."
+)
+
+# Launch the interface
+iface.launch()