Spaces:
Runtime error
Runtime error
File size: 7,991 Bytes
c3637f5 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 |
"""
Intent Classification Model Evaluator
This script analyzes the CSV data produced by the Flask app to generate
evaluation metrics and visualizations.
"""
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import os
import argparse
from datetime import datetime
def load_evaluation_data(csv_path="model_evaluation.csv"):
"""Load the CSV data and do basic preprocessing"""
if not os.path.exists(csv_path):
print(f"Error: File {csv_path} not found")
return None
# Load the data
df = pd.read_csv(csv_path)
# Convert string boolean to actual boolean
#df['is_ood'] = df['is_ood'].apply(lambda x: x.lower() == 'true')
df['is_ood'] = df['is_ood'].apply(lambda x: str(x).lower() == 'true')
# Convert timestamp to datetime
df['timestamp'] = pd.to_datetime(df['timestamp'])
print(f"Loaded {len(df)} evaluation records")
return df
def generate_basic_stats(df):
"""Generate basic statistics from the evaluation data"""
if df is None or len(df) == 0:
return "No data available for analysis"
stats = {
"total_queries": len(df),
"unique_queries": df['input_text'].nunique(),
"in_distribution_count": (~df['is_ood']).sum(),
"out_of_distribution_count": df['is_ood'].sum(),
"ood_percentage": df['is_ood'].mean() * 100,
"avg_confidence": df['confidence'].mean(),
"avg_energy_score": df['energy_score'].mean(),
"top_intents": df['predicted_intent'].value_counts().head(10).to_dict()
}
# Calculate metrics grouped by detection method
method_stats = df.groupby('detection_method').agg({
'is_ood': ['mean', 'count'],
'confidence': ['mean', 'std'],
'energy_score': ['mean', 'std']
})
return stats, method_stats
def plot_distributions(df, output_dir="evaluation_plots"):
"""Create plots for analyzing the model performance"""
if df is None or len(df) == 0:
print("No data available for plotting")
return
# Create output directory if it doesn't exist
os.makedirs(output_dir, exist_ok=True)
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
# Plot 1: Confidence Distribution
plt.figure(figsize=(10, 6))
plt.hist(df[~df['is_ood']]['confidence'], bins=20, alpha=0.7, label='In-Distribution')
plt.hist(df[df['is_ood']]['confidence'], bins=20, alpha=0.7, label='Out-of-Distribution')
plt.xlabel('Confidence Score')
plt.ylabel('Count')
plt.title('Confidence Score Distribution')
plt.legend()
plt.savefig(os.path.join(output_dir, f"{timestamp}_confidence_distribution.png"))
# Plot 2: Energy Score Distribution
plt.figure(figsize=(10, 6))
plt.hist(df[~df['is_ood']]['energy_score'], bins=20, alpha=0.7, label='In-Distribution')
plt.hist(df[df['is_ood']]['energy_score'], bins=20, alpha=0.7, label='Out-of-Distribution')
plt.xlabel('Energy Score')
plt.ylabel('Count')
plt.title('Energy Score Distribution')
plt.legend()
plt.savefig(os.path.join(output_dir, f"{timestamp}_energy_distribution.png"))
# Plot 3: Intent Distribution (Top 10)
intent_counts = df['predicted_intent'].value_counts().head(10)
plt.figure(figsize=(12, 6))
intent_counts.plot(kind='bar')
plt.xlabel('Intent')
plt.ylabel('Count')
plt.title('Top 10 Predicted Intents')
plt.xticks(rotation=45, ha='right')
plt.tight_layout()
plt.savefig(os.path.join(output_dir, f"{timestamp}_intent_distribution.png"))
# Plot 4: OOD Detection Method Comparison
plt.figure(figsize=(10, 6))
method_ood = df.groupby('detection_method')['is_ood'].mean() * 100
method_ood.plot(kind='bar')
plt.xlabel('Detection Method')
plt.ylabel('OOD Percentage')
plt.title('OOD Detection Rate by Method')
plt.savefig(os.path.join(output_dir, f"{timestamp}_ood_by_method.png"))
print(f"Plots saved to {output_dir} directory")
def analyze_inputs(df):
"""Analyze input texts for patterns"""
if df is None or len(df) == 0:
return "No data available for analysis"
# Basic text statistics
df['text_length'] = df['input_text'].apply(len)
df['word_count'] = df['input_text'].apply(lambda x: len(x.split()))
text_stats = {
"avg_text_length": df['text_length'].mean(),
"avg_word_count": df['word_count'].mean(),
"max_text_length": df['text_length'].max(),
"min_text_length": df['text_length'].min()
}
# Analyze correlation between text length and predictions
length_vs_ood = df.groupby(pd.cut(df['text_length'], 10))['is_ood'].mean()
length_vs_confidence = df.groupby(pd.cut(df['text_length'], 10))['confidence'].mean()
print("\nInput Text Analysis:")
print(f"Average text length: {text_stats['avg_text_length']:.1f} characters")
print(f"Average word count: {text_stats['avg_word_count']:.1f} words")
return text_stats, length_vs_ood, length_vs_confidence
def suggest_thresholds(df):
"""Analyze the data to suggest optimal thresholds for OOD detection"""
if df is None or len(df) == 0 or len(df['is_ood'].unique()) < 2:
return "Insufficient data for threshold analysis - need both OOD and non-OOD examples"
# Simple suggestion based on average values
suggested_energy = np.mean([
df[df['is_ood']]['energy_score'].mean(),
df[~df['is_ood']]['energy_score'].mean()
])
suggested_msp = np.mean([
df[df['is_ood']]['confidence'].mean(),
df[~df['is_ood']]['confidence'].mean()
])
print("\nThreshold Suggestions:")
print(f"Current data suggests an energy threshold around: {suggested_energy:.4f}")
print(f"Current data suggests an MSP threshold around: {suggested_msp:.4f}")
print("Note: These are rough estimates. For proper threshold tuning,")
print("you should use a dedicated validation set and ROC curve analysis.")
return suggested_energy, suggested_msp
def main():
parser = argparse.ArgumentParser(description="Analyze intent classification evaluation data")
parser.add_argument('--csv', default='model_evaluation.csv', help='Path to the evaluation CSV file')
parser.add_argument('--plots', default='evaluation_plots', help='Directory to save plots')
parser.add_argument('--no-plots', action='store_true', help='Skip generating plots')
args = parser.parse_args()
print(f"Loading data from {args.csv}...")
df = load_evaluation_data(args.csv)
if df is not None and len(df) > 0:
print("\n===== BASIC STATISTICS =====")
stats, method_stats = generate_basic_stats(df)
print(f"Total queries: {stats['total_queries']}")
print(f"In-distribution queries: {stats['in_distribution_count']} ({100-stats['ood_percentage']:.1f}%)")
print(f"Out-of-distribution queries: {stats['out_of_distribution_count']} ({stats['ood_percentage']:.1f}%)")
print(f"Average confidence score: {stats['avg_confidence']:.4f}")
print(f"Average energy score: {stats['avg_energy_score']:.4f}")
print("\nTop predicted intents:")
for intent, count in list(stats['top_intents'].items())[:5]:
print(f" - {intent}: {count}")
print("\n===== DETECTION METHOD COMPARISON =====")
print(method_stats)
# Analyze input texts
analyze_inputs(df)
# Suggest threshold values
suggest_thresholds(df)
# Generate plots if not disabled
if not args.no_plots:
plot_distributions(df, args.plots)
print("\nAnalysis complete!")
if __name__ == "__main__":
main() |