Spaces:
Sleeping
Sleeping
| import streamlit as st | |
| import pandas as pd | |
| import numpy as np | |
| import plotly.express as px | |
| import plotly.graph_objects as go | |
| # Set page config | |
| st.set_page_config( | |
| page_title="LLM Evaluation Framework", | |
| page_icon="🤖", | |
| layout="wide" | |
| ) | |
| # Title and description | |
| st.title("🤖 LLM Quantitative Evaluation Framework") | |
| st.markdown("Data-driven decision making for Large Language Model selection") | |
| # Model data | |
| models_data = { | |
| "Model": ["GPT-4 Turbo", "Claude 3 Opus", "Claude 3 Sonnet", "Gemini Pro", "Llama 2 70B", "Mistral 7B"], | |
| "Provider": ["OpenAI", "Anthropic", "Anthropic", "Google", "Meta", "Mistral AI"], | |
| "Open Source": [False, False, False, False, True, True], | |
| "Parameters (B)": [1700, 500, 200, 340, 70, 7], | |
| "Context Length (K)": [128, 200, 200, 32, 4, 8], | |
| "Input Cost ($/1K tokens)": [0.01, 0.015, 0.003, 0.0005, 0.0007, 0.0002], | |
| "Output Cost ($/1K tokens)": [0.03, 0.075, 0.015, 0.0015, 0.0009, 0.0002], | |
| "Speed (tokens/s)": [40, 35, 45, 50, 30, 60], | |
| "Latency (s)": [2.5, 3.0, 2.0, 1.8, 4.0, 1.5], | |
| "Uptime (%)": [99.9, 99.8, 99.8, 99.9, 95.0, 94.0], | |
| "Rate Limit (req/min)": [500, 400, 600, 1000, 200, 100], | |
| "Knowledge Cutoff": ["2023-04", "2023-08", "2023-08", "2023-11", "2023-07", "2023-09"] | |
| } | |
| df = pd.DataFrame(models_data) | |
| # Sidebar for weights | |
| st.sidebar.header("🎯 Evaluation Criteria Weights") | |
| st.sidebar.markdown("Adjust the importance of each factor (total should equal 100%)") | |
| weights = {} | |
| weights['performance'] = st.sidebar.slider("Performance", 0, 50, 25) | |
| weights['cost'] = st.sidebar.slider("Cost Efficiency", 0, 50, 25) | |
| weights['speed'] = st.sidebar.slider("Speed", 0, 50, 20) | |
| weights['reliability'] = st.sidebar.slider("Reliability", 0, 50, 15) | |
| weights['compliance'] = st.sidebar.slider("Compliance/Open Source", 0, 50, 10) | |
| weights['integration'] = st.sidebar.slider("Integration Ease", 0, 50, 5) | |
| total_weights = sum(weights.values()) | |
| st.sidebar.write(f"**Total: {total_weights}%**") | |
| if total_weights != 100: | |
| st.sidebar.warning("⚠️ Weights should total 100%") | |
| # Usage scenario | |
| st.sidebar.header("📊 Usage Scenario") | |
| monthly_requests = st.sidebar.number_input("Monthly Requests", value=100000, step=10000) | |
| avg_input_tokens = st.sidebar.number_input("Avg Input Tokens", value=500, step=50) | |
| avg_output_tokens = st.sidebar.number_input("Avg Output Tokens", value=200, step=50) | |
| # Scoring functions | |
| def calculate_performance_score(row): | |
| param_score = min((row['Parameters (B)'] / 1700) * 100, 100) | |
| context_score = min((row['Context Length (K)'] / 200) * 100, 100) | |
| freshness_score = 100 if row['Knowledge Cutoff'] >= "2023-08" else 70 | |
| return param_score * 0.4 + context_score * 0.4 + freshness_score * 0.2 | |
| def calculate_cost_score(row): | |
| monthly_cost = monthly_requests * ( | |
| (avg_input_tokens / 1000) * row['Input Cost ($/1K tokens)'] + | |
| (avg_output_tokens / 1000) * row['Output Cost ($/1K tokens)'] | |
| ) | |
| max_cost = 5000 | |
| return max(0, 100 - (monthly_cost / max_cost) * 100) | |
| def calculate_speed_score(row): | |
| speed_score = (row['Speed (tokens/s)'] / 60) * 50 | |
| latency_score = max(0, 50 - (row['Latency (s)'] / 5) * 50) | |
| return speed_score + latency_score | |
| def calculate_reliability_score(row): | |
| uptime_score = (row['Uptime (%)'] / 100) * 60 | |
| rate_limit_score = min((row['Rate Limit (req/min)'] / 1000) * 40, 40) | |
| return uptime_score + rate_limit_score | |
| def calculate_compliance_score(row): | |
| open_source_bonus = 40 if row['Open Source'] else 0 | |
| return open_source_bonus + 60 | |
| def calculate_integration_score(row): | |
| api_score = 70 if not row['Open Source'] else 30 | |
| support_score = 30 if row['Provider'] in ["OpenAI", "Google"] else 20 | |
| return min(api_score + support_score, 100) | |
| # Calculate scores | |
| df['Performance Score'] = df.apply(calculate_performance_score, axis=1) | |
| df['Cost Score'] = df.apply(calculate_cost_score, axis=1) | |
| df['Speed Score'] = df.apply(calculate_speed_score, axis=1) | |
| df['Reliability Score'] = df.apply(calculate_reliability_score, axis=1) | |
| df['Compliance Score'] = df.apply(calculate_compliance_score, axis=1) | |
| df['Integration Score'] = df.apply(calculate_integration_score, axis=1) | |
| # Calculate weighted overall score | |
| if total_weights > 0: | |
| df['Overall Score'] = ( | |
| df['Performance Score'] * weights['performance'] / 100 + | |
| df['Cost Score'] * weights['cost'] / 100 + | |
| df['Speed Score'] * weights['speed'] / 100 + | |
| df['Reliability Score'] * weights['reliability'] / 100 + | |
| df['Compliance Score'] * weights['compliance'] / 100 + | |
| df['Integration Score'] * weights['integration'] / 100 | |
| ) * (100 / total_weights) | |
| else: | |
| df['Overall Score'] = 0 | |
| # Sort by overall score | |
| df_sorted = df.sort_values('Overall Score', ascending=False).reset_index(drop=True) | |
| # Calculate monthly costs | |
| df_sorted['Monthly Cost ($)'] = monthly_requests * ( | |
| (avg_input_tokens / 1000) * df_sorted['Input Cost ($/1K tokens)'] + | |
| (avg_output_tokens / 1000) * df_sorted['Output Cost ($/1K tokens)'] | |
| ) | |
| # Main content area | |
| col1, col2 = st.columns([2, 1]) | |
| with col1: | |
| st.header("🏆 Model Rankings") | |
| # Display top 3 models with medals | |
| medals = ["🥇", "🥈", "🥉"] | |
| for i in range(min(3, len(df_sorted))): | |
| with st.container(): | |
| st.markdown(f""" | |
| <div style="border: 2px solid {'gold' if i==0 else 'silver' if i==1 else '#CD7F32'}; | |
| border-radius: 10px; padding: 15px; margin: 10px 0; | |
| background-color: {'#FFF8DC' if i==0 else '#F8F8FF' if i==1 else '#FDF5E6'}"> | |
| <h3>{medals[i]} {df_sorted.iloc[i]['Model']} - {df_sorted.iloc[i]['Provider']}</h3> | |
| <p><strong>Overall Score: {df_sorted.iloc[i]['Overall Score']:.1f}/100</strong></p> | |
| <p>Monthly Cost: ${df_sorted.iloc[i]['Monthly Cost ($)']:.2f} | | |
| Parameters: {df_sorted.iloc[i]['Parameters (B)']}B | | |
| Context: {df_sorted.iloc[i]['Context Length (K)']}K tokens</p> | |
| </div> | |
| """, unsafe_allow_html=True) | |
| with col2: | |
| st.header("💰 Cost Analysis") | |
| # Cost comparison chart | |
| fig_cost = px.bar( | |
| df_sorted, | |
| x='Monthly Cost ($)', | |
| y='Model', | |
| orientation='h', | |
| title="Monthly Cost Comparison", | |
| color='Monthly Cost ($)', | |
| color_continuous_scale='RdYlGn_r' | |
| ) | |
| fig_cost.update_layout(height=400) | |
| st.plotly_chart(fig_cost, use_container_width=True) | |
| # Detailed comparison table | |
| st.header("📊 Detailed Comparison") | |
| display_cols = ['Model', 'Provider', 'Overall Score', 'Monthly Cost ($)', | |
| 'Performance Score', 'Cost Score', 'Speed Score', | |
| 'Reliability Score', 'Compliance Score', 'Integration Score'] | |
| st.dataframe(df_sorted[display_cols].round(1), use_container_width=True) | |
| # Radar chart for top 3 models | |
| st.header("🎯 Multi-Dimensional Analysis") | |
| categories = ['Performance', 'Cost', 'Speed', 'Reliability', 'Compliance', 'Integration'] | |
| fig_radar = go.Figure() | |
| colors = ['gold', 'silver', '#CD7F32'] | |
| for i in range(min(3, len(df_sorted))): | |
| model = df_sorted.iloc[i] | |
| values = [ | |
| model['Performance Score'], | |
| model['Cost Score'], | |
| model['Speed Score'], | |
| model['Reliability Score'], | |
| model['Compliance Score'], | |
| model['Integration Score'] | |
| ] | |
| fig_radar.add_trace(go.Scatterpolar( | |
| r=values, | |
| theta=categories, | |
| fill='toself', | |
| name=model['Model'], | |
| line_color=colors[i] | |
| )) | |
| fig_radar.update_layout( | |
| polar=dict( | |
| radialaxis=dict( | |
| visible=True, | |
| range=[0, 100] | |
| )), | |
| showlegend=True, | |
| title="Top 3 Models - Multi-Dimensional Comparison" | |
| ) | |
| st.plotly_chart(fig_radar, use_container_width=True) | |
| # Methodology | |
| st.header("🔬 Scoring Methodology") | |
| st.markdown(""" | |
| **Performance Score (0-100):** | |
| - Parameters: 40% weight (normalized to GPT-4's 1.7T) | |
| - Context Length: 40% weight (normalized to 200K tokens) | |
| - Knowledge Freshness: 20% weight (post-Aug 2023 = 100, else 70) | |
| **Cost Efficiency Score (0-100):** | |
| - Based on total monthly cost for your usage scenario | |
| - Normalized against $5,000/month baseline | |
| - Higher score = lower cost | |
| **Speed Score (0-100):** | |
| - Tokens/second: 50% weight (normalized to 60 tok/s) | |
| - Latency (inverse): 50% weight (normalized to 5s max) | |
| **Reliability Score (0-100):** | |
| - Uptime percentage: 60% weight | |
| - Rate limits: 40% weight (normalized to 1000 req/min) | |
| **Compliance Score (0-100):** | |
| - Open source availability: 40 points | |
| - License permissiveness: 60 points | |
| **Integration Score (0-100):** | |
| - API availability: 70 points (closed source) or 30 points (open source) | |
| - Provider support quality: 30 points | |
| """) |