Update app.py

app.py

@@ -7,6 +7,8 @@ import json
 import re
 import os
 from config import DATASETS, MODELS
+import matplotlib.pyplot as plt
+import altair as alt
 
 load_dotenv()
 client = Together(api_key=os.getenv('TOGETHERAI_API_KEY'))
@@ -46,7 +48,6 @@ def get_model_response(question, options, prompt_template, model_name):
             model=model_config["model_id"],
             messages=[{"role": "user", "content": prompt}]
         )
-        
         response_text = response.choices[0].message.content.strip()
         json_match = re.search(r'\{.*\}', response_text, re.DOTALL)
         json_response = json.loads(json_match.group(0))
@@ -57,7 +58,6 @@ def get_model_response(question, options, prompt_template, model_name):
             return f"Error: Answer '{answer}' does not match any options"
         
         return answer
-            
     except Exception as e:
         return f"Error: {str(e)}"
 
@@ -70,6 +70,14 @@ def main():
     st.set_page_config(page_title="LLM Benchmarking in Healthcare", layout="wide")
     st.title("LLM Benchmarking in Healthcare")
 
+    if 'all_results' not in st.session_state:
+        st.session_state.all_results = {}
+    if 'detailed_model' not in st.session_state:
+        st.session_state.detailed_model = None
+    if 'detailed_dataset' not in st.session_state:
+        st.session_state.detailed_dataset = None
+    if 'last_evaluated_dataset' not in st.session_state:
+        st.session_state.last_evaluated_dataset = None
     col1, col2 = st.columns(2)
     with col1:
         selected_dataset = st.selectbox(
@@ -78,12 +86,15 @@ def main():
             help="Choose the dataset to evaluate on"
         )
     with col2:
-        selected_model = st.selectbox(
-            "Select Model",
+        selected_model = st.multiselect(
+            "Select Model(s)",
             options=list(MODELS.keys()),
-            help="Choose the model to evaluate"
+            default=[list(MODELS.keys())[0]],
+            help="Choose one or more models to evaluate."
         )
 
+    models_to_evaluate = selected_model
+
     default_prompt = '''You are a medical AI assistant. Please answer the following multiple choice question.
 
 Question: {question}
@@ -144,80 +155,165 @@ Important:
             st.error("Please set the TOGETHERAI_API_KEY in your .env file")
             return
 
-        progress_bar = st.progress(0)
-        status_text = st.empty()
+        progress_container = st.container()
+        with progress_container:
+            progress_bar = st.progress(0)
+            status_text = st.empty()
+            substatus_text = st.empty() 
+        
         results_container = st.container()
+        all_results = {}
         
-        results = []
-        for i in range(num_questions):
-            question = questions[i]
-            progress = (i + 1) / num_questions
-            progress_bar.progress(progress)
-            status_text.text(f"Evaluating question {i + 1}/{num_questions}")
-
-            model_response = get_model_response(
-                question['question'], 
-                question['options'],
-                prompt_template,
-                selected_model
-            )
+        total_iterations = len(models_to_evaluate) * num_questions
+        current_iteration = 0
+        
+        for model_name in models_to_evaluate:
+            substatus_text.markdown(f"<small>Evaluating model: {model_name} on {selected_dataset}</small>", unsafe_allow_html=True)
             
-            options_text = "\n".join([f"{chr(65+i)}. {opt}" for i, opt in enumerate(question['options'])])
-            formatted_prompt = prompt_template.replace("{question}", question['question']).replace("{options}", options_text)
-            raw_response = client.chat.completions.create(
-                model=MODELS[selected_model]["model_id"],
-                messages=[{"role": "user", "content": formatted_prompt}]
-            ).choices[0].message.content.strip()
+            results = []
+            for i in range(num_questions):
+                question = questions[i]
+                current_iteration += 1
+                progress = current_iteration / total_iterations
+                progress_bar.progress(progress)
+                status_text.text(f"Progress: {current_iteration}/{total_iterations} evaluations")
+
+                model_response = get_model_response(
+                    question['question'], 
+                    question['options'],
+                    prompt_template,
+                    model_name
+                )
+                
+                options_text = "\n".join([f"{chr(65+i)}. {opt}" for i, opt in enumerate(question['options'])])
+                formatted_prompt = prompt_template.replace("{question}", question['question']).replace("{options}", options_text)
+                raw_response = client.chat.completions.create(
+                    model=MODELS[model_name]["model_id"],
+                    messages=[{"role": "user", "content": formatted_prompt}],
+                    temperature=0.7
+                ).choices[0].message.content.strip()
+                
+                is_correct = evaluate_response(model_response, question['correct_answer'])
+                
+                results.append({
+                    'question': question['question'],
+                    'options': question['options'],
+                    'model_response': model_response,
+                    'raw_llm_response': raw_response,
+                    'prompt_sent': formatted_prompt,
+                    'correct_answer': question['correct_answer'],
+                    'subject': question['subject_name'],
+                    'is_correct': is_correct,
+                    'explanation': question['explanation']
+                })
             
-            is_correct = evaluate_response(model_response, question['correct_answer'])
+            all_results[model_name] = results
+        st.session_state.all_results = all_results
+        st.session_state.last_evaluated_dataset = selected_dataset
+
+
+        if st.session_state.detailed_model is None and all_results:
+            st.session_state.detailed_model = list(all_results.keys())[0]
+        if st.session_state.detailed_dataset is None:
+            st.session_state.detailed_dataset = selected_dataset
+
+        st.rerun() 
+
+    if st.session_state.all_results:
+        st.subheader("Evaluation Results")
+        
+        model_metrics = {}
+        for model_name, results in st.session_state.all_results.items():
+            df = pd.DataFrame(results)
+            metrics = {
+                'Accuracy': df['is_correct'].mean(),
+            }
+            model_metrics[model_name] = metrics
+
+        metrics_df = pd.DataFrame(model_metrics).T
+        
+        st.subheader("Model Performance Comparison")
+        
+        accuracy_chart = alt.Chart(
+            metrics_df.reset_index().melt(id_vars=['index'], value_vars=['Accuracy'])
+        ).mark_bar().encode(
+            x=alt.X('index:N', title=None, axis=None), 
+            y=alt.Y('value:Q', title='Accuracy', scale=alt.Scale(domain=[0, 1])),
+            color='index:N'
+        ).properties(
+            height=300,
+            title={
+                "text": "Model Accuracy",
+                "baseline": "bottom",
+                "orient": "bottom",  
+                "dy": 20              
+            }
+        )
+        st.altair_chart(accuracy_chart, use_container_width=True)
+
+    if st.session_state.all_results:
+        st.subheader("Detailed Results")
+        
+        def update_model():
+            st.session_state.detailed_model = st.session_state.model_select
             
-            results.append({
-                'question': question['question'],
-                'options': question['options'],
-                'model_response': model_response,
-                'raw_llm_response': raw_response,
-                'prompt_sent': formatted_prompt,
-                'correct_answer': question['correct_answer'],
-                'subject': question['subject_name'],
-                'is_correct': is_correct,
-                'explanation': question['explanation']
-            })
-
-        with results_container:
-            st.subheader("Evaluation Results")
+        def update_dataset():
+            st.session_state.detailed_dataset = st.session_state.dataset_select
+
+        col1, col2 = st.columns(2)
+        with col1:
+            selected_model_details = st.selectbox(
+                "Select model",
+                options=list(st.session_state.all_results.keys()),
+                key="model_select",
+                on_change=update_model,
+                index=list(st.session_state.all_results.keys()).index(st.session_state.detailed_model) 
+                    if st.session_state.detailed_model in st.session_state.all_results else 0
+            )
+        
+        with col2:
+            selected_dataset_details = st.selectbox(
+                "Select dataset",
+                options=[st.session_state.last_evaluated_dataset], 
+                key="dataset_select",
+                on_change=update_dataset
+            )
+
+        if selected_model_details in st.session_state.all_results:
+            results = st.session_state.all_results[selected_model_details]
             df = pd.DataFrame(results)
             accuracy = df['is_correct'].mean()
+            
             st.metric("Accuracy", f"{accuracy:.2%}")
             
             for idx, result in enumerate(results):
-                st.markdown("---")
-                st.subheader(f"Question {idx + 1} - {result['subject']}")
-                
-                st.write("Question:", result['question'])
-                st.write("Options:")
-                for i, opt in enumerate(result['options']):
-                    st.write(f"{chr(65+i)}. {opt}")
-                
-                col1, col2 = st.columns(2)
-                with col1:
-                    with st.expander("Show Prompt"):
+                with st.expander(f"Question {idx + 1} - {result['subject']}"):
+                    st.write("Question:", result['question'])
+                    st.write("Options:")
+                    for i, opt in enumerate(result['options']):
+                        st.write(f"{chr(65+i)}. {opt}")
+                    
+                    col1, col2 = st.columns(2)
+                    with col1:
+                        st.write("Prompt Used:")
                         st.code(result['prompt_sent'])
-                with col2:
-                    with st.expander("Show Raw Response"):
+                    with col2:
+                        st.write("Raw Response:")
                         st.code(result['raw_llm_response'])
-                
-                col1, col2 = st.columns(2)
-                with col1:
-                    st.write("Correct Answer:", result['correct_answer'])
-                    st.write("Model Answer:", result['model_response'])
-                with col2:
-                    if result['is_correct']:
-                        st.success("Correct!")
-                    else:
-                        st.error("Incorrect")
-                
-                with st.expander("Show Explanation"):
-                    st.write(result['explanation'])
+                    
+                    col1, col2 = st.columns(2)
+                    with col1:
+                        st.write("Correct Answer:", result['correct_answer'])
+                        st.write("Model Answer:", result['model_response'])
+                    with col2:
+                        if result['is_correct']:
+                            st.success("Correct!")
+                        else:
+                            st.error("Incorrect")
+                    
+                    st.write("Explanation:", result['explanation'])
+        else:
+            st.info(f"No results available for {selected_model_details} on {selected_dataset_details}. Please run the evaluation first.")
 
 if __name__ == "__main__":
     main()