Enhanced evaluation system

gradio_full_llm_eval.py

@@ -1,336 +1,254 @@
+# gradio_full_llm_eval.py – Final Updated Version with ATS Scoring and Visualized UI
 import gradio as gr
 import os
 import pandas as pd
-import plotly.graph_objects as go
 import plotly.express as px
-from datetime import datetime
+import plotly.graph_objects as go
+import plotly.io as pio
+import zipfile
 import json
+from datetime import datetime
+from dotenv import load_dotenv
 
-# Import modules from existing files
-from response_generator import generate_all_responses
-from round_robin_evaluator import comprehensive_round_robin_evaluation, save_comprehensive_results
+from response_generator import generate_all_responses_with_reasoning
+from round_robin_evaluator import comprehensive_round_robin_evaluation
 from realtime_detector import is_realtime_prompt
 from search_fallback import get_google_snippets
-from llm_prompt_eval_analysis import generate_visualizations, analyze_evaluation_data
 
-# Load environment variables
-from dotenv import load_dotenv
 load_dotenv()
+pio.kaleido.scope.default_format = "png"
 
-def check_api_keys():
-    """Check if all required API keys are available."""
-    keys_status = {}
-    
-    # Check OpenAI
-    openai_key = os.getenv("OPENAI_API_KEY")
-    keys_status["OpenAI (GPT-4)"] = "Available" if openai_key else "Missing"
-    
-    # Check Claude
-    claude_key = os.getenv("CLAUDE_API_KEY")
-    keys_status["Claude 3"] = "Available" if claude_key else "Missing"
-    
-    # Check Gemini
-    gemini_key = os.getenv("GEMINI_API_KEY")
-    keys_status["Gemini 1.5"] = "Available" if gemini_key else "Missing"
-    
-    # Check Google Search (optional)
-    google_key = os.getenv("GOOGLE_API_KEY")
-    google_cse = os.getenv("GOOGLE_CSE_ID")
-    keys_status["Google Search"] = "Available" if (google_key and google_cse) else "Missing"
-    
-    return keys_status
-
-def process_prompt(prompt, enable_realtime_detection, enable_evaluation, enable_analysis):
-    """Process a prompt through the complete pipeline."""
-    if not prompt.strip():
-        return "Please enter a prompt.", None, None, None, None, None
-    
-    results = {
-        "prompt": prompt,
-        "responses": {},
-        "evaluation": None,
-        "analysis": None,
-        "search_results": None,
-        "is_realtime": False
-    }
-    
-    # Step 1: Check if real-time detection is needed
-    if enable_realtime_detection:
-        try:
-            results["is_realtime"] = is_realtime_prompt(prompt)
-            if results["is_realtime"]:
-                # Get Google search results
-                search_results = get_google_snippets(prompt)
-                results["search_results"] = search_results
-                # Enhance prompt with search results
-                enhanced_prompt = f"{prompt}\n\nRecent information: {search_results}"
-            else:
-                enhanced_prompt = prompt
-        except Exception as e:
-            print(f"Real-time detection error: {e}")
-            enhanced_prompt = prompt
-    else:
-        enhanced_prompt = prompt
-    
-    # Step 2: Generate responses from all models
+metrics = ['helpfulness', 'correctness', 'coherence', 'tone_score',
+           'accuracy', 'relevance', 'completeness', 'clarity']
+
+def extract_text_from_resume(file):
+    ext = os.path.splitext(file.name)[1].lower()
+    if ext == ".pdf":
+        import fitz
+        with fitz.open(file.name) as doc:
+            return "\n".join(page.get_text() for page in doc)
+    elif ext == ".docx":
+        import docx
+        doc = docx.Document(file.name)
+        return "\n".join(p.text for p in doc.paragraphs)
+    elif ext == ".txt":
+        return file.read().decode('utf-8')
+    return ""
+
+def ats_score_advanced(response, resume, jd):
+    prompt = f"""
+You are a professional ATS scoring engine. Compare the generated response to the candidate's resume and job description using:
+1. Keyword Matching
+2. Section Weighting
+3. Semantic Similarity
+4. Recency/Frequency
+5. Penalty Detection
+6. Aggregation
+
+Resume:
+{resume}
+
+Job Description:
+{jd}
+
+Response:
+{response}
+
+Return JSON:
+{{"ats_score": <0-100>, "strengths": ["..."], "gaps": ["..."], "suggestions": ["..."]}}
+"""
+    from openai import OpenAI
+    openai_client = OpenAI(api_key=os.getenv("OPENAI_API_KEY"))
     try:
-        responses = generate_all_responses(enhanced_prompt)
-        results["responses"] = responses
-    except Exception as e:
-        return f"Error generating responses: {e}", None, None, None, None, None
-    
-    # Step 3: Perform evaluation if requested
-    if enable_evaluation and responses:
-        try:
-            evaluation_results = comprehensive_round_robin_evaluation(responses, prompt)
-            results["evaluation"] = evaluation_results
-            
-            # Save results
-            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-            csv_file = save_comprehensive_results(evaluation_results, prompt, timestamp)
-            
-        except Exception as e:
-            print(f"Evaluation error: {e}")
-    
-    # Step 4: Generate analysis if requested
-    if enable_analysis and results["evaluation"]:
-        try:
-            # Create a temporary DataFrame for analysis
-            analysis_data = []
-            for model, data in results["evaluation"].items():
-                for evaluator, eval_data in data.get('evaluations', {}).items():
+        res = openai_client.chat.completions.create(
+            model="gpt-4",
+            messages=[{"role": "user", "content": prompt}],
+            temperature=0
+        )
+        return json.loads(res.choices[0].message.content.strip())
+    except:
+        return {"ats_score": 50, "strengths": [], "gaps": [], "suggestions": ["Check formatting."]}
+
+def create_visualizations(df, results_dir):
+    image_files = []
+    summary = df.groupby('target_model')[metrics].mean().reset_index()
+
+    heatmap = px.imshow(summary[metrics].values, x=metrics, y=summary['target_model'],
+                        labels=dict(x="Metric", y="Model", color="Score"),
+                        title="Heatmap: Metrics Across Models", color_continuous_scale='Viridis')
+    heatmap_path = os.path.join(results_dir, "heatmap.png")
+    heatmap.write_image(heatmap_path)
+    image_files.append(heatmap_path)
+
+    radar = go.Figure()
+    for _, row in summary.iterrows():
+        radar.add_trace(go.Scatterpolar(r=list(row[metrics]), theta=metrics, fill='toself', name=row['target_model']))
+    radar.update_layout(title="Radar Chart: Model Score Profiles", polar=dict(radialaxis=dict(visible=True, range=[0, 1])))
+    radar_path = os.path.join(results_dir, "radar.png")
+    radar.write_image(radar_path)
+    image_files.append(radar_path)
+
+    bar = px.bar(summary.melt(id_vars='target_model'), x='variable', y='value', color='target_model', barmode='group',
+                 title="Bar Chart: Metric Comparison")
+    bar_path = os.path.join(results_dir, "barchart.png")
+    bar.write_image(bar_path)
+    image_files.append(bar_path)
+
+    return (heatmap, radar, bar), image_files
+
+def format_ats_feedback(score, strengths, gaps, suggestions):
+    color = "🟢" if score >= 75 else "🟡" if score >= 50 else "🔴"
+    return f"""
+### ATS Match Score: ~{score}% {color}
+
+#### **Strengths / High Matches:**
+{chr(10).join([f"* {s}" for s in strengths]) if strengths else "* None found."}
+
+#### **Partial or Missing:**
+{chr(10).join([f"* {g}" for g in gaps]) if gaps else "* None mentioned."}
+
+#### **How to Improve ATS Score:**
+{chr(10).join([f"1. {s}" for s in suggestions]) if suggestions else "1. Add missing skills."}
+"""
+
+def process_prompt(prompt, enable_realtime, enable_eval, enable_analysis, user_file, model_selection):
+    selected_models = [m for m, enabled in zip(["GPT-4", "Claude 3", "Gemini 1.5"], model_selection) if enabled]
+    resume_text = ""
+    batch_mode = user_file and user_file.name.endswith(".csv")
+    resume_mode = user_file and user_file.name.lower().endswith(('.pdf', '.docx', '.txt'))
+
+    prompts = [prompt]
+    ats_summary_texts = []
+    search_results = ""
+
+    if batch_mode:
+        df_batch = pd.read_csv(user_file.name)
+        prompts = df_batch['prompt'].dropna().tolist()
+    elif resume_mode:
+        resume_text = extract_text_from_resume(user_file)
+
+    all_rows, all_charts = [], []
+    zip_path, ats_table_markdown = None, ""
+
+    for prompt_text in prompts:
+        search_results = get_google_snippets(prompt_text) if enable_realtime and is_realtime_prompt(prompt_text) else ""
+        final_prompt = f"{prompt_text}\n\nRecent info: {search_results}" if search_results else prompt_text
+        responses = generate_all_responses_with_reasoning(final_prompt, selected_models)
+
+        ats_rows = []
+        for model in responses:
+            model_resp = responses[model]['response']
+            if resume_text:
+                ats_result = ats_score_advanced(model_resp, resume_text, prompt_text)
+                feedback = format_ats_feedback(ats_result['ats_score'], ats_result.get('strengths', []), ats_result.get('gaps', []), ats_result.get('suggestions', []))
+                responses[model]['ats_embed'] = f"###  Response\n\n{model_resp}\n\n---\n\n###  ATS Evaluation\n\n{feedback}"
+                ats_rows.append(f"| {model} | {ats_result['ats_score']} | {', '.join(ats_result.get('strengths', []))} | {', '.join(ats_result.get('suggestions', []))} |")
+            else:
+                responses[model]['ats_embed'] = f"###  Response\n\n{model_resp}\n\n---\n\n**Explainability:**\n{responses[model]['reasoning']}"
+        if ats_rows:
+            ats_table_markdown = "| Model | Score | Strengths | Suggestions |\n|-------|-------|-----------|-------------|\n" + "\n".join(ats_rows)
+
+        if enable_eval:
+            compact = {k: v['response'] for k, v in responses.items()}
+            eval_result = comprehensive_round_robin_evaluation(compact, final_prompt)
+            for model, data in eval_result.items():
+                for evaluator, scores in data['evaluations'].items():
                     row = {
+                        'prompt': prompt_text,
                         'target_model': model,
                         'evaluator': evaluator,
-                        'helpfulness': eval_data.get('helpfulness', 0.5),
-                        'correctness': eval_data.get('correctness', 0.5),
-                        'coherence': eval_data.get('coherence', 0.5),
-                        'clarity': eval_data.get('clarity', 0.5),
-                        'response': data.get('response', '')
+                        'response': responses[model]['response'],
+                        'explainability': responses[model]['reasoning']
                     }
-                    analysis_data.append(row)
-            
-            if analysis_data:
-                df = pd.DataFrame(analysis_data)
-                results["analysis"] = df
-                
-        except Exception as e:
-            print(f"Analysis error: {e}")
-    
-    return format_results(results)
-
-def format_results(results):
-    """Format results for Gradio display."""
-    prompt = results["prompt"]
-    responses = results["responses"]
-    evaluation = results["evaluation"]
-    analysis = results["analysis"]
-    search_results = results["search_results"]
-    is_realtime = results["is_realtime"]
-    
-    # Format responses
-    responses_text = ""
-    if responses:
-        responses_text = "MODEL RESPONSES:\n" + "="*50 + "\n"
-        for model, response in responses.items():
-            responses_text += f"\n{model}:\n{'-'*20}\n{response}\n"
-    else:
-        responses_text = "No responses generated. Check API keys."
-    
-    # Format evaluation results
-    evaluation_text = ""
-    if evaluation:
-        evaluation_text = "EVALUATION RESULTS:\n" + "="*50 + "\n"
-        for model, data in evaluation.items():
-            avg_scores = data.get('average_scores', {})
-            evaluation_text += f"\n{model} Average Scores:\n"
-            for metric, score in avg_scores.items():
-                evaluation_text += f"  {metric}: {score}\n"
-            evaluation_text += f"  Evaluated by: {list(data.get('evaluations', {}).keys())}\n"
-    else:
-        evaluation_text = "No evaluation performed."
-    
-    # Format search results
-    search_text = ""
-    if search_results and is_realtime:
-        search_text = "REAL-TIME SEARCH RESULTS:\n" + "="*50 + "\n"
-        search_text += search_results
-    elif is_realtime:
-        search_text = "Real-time query detected but search results unavailable."
-    else:
-        search_text = "Not a real-time query."
-    
-    # Create visualizations
-    charts = []
-    if analysis is not None and not analysis.empty:
-        charts = create_visualizations(analysis)
-    
-    return responses_text, evaluation_text, search_text, charts
-
-def create_visualizations(df):
-    """Create Plotly visualizations for the analysis."""
-    charts = []
-    
-    try:
-        # 1. Model Performance Comparison
-        if 'target_model' in df.columns:
-            metrics = ['helpfulness', 'correctness', 'coherence', 'clarity']
-            
-            for metric in metrics:
-                if metric in df.columns:
-                    fig = px.box(df, x='target_model', y=metric, 
-                               title=f'{metric.title()} Scores by Model',
-                               color='target_model')
-                    fig.update_layout(showlegend=False)
-                    charts.append(fig)
-        
-        # 2. Evaluator Bias Analysis
-        if 'evaluator' in df.columns:
-            metrics = ['helpfulness', 'correctness', 'coherence', 'clarity']
-            
-            for metric in metrics:
-                if metric in df.columns:
-                    fig = px.box(df, x='evaluator', y=metric,
-                               title=f'{metric.title()} Scores by Evaluator',
-                               color='evaluator')
-                    fig.update_layout(showlegend=False)
-                    charts.append(fig)
-        
-        # 3. Heatmap of Cross-Evaluations
-        if 'target_model' in df.columns and 'evaluator' in df.columns and 'helpfulness' in df.columns:
-            pivot_data = df.pivot_table(
-                values='helpfulness',
-                index='target_model',
-                columns='evaluator',
-                aggfunc='mean'
-            ).fillna(0)
-            
-            fig = px.imshow(pivot_data.values,
-                           x=pivot_data.columns,
-                           y=pivot_data.index,
-                           title='Cross-Evaluation Heatmap (Helpfulness)',
-                           color_continuous_scale='RdYlBu_r',
-                           aspect='auto')
-            fig.update_layout(xaxis_title='Evaluator', yaxis_title='Target Model')
-            charts.append(fig)
-    
-    except Exception as e:
-        print(f"Visualization error: {e}")
-    
-    return charts
-
-def export_results(responses_text, evaluation_text, search_text):
-    """Export results to a text file."""
-    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-    filename = f"results/export_{timestamp}.txt"
-    
-    os.makedirs("results", exist_ok=True)
-    
-    with open(filename, 'w', encoding='utf-8') as f:
-        f.write("LLM COMPARISON RESULTS\n")
-        f.write("="*50 + "\n")
-        f.write(f"Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n\n")
-        
-        f.write(responses_text + "\n\n")
-        f.write(evaluation_text + "\n\n")
-        f.write(search_text + "\n\n")
-    
-    return f"Results exported to {filename}"
-
-# Create Gradio interface
+                    row.update({k: scores.get(k, 0.5) for k in metrics})
+                    row.update({f"avg_{k}": data['average_scores'].get(k, 0.5) for k in metrics})
+                    all_rows.append(row)
+
+    df_all = pd.DataFrame(all_rows)
+    if not df_all.empty:
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        results_dir = f"results/batch_{timestamp}"
+        os.makedirs(results_dir, exist_ok=True)
+        csv_path = os.path.join(results_dir, "evaluation.csv")
+        df_all.to_csv(csv_path, index=False)
+        (heatmap, radar, bar), chart_paths = create_visualizations(df_all, results_dir)
+        all_charts = [heatmap, radar, bar]
+        zip_path = os.path.join(results_dir, "bundle.zip")
+        with zipfile.ZipFile(zip_path, 'w') as zipf:
+            zipf.write(csv_path, arcname="evaluation.csv")
+            for chart in chart_paths:
+                zipf.write(chart, arcname=os.path.basename(chart))
+        if batch_mode:
+            df_batch['ATS Summary'] = ats_summary_texts
+            df_batch.to_csv(os.path.join(results_dir, "batch_prompts_output.csv"), index=False)
+            zipf.write(os.path.join(results_dir, "batch_prompts_output.csv"), arcname="batch_prompts_output.csv")
+
+    return tuple(
+        responses[model].get('ats_embed', responses[model]['response']) for model in ["GPT-4", "Claude 3", "Gemini 1.5"]
+    ) + (
+        search_results or "N/A",
+        *all_charts,
+        df_all[['target_model', 'evaluator'] + metrics] if not df_all.empty else pd.DataFrame(),
+        ats_table_markdown,
+        zip_path
+    )
+
+def download_results(path):
+    return path if path and os.path.exists(path) else None
+
 def create_interface():
-    """Create the Gradio interface."""
-    
-    # Check API keys
-    api_status = check_api_keys()
-    api_status_text = "API KEY STATUS:\n" + "="*30 + "\n"
-    for service, status in api_status.items():
-        api_status_text += f"{service}: {status}\n"
-    
-    with gr.Blocks(title="LLM Comparison Hub", theme=gr.themes.Soft()) as interface:
-        gr.Markdown("# LLM Comparison Hub")
-        gr.Markdown("Compare responses from GPT-4, Claude 3, and Gemini 1.5 with comprehensive evaluation and analysis.")
-        
+    with gr.Blocks(title="LLM Comparison Hub") as demo:
+        gr.Markdown("""
+# LLM Comparison Hub
+This app compares LLM responses using round-robin evaluations, with real-time query detection and comprehensive analysis.
+
+**How to use:**
+- Enter a prompt (JD or query)
+- Upload a resume (PDF/DOCX/TXT) or a CSV with prompts
+- Select models
+- Click evaluate
+
+**Features:**
+- Real-time web search fallback
+- Resume vs JD ATS scoring (optional)
+- Batch CSV prompt evaluation
+- Visualizations (Heatmap, Radar, Bar)
+- ZIP export of all results
+""")
         with gr.Row():
-            with gr.Column(scale=2):
-                # Input section
-                gr.Markdown("## Input")
-                prompt_input = gr.Textbox(
-                    label="Enter your prompt",
-                    placeholder="Type your question or prompt here...",
-                    lines=4
-                )
-                
-                with gr.Row():
-                    realtime_checkbox = gr.Checkbox(label="Enable real-time detection", value=True)
-                    evaluation_checkbox = gr.Checkbox(label="Enable evaluation", value=True)
-                    analysis_checkbox = gr.Checkbox(label="Enable analysis", value=True)
-                
-                process_btn = gr.Button("Process Prompt", variant="primary")
-                
-                # API status
-                gr.Markdown("## API Status")
-                api_status_display = gr.Textbox(
-                    value=api_status_text,
-                    label="API Keys",
-                    lines=len(api_status) + 3,
-                    interactive=False
-                )
-            
-            with gr.Column(scale=3):
-                # Output section
-                gr.Markdown("## Results")
-                
+            with gr.Column():
+                prompt = gr.Textbox(label="Enter Prompt", lines=4)
+                user_file = gr.File(label="Upload Resume or CSV", file_types=[".pdf", ".docx", ".txt", ".csv"])
+                model_selector = gr.CheckboxGroup(label="Select Models", choices=["GPT-4", "Claude 3", "Gemini 1.5"], value=["GPT-4", "Claude 3", "Gemini 1.5"])
+                enable_realtime = gr.Checkbox(label="Enable real-time detection", value=True)
+                enable_eval = gr.Checkbox(label="Enable evaluation", value=True)
+                enable_analysis = gr.Checkbox(label="Enable analysis", value=True)
+                submit = gr.Button("Run Evaluation")
+
+            with gr.Column():
                 with gr.Tabs():
-                    with gr.TabItem("Responses"):
-                        responses_output = gr.Textbox(
-                            label="Model Responses",
-                            lines=15,
-                            interactive=False
-                        )
-                    
-                    with gr.TabItem("Evaluation"):
-                        evaluation_output = gr.Textbox(
-                            label="Evaluation Results",
-                            lines=15,
-                            interactive=False
-                        )
-                    
-                    with gr.TabItem("Search Results"):
-                        search_output = gr.Textbox(
-                            label="Real-time Search Results",
-                            lines=10,
-                            interactive=False
-                        )
-                    
-                    with gr.TabItem("Visualizations"):
-                        charts_output = gr.Plot(label="Analysis Charts")
-                
-                # Export button
-                export_btn = gr.Button("Export Results")
-                export_output = gr.Textbox(label="Export Status", interactive=False)
-        
-        # Event handlers
-        process_btn.click(
+                    with gr.Tab("GPT-4"): gpt_out = gr.Markdown()
+                    with gr.Tab("Claude 3"): claude_out = gr.Markdown()
+                    with gr.Tab("Gemini 1.5"): gemini_out = gr.Markdown()
+                    with gr.Tab("Evaluation Table"): df_out = gr.Dataframe()
+                    with gr.Tab("ATS Evaluation"): ats_summary = gr.Markdown()
+                    with gr.Tab("Search Results"): search_out = gr.Markdown()
+                    with gr.Tab("Visualizations"):
+                        heatmap_plot = gr.Plot()
+                        radar_plot = gr.Plot()
+                        bar_plot = gr.Plot()
+                export_btn = gr.Button("Download ZIP Bundle")
+                zip_output = gr.File(file_types=[".zip"], interactive=False, visible=True)
+
+        submit.click(
             fn=process_prompt,
-            inputs=[prompt_input, realtime_checkbox, evaluation_checkbox, analysis_checkbox],
-            outputs=[responses_output, evaluation_output, search_output, charts_output]
+            inputs=[prompt, enable_realtime, enable_eval, enable_analysis, user_file, model_selector],
+            outputs=[gpt_out, claude_out, gemini_out, search_out, heatmap_plot, radar_plot, bar_plot, df_out, ats_summary, zip_output]
         )
-        
-        export_btn.click(
-            fn=export_results,
-            inputs=[responses_output, evaluation_output, search_output],
-            outputs=[export_output]
-        )
-    
-    return interface
+        export_btn.click(download_results, inputs=[zip_output], outputs=[zip_output])
+
+    return demo
 
 if __name__ == "__main__":
-    # Create and launch the interface
-    interface = create_interface()
-    interface.launch(
-        server_name="0.0.0.0",
-        server_port=7860,
-        share=False,
-        debug=True
-    ) 
+    app = create_interface()
+    app.launch(server_name="0.0.0.0", server_port=7860, share=False, debug=True)

information

@@ -0,0 +1,136 @@
+LLM-Compare-Hub Project File Structure and Use Cases
+====================================================
+
+Core Application Files
+---------------------
+
+gradio_full_llm_eval.py (1.0B)
+- Use Case: Main Gradio web interface
+- Function: Orchestrates the entire application, provides user-friendly web UI
+- Features: Prompt input, response display, evaluation results, analysis generation
+- Status: Modular - delegates all logic to other files
+
+response_generator.py (8.6KB)
+- Use Case: LLM response generation and comparison
+- Function: Generates responses from GPT-4, Claude 3, and Gemini 1.5
+- Features: Side-by-side response comparison, batch processing, optional evaluation
+- Status: Standalone tool + used by Gradio app
+
+round_robin_evaluator.py (9.3KB)
+- Use Case: Comprehensive model evaluation system
+- Function: Each model evaluates all other models (GPT-4 evaluates Claude/Gemini, etc.)
+- Features: Multi-metric scoring, CSV export, detailed reasoning
+- Status: Core evaluation engine used by all other components
+
+llm_prompt_eval_analysis.py (12KB)
+- Use Case: Data analysis and visualization
+- Function: Analyzes evaluation results, generates charts and reports
+- Features: Statistical analysis, correlation matrices, performance comparisons
+- Status: Standalone analysis tool + used by Gradio app
+
+llm_response_logger.py (6.3KB)
+- Use Case: Quick testing and logging tool
+- Function: Rapid testing of all models with single or batch prompts
+- Features: Quick evaluation, CSV export, batch processing
+- Status: Standalone testing tool + used by Gradio app
+
+Supporting Modules
+-----------------
+
+realtime_detector.py (923B)
+- Use Case: Real-time query detection
+- Function: Determines if a prompt needs current information
+- Features: Uses GPT-3.5-turbo to classify real-time vs. general queries
+- Status: Utility module used by response generation
+
+search_fallback.py (1.6KB)
+- Use Case: Google search integration
+- Function: Fetches current information for real-time queries
+- Features: Google Custom Search API integration, result formatting
+- Status: Utility module used by response generation
+
+Configuration & Documentation
+----------------------------
+
+requirements.txt (232B)
+- Use Case: Python dependencies
+- Function: Lists all required packages and versions
+- Features: Gradio, OpenAI, Anthropic, Google AI, pandas, matplotlib, etc.
+- Status: Essential for project setup
+
+.env (not shown - should exist)
+- Use Case: API key configuration
+- Function: Stores all API keys securely
+- Features: OpenAI, Claude, Gemini, Google Search API keys
+- Status: Essential for functionality
+
+.gitignore (661B)
+- Use Case: Git version control
+- Function: Excludes sensitive files from version control
+- Features: API keys, results, cache files, etc.
+- Status: Essential for security
+
+README.md (4.1KB)
+- Use Case: Project documentation
+- Function: Setup instructions, usage guide, feature descriptions
+- Features: Installation, configuration, usage examples
+- Status: Essential for users and collaborators
+
+Testing & Development
+--------------------
+
+test_standalone_tools.py (4.1KB)
+- Use Case: Testing and demonstration
+- Function: Shows how to use all standalone tools
+- Features: Quick start guide, sample prompts, tool explanations
+- Status: Development/testing tool
+
+__pycache__/ (directory)
+- Use Case: Python cache
+- Function: Stores compiled Python bytecode
+- Features: Improves import performance
+- Status: Auto-generated, can be deleted
+
+Generated Files (when running the app)
+-------------------------------------
+
+results/ (directory - created when needed)
+- Use Case: Evaluation results storage
+- Function: Stores CSV files with evaluation data
+- Features: Timestamped files, comprehensive evaluation data
+- Status: Auto-generated during evaluation
+
+analysis_results/ (directory - created when needed)
+- Use Case: Analysis output storage
+- Function: Stores charts, reports, and visualizations
+- Features: Performance charts, correlation matrices, analysis reports
+- Status: Auto-generated during analysis
+
+Project Summary
+==============
+
+Your project has a clean, modular architecture with:
+- 4 core functional modules (response generation, evaluation, analysis, logging)
+- 2 utility modules (real-time detection, search integration)
+- 1 main interface (Gradio web app)
+- Complete configuration (requirements, environment, documentation)
+- Testing tools for development and demonstration
+
+All files serve specific purposes and work together to provide a comprehensive LLM comparison and evaluation system.
+
+Key Features:
+- Multi-model response generation (GPT-4, Claude 3, Gemini 1.5)
+- Comprehensive round-robin evaluation system
+- Real-time query detection and search integration
+- Advanced data analysis and visualization
+- Batch processing capabilities
+- Clean, production-ready code without emojis
+- Modular architecture for maintainability
+- Complete web interface via Gradio
+- Standalone tools for automation and testing
+
+Usage:
+1. Set up API keys in .env file
+2. Install dependencies: pip install -r requirements.txt
+3. Run main app: python gradio_full_llm_eval.py
+4. Or use standalone tools for specific tasks 

requirements.txt

@@ -10,4 +10,7 @@ python-dotenv>=1.0.0
 requests>=2.31.0
 tqdm>=4.65.0
 scikit-learn>=1.3.0
-plotly>=5.18.0 
+plotly>=5.18.0
+kaleido>=0.2.1
+PyMuPDF>=1.23.0
+python-docx>=1.1.0

response_generator.py

@@ -1,11 +1,8 @@
-import csv
 import os
 from dotenv import load_dotenv
 from openai import OpenAI
 import anthropic
 import google.generativeai as genai
-from round_robin_evaluator import comprehensive_round_robin_evaluation, save_comprehensive_results
-from datetime import datetime
 
 # Load API keys from .env
 load_dotenv()
@@ -14,20 +11,50 @@ anthropic_client = anthropic.Anthropic(api_key=os.getenv("CLAUDE_API_KEY"))
 genai.configure(api_key=os.getenv("GEMINI_API_KEY"))
 
 def get_gpt4_response(prompt):
-    """Get response from GPT-4."""
     try:
+        if "Recent info:" in prompt:
+            user_prompt, realtime_info = prompt.split("Recent info:", 1)
+            messages = [
+                {
+                    "role": "system",
+                    "content": (
+                        "You are an expert ATS evaluator. You are comparing a job description (JD) and a resume to produce an ATS score. "
+                        "Highlight matches, gaps, suggestions for improvement, and an overall score."
+                    )
+                },
+                {"role": "user", "content": user_prompt.strip()},
+                {
+                    "role": "user",
+                    "content": (
+                        f"Here is some recent real-time context for your reference:\n\n{realtime_info.strip()}\n\n"
+                        "Based on this, tailor your response as if the data is accurate."
+                    )
+                }
+            ]
+        else:
+            messages = [
+                {
+                    "role": "system",
+                    "content": (
+                        "You are an expert ATS evaluator. You are comparing a job description (JD) and a resume to produce an ATS score. "
+                        "Highlight matches, gaps, suggestions for improvement, and an overall score."
+                    )
+                },
+                {"role": "user", "content": prompt}
+            ]
+
         response = openai_client.chat.completions.create(
             model="gpt-4",
-            messages=[{"role": "user", "content": prompt}],
+            messages=messages,
             temperature=0.7
         )
         return response.choices[0].message.content
+
     except Exception as e:
         print(f"Error with GPT-4: {e}")
-        return None
+        return "GPT-4 failed."
 
 def get_claude_response(prompt):
-    """Get response from Claude."""
     try:
         response = anthropic_client.messages.create(
             model="claude-3-opus-20240229",
@@ -38,226 +65,39 @@ def get_claude_response(prompt):
         return response.content[0].text
     except Exception as e:
         print(f"Error with Claude 3: {e}")
-        return None
+        return "Claude 3 failed."
 
 def get_gemini_response(prompt):
-    """Get response from Gemini."""
     try:
         model = genai.GenerativeModel("gemini-1.5-pro")
         response = model.generate_content(prompt)
         return response.text
     except Exception as e:
         print(f"Error with Gemini: {e}")
-        return None
+        return "Gemini 1.5 failed."
 
-def display_responses_side_by_side(responses, prompt):
-    """Display responses in a formatted side-by-side comparison."""
-    print("\n" + "="*80)
-    print(f"PROMPT: {prompt}")
-    print("="*80)
-    
-    models = list(responses.keys())
-    if len(models) == 0:
-        print("No responses to display")
-        return
-    
-    # Display responses
-    for i, model in enumerate(models, 1):
-        response = responses[model]
-        print(f"\n{i}. {model} RESPONSE:")
-        print("-" * 40)
-        print(response)
-        print("-" * 40)
-        print(f"Length: {len(response)} characters")
-        print()
+def generate_all_responses_with_reasoning(prompt, selected_models=None):
+    all_models = {
+        "GPT-4": get_gpt4_response,
+        "Claude 3": get_claude_response,
+        "Gemini 1.5": get_gemini_response
+    }
+    models_to_use = selected_models if selected_models else list(all_models.keys())
 
-def generate_and_compare_responses():
-    """Generate responses from all models and display comparison."""
-    print("=== Response Generator - Model Comparison Tool ===\n")
-    
-    # Get prompt from user
-    prompt = input("Enter your prompt: ")
-    if not prompt.strip():
-        print("No prompt provided. Exiting.")
-        return
-    
-    print(f"\nGenerating responses for: '{prompt}'")
-    print("=" * 60)
-    
-    # Collect responses from all models
     responses = {}
-    
-    print("\n1. Generating GPT-4 response...")
-    gpt_response = get_gpt4_response(prompt)
-    if gpt_response:
-        responses['GPT-4'] = gpt_response
-        print("GPT-4 response generated")
-    else:
-        print("GPT-4 failed")
-    
-    print("\n2. Generating Claude response...")
-    claude_response = get_claude_response(prompt)
-    if claude_response:
-        responses['Claude 3'] = claude_response
-        print("Claude response generated")
-    else:
-        print("Claude failed")
-    
-    print("\n3. Generating Gemini response...")
-    gemini_response = get_gemini_response(prompt)
-    if gemini_response:
-        responses['Gemini 1.5'] = gemini_response
-        print("Gemini response generated")
-    else:
-        print("Gemini failed")
-    
-    if not responses:
-        print("\nNo models generated responses. Check your API keys.")
-        return
-    
-    print(f"\nSuccessfully generated {len(responses)} responses")
-    
-    # Display side-by-side comparison
-    display_responses_side_by_side(responses, prompt)
-    
-    # Ask if user wants evaluation
-    evaluate = input("\nDo you want to evaluate these responses? (y/n): ").strip().lower()
-    
-    if evaluate in ['y', 'yes']:
-        print("\n4. Performing comprehensive evaluation...")
+    for model_name in models_to_use:
+        fetch_fn = all_models[model_name]
         try:
-            comprehensive_results = comprehensive_round_robin_evaluation(responses, prompt)
-            
-            # Save results
-            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-            csv_file = save_comprehensive_results(comprehensive_results, prompt, timestamp)
-            
-            if csv_file:
-                print(f"Evaluation results saved to: {csv_file}")
-            
-            # Display evaluation summary
-            print("\n=== EVALUATION SUMMARY ===")
-            for model, data in comprehensive_results.items():
-                avg_scores = data.get('average_scores', {})
-                print(f"\n{model} Scores:")
-                print(f"  Helpfulness: {avg_scores.get('helpfulness', 'N/A')}")
-                print(f"  Correctness: {avg_scores.get('correctness', 'N/A')}")
-                print(f"  Coherence: {avg_scores.get('coherence', 'N/A')}")
-                print(f"  Clarity: {avg_scores.get('clarity', 'N/A')}")
-                print(f"  Evaluated by: {list(data.get('evaluations', {}).keys())}")
-            
+            response = fetch_fn(prompt)
+            reason_prompt = (
+                f"Why did you generate this response to the prompt:\n\n"
+                f"\"{prompt}\"\n\n"
+                f"Your Response:\n\"{response}\"\n\n"
+                "Explain your reasoning behind structuring or phrasing it that way."
+            )
+            reasoning = fetch_fn(reason_prompt)
+            responses[model_name] = {"response": response, "reasoning": reasoning}
         except Exception as e:
-            print(f"Evaluation failed: {e}")
-    
-    print("\n=== Response generation completed ===")
+            responses[model_name] = {"response": "Failed", "reasoning": str(e)}
 
-def batch_generate_from_file(filename):
-    """Generate responses for multiple prompts from a file."""
-    if not os.path.exists(filename):
-        print(f"File {filename} not found.")
-        return
-    
-    print(f"=== Batch Response Generation from {filename} ===")
-    
-    with open(filename, 'r', encoding='utf-8') as f:
-        prompts = [line.strip() for line in f if line.strip()]
-    
-    print(f"Found {len(prompts)} prompts to process")
-    
-    all_results = []
-    
-    for i, prompt in enumerate(prompts, 1):
-        print(f"\n--- Processing Prompt {i}/{len(prompts)} ---")
-        print(f"Prompt: {prompt}")
-        
-        # Generate responses
-        responses = {}
-        
-        gpt_response = get_gpt4_response(prompt)
-        if gpt_response:
-            responses['GPT-4'] = gpt_response
-        
-        claude_response = get_claude_response(prompt)
-        if claude_response:
-            responses['Claude 3'] = claude_response
-        
-        gemini_response = get_gemini_response(prompt)
-        if gemini_response:
-            responses['Gemini 1.5'] = gemini_response
-        
-        if responses:
-            # Display comparison
-            display_responses_side_by_side(responses, prompt)
-            
-            # Evaluate
-            try:
-                comprehensive_results = comprehensive_round_robin_evaluation(responses, prompt)
-                timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-                csv_file = save_comprehensive_results(comprehensive_results, prompt, f"{timestamp}_batch_{i}")
-                print(f"Results saved for prompt {i}")
-                all_results.append((prompt, comprehensive_results))
-            except Exception as e:
-                print(f"Evaluation failed for prompt {i}: {e}")
-        else:
-            print(f"No responses for prompt {i}")
-    
-    # Save summary
-    if all_results:
-        summary_file = f"results/batch_summary_{datetime.now().strftime('%Y%m%d_%H%M%S')}.txt"
-        os.makedirs("results", exist_ok=True)
-        
-        with open(summary_file, 'w', encoding='utf-8') as f:
-            f.write("BATCH RESPONSE GENERATION SUMMARY\n")
-            f.write("=" * 50 + "\n\n")
-            
-            for prompt, results in all_results:
-                f.write(f"PROMPT: {prompt}\n")
-                f.write("-" * 30 + "\n")
-                
-                for model, data in results.items():
-                    avg_scores = data.get('average_scores', {})
-                    f.write(f"{model}:\n")
-                    f.write(f"  Helpfulness: {avg_scores.get('helpfulness', 'N/A')}\n")
-                    f.write(f"  Correctness: {avg_scores.get('correctness', 'N/A')}\n")
-                    f.write(f"  Coherence: {avg_scores.get('coherence', 'N/A')}\n")
-                    f.write(f"  Clarity: {avg_scores.get('clarity', 'N/A')}\n\n")
-                
-                f.write("\n" + "="*50 + "\n\n")
-        
-        print(f"\nBatch summary saved to: {summary_file}")
-    
-    print("\n=== Batch generation completed ===")
-
-def generate_all_responses(prompt):
-    """Generate responses from all models for a given prompt."""
-    responses = {}
-    
-    # Generate responses from all models
-    gpt_response = get_gpt4_response(prompt)
-    if gpt_response:
-        responses['GPT-4'] = gpt_response
-    
-    claude_response = get_claude_response(prompt)
-    if claude_response:
-        responses['Claude 3'] = claude_response
-    
-    gemini_response = get_gemini_response(prompt)
-    if gemini_response:
-        responses['Gemini 1.5'] = gemini_response
-    
     return responses
-
-if __name__ == "__main__":
-    print("=== Response Generator Tool ===")
-    print("1. Interactive mode")
-    print("2. Batch mode from file")
-    
-    choice = input("Choose mode (1 or 2): ").strip()
-    
-    if choice == "1":
-        generate_and_compare_responses()
-    elif choice == "2":
-        filename = input("Enter filename with prompts (one per line): ").strip()
-        batch_generate_from_file(filename)
-    else:
-        print("Invalid choice. Exiting.")

round_robin_evaluator.py

@@ -5,6 +5,7 @@ import google.generativeai as genai
 from dotenv import load_dotenv
 import csv
 import json
+import re
 
 # Load environment variables
 load_dotenv()
@@ -12,6 +13,16 @@ openai_client = OpenAI(api_key=os.getenv("OPENAI_API_KEY"))
 anthropic_client = anthropic.Anthropic(api_key=os.getenv("CLAUDE_API_KEY"))
 genai.configure(api_key=os.getenv("GEMINI_API_KEY"))
 
+def safe_parse_json(text):
+    """Extract and parse JSON from a possibly noisy LLM output."""
+    try:
+        match = re.search(r'{.*}', text, re.DOTALL)
+        if match:
+            return json.loads(match.group())
+    except Exception as e:
+        print(f"[Safe JSON Parse Error] {e}")
+    return None
+
 def evaluate_response(evaluator_model, prompt, target_model, response_text):
     """Evaluate a response using the specified evaluator model."""
     evaluation_prompt = (
@@ -19,26 +30,20 @@ def evaluate_response(evaluator_model, prompt, target_model, response_text):
         f"Here is the original prompt: \"{prompt}\"\n"
         f"Here is the response from {target_model}: \"{response_text}\"\n\n"
         f"Evaluate this response on the following criteria from 0 (worst) to 1 (best):\n"
-        f"- Helpfulness: How useful and informative is the response?\n"
-        f"- Correctness: How accurate and factually correct is the response?\n"
-        f"- Coherence: How well-structured and logical is the response?\n"
-        f"- Tone: How appropriate and professional is the tone?\n"
-        f"- Accuracy: How precise and detailed is the information?\n"
-        f"- Relevance: How well does the response address the prompt?\n"
-        f"- Completeness: How comprehensive is the response?\n"
-        f"- Clarity: How clear and easy to understand is the response?\n\n"
-        f"Return the result in this exact JSON format:\n\n"
+        f"- Helpfulness\n- Correctness\n- Coherence\n- Tone\n- Accuracy\n"
+        f"- Relevance\n- Completeness\n- Clarity\n\n"
+        f"Return ONLY a valid JSON object with the following keys:\n"
         f"{{\n"
-        f"  \"helpfulness\": <0-1>,\n"
-        f"  \"correctness\": <0-1>,\n"
-        f"  \"coherence\": <0-1>,\n"
-        f"  \"tone_score\": <0-1>,\n"
-        f"  \"accuracy\": <0-1>,\n"
-        f"  \"relevance\": <0-1>,\n"
-        f"  \"completeness\": <0-1>,\n"
-        f"  \"clarity\": <0-1>,\n"
-        f"  \"reasoning\": \"detailed explanation for the scores\",\n"
-        f"  \"notes\": \"additional observations about the response\"\n"
+        f"  \"helpfulness\": <float>,\n"
+        f"  \"correctness\": <float>,\n"
+        f"  \"coherence\": <float>,\n"
+        f"  \"tone_score\": <float>,\n"
+        f"  \"accuracy\": <float>,\n"
+        f"  \"relevance\": <float>,\n"
+        f"  \"completeness\": <float>,\n"
+        f"  \"clarity\": <float>,\n"
+        f"  \"reasoning\": \"explanation\",\n"
+        f"  \"notes\": \"additional remarks\"\n"
         f"}}"
     )
 
@@ -65,106 +70,70 @@ def evaluate_response(evaluator_model, prompt, target_model, response_text):
         else:
             print(f"Unknown evaluator model: {evaluator_model}")
             return None
-        
-        # Try to parse JSON response
-        try:
-            if isinstance(result, str):
-                parsed = json.loads(result)
-            else:
-                parsed = result
+
+        parsed = safe_parse_json(result)
+        if parsed:
             return parsed
-        except json.JSONDecodeError:
+        else:
             print(f"Failed to parse JSON from {evaluator_model} evaluation")
             return None
-            
+
     except Exception as e:
         print(f"Error in {evaluator_model} evaluation: {str(e)}")
         return None
 
 def comprehensive_round_robin_evaluation(responses_dict, prompt):
-    """
-    Perform comprehensive round-robin evaluation where each model evaluates all other models.
-    
-    Args:
-        responses_dict: Dictionary with model names as keys and response texts as values
-        prompt: The original prompt
-    
-    Returns:
-        Dictionary with comprehensive evaluation results
-    """
     print("\nStarting comprehensive round-robin evaluation...")
-    
-    # Define the evaluation matrix
+
     evaluation_matrix = {
         "GPT-4": ["Claude 3", "Gemini 1.5"],
-        "Claude 3": ["GPT-4", "Gemini 1.5"], 
+        "Claude 3": ["GPT-4", "Gemini 1.5"],
         "Gemini 1.5": ["GPT-4", "Claude 3"]
     }
-    
-    # Initialize results structure
+
     comprehensive_results = {}
-    
-    # For each model, collect evaluations from other models
+
     for target_model, response_text in responses_dict.items():
         print(f"\nCollecting evaluations for {target_model}...")
-        
-        # Initialize target model data
         comprehensive_results[target_model] = {
             'response': response_text,
             'evaluations': {},
             'average_scores': {}
         }
-        
-        # Get evaluations from other models
-        evaluators = evaluation_matrix[target_model]
-        for evaluator in evaluators:
+
+        for evaluator in evaluation_matrix[target_model]:
             print(f"  {evaluator} evaluating {target_model}...")
             evaluation = evaluate_response(evaluator, prompt, target_model, response_text)
-            
             if evaluation:
                 comprehensive_results[target_model]['evaluations'][evaluator] = evaluation
                 print(f"    {evaluator} evaluation completed")
             else:
                 print(f"    {evaluator} evaluation failed")
-        
-        # Calculate average scores across all evaluators
+
         if comprehensive_results[target_model]['evaluations']:
-            metrics = ['helpfulness', 'correctness', 'coherence', 'tone_score', 
-                      'accuracy', 'relevance', 'completeness', 'clarity']
-            
+            metrics = ['helpfulness', 'correctness', 'coherence', 'tone_score',
+                       'accuracy', 'relevance', 'completeness', 'clarity']
             for metric in metrics:
-                scores = []
-                for evaluator, eval_data in comprehensive_results[target_model]['evaluations'].items():
-                    if metric in eval_data and isinstance(eval_data[metric], (int, float)):
-                        scores.append(eval_data[metric])
-                
-                if scores:
-                    avg_score = sum(scores) / len(scores)
-                    comprehensive_results[target_model]['average_scores'][metric] = round(avg_score, 3)
-                else:
-                    comprehensive_results[target_model]['average_scores'][metric] = 0.5
-    
+                scores = [
+                    eval_data[metric]
+                    for eval_data in comprehensive_results[target_model]['evaluations'].values()
+                    if metric in eval_data and isinstance(eval_data[metric], (int, float))
+                ]
+                comprehensive_results[target_model]['average_scores'][metric] = round(sum(scores) / len(scores), 3) if scores else 0.5
+
     print(f"\nComprehensive evaluation completed for {len(comprehensive_results)} models")
     return comprehensive_results
 
 def save_comprehensive_results(comprehensive_results, prompt, timestamp=None):
-    """Save comprehensive evaluation results to CSV."""
     if timestamp is None:
-        from datetime import datetime
         timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-    
+
     filename = f"results/comprehensive_eval_{timestamp}.csv"
-    
-    # Ensure results directory exists
     os.makedirs("results", exist_ok=True)
-    
-    # Prepare data for CSV
+
     rows = []
     for model, data in comprehensive_results.items():
-        # Get average scores
         avg_scores = data.get('average_scores', {})
-        
-        # Create row for each evaluator
         for evaluator, evaluation in data.get('evaluations', {}).items():
             row = {
                 'timestamp': timestamp,
@@ -192,44 +161,15 @@ def save_comprehensive_results(comprehensive_results, prompt, timestamp=None):
                 'avg_clarity': avg_scores.get('clarity', 0.5)
             }
             rows.append(row)
-    
-    # Write to CSV
+
     if rows:
         fieldnames = list(rows[0].keys())
-        with open(filename, 'w', newline='', encoding='utf-8') as csvfile:
-            writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
+        with open(filename, 'w', newline='', encoding='utf-8') as f:
+            writer = csv.DictWriter(f, fieldnames=fieldnames)
             writer.writeheader()
             writer.writerows(rows)
-        
         print(f"Results saved to {filename}")
         return filename
     else:
         print("No results to save")
         return None
-
-def round_robin_evaluate_and_log(responses):
-    """Legacy function for backward compatibility."""
-    print("This function is deprecated. Use comprehensive_round_robin_evaluation instead.")
-    return comprehensive_round_robin_evaluation(responses, "Legacy prompt")
-
-if __name__ == "__main__":
-    # Test the evaluation system
-    test_responses = {
-        "GPT-4": "This is a test response from GPT-4.",
-        "Claude 3": "This is a test response from Claude 3.",
-        "Gemini 1.5": "This is a test response from Gemini 1.5."
-    }
-    
-    test_prompt = "What is artificial intelligence?"
-    
-    print("Testing round-robin evaluation system...")
-    results = comprehensive_round_robin_evaluation(test_responses, test_prompt)
-    
-    if results:
-        print("\nTest completed successfully!")
-        for model, data in results.items():
-            print(f"\n{model} average scores:")
-            for metric, score in data.get('average_scores', {}).items():
-                print(f"  {metric}: {score}")
-    else:
-        print("Test failed!")