Update app.py

app.py

@@ -1,46 +1,52 @@
 import streamlit as st
-import warnings
-from sentence_transformers import SentenceTransformer
-from scipy.spatial.distance import cosine
-from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+import seaborn as sns
+from sklearn.feature_extraction.text import TfidfVectorizer
 from sklearn.metrics import pairwise_distances
-from nltk.translate.bleu_score import sentence_bleu
 from rouge_score import rouge_scorer
-import numpy as np
+import gensim.downloader as api
+from sentence_transformers import SentenceTransformer
+from scipy.spatial.distance import cosine
 import PyPDF2
-import seaborn as sns
-import matplotlib.pyplot as plt
-import pandas as pd
+import spacy
 from difflib import SequenceMatcher
-import streamlit_shadcn_ui as ui
-import bert_score
-import gensim.downloader as api
 
-# Suppress specific FutureWarning from transformers
-warnings.filterwarnings("ignore", category=FutureWarning, message=".*clean_up_tokenization_spaces.*")
+# Load spaCy model
+nlp = spacy.load('en_core_web_sm')
+
+# Load stop words from spaCy
+stop_words = set(nlp.Defaults.stop_words)
 
 # Initialize models
-model = SentenceTransformer('all-mpnet-base-v2')
-tfidf_vectorizer = TfidfVectorizer()
+@st.cache_resource
+def load_models():
+    model = SentenceTransformer('all-mpnet-base-v2')
+    tfidf_vectorizer = TfidfVectorizer()
+    word2vec_model = api.load("word2vec-google-news-300")  # Load Word2Vec model
+    return model, tfidf_vectorizer, word2vec_model
+
+model, tfidf_vectorizer, word2vec_model = load_models()
 
 # Initialize session state for results table if not already present
 if 'results_df' not in st.session_state:
     st.session_state.results_df = pd.DataFrame(columns=[
-        "LLM1", "LLM2", 
-        "Context Similarity (%)", 
-        "Levenshtein Similarity (%)", 
-        "Jaccard Similarity (%)",
-        "BLEU Score", 
-        "ROUGE-L (%)",
-        "BERTScore (%)",
-        "WMD"
+        "LLM1", "LLM2", "Paraphrasing Similarity (%)", 
+        "Direct Text Comparison (%)", "Summarization Similarity (%)", 
+        "Combined Similarity (%)"
     ])
 
-# Function to chunk text into smaller parts
+# Initialize session state for radar chart data if not already present
+if 'radar_chart_data' not in st.session_state:
+    st.session_state.radar_chart_data = []
+
+# Functions
+@st.cache_data
 def chunk_text(text, chunk_size=500):
     return [text[i:i+chunk_size] for i in range(0, len(text), chunk_size)]
 
-# Function to create embeddings
+@st.cache_data
 def create_embeddings(chunks):
     try:
         embeddings = model.encode(chunks, show_progress_bar=False)
@@ -49,7 +55,7 @@ def create_embeddings(chunks):
         st.error(f"Error creating embeddings: {e}")
         return np.array([])
 
-# Function to calculate similarity ratio and find matches
+@st.cache_data
 def calculate_similarity_ratio_and_find_matches(embeddings1, embeddings2):
     try:
         similarities = np.dot(embeddings1, embeddings2.T)  # Dot product
@@ -60,11 +66,13 @@ def calculate_similarity_ratio_and_find_matches(embeddings1, embeddings2):
         st.error(f"Error calculating similarity ratio: {e}")
         return np.array([]), 0
 
-# Function to calculate word similarity ratio
+@st.cache_data
 def calculate_word_similarity_ratio(text1, text2):
     try:
-        words1 = text1.split()
-        words2 = text2.split()
+        doc1 = nlp(text1)
+        doc2 = nlp(text2)
+        words1 = [token.text for token in doc1 if not token.is_stop and not token.is_punct]
+        words2 = [token.text for token in doc2 if not token.is_stop and not token.is_punct]
         
         if not words1 or not words2:
             return 0
@@ -73,7 +81,7 @@ def calculate_word_similarity_ratio(text1, text2):
         word_embeddings2 = model.encode(words2)
         
         similarities = np.array([
-            max([1 - cosine(emb1, emb2) for emb2 in word_embeddings2], default=0) 
+            max([1 - cosine(emb1, emb2) for emb2 in word_embeddings2], default=0)
             for emb1 in word_embeddings1
         ])
         
@@ -83,27 +91,32 @@ def calculate_word_similarity_ratio(text1, text2):
         st.error(f"Error calculating word similarity ratio: {e}")
         return 0
 
-# Function to calculate BLEU score
+@st.cache_data
 def calculate_bleu_score(reference, candidate):
+    from nltk.translate.bleu_score import sentence_bleu
     return sentence_bleu([reference.split()], candidate.split())
 
-# Function to calculate ROUGE-L score
+@st.cache_data
 def calculate_rouge_l_score(reference, candidate):
     scorer = rouge_scorer.RougeScorer(['rougeL'], use_stemmer=True)
     scores = scorer.score(reference, candidate)
     return scores['rougeL'].fmeasure * 100
 
-# Function to calculate BERTScore
+@st.cache_data
 def calculate_bertscore(reference, candidate):
+    import bert_score
     P, R, F1 = bert_score.score([candidate], [reference], model_type='bert-base-uncased')
     return F1.mean().item() * 100
 
-# Function to calculate WMD
+@st.cache_data
 def calculate_wmd(reference, candidate):
-    model = api.load("word2vec-google-news-300")
-    return model.wmdistance(reference.split(), candidate.split())
+    doc1 = nlp(reference.lower())
+    doc2 = nlp(candidate.lower())
+    reference_tokens = [token.text for token in doc1 if not token.is_stop and not token.is_punct]
+    candidate_tokens = [token.text for token in doc2 if not token.is_stop and not token.is_punct]
+    return word2vec_model.wmdistance(reference_tokens, candidate_tokens)
 
-# Function to extract text from PDF
+@st.cache_data
 def extract_pdf_text(pdf_file):
     try:
         reader = PyPDF2.PdfReader(pdf_file)
@@ -115,128 +128,200 @@ def extract_pdf_text(pdf_file):
         st.error(f"Error extracting text from PDF: {e}")
         return ""
 
-# Function to calculate Levenshtein distance
+@st.cache_data
 def calculate_levenshtein_ratio(text1, text2):
     return SequenceMatcher(None, text1, text2).ratio()
 
-# Function to calculate Jaccard similarity
+@st.cache_data
 def calculate_jaccard_similarity(text1, text2):
+    from sklearn.feature_extraction.text import CountVectorizer
     vectorizer = CountVectorizer(binary=True).fit_transform([text1, text2])
     vectors = vectorizer.toarray()
-    # Compute the intersection and union for Jaccard Similarity
     intersection = np.sum(np.minimum(vectors[0], vectors[1]))
     union = np.sum(np.maximum(vectors[0], vectors[1]))
     return intersection / union if union != 0 else 0
 
-# Function to calculate TF-IDF cosine similarity
+@st.cache_data
 def calculate_tfidf_cosine_similarity(text1, text2):
     tfidf_matrix = tfidf_vectorizer.fit_transform([text1, text2])
     return 1 - pairwise_distances(tfidf_matrix, metric='cosine')[0, 1]
 
-# Streamlit UI
-st.sidebar.title("LLM Details")
-llm1_name = st.sidebar.text_input("What is LLM1?", "LLM1")
-llm2_name = st.sidebar.text_input("What is LLM2?", "LLM2")
+@st.cache_data
+def calculate_paraphrasing_similarity(text1, text2):
+    try:
+        chunks_1 = chunk_text(text1)
+        chunks_2 = chunk_text(text2)
+        embeddings_1 = create_embeddings(chunks_1)
+        embeddings_2 = create_embeddings(chunks_2)
+        
+        if embeddings_1.size > 0 and embeddings_2.size > 0:
+            similarities, average_similarity = calculate_similarity_ratio_and_find_matches(embeddings_1, embeddings_2)
+            return average_similarity * 100
+        return 0
+    except Exception as e:
+        st.error(f"Error calculating paraphrasing similarity: {e}")
+        return 0
+
+@st.cache_data
+def calculate_direct_text_comparison_similarity(text1, text2):
+    try:
+        levenshtein_ratio = calculate_levenshtein_ratio(text1, text2) * 100
+        jaccard_similarity = calculate_jaccard_similarity(text1, text2) * 100
+        tfidf_cosine_similarity = calculate_tfidf_cosine_similarity(text1, text2) * 100
+        bleu_score = calculate_bleu_score(text1, text2) * 100
+        rouge_l_score = calculate_rouge_l_score(text1, text2)
+        bertscore = calculate_bertscore(text1, text2)
+        
+        return (levenshtein_ratio * 0.1 + jaccard_similarity * 0.2 + 
+                tfidf_cosine_similarity * 0.2 + bleu_score * 0.2 + 
+                rouge_l_score * 0.2 + bertscore * 0.2) / 1.1
+    except Exception as e:
+        st.error(f"Error calculating direct text comparison similarity: {e}")
+        return 0
+
+@st.cache_data
+def calculate_summarization_similarity(text1, text2):
+    try:
+        wmd = calculate_wmd(text1, text2)
+        return (1 - wmd) * 100
+    except Exception as e:
+        st.error(f"Error calculating summarization similarity: {e}")
+        return 0
 
+# Streamlit UI
 st.title("Text-Based Similarity Comparison")
 
-# Create two columns for text input
-col1, col2 = st.columns(2)
+# Create a two-column layout for input
+col1, col2 = st.columns([2, 1])
 
 with col1:
-    st.write(f"**{llm1_name} response**")
-    upload_pdf_1 = st.file_uploader(f"Upload PDF for {llm1_name} response", type="pdf", key="pdf1")
-    if upload_pdf_1:
-        text_input_1 = extract_pdf_text(upload_pdf_1)
-    else:
-        text_input_1 = st.text_area(f" Text for {llm1_name}", height=150, key="text1")
+    st.sidebar.title("LLM Details")
+    llm1_name = st.sidebar.text_input("What is LLM1?", "LLM1")
+    llm2_name = st.sidebar.text_input("What is LLM2?", "LLM2")
+    
+    st.write("## Input")
+    
+    # Create two columns for text input
+    input_col1, input_col2 = st.columns(2)
+    
+    with input_col1:
+        st.write(f"{llm1_name} response")
+        upload_pdf_1 = st.file_uploader(f"Upload PDF for {llm1_name} response", type="pdf", key="pdf1")
+        if upload_pdf_1:
+            text_input_1 = extract_pdf_text(upload_pdf_1)
+        else:
+            text_input_1 = st.text_area(f"Text for {llm1_name}", height=150, key="text1")
+    
+    with input_col2:
+        st.write(f"{llm2_name} response")
+        upload_pdf_2 = st.file_uploader(f"Upload PDF for {llm2_name} response", type="pdf", key="pdf2")
+        if upload_pdf_2:
+            text_input_2 = extract_pdf_text(upload_pdf_2)
+        else:
+            text_input_2 = st.text_area(f"Text for {llm2_name}", height=150, key="text2")
 
-with col2:
-    st.write(f"**{llm2_name} response**")
-    upload_pdf_2 = st.file_uploader(f"Upload PDF for {llm2_name} response", type="pdf", key="pdf2")
-    if upload_pdf_2:
-        text_input_2 = extract_pdf_text(upload_pdf_2)
-    else:
-        text_input_2 = st.text_area(f" Text for {llm2_name}", height=150, key="text2")
-
-if (text_input_1 and text_input_2) or (upload_pdf_1 and upload_pdf_2):
-    if st.button("Submit"):
-        # Process texts
-        chunks_1 = chunk_text(text_input_1)
-        chunks_2 = chunk_text(text_input_2)
-        embeddings_1 = create_embeddings(chunks_1)
-        embeddings_2 = create_embeddings(chunks_2)
+    if (text_input_1 and text_input_2) or (upload_pdf_1 and upload_pdf_2):
+        if st.button("Submit"):
+            # Calculate similarity metrics
+            paraphrasing_similarity = calculate_paraphrasing_similarity(text_input_1, text_input_2)
+            direct_text_comparison_similarity = calculate_direct_text_comparison_similarity(text_input_1, text_input_2)
+            summarization_similarity = calculate_summarization_similarity(text_input_1, text_input_2)
 
-        # Calculate and display similarity ratio
-        if embeddings_1.size > 0 and embeddings_2.size > 0:
-            similarities, similarity_ratio = calculate_similarity_ratio_and_find_matches(embeddings_1, embeddings_2)
-
-            # Calculate word similarity ratios for chunks
-            word_similarities = []
-            min_chunks = min(len(chunks_1), len(chunks_2))
-            for i in range(min_chunks):
-                word_similarity_ratio = calculate_word_similarity_ratio(chunks_1[i], chunks_2[i])
-                word_similarities.append(word_similarity_ratio * 100)
-
-            # Calculate additional metrics
-            levenshtein_ratio = calculate_levenshtein_ratio(text_input_1, text_input_2) * 100
-            jaccard_similarity = calculate_jaccard_similarity(text_input_1, text_input_2) * 100
-            tfidf_cosine_similarity = calculate_tfidf_cosine_similarity(text_input_1, text_input_2) * 100
-            bleu_score = calculate_bleu_score(text_input_1, text_input_2) * 100
-            rouge_l_score = calculate_rouge_l_score(text_input_1, text_input_2)
-            bertscore = calculate_bertscore(text_input_1, text_input_2)
-            wmd = calculate_wmd(text_input_1, text_input_2)
+            # Combine all metrics into a single similarity score
+            total_similarity = (paraphrasing_similarity * 0.33 + 
+                                direct_text_comparison_similarity * 0.33 + 
+                                summarization_similarity * 0.33)
 
             # Update results table in session state
             new_row = pd.Series({
                 "LLM1": llm1_name,
                 "LLM2": llm2_name,
-                "Context Similarity (%)": similarity_ratio * 100,
-                "Levenshtein Similarity (%)": levenshtein_ratio,
-                "Jaccard Similarity (%)": jaccard_similarity,
-                "BLEU Score": bleu_score,
-                "ROUGE-L (%)": rouge_l_score,
-                "BERTScore (%)": bertscore,
-                "WMD": wmd
+                "Paraphrasing Similarity (%)": paraphrasing_similarity,
+                "Direct Text Comparison (%)": direct_text_comparison_similarity,
+                "Summarization Similarity (%)": summarization_similarity,
+                "Combined Similarity (%)": total_similarity
             })
-
             st.session_state.results_df = pd.concat([st.session_state.results_df, new_row.to_frame().T], ignore_index=True)
 
-            # Display metrics
+            # Add new data for radar chart
+            st.session_state.radar_chart_data.append({
+                "name": f"{llm1_name} vs {llm2_name}",
+                "paraphrasing_similarity": paraphrasing_similarity,
+                "direct_text_comparison_similarity": direct_text_comparison_similarity,
+                "summarization_similarity": summarization_similarity
+            })
+
+            # Display metrics with large and bold text
             st.subheader("Results")
-            st.write(f"**Context Similarity:** {similarity_ratio * 100:.2f}%")
-            st.write(f"**Levenshtein Similarity:** {levenshtein_ratio:.2f}%")
-            st.write(f"**Jaccard Similarity:** {jaccard_similarity:.2f}%")
-            st.write(f"**TF-IDF Cosine Similarity:** {tfidf_cosine_similarity:.2f}%")
-            st.write(f"**BLEU Score:** {bleu_score:.2f}")
-            st.write(f"**ROUGE-L Score:** {rouge_l_score:.2f}%")
-            st.write(f"**BERTScore:** {bertscore:.2f}%")
-            st.write(f"**Word Mover's Distance (WMD):** {wmd:.4f}")
-
-            # Visualize the data
-            st.subheader("Metrics Comparison")
-            sns.set(style="whitegrid")
-            fig, ax = plt.subplots(figsize=(10, 6))
-            sns.barplot(data=st.session_state.results_df.drop(columns=["LLM1", "LLM2", "WMD"]), palette="viridis")
-            plt.xticks(rotation=45)
-            plt.title("Text Similarity Metrics")
-            plt.tight_layout()
-            st.pyplot(fig)
-
-            st.subheader("Similarity Over Chunks")
-            fig, ax = plt.subplots(figsize=(10, 6))
-            ax.plot(word_similarities, marker='o', linestyle='-', color='b', label='Word Similarity')
-            ax.axhline(similarity_ratio * 100, color='r', linestyle='--', label='Context Similarity')
-            plt.xlabel("Chunk Index")
-            plt.ylabel("Similarity (%)")
-            plt.legend()
-            plt.title("Similarity across Text Chunks")
-            plt.tight_layout()
-            st.pyplot(fig)
-
-            # Display results dataframe
-            st.subheader("Detailed Results Table")
-            st.write(st.session_state.results_df)
-
-else:
-    st.warning("Please enter both responses or upload PDF files.")
+            st.markdown(f"Paraphrasing Similarity: {paraphrasing_similarity:.2f}%", unsafe_allow_html=True)
+            st.markdown(f"Direct Text Comparison Similarity: {direct_text_comparison_similarity:.2f}%", unsafe_allow_html=True)
+            st.markdown(f"Summarization Similarity: {summarization_similarity:.2f}%", unsafe_allow_html=True)
+            st.markdown(f"Combined Similarity Score: {total_similarity:.2f}%", unsafe_allow_html=True)
+
+with col2:
+    st.write("## Metrics and Results")
+    
+    # Display radar chart
+    if st.session_state.radar_chart_data:
+        st.subheader("Metrics Comparison")
+        labels = ["Paraphrasing Similarity", "Direct Text Comparison Similarity", "Summarization Similarity"]
+        num_vars = len(labels)
+        angles = np.linspace(0, 2 * np.pi, num_vars, endpoint=False).tolist()
+        angles += angles[:1]
+        
+        fig, ax = plt.subplots(figsize=(6, 6), subplot_kw=dict(polar=True))
+        
+        # Plot each response with a different color
+        color_palette = sns.color_palette("husl", len(st.session_state.radar_chart_data))
+        for idx, data in enumerate(st.session_state.radar_chart_data):
+            values = [
+                data["paraphrasing_similarity"], 
+                data["direct_text_comparison_similarity"], 
+                data["summarization_similarity"]
+            ]
+            values += values[:1]
+            ax.fill(angles, values, color=color_palette[idx], alpha=0.25, label=data["name"])
+            ax.plot(angles, values, color=color_palette[idx], linewidth=2, linestyle='solid')
+        
+        ax.set_yticklabels([])
+        ax.set_xticks(angles[:-1])
+        ax.set_xticklabels(labels)
+        plt.title("Radar Chart of Similarity Metrics")
+        plt.legend(loc='upper right', bbox_to_anchor=(1.3, 1.1))
+        st.pyplot(fig)
+
+    # Display metrics sliders beside the radar chart
+    if st.session_state.radar_chart_data:
+        st.subheader("Metrics Position")
+        metrics = st.session_state.radar_chart_data[-1]
+        for metric_name in ["paraphrasing_similarity", "direct_text_comparison_similarity", "summarization_similarity"]:
+            st.slider(
+                metric_name.replace("_", " ").title(), 
+                0, 100, 
+                int(metrics[metric_name]), 
+                key=metric_name, 
+                disabled=True,  # Make the slider non-editable
+                format="%.0f"  # Format the slider value to be an integer
+            )
+
+# Create a three-column layout for the results table and action buttons
+results_col, actions_col = st.columns([2, 1])
+
+with results_col:
+    st.write("## Detailed Results Table")
+    if not st.session_state.results_df.empty:
+        st.write(st.session_state.results_df)
+        
+        # Download the results as a CSV file
+        csv_data = st.session_state.results_df.to_csv(index=False).encode('utf-8')
+        st.download_button(label="Download Results as CSV", data=csv_data, file_name='similarity_results.csv', mime='text/csv')
+
+with actions_col:
+    if st.button("Reset Table"):
+        st.session_state.results_df = pd.DataFrame(columns=[
+            "LLM1", "LLM2", "Paraphrasing Similarity (%)", 
+            "Direct Text Comparison (%)", "Summarization Similarity (%)", 
+            "Combined Similarity (%)"
+        ])
+        st.session_state.radar_chart_data = []
+        st.write("Results table has been reset.")