optim + logging

app.py

@@ -1,7 +1,9 @@
 import streamlit as st
 import pandas as pd
+import numpy as np
 import json
 import os
+import time
 import zipfile
 from sentence_transformers import SentenceTransformer, util
 from loguru import logger
@@ -26,24 +28,48 @@ def load_data():
 df = load_data()
 
 display_columns = ["uuid","problem", "source", "question_type", "problem_type"]
-df_filtered = df[display_columns]
+df = df[display_columns]
 
 # ================== FUNCTION DEFINITIONS ==================
 def compute_embeddings(problems):
     """Compute sentence embeddings."""
     return model.encode(problems, normalize_embeddings=True)
 
+
 def find_similar_problems(df, similarity_threshold=0.9):
-    """Find similar problems using cosine similarity."""
+    """Find similar problems using cosine similarity"""
+    
+    st.status("🔄 Computing problem embeddings...")
+    start_time = time.time()
     embeddings = compute_embeddings(df['problem'].tolist())
+    st.success("✅ Embeddings computed!", icon="✅")
+    
+    st.status("🔄 Computing cosine similarity matrix...")
     similarity_matrix = util.cos_sim(embeddings, embeddings).numpy()
-    pairs = []
-    for i in range(len(df)):
-        for j in range(i + 1, len(df)):
-            score = similarity_matrix[i][j]
-            if score > similarity_threshold:
-                pairs.append((df.iloc[i]["uuid"], df.iloc[j]["uuid"], float(score)))
-    return sorted(pairs, key=lambda x: x[2], reverse=True)  # Sort by similarity score
+    st.success("✅ Similarity matrix computed!", icon="✅")
+
+    num_problems = len(df)
+    upper_triangle_indices = np.triu_indices(num_problems, k=1)
+    
+    st.status("🔄 Filtering similar problems...")
+    i_indices, j_indices = upper_triangle_indices
+    similarity_scores = similarity_matrix[i_indices, j_indices]
+
+    mask = similarity_scores > similarity_threshold
+    filtered_i = i_indices[mask]
+    filtered_j = j_indices[mask]
+    filtered_scores = similarity_scores[mask]
+
+    pairs = [
+        (df.iloc[i]["uuid"], df.iloc[j]["uuid"], float(score))
+        for i, j, score in zip(filtered_i, filtered_j, filtered_scores)
+    ]
+    
+    sorted_pairs = sorted(pairs, key=lambda x: x[2], reverse=True)
+    
+    st.success(f"✅ Analysis complete! Found {len(sorted_pairs)} similar problems in {time.time() - start_time:.2f}s", icon="🎉")
+
+    return sorted_pairs
 
 def analyze_clusters(df, similarity_threshold=0.9):
     """Analyze duplicate problem clusters."""
@@ -81,7 +107,7 @@ similarity_threshold = st.sidebar.slider(
 
 # Display first 5 rows of dataset
 st.subheader("📄 Explore the Dataset")
-st.dataframe(df_filtered.head(5))
+st.dataframe(df.head(5))
 
 if st.sidebar.button("Run Deduplication Analysis"):
     with st.spinner("Analyzing..."):