serialization issue

app.py

@@ -3,7 +3,6 @@ import pandas as pd
 import json
 import os
 from sentence_transformers import SentenceTransformer, util
-from openai import OpenAI
 from loguru import logger
 
 # ================== CONFIGURATION ==================
@@ -31,50 +30,49 @@ def find_similar_problems(df, similarity_threshold=0.9):
     """Find similar problems using cosine similarity."""
     embeddings = compute_embeddings(df['problem'].tolist())
     similarity_matrix = util.cos_sim(embeddings, embeddings).numpy()
-
     clusters = {}
     for i in range(len(df)):
         current_uuid = df["uuid"][i]
         similar_items = [
-            (df["uuid"][j], similarity_matrix[i][j])
+            (df["uuid"][j], float(similarity_matrix[i][j]))  # Convert float32 to float
             for j in range(i + 1, len(df))
             if similarity_matrix[i][j] > similarity_threshold
         ]
-
         if similar_items:
             clusters[current_uuid] = similar_items
-
     return clusters
 
 def analyze_clusters(df, similarity_threshold=0.9):
     """Analyze duplicate problem clusters."""
     clusters = find_similar_problems(df, similarity_threshold)
     detailed_analysis = {}
-
     for key, values in clusters.items():
         base_row = df[df["uuid"] == key].iloc[0]
         cluster_details = []
-
         for val, score in values:
             comparison_row = df[df["uuid"] == val].iloc[0]
             
             column_differences = {}
             for col in df.columns:
                 if col != "uuid":
+                    base_val = base_row[col]
+                    comp_val = comparison_row[col]
+                    # Convert numpy types to native Python types
+                    if hasattr(base_val, 'item'):
+                        base_val = base_val.item()
+                    if hasattr(comp_val, 'item'):
+                        comp_val = comp_val.item()
                     column_differences[col] = {
-                        'base': base_row[col],
-                        'comparison': comparison_row[col],
-                        'match': base_row[col] == comparison_row[col]
+                        'base': base_val,
+                        'comparison': comp_val,
+                        'match': bool(base_val == comp_val)  # Convert numpy bool to Python bool
                     }
-
             cluster_details.append({
                 'uuid': val,
-                'similarity_score': score,
+                'similarity_score': float(score),  # Convert float32 to float
                 'column_differences': column_differences,
             })
-
         detailed_analysis[key] = cluster_details
-
     return detailed_analysis
 
 # ================== STREAMLIT UI ==================
@@ -88,22 +86,18 @@ similarity_threshold = st.sidebar.slider(
 if st.sidebar.button("Run Deduplication Analysis"):
     with st.spinner("Analyzing..."):
         results = analyze_clusters(df, similarity_threshold)
-
     st.success("Analysis Complete!")
-
+    
     st.subheader("📊 Duplicate Problem Clusters")
     for base_uuid, cluster in results.items():
         base_problem = df[df["uuid"] == base_uuid]["problem"].values[0]
         st.markdown(f"### Problem: {base_problem}")
-
         for entry in cluster:
             similar_problem = df[df["uuid"] == entry["uuid"]]["problem"].values[0]
             st.write(f"**Similar to:** {similar_problem}")
             st.write(f"**Similarity Score:** {entry['similarity_score']:.4f}")
-
             with st.expander("Show Column Differences"):
                 st.json(entry["column_differences"])
-
             st.markdown("---")
 
     # Export results