1.22 print debug

app.py

@@ -186,51 +186,30 @@ class NewsProcessor:
         if df.empty:
             return pd.DataFrame(columns=['cluster_id', 'datetime', 'company', 'relevance_score', 'text', 'cluster_size'])
             
-        df = df.sort_values('datetime')
+        df = df.copy()  # Make a copy to preserve original indices
         
-        # First, filter out news where the company isn't the main subject
-        relevance_results = []
-        for idx, row in df.iterrows():
-            title = row['title'] if 'title' in row else ''
-            is_main, score = self.is_company_main_subject(title, row['text'], row['company'])
-            if is_main:
-                relevance_results.append({
-                    'idx': idx,
-                    'relevance_score': score
-                })
-        
-        if not relevance_results:
-            return pd.DataFrame(columns=['cluster_id', 'datetime', 'company', 'relevance_score', 'text', 'cluster_size'])
-            
-        relevant_indices = [r['idx'] for r in relevance_results]
-        relevance_scores = {r['idx']: r['relevance_score'] for r in relevance_results}
-        
-        df_filtered = df.loc[relevant_indices].copy()
-        df_filtered['relevance_score'] = df_filtered.index.map(relevance_scores)
-        
-        # Continue with clustering logic...
         clusters = []
         processed = set()
         
-        for i in tqdm(range(len(df_filtered)), total=len(df_filtered)):
-            if i in processed:
+        for idx in df.index:  # Iterate over original indices
+            if idx in processed:
                 continue
                 
-            row1 = df_filtered.iloc[i]
-            cluster = [df_filtered.index[i]]
-            processed.add(i)
+            row1 = df.loc[idx]
+            cluster = [idx]  # Store original index
+            processed.add(idx)
             
             if not pd.isna(row1['text']):
                 text1_embedding = self.encode_text(row1['text'])
                 
                 if progress_bar:
-                    progress_bar.progress(len(processed) / len(df_filtered))
+                    progress_bar.progress(len(processed) / len(df))
                 
-                for j in range(len(df_filtered)):
-                    if j in processed:
+                for other_idx in df.index:  # Iterate over original indices
+                    if other_idx in processed:
                         continue
                         
-                    row2 = df_filtered.iloc[j]
+                    row2 = df.loc[other_idx]
                     if pd.isna(row2['text']):
                         continue
                         
@@ -242,12 +221,12 @@ class NewsProcessor:
                     similarity = np.dot(text1_embedding, text2_embedding)
                     
                     if similarity >= self.similarity_threshold:
-                        cluster.append(df_filtered.index[j])
-                        processed.add(j)
+                        cluster.append(other_idx)
+                        processed.add(other_idx)
             
             clusters.append(cluster)
         
-        # Create result DataFrame
+        # Create result DataFrame preserving original indices
         result_data = []
         for cluster_id, cluster_indices in enumerate(clusters, 1):
             cluster_rows = df.loc[cluster_indices]
@@ -256,12 +235,12 @@ class NewsProcessor:
                     'cluster_id': cluster_id,
                     'datetime': df.loc[idx, 'datetime'],
                     'company': df.loc[idx, 'company'],
-                    'relevance_score': relevance_scores[idx],
                     'text': df.loc[idx, 'text'],
                     'cluster_size': len(cluster_indices)
                 })
         
-        return pd.DataFrame(result_data)
+        result_df = pd.DataFrame(result_data, index=sum(clusters, []))  # Use original indices
+        return result_df
 
 class NewsDeduplicator:
     def __init__(self, fuzzy_threshold=85):
@@ -322,7 +301,7 @@ def create_download_link(df: pd.DataFrame, filename: str) -> str:
 
 
 def main():
-    st.title("кластеризуем новости v.1.21 print debug")
+    st.title("кластеризуем новости v.1.22 + print debug")
     st.write("Upload Excel file with columns: company, datetime, text")
     
     uploaded_file = st.file_uploader("Choose Excel file", type=['xlsx'])
@@ -360,61 +339,54 @@ def main():
                     # Step 1: Deduplicate
                     deduplicator = NewsDeduplicator(fuzzy_threshold)
                     dedup_df = deduplicator.deduplicate(df, progress_bar)
-                    st.write("\nAfter deduplication:")
-                    st.write(f"dedup_df indices: {dedup_df.index.tolist()}")
-                    st.success(f"Removed {len(df) - len(dedup_df)} duplicates")
-                    
-                    # Preserve all columns from original DataFrame in dedup_df
-                    dedup_df_full = df_original.loc[dedup_df.index].copy()
-                    st.write(f"dedup_df_full indices: {dedup_df_full.index.tolist()}")
+                    st.write("\nDeduplication Results:")
+                    st.write(f"Original indices: {df.index.tolist()}")
+                    st.write(f"Dedup indices: {dedup_df.index.tolist()}")
+                    st.write(f"Sample from dedup_df:")
+                    st.write(dedup_df[['company', 'text']].head())
                     
                     # Step 2: Cluster deduplicated news
                     processor = NewsProcessor(similarity_threshold, time_threshold)
                     result_df = processor.process_news(dedup_df, progress_bar)
-                    st.write("\nAfter clustering:")
-                    st.write(f"result_df indices: {result_df.index.tolist()}")
+                    st.write("\nClustering Results:")
+                    st.write(f"Result df indices: {result_df.index.tolist()}")
+                    
+                    # Display cluster information
+                    if len(result_df) > 0:
+                        st.write("\nCluster Details:")
+                        for cluster_id in result_df['cluster_id'].unique():
+                            cluster_mask = result_df['cluster_id'] == cluster_id
+                            if sum(cluster_mask) > 1:  # Only show multi-item clusters
+                                cluster_indices = result_df[cluster_mask].index.tolist()
+                                st.write(f"\nCluster {cluster_id}:")
+                                st.write(f"Indices: {cluster_indices}")
+                                # Show texts for verification
+                                for idx in cluster_indices:
+                                    text_length = len(str(dedup_df.loc[idx, 'text']))
+                                    st.write(f"Index {idx} - Length {text_length}:")
+                                    st.write(str(dedup_df.loc[idx, 'text'])[:100] + '...')
                     
-                    # Initialize set of indices to delete
+                    # Process clusters for deletion
                     indices_to_delete = set()
                     
-                    # Find rows to delete from multi-item clusters
                     if len(result_df) > 0:
-                        # Get all multi-item clusters
-                        multi_clusters = result_df[result_df['cluster_size'] > 1]['cluster_id'].unique()
-                        st.write(f"\nMulti-clusters found: {multi_clusters.tolist()}")
-                        
-                        # For each multi-item cluster
-                        for cluster_id in multi_clusters:
-                            st.write(f"\nProcessing cluster {cluster_id}:")
-                            # Get rows in this cluster
+                        for cluster_id in result_df['cluster_id'].unique():
                             cluster_mask = result_df['cluster_id'] == cluster_id
-                            cluster_rows = result_df[cluster_mask]
-                            st.write(f"Cluster rows indices: {cluster_rows.index.tolist()}")
-                            
-                            # Get their original indices from dedup_df_full
-                            original_indices = dedup_df.index[cluster_rows.index]
-
-                            #original_indices = dedup_df_full.index[cluster_rows.index - 1] -it was wrong!
-                            st.write(f"Original indices: {original_indices.tolist()}")
-                            
-                            # Find the row with longest text among these indices
-                            text_lengths = dedup_df_full.loc[original_indices, text_column].fillna('').str.len()
-                            st.write(f"Text lengths: {text_lengths.to_dict()}")
-                            longest_text_idx = text_lengths.idxmax()
-                            st.write(f"Longest text index: {longest_text_idx}")
-                            
-                            # Add all other indices to delete set
-                            new_indices_to_delete = set(original_indices) - {longest_text_idx}
-                            indices_to_delete.update(new_indices_to_delete)
-                            st.write(f"Indices to delete from this cluster: {new_indices_to_delete}")
+                            if sum(cluster_mask) > 1:
+                                cluster_indices = result_df[cluster_mask].index.tolist()
+                                text_lengths = dedup_df.loc[cluster_indices, 'text'].fillna('').str.len()
+                                longest_text_idx = text_lengths.idxmax()
+                                indices_to_delete.update(set(cluster_indices) - {longest_text_idx})
                     
-                    st.write(f"\nFinal indices to delete: {sorted(list(indices_to_delete))}")
+                    st.write("\nDeletion Summary:")
+                    st.write(f"Indices to delete: {sorted(list(indices_to_delete))}")
                     
-                    # Create final declustered DataFrame by removing identified rows
-                    declustered_df = dedup_df_full.copy()
+                    # Create final DataFrame
+                    declustered_df = dedup_df.copy()
                     if indices_to_delete:
                         declustered_df = declustered_df.drop(index=list(indices_to_delete))
-                        st.write(f"\nFinal kept indices: {sorted(declustered_df.index.tolist())}")
+                    
+                    st.write(f"Final indices kept: {sorted(declustered_df.index.tolist())}")
 
                     
                     # Print statistics