1.19

app.py

@@ -322,15 +322,15 @@ def create_download_link(df: pd.DataFrame, filename: str) -> str:
 
 
 def main():
-    st.title("кластеризуем новости v.1.17")
+    st.title("кластеризуем новости v.1.19")
     st.write("Upload Excel file with columns: company, datetime, text")
     
     uploaded_file = st.file_uploader("Choose Excel file", type=['xlsx'])
     
     if uploaded_file:
         try:
-            # Read all columns from original sheet and reset index
-            df_original = pd.read_excel(uploaded_file, sheet_name='Публикации').reset_index(drop=True)
+            # Read all columns from original sheet
+            df_original = pd.read_excel(uploaded_file, sheet_name='Публикации')
             st.write("Available columns:", df_original.columns.tolist())
             
             # Create working copy with required columns
@@ -360,56 +360,50 @@ def main():
                     # Step 1: Deduplicate
                     deduplicator = NewsDeduplicator(fuzzy_threshold)
                     dedup_df = deduplicator.deduplicate(df, progress_bar)
-                    dedup_df = dedup_df.reset_index(drop=True)  # Reset index after deduplication
                     st.success(f"Removed {len(df) - len(dedup_df)} duplicates")
                     
-                    # Preserve all columns from original DataFrame in dedup_df and reset index
-                    dedup_df_full = df_original.loc[dedup_df.index].copy().reset_index(drop=True)
-                    dedup_df.index = dedup_df_full.index  # Ensure indices match
+                    # Preserve all columns from original DataFrame in dedup_df
+                    dedup_df_full = df_original.loc[dedup_df.index].copy()
                     
                     # Step 2: Cluster deduplicated news
                     processor = NewsProcessor(similarity_threshold, time_threshold)
                     result_df = processor.process_news(dedup_df, progress_bar)
                     
+                    # Initialize set of indices to delete
+                    indices_to_delete = set()
+                    
+                    # Find rows to delete from multi-item clusters
                     if len(result_df) > 0:
-                        # Ensure result_df index matches dedup_df_full
-                        result_df.index = dedup_df_full.index[result_df.index]
-                        
-                        # Initialize set of indices to delete
-                        indices_to_delete = set()
-                        
-                        # Find rows to delete from multi-item clusters
+                        # Get all multi-item clusters
                         multi_clusters = result_df[result_df['cluster_size'] > 1]['cluster_id'].unique()
                         
+                        # For each multi-item cluster
                         for cluster_id in multi_clusters:
-                            # Get indices of all rows in this cluster
+                            # Get rows in this cluster
                             cluster_mask = result_df['cluster_id'] == cluster_id
-                            cluster_indices = result_df[cluster_mask].index.tolist()
+                            cluster_rows = result_df[cluster_mask]
                             
-                            # Get their text lengths
-                            text_lengths = dedup_df_full.loc[cluster_indices, text_column].fillna('').str.len()
+                            # Get their original indices from dedup_df_full
+                            original_indices = dedup_df_full.index[cluster_rows.index - 1]
                             
-                            # Find index with longest text
+                            # Find the row with longest text among these indices
+                            text_lengths = dedup_df_full.loc[original_indices, text_column].fillna('').str.len()
                             longest_text_idx = text_lengths.idxmax()
                             
                             # Add all other indices to delete set
-                            indices_to_delete.update(set(cluster_indices) - {longest_text_idx})
-                        
-                        # Create final declustered DataFrame by removing identified rows
-                        declustered_df = dedup_df_full.copy()
-                        if indices_to_delete:
-                            declustered_df = declustered_df.drop(index=list(indices_to_delete))
-                    else:
-                        declustered_df = dedup_df_full.copy()
-                        indices_to_delete = set()
-                        multi_clusters = []
+                            indices_to_delete.update(set(original_indices) - {longest_text_idx})
+                    
+                    # Create final declustered DataFrame by removing identified rows
+                    declustered_df = dedup_df_full.copy()
+                    if indices_to_delete:
+                        declustered_df = declustered_df.drop(index=list(indices_to_delete))
                     
                     # Print statistics
                     st.success(f"""
                         Processing results:
                         - Original rows: {len(df_original)}
                         - After deduplication: {len(dedup_df_full)}
-                        - Multi-item clusters found: {len(multi_clusters)}
+                        - Multi-item clusters found: {len(multi_clusters) if len(result_df) > 0 else 0}
                         - Rows removed from clusters: {len(indices_to_delete)}
                         - Final rows kept: {len(declustered_df)}
                     """)