1.15

app.py

@@ -320,8 +320,9 @@ def create_download_link(df: pd.DataFrame, filename: str) -> str:
     b64 = base64.b64encode(excel_buffer.read()).decode()
     return f'<a href="data:application/vnd.openxmlformats-officedocument.spreadsheetml.sheet;base64,{b64}" download="{filename}">Download {filename}</a>'
 
+
 def main():
-    st.title("кластеризуем новости v.1.14")
+    st.title("кластеризуем новости v.1.15")
     st.write("Upload Excel file with columns: company, datetime, text")
     
     uploaded_file = st.file_uploader("Choose Excel file", type=['xlsx'])
@@ -361,63 +362,75 @@ def main():
                     dedup_df = deduplicator.deduplicate(df, progress_bar)
                     st.success(f"Removed {len(df) - len(dedup_df)} duplicates")
                     
-                    # Preserve all columns from original DataFrame in dedup_df
+                    # Get working copy of deduplicated data with all columns
                     dedup_df_full = df_original.loc[dedup_df.index].copy()
                     
+                    # Create working copy for clustering with required columns
+                    working_df = dedup_df_full[[company_column, datetime_column, title_column, text_column]].copy()
+                    working_df.columns = ['company', 'datetime', 'title', 'text']
+                    
                     # Step 2: Cluster deduplicated news
                     processor = NewsProcessor(similarity_threshold, time_threshold)
-                    result_df = processor.process_news(dedup_df, progress_bar)
+                    result_df = processor.process_news(working_df, progress_bar)
                     
-                    # Initialize set of indices to delete
+                    # Create a dictionary to store indices to delete for each cluster
                     indices_to_delete = set()
                     
-                    # Find rows to delete from multi-item clusters
-                    multi_clusters = result_df[result_df['cluster_size'] > 1]['cluster_id'].unique()
-                    
-                    for cluster_id in multi_clusters:
-                        # Get all rows in this cluster
-                        cluster_indices = result_df[result_df['cluster_id'] == cluster_id].index
-                        
-                        # Find the row with longest text
-                        text_lengths = dedup_df_full.loc[cluster_indices, text_column].str.len()
-                        longest_text_idx = text_lengths.idxmax()
-                        
-                        # Add all other rows from this cluster to delete list
-                        indices_to_delete.update(set(cluster_indices) - {longest_text_idx})
+                    # Process clusters with more than 1 member
+                    if not result_df.empty:
+                        for cluster_id in result_df['cluster_id'].unique():
+                            cluster_rows = result_df[result_df['cluster_id'] == cluster_id]
+                            
+                            if len(cluster_rows) > 1:  # Only process multi-member clusters
+                                # Get indices of all rows in this cluster
+                                cluster_indices = cluster_rows.index
+                                
+                                # Find the row with the longest text
+                                text_lengths = working_df.loc[cluster_indices, 'text'].str.len()
+                                longest_text_idx = text_lengths.idxmax()
+                                
+                                # Add all other indices from this cluster to deletion set
+                                cluster_indices_to_delete = set(cluster_indices) - {longest_text_idx}
+                                indices_to_delete.update(cluster_indices_to_delete)
                     
-                    # Create final declustered DataFrame by removing identified rows
-                    declustered_df = dedup_df_full.drop(index=list(indices_to_delete))
+                    # Create final declustered DataFrame by dropping the identified rows
+                    declustered_df = dedup_df_full.copy()
+                    if indices_to_delete:
+                        declustered_df = declustered_df.drop(index=list(indices_to_delete))
                     
-                    # Print statistics
+                    # Print statistics for verification
                     st.success(f"""
                         Processing results:
                         - Original rows: {len(df_original)}
                         - After deduplication: {len(dedup_df_full)}
-                        - Multi-item clusters found: {len(multi_clusters)}
-                        - Rows removed from clusters: {len(indices_to_delete)}
+                        - Rows in clusters (any size): {len(result_df) if not result_df.empty else 0}
+                        - Rows removed from multi-member clusters: {len(indices_to_delete)}
                         - Final rows after declustering: {len(declustered_df)}
                     """)
                     
+                    # Add debugging information
+                    if not result_df.empty:
+                        multi_clusters = len(result_df[result_df['cluster_size'] > 1]['cluster_id'].unique())
+                        st.write(f"Number of multi-member clusters found: {multi_clusters}")
+                        
+                        # Show cluster sizes
+                        cluster_sizes = result_df['cluster_size'].value_counts().sort_index()
+                        st.write("Cluster size distribution:")
+                        st.write(cluster_sizes)
+                    
                     # Download buttons for all results
                     st.subheader("Download Results")
                     st.markdown(create_download_link(dedup_df_full, "deduplicated_news.xlsx"), unsafe_allow_html=True)
-                    st.markdown(create_download_link(result_df, "clustered_news.xlsx"), unsafe_allow_html=True)
                     st.markdown(create_download_link(declustered_df, "declustered_news.xlsx"), unsafe_allow_html=True)
                     
-                    st.subheader("Cluster Statistics")
-                    cluster_stats = result_df.groupby('cluster_id').agg({
-                        'cluster_size': 'first',
-                        'company': 'first',
-                        'relevance_score': 'mean'
-                    }).rename(columns={'relevance_score': 'avg_relevance'})
-                    st.dataframe(cluster_stats)
-                    
-                    st.subheader("Largest Clusters")
-                    largest_clusters = result_df[result_df['cluster_size'] > 1].sort_values(
-                        ['cluster_size', 'cluster_id', 'datetime'],
-                        ascending=[False, True, True]
-                    )
-                    st.dataframe(largest_clusters)
+                    # Show cluster statistics if clusters were found
+                    if not result_df.empty:
+                        st.subheader("Largest Clusters")
+                        largest_clusters = result_df[result_df['cluster_size'] > 1].sort_values(
+                            ['cluster_size', 'cluster_id', 'datetime'],
+                            ascending=[False, True, True]
+                        )
+                        st.dataframe(largest_clusters)
                     
                 except Exception as e:
                     st.error(f"Error: {str(e)}")