Update app.py

app.py

@@ -181,7 +181,7 @@ def improve_summary_generation(text, model, tokenizer):
                 "length_penalty": 1.5,
                 "no_repeat_ngram_size": 3,
                 "temperature": 0.7,
-                "repetition_penalty": 1.5  # Increased to reduce repetition
+                "repetition_penalty": 1.5
             }
         )
     
@@ -214,12 +214,6 @@ def improve_summary_generation(text, model, tokenizer):
 
 def validate_summary(summary, original_text):
     """Validate summary content against original text"""
-    import re
-    
-    # Don't validate empty summaries
-    if not summary or not original_text:
-        return False
-        
     # Check for age inconsistencies
     age_mentions = re.findall(r'(\d+\.?\d*)\s*years?', summary.lower())
     if len(age_mentions) > 1:  # Multiple age mentions
@@ -237,72 +231,8 @@ def validate_summary(summary, original_text):
     if summary_words < 20 or summary_words > original_words * 0.8:
         return False
     
-    # Check for common error patterns
-    error_patterns = [
-        r'mean.*mean',
-        r'median.*median',
-        r'results.*results',
-        r'conclusion.*conclusion',
-        r'significance.*significance'
-    ]
-    
-    for pattern in error_patterns:
-        if len(re.findall(pattern, summary.lower())) > 1:
-            return False
-    
     return True
 
-def post_process_summary(summary):
-    """Enhanced post-processing to catch common errors"""
-    if not summary:
-        return summary
-    
-    # Remove contradictory age statements
-    age_statements = []
-    lines = summary.split('.')
-    cleaned_lines = []
-    for line in lines:
-        if "age" not in line.lower():
-            cleaned_lines.append(line)
-        elif not age_statements:  # Only keep first age statement
-            age_statements.append(line)
-            cleaned_lines.append(line)
-    
-    # Remove redundant statements
-    seen_content = set()
-    unique_lines = []
-    for line in cleaned_lines:
-        # Skip empty lines
-        if not line.strip():
-            continue
-            
-        # Normalize for comparison
-        line_core = ' '.join(sorted(line.lower().split()))
-        
-        # Check for near-duplicates
-        duplicate = False
-        for seen in seen_content:
-            if line_core in seen or seen in line_core:
-                duplicate = True
-                break
-                
-        if not duplicate:
-            seen_content.add(line_core)
-            unique_lines.append(line)
-    
-    # Join sentences with proper spacing and punctuation
-    cleaned_summary = '. '.join(s.strip() for s in unique_lines if s.strip())
-    if cleaned_summary and not cleaned_summary.endswith('.'):
-        cleaned_summary += '.'
-    
-    # Additional cleaning
-    cleaned_summary = cleaned_summary.replace(" and and ", " and ")
-    cleaned_summary = cleaned_summary.replace("results showed", "")
-    cleaned_summary = cleaned_summary.replace("results indicated", "")
-    cleaned_summary = cleaned_summary.replace("  ", " ")
-    
-    return cleaned_summary
-
 def generate_focused_summary(question, abstracts, model, tokenizer):
     """Generate focused summary based on question"""
     # Preprocess each abstract
@@ -327,22 +257,63 @@ def generate_focused_summary(question, abstracts, model, tokenizer):
     
     return tokenizer.decode(summary_ids[0], skip_special_tokens=True)
 
-
-def validate_summary(summary, original_text):
-    """Validate summary content against original text"""
-    # Check for age inconsistencies
-    age_mentions = re.findall(r'(\d+\.?\d*)\s*years?', summary.lower())
-    if len(age_mentions) > 1:  # Multiple age mentions
-        return False
-    
-    # Check for repetitive sentences
-    sentences = summary.split('.')
-    unique_sentences = set(s.strip().lower() for s in sentences if s.strip())
-    if len(sentences) - len(unique_sentences) > 1:  # More than one duplicate
-        return False
-    
-    return True
+def create_filter_controls(df, sort_column):
+    """Create appropriate filter controls based on the selected column"""
+    filtered_df = df.copy()
+    
+    if sort_column == 'Publication Year':
+        # Year range slider
+        year_min = int(df['Publication Year'].min())
+        year_max = int(df['Publication Year'].max())
+        col1, col2 = st.columns(2)
+        with col1:
+            start_year = st.number_input('From Year', 
+                min_value=year_min, 
+                max_value=year_max,
+                value=year_min)
+        with col2:
+            end_year = st.number_input('To Year', 
+                min_value=year_min, 
+                max_value=year_max,
+                value=year_max)
+        filtered_df = filtered_df[
+            (filtered_df['Publication Year'] >= start_year) & 
+            (filtered_df['Publication Year'] <= end_year)
+        ]
+        
+    elif sort_column == 'Authors':
+        # Multi-select for authors
+        unique_authors = sorted(set(
+            author.strip()
+            for authors in df['Authors'].dropna()
+            for author in authors.split(';')
+        ))
+        selected_authors = st.multiselect(
+            'Select Authors',
+            unique_authors
+        )
+        if selected_authors:
+            filtered_df = filtered_df[
+                filtered_df['Authors'].apply(
+                    lambda x: any(author in str(x) for author in selected_authors)
+                )
+            ]
+            
+    elif sort_column == 'Source Title':
+        # Multi-select for source titles
+        unique_sources = sorted(df['Source Title'].unique())
+        selected_sources = st.multiselect(
+            'Select Sources',
+            unique_sources
+        )
+        if selected_sources:
+            filtered_df = filtered_df[filtered_df['Source Title'].isin(selected_sources)]
+            
+    elif sort_column == 'Article Title':
+        # Only alphabetical sorting, no filtering
+        pass
     
+    return filtered_df
 
 def main():
     st.title("🔬 Biomedical Papers Analysis")
@@ -391,46 +362,59 @@ def main():
                 if st.session_state.summaries is None:
                     try:
                         with st.spinner("Generating individual paper summaries..."):
-                            # Load summarization model
                             model, tokenizer = load_model("summarize")
-                            
-                            # Generate summaries for each abstract
                             summaries = []
                             progress_bar = st.progress(0)
                             
                             for idx, abstract in enumerate(df['Abstract']):
-                                # Replace this line
-                                # summary = generate_summary(abstract, model, tokenizer)
-                                # With this line
                                 summary = improve_summary_generation(abstract, model, tokenizer)
                                 summaries.append(summary)
                                 progress_bar.progress((idx + 1) / len(df))
                             
-                            # Store summaries in session state
                             st.session_state.summaries = summaries
-                            
-                            # Cleanup
                             cleanup_model(model, tokenizer)
                             progress_bar.empty()
                             
                     except Exception as e:
                         st.error(f"Error generating summaries: {str(e)}")
-                        st.session_state.processing_started = False  # Reset to allow retry
+                        st.session_state.processing_started = False
                 
-                # Display summaries with improved sorting
+                # Display summaries with improved sorting and filtering
                 if st.session_state.summaries is not None:
                     col1, col2 = st.columns(2)
                     with col1:
                         sort_options = ['Article Title', 'Authors', 'Publication Year', 'Source Title']
-                        sort_column = st.selectbox("Sort by:", sort_options)
+                        sort_column = st.selectbox("Sort/Filter by:", sort_options)
                     with col2:
-                        ascending = st.checkbox("Ascending order", True)
+                        # Only show A-Z/Z-A option for Article Title
+                        if sort_column == 'Article Title':
+                            ascending = st.radio(
+                                "Sort order",
+                                ["A to Z", "Z to A"],
+                                horizontal=True
+                            ) == "A to Z"
+                        else:
+                            ascending = True  # Default for other columns
                     
-                    # Create display dataframe with formatted year
+                    # Create display dataframe
                     display_df = df.copy()
                     display_df['Summary'] = st.session_state.summaries
                     display_df['Publication Year'] = display_df['Publication Year'].astype(int)
-                    sorted_df = display_df.sort_values(by=sort_column, ascending=ascending)
+                    
+                    # Apply filters
+                    filtered_df = create_filter_controls(display_df, sort_column)
+                    
+                    if sort_column == 'Article Title':
+                        # Sort alphabetically
+                        sorted_df = filtered_df.sort_values(by=sort_column, ascending=ascending)
+                    else:
+                        # Keep original order for other columns after filtering		
+		  	# Keep original order for other columns after filtering
+                        sorted_df = filtered_df
+                    
+                    # Show number of filtered results
+                    if len(sorted_df) != len(display_df):
+                        st.write(f"Showing {len(sorted_df)} of {len(display_df)} papers")
                     
                     # Apply custom styling
                     st.markdown("""
@@ -463,7 +447,7 @@ def main():
                     </style>
                     """, unsafe_allow_html=True)
                     
-                    # Display papers in side-by-side layout
+                    # Display papers using the filtered and sorted dataframe
                     for _, row in sorted_df.iterrows():
                         paper_info_cols = st.columns([1, 1])
                         
@@ -489,7 +473,7 @@ def main():
                             </div>
                             """, unsafe_allow_html=True)
                         
-                        # Add some spacing between papers
+                        # Add spacing between papers
                         st.markdown("<div style='margin-bottom: 20px;'></div>", unsafe_allow_html=True)
                 
                 # Question-focused Summary Section (only if question provided)