back to normal

app.py

@@ -2,36 +2,15 @@
 import streamlit as st
 import os
 import sys
-import time
 import logging
-import concurrent.futures # Use concurrent.futures explicitly
-from concurrent.futures import ThreadPoolExecutor, Future # Import Future
-import shutil # Import shutil for clearing directory
-import queue # Import the queue module
 
 # Add the project root to the sys.path to allow importing modules like config, document_processing, etc.
 # This assumes app.py is in the project root directory.
-project_root = os.path.abspath(os.path.join(os.path.dirname(__file__)))
-if project_root not in sys.path:
-    sys.path.append(project_root)
-
-# Initialize ThreadPoolExecutor - reuse it across reruns
-# Max workers can be adjusted based on expected load and I/O vs CPU bound tasks
-if 'executor' not in st.session_state:
-    st.session_state.executor = ThreadPoolExecutor(max_workers=os.cpu_count()*2) # Adjust max_workers as needed
-
-# Initialize a thread-safe queue for communication from background threads
-# This queue will store updates about file processing status
-if 'update_queue' not in st.session_state:
-    st.session_state.update_queue = queue.Queue()
-
+# Adjust the path if your app.py is in a subdirectory.
 
 try:
-    # Import the necessary functions and constants from your main script
-    # Ensure your main.py has the clear_upload_directory function and TEMP_UPLOAD_DIR constant
-    # and that summarize_extracted_documents accepts the update_queue argument.
-    from main import process_uploaded_files, setup_retrieval_system, summarize_extracted_documents, clear_upload_directory, TEMP_UPLOAD_DIR
-
+    # Import the necessary functions from your main script    print("YES1")
+    from main import process_uploaded_files, setup_retrieval_system, summarize_extracted_documents
     # Configure Streamlit's logging to match your application's settings
     logging.basicConfig(level='INFO', format="%(asctime)s - %(name)s - %(levelname)s - %(message)s")
     logger = logging.getLogger(__name__)
@@ -51,7 +30,7 @@ except ImportError as e:
 # --- Streamlit App Configuration ---
 st.set_page_config(
     page_title="Aya Insight Document Summarizer",
-    page_icon="塘",
+    page_icon="📄",
     layout="wide"
 )
 
@@ -59,30 +38,21 @@ st.set_page_config(
 # Initialize session state variables if they don't exist
 if 'api_key_entered' not in st.session_state:
     st.session_state.api_key_entered = False
-# Initialize summary_results as a dictionary to store results per file
-# Key: filename, Value: {'status': str, 'summary': str | None, 'error': str | None, 'success': bool, ...}
-if 'summary_results' not in st.session_state or st.session_state.summary_results is None:
-    st.session_state.summary_results = {}
+if 'summary_results' not in st.session_state:
+    st.session_state.summary_results = None
 if 'selected_filename' not in st.session_state:
     st.session_state.selected_filename = None
-# Add state variables to track the background task for each file
-# Key: filename, Value: concurrent.futures.Future object
-if 'processing_futures' not in st.session_state:
-    st.session_state.processing_futures = {}
-# Overall summarizing flag
-if 'summarizing' not in st.session_state:
-    st.session_state.summarizing = False
-if 'overall_error' not in st.session_state: # For errors in initial steps (extraction, setup)
-     st.session_state.overall_error = None
 
 
 # --- API Key Input Section ---
 if not st.session_state.api_key_entered:
-    st.title("白 Enter Your Cohere API Key to Unlock")
+    st.title("🔒 Enter Your Cohere API Key to Unlock")
     api_key = st.text_input("Cohere API Key", type="password", help="Enter your Cohere API key to use the summarization service.")
 
     if st.button("Unlock"):
         if api_key:
+            # Basic validation: Just check if it's not empty.
+            # For a real application, you might want to validate by making a small API call.
             os.environ["COHERE_API_KEY"] = api_key # Set the environment variable
             st.session_state.api_key_entered = True
             st.success("API Key accepted. You can now upload documents.")
@@ -92,7 +62,7 @@ if not st.session_state.api_key_entered:
 
 # --- Main Application Content (Unlocked) ---
 if st.session_state.api_key_entered and modules_loaded:
-    st.title("塘 Aya Insight Document Summarizer")
+    st.title("📄 Aya Insight Document Summarizer")
     st.markdown("""
     Upload one or more PDF or image files to get a structured summary for each document.
     """)
@@ -106,338 +76,140 @@ if st.session_state.api_key_entered and modules_loaded:
     )
 
     # --- Summarize Button and Logic ---
-    # Disable the button if summarization is already in progress
     if uploaded_files: # Only show button if files are uploaded
         st.info(f"You have uploaded {len(uploaded_files)} file(s).")
 
-        # Use a unique key for the button based on the summarizing state
-        button_key = "summarize_button_processing" if st.session_state.summarizing else "summarize_button_ready"
-
-        if st.button("Generate Summaries", key=button_key, disabled=st.session_state.summarizing):
+        if st.button("Generate Summaries", key="summarize_button"):
             st.session_state.selected_filename = None # Reset selected file on new summary generation
-            st.session_state.summary_results = {} # Clear previous results before starting new process
-            st.session_state.processing_futures = {} # Clear previous futures
-            st.session_state.summarizing = True # Set summarizing flag
-            st.session_state.overall_error = None # Clear previous overall errors
-            st.session_state.update_queue = queue.Queue() # Create a new queue for this batch
-
             if not uploaded_files:
                 st.warning("Please upload at least one file before generating summaries.")
-                st.session_state.summarizing = False # Reset flag if no files
-                st.rerun() # Rerun to clear spinner if no files
             else:
                 st.subheader("Processing Documents...")
-                # Clear the temporary upload directory from previous runs
-                clear_upload_directory()
-                logger.info("Cleared previous upload directory content.")
-
-                # --- Initial Processing (Extraction and Retrieval Setup) ---
-                # This part still runs in the main Streamlit thread
-                logger.info(f"Calling process_uploaded_files with {len(uploaded_files)} files.")
-                try:
-                    # Save files and perform initial extraction
-                    extraction_results = process_uploaded_files(uploaded_files)
-                    logger.info(f"Finished document extraction. {len(extraction_results)} results obtained.")
-
-                    # Initialize status for all uploaded files (even those that might fail extraction)
-                    # This ensures all uploaded files get a tile.
-                    uploaded_filenames = [f.name for f in uploaded_files]
-                    for filename in uploaded_filenames:
-                         # Find the corresponding extraction result to get potential early errors
-                         initial_res = next((res for res in extraction_results if res.get('filename') == filename), {})
-                         status = 'waiting'
-                         error = initial_res.get('error')
-                         if error:
-                             status = 'extraction_error'
-                             st.warning(f"Extraction failed for {filename}: {error}")
-
-                         st.session_state.summary_results[filename] = {
-                             'status': status,
-                             'summary': None,
-                             'error': error,
-                             'success': False
-                         }
-
-                    # Filter for files that had successful extraction (have text)
-                    summarizable_extraction_results = [res for res in extraction_results if res.get('text')]
-
-                    if not summarizable_extraction_results:
-                        st.error("No text could be extracted from the uploaded files. Please check the file formats.")
-                        logger.error("No text extracted from any uploaded file.")
-                        st.session_state.summarizing = False # Reset flag
-                        st.rerun() # Rerun to clear spinner and show error
-                        # st.stop() # Don't stop, allow user to try again
-
-
-                    # Setup retrieval system (Vector Store and Embedding)
-                    logger.info("Calling setup_retrieval_system.")
-                    # setup_retrieval_system needs results with text
-                    extraction_results_with_chunks, retriever = setup_retrieval_system(summarizable_extraction_results)
-                    logger.info("Retriever system setup complete.")
-
-                    # Check if chunking was successful for summarizable files
-                    final_summarizable_results = []
-                    for res in extraction_results_with_chunks:
-                         filename = res.get('filename', 'unknown')
-                         if res.get('chunk_size', 0) > 0:
-                             final_summarizable_results.append(res)
-                             # Update status for files that are ready for summarization
-                             if filename in st.session_state.summary_results:
-                                  st.session_state.summary_results[filename]['status'] = 'queued' # Ready for background task
-                         else:
-                             # Update status for files that failed chunking
-                             if filename in st.session_state.summary_results and st.session_state.summary_results[filename]['status'] == 'waiting':
-                                  st.session_state.summary_results[filename].update({
-                                      'status': 'chunking_error',
-                                      'error': 'Could not create text chunks',
-                                      'success': False
-                                  })
-                                  st.warning(f"Chunking failed for {filename}.")
-
-
-                    if not final_summarizable_results:
-                         st.error("No usable content chunks were created from the extracted text. Summarization cannot proceed.")
-                         logger.error("No usable content chunks created.")
-                         st.session_state.summarizing = False # Reset flag
-                         st.rerun()
-                         # st.stop()
-
-
-                    # --- Submit Individual Summarization Tasks to Background ---
-                    logger.info(f"Submitting {len(final_summarizable_results)} summarization tasks to ThreadPoolExecutor.")
-
-                    # Submit the synchronous summarization task for each document to the background thread
-                    # Pass the queue object to the background function
-                    for doc_result in final_summarizable_results:
-                         filename = doc_result.get('filename', 'unknown')
-                         # Pass necessary data and the queue object to the background task
-                         # The summarize_extracted_documents in main.py is expected to handle a list of one document
-                         future = st.session_state.executor.submit(
-                            summarize_extracted_documents, # The synchronous function to run
-                            [doc_result], # Pass a list with a single document result
-                            retriever,
-                            st.session_state.update_queue # Pass the queue
-                         )
-                         st.session_state.processing_futures[filename] = future
-                         # Update initial status in main thread's session state immediately
-                         if filename in st.session_state.summary_results:
-                              st.session_state.summary_results[filename]['status'] = 'processing' # Initial status while task is running
-
-                    logger.info(f"Individual summarization tasks submitted to background for {len(st.session_state.processing_futures)} files.")
-
-                    # Rerun immediately to show the spinner and initial state with 'processing' tiles
-                    st.rerun()
-
-                except FileNotFoundError as fnf_error:
-                    st.error(f"Configuration Error: {fnf_error}. Please check your environment settings.")
-                    logger.error(f"Configuration Error during Streamlit process: {fnf_error}", exc_info=True)
-                    st.session_state.summarizing = False # Reset flag on error
-                    st.session_state.overall_error = str(fnf_error)
-                    st.session_state.processing_futures = {} # Clear futures on error
-                    st.rerun() # Rerun to show error and reset state
-                except Exception as e:
-                    st.error(f"An unexpected error occurred during initial processing: {e}")
-                    logger.error(f"An unexpected error occurred during initial Streamlit process: {e}", exc_info=True)
-                    st.session_state.summarizing = False # Reset flag on error
-                    st.session_state.overall_error = str(e)
-                    st.session_state.processing_futures = {} # Clear futures on error
-                    st.rerun() # Rerun to show error and reset state
-
-
-    # --- Process Queue and Check Background Task Status ---
-    # This block runs on every Streamlit rerun
-    # Process any updates from the background queue
-    updated_from_queue = False
-    try:
-        while not st.session_state.update_queue.empty():
-            update = st.session_state.update_queue.get_nowait()
-            filename = update.get('filename')
-            status = update.get('status')
-            result_data = update.get('result_data')
-
-            if filename and filename in st.session_state.summary_results:
-                logger.debug(f"Main thread processing queue update for {filename}: status={status}")
-                st.session_state.summary_results[filename]['status'] = status
-                if result_data:
-                    # Update other fields if result data is provided (e.g., summary, error, success)
-                    st.session_state.summary_results[filename].update(result_data)
-                updated_from_queue = True
-            st.session_state.update_queue.task_done() # Mark the task as done in the queue
-
-    except queue.Empty:
-        # This is expected when the queue is empty
-        pass
-    except Exception as e:
-        logger.error(f"Error processing queue: {e}", exc_info=True)
-        st.error(f"An error occurred while processing background updates: {e}")
-
-
-    # Check completed futures to know when tasks are done and potentially trigger rerun
-    completed_futures_filenames = [filename for filename, future in st.session_state.processing_futures.items() if future.done()]
-
-    if completed_futures_filenames:
-        logger.debug(f"Found {len(completed_futures_filenames)} completed futures.")
-        rerun_needed = False
-        for filename in completed_futures_filenames:
-            # We don't need to get the result here as the queue callback
-            # already put the final status/result into the queue and updated session_state.
-            # We just need to know the future is done to remove it.
-            try:
-                # Optional: Check for exceptions in the future if the queue callback
-                # didn't handle all error reporting scenarios.
-                # future.result() # This would re-raise exceptions from the background task
-                pass
-            except Exception as e:
-                 logger.error(f"Exception in completed future for {filename} (already reported via queue): {e}", exc_info=True)
-                 # The queue processing should have updated the status to 'error' already.
-
-            # Remove the completed future
-            if filename in st.session_state.processing_futures:
-                del st.session_state.processing_futures[filename]
-                rerun_needed = True # Indicate that UI might need update if queue updates didn't trigger it
-
-        # Trigger a rerun if any futures completed or if queue was processed
-        if rerun_needed or updated_from_queue:
-             logger.info("Completed futures processed or queue updated, triggering rerun.")
-             st.rerun()
-
-    # Check if all futures are done and no more are being added
-    if not st.session_state.processing_futures and st.session_state.summarizing:
-        logger.info("All background tasks completed.")
-        st.session_state.summarizing = False # Reset overall summarizing flag
-        # Trigger a final rerun to show completion status if not already triggered
-        if not updated_from_queue: # Avoid double rerun if queue processing already did it
-             st.rerun()
+                all_summary_results = [] # To store results for display
+
+                # Use a spinner to indicate processing
+                with st.spinner("Processing documents and generating summaries... This may take a few minutes depending on file size and number."):
+                    try:
+                        # Step 1: Process uploaded files (Extraction)
+                        logger.info(f"Calling process_uploaded_files with {len(uploaded_files)} files.")
+                        extraction_results = process_uploaded_files(uploaded_files)
+                        logger.info(f"Finished document extraction. {len(extraction_results)} results obtained.")
+
+                        # Check if any files were successfully extracted
+                        if not any(res.get('text') for res in extraction_results):
+                            st.error("No text could be extracted from the uploaded files. Please check the file formats.")
+                            logger.error("No text extracted from any uploaded file.")
+                            st.session_state.summary_results = [] # Store empty results
+                            # st.stop() # Don't stop, allow user to try again
+
+                        # Step 2: Setup retrieval system (Vector Store and Embedding)
+                        logger.info("Calling setup_retrieval_system.")
+                        extraction_results_with_chunks, retriever = setup_retrieval_system(extraction_results)
+                        logger.info("Retriever system setup complete.")
+
+                        # Step 3: Summarize the extracted documents
+                        logger.info("Calling summarize_extracted_documents.")
+                        summary_results = summarize_extracted_documents(extraction_results_with_chunks, retriever)
+                        logger.info(f"Finished summarization. {len(summary_results)} summary results obtained.")
+
+                        st.session_state.summary_results = summary_results # Store results in session state
+
+                    except FileNotFoundError as fnf_error:
+                        st.error(f"Configuration Error: {fnf_error}. Please check your environment settings.")
+                        logger.error(f"Configuration Error during Streamlit process: {fnf_error}", exc_info=True)
+                        st.session_state.summary_results = [] # Store empty results on error
+                    except Exception as e:
+                        st.error(f"An unexpected error occurred during processing: {e}")
+                        logger.error(f"An unexpected error occurred during Streamlit process: {e}", exc_info=True)
+                        st.session_state.summary_results = [] # Store empty results on error
 
 
     # --- Display Document Tiles and Summaries ---
-    # This section reads from the incrementally updated st.session_state.summary_results.
-    # It will update whenever Streamlit reruns.
-    if st.session_state.summary_results is not None: # Check if the dictionary exists
-        st.subheader("Processing Status and Summaries:")
-
-        if not st.session_state.summary_results and not st.session_state.summarizing and st.session_state.overall_error is None:
-            st.info("Upload files and click 'Generate Summaries' to begin.")
-        elif st.session_state.summarizing or st.session_state.summary_results:
-             # Display an overall status message
-             total_files = len(st.session_state.summary_results)
-             # Count files that have reached a final state
-             completed_count = sum(1 for res in st.session_state.summary_results.values() if res.get('status') in ['completed', 'error', 'skipped', 'extraction_error', 'no_text', 'chunking_error'])
-             # Count files that are currently in an intermediate processing state
-             processing_count = sum(1 for res in st.session_state.summary_results.values() if res.get('status') in ['waiting', 'queued', 'processing', 'summarizing'])
-
-             if st.session_state.summarizing:
-                  st.info(f"Processing... {processing_count} in progress, {completed_count} completed or skipped out of {total_files} files.")
-             elif st.session_state.overall_error:
-                  st.error(f"An error occurred during initial processing: {st.session_state.overall_error}")
-                  st.warning(f"{completed_count} files processed with status out of {total_files} attempted.")
-             else:
-                  successful_summaries = sum(1 for res in st.session_state.summary_results.values() if res.get('status') == 'completed')
-                  st.success(f"Finished processing. {successful_summaries} summaries successfully generated out of {total_files} files.")
-                  if completed_count < total_files:
-                       st.warning(f"Some files ({total_files - completed_count}) did not complete processing.")
-
-
-        # Display files as a grid of clickable tiles if there are results
-        if st.session_state.summary_results:
+    if st.session_state.summary_results is not None:
+        st.subheader("Summaries:")
+
+        if not st.session_state.summary_results:
+            st.info("No summaries were generated. Upload files and click 'Generate Summaries'.")
+        else:
+            # Display files as a grid of clickable tiles
             files_per_row = 3
-            # Use the keys (filenames) for consistent ordering
-            filenames = list(st.session_state.summary_results.keys())
-            rows = len(filenames) // files_per_row + (len(filenames) % files_per_row > 0)
+            rows = len(st.session_state.summary_results) // files_per_row + (len(st.session_state.summary_results) % files_per_row > 0)
+
+            # Create a list of filenames for easy access
+            filenames = [res.get('filename', f'File {i+1}') for i, res in enumerate(st.session_state.summary_results)]
 
             for i in range(rows):
                 cols = st.columns(files_per_row)
                 for j in range(files_per_row):
                     file_index = i * files_per_row + j
-                    if file_index < len(filenames):
-                        filename = filenames[file_index]
-                        result = st.session_state.summary_results.get(filename, {}) # Get result safely
-                        status = result.get('status', 'unknown')
+                    if file_index < len(st.session_state.summary_results):
+                        result = st.session_state.summary_results[file_index]
+                        filename = result.get('filename', f'File {file_index+1}')
                         is_selected = st.session_state.selected_filename == filename
 
-                        # Determine tile appearance based on status
-                        if status == 'completed':
-                            button_label = f"塘 {filename} (Completed)"
-                            # Use a custom style or expander for completed tiles if needed,
-                            # or rely on the selection below to show the summary.
-                        elif status in ['error', 'skipped', 'extraction_error', 'no_text', 'chunking_error']:
-                            button_label = f"☒ {filename} ({status.replace('_', ' ').title()})" # e.g., "Extraction Error"
-                        elif status in ['waiting', 'queued', 'processing', 'summarizing']:
-                             button_label = f"時計 {filename} ({status.replace('_', ' ').title()})..."
-                        else:
-                            button_label = f"ファイル {filename} (Status: {status})"
-
-                        # Use a standard button within the column
-                        # Add a unique key based on the filename and whether it's selected
-                        button_key = f"tile_button_{filename}_{'selected' if is_selected else 'normal'}"
-                        # Use beta_container or just cols[j]
+                        # Create a tile using a button or markdown link
+                        # Using a button inside a column for simplicity
                         with cols[j]:
-                            # Make the button disabled if it's in a processing state
-                            button_disabled = status in ['waiting', 'queued', 'processing', 'summarizing']
-                            if st.button(button_label, key=button_key, use_container_width=True, disabled=button_disabled):
-                                # Only allow selecting tiles that have finished processing (completed, error, skipped, etc.)
-                                if not button_disabled:
-                                    st.session_state.selected_filename = filename
-                                    logger.info(f"Selected file: {filename}")
-                                else:
-                                     # This block is technically unreachable because the button is disabled,
-                                     # but good practice to handle.
-                                     st.info(f"Cannot select '{filename}' while it is still processing or waiting.")
-                                     st.session_state.selected_filename = None # Deselect if a processing tile was somehow clicked
-
-
-            # Display summary/details of the selected file below the tiles
+                            # Add a border or highlight if selected
+                            tile_style = "border: 2px solid lightgrey; padding: 10px; margin: 5px; text-align: center; cursor: pointer;"
+                            if is_selected:
+                                tile_style = "border: 2px solid steelblue; padding: 10px; margin: 5px; text-align: center; cursor: pointer; background-color: #e6f3ff;" # Highlight color
+
+                            # Use markdown with HTML to create the clickable tile appearance
+                            # When clicked, set the selected filename in session state
+                            st.markdown(
+                                f"""
+                                <div style="{tile_style}" onclick="document.getElementById('hidden_button_{file_index}').click()">
+                                    📄<br>
+                                    <strong>{filename}</strong>
+                                </div>
+                                <button id="hidden_button_{file_index}" style="display: none;" onclick="document.getElementById('hidden_button_{file_index}').click()"></button>
+                                """,
+                                unsafe_allow_html=True
+                            )
+                            # Streamlit buttons don't work directly with markdown clicks like this easily.
+                            # A simpler approach is to use a standard button and handle the click.
+                            # Let's use a standard button instead of complex markdown/JS.
+
+                            # Alternative using a standard button:
+                            if st.button(f"📄 {filename}", key=f"tile_button_{file_index}"):
+                                st.session_state.selected_filename = filename
+                                logger.info(f"Selected file: {filename}")
+                                st.rerun() # Rerun to display the summary
+
+
+            # Display summary of the selected file
             if st.session_state.selected_filename:
                 st.markdown("---") # Separator
-                st.subheader(f"Details for: {st.session_state.selected_filename}")
-
-                selected_result = st.session_state.summary_results.get(st.session_state.selected_filename)
-
-                if selected_result:
-                    status = selected_result.get('status', 'unknown')
-                    st.markdown(f"**Status:** {status.replace('_', ' ').title()}")
-
-                    if status == 'completed':
-                        summary = selected_result.get('summary')
-                        if summary:
-                            st.markdown("---")
-                            st.markdown("### Summary:")
-                            st.markdown(summary) # Render markdown summary
-                        else:
-                            st.info(f"Summary is empty for {st.session_state.selected_filename}.")
-                    elif selected_result.get('error'):
-                         st.error(f"Error details: {selected_result.get('error')}")
-                    elif status == 'skipped':
-                         st.info("This file was skipped due to initial processing issues.")
-                    elif status == 'no_text':
-                         st.info("No text could be extracted from this file.")
-                    elif status == 'chunking_error':
-                         st.info("Could not create usable text chunks from this file.")
+                st.subheader(f"Summary for: {st.session_state.selected_filename}")
+
+                # Find the summary for the selected file
+                selected_summary = None
+                selected_result = None
+                for result in st.session_state.summary_results:
+                    if result.get('filename') == st.session_state.selected_filename:
+                        selected_summary = result.get('summary')
+                        selected_result = result
+                        break
+
+                if selected_summary:
+                    if selected_result.get('success'):
+                         st.markdown(selected_summary) # Render markdown summary
                     else:
-                         st.info(f"Details are not available for '{st.session_state.selected_filename}' yet as it has status: {status.replace('_', ' ').title()}")
+                         st.error(f"Could not load summary for {st.session_state.selected_filename}: {selected_result.get('error', 'Unknown error')}")
+                else:
+                    st.info(f"Summary not available for {st.session_state.selected_filename}.")
 
+            # Display overall processing status
+            successful_count = sum(res.get('success', False) for res in st.session_state.summary_results)
+            total_files = len(st.session_state.summary_results)
+            st.markdown(f"---") # Final separator
+            st.success(f"Processed {total_files} files. Successfully summarized {successful_count}.")
+            if successful_count < total_files:
+                st.warning("Some files could not be processed or summarized. See error messages above.")
 
-# --- Message if modules failed to load ---
-if not modules_loaded:
-     st.error("Application modules failed to load. Please check your environment and project setup.")
 
 # --- Message if API Key is not entered and modules loaded ---
-if not st.session_state.api_key_entered and modules_loaded and not st.session_state.summarizing:
+if not st.session_state.api_key_entered:
     st.info("Enter your Cohere API Key above to unlock the application functionality.")
 
-# --- Keep UI updated by rerunning periodically if processing ---
-# This is a simple mechanism to trigger reruns to check background task status and queue.
-# Adjust sleep time as needed, shorter times make it more responsive but use more CPU.
-# Rerun if there are active processing futures or if the queue is not empty.
-if st.session_state.processing_futures or not st.session_state.update_queue.empty():
-     logger.debug(f"Active processing futures ({len(st.session_state.processing_futures)}) or queue not empty, sleeping briefly to trigger rerun.")
-     time.sleep(0.5) # Sleep for a short duration
-     st.rerun() # Rerun the app to check for completed futures and process queue
-elif st.session_state.summarizing:
-    # If summarizing is true but no futures are active and queue is empty,
-    # it means initial steps might be running or processing just finished.
-    # Rerun once more to update final status.
-     logger.debug("Summarizing is True but no active futures and queue is empty, triggering final rerun.")
-     st.session_state.summarizing = False # Ensure flag is reset
-     st.rerun()
-

main.py

@@ -1,64 +1,26 @@
-import time
-import logging
+import logging  # Import logging
 import os
 import tempfile
-import shutil  # Import shutil for directory cleaning
-from typing import List, Dict, Any, Tuple, Optional
-from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed
-import queue # Import the queue module
-
-# Import necessary modules from your project
-# Assuming config, document_processing, retrieval, summarization are accessible via sys.path
-# You might need to adjust these imports based on your actual project structure
-try:
-    from app.config.settings import DOCS_FOLDER # Keep for local runs if needed
-except ImportError:
-    # Define a default if config.config is not available
-    DOCS_FOLDER = "docs" # Default folder name
-
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from typing import List, Dict, Any, Tuple
 
+from app.config.settings import DOCS_FOLDER
+# Import classes from the renamed modules
 from app.document_processing.extractors import DocumentProcessorAdapter
 from app.retrieval.vector_store import Retriever
+from app.summarization.output import SummaryOutputManager
 from app.summarization.summarizer import DocumentSummarizer
 
-# Configure module-specific logger
+# Configure logging for the main script
 logger = logging.getLogger(__name__)
 
-# Add a simple print statement to confirm module loading
-logger.info("main.py is being loaded.")
-
-# Define a persistent temporary directory relative to the project root
-# This assumes app.py and main.py are in the project root
-PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__)))
-TEMP_UPLOAD_DIR = os.path.join(PROJECT_ROOT, 'temp_uploads')
-os.makedirs(TEMP_UPLOAD_DIR, exist_ok=True) # Ensure the directory exists
-
-def clear_upload_directory():
-    """Clears all files from the persistent temporary upload directory."""
-    if os.path.exists(TEMP_UPLOAD_DIR):
-        logger.info(f"Clearing temporary upload directory: {TEMP_UPLOAD_DIR}")
-        # Iterate through all items in the directory
-        for item in os.listdir(TEMP_UPLOAD_DIR):
-            item_path = os.path.join(TEMP_UPLOAD_DIR, item)
-            try:
-                # Check if it's a file or a symbolic link and remove it
-                if os.path.isfile(item_path) or os.path.islink(item_path):
-                    os.unlink(item_path)
-                    logger.debug(f"Deleted file/link: {item_path}")
-                # Check if it's a directory and remove it and its contents
-                elif os.path.isdir(item_path):
-                    shutil.rmtree(item_path)
-                    logger.debug(f"Deleted directory: {item_path}")
-            except Exception as e:
-                logger.error(f"Error deleting {item_path}: {e}", exc_info=True)
-    else:
-         logger.info(f"Temporary upload directory does not exist, no need to clear: {TEMP_UPLOAD_DIR}")
 
 
 def process_uploaded_files(uploaded_files) -> List[Dict[str, Any]]:
     """
     Processes a list of files uploaded via Streamlit.
-    Saves them into a persistent temporary folder and uses the DocumentProcessorAdapter
+    Saves them temporarily into a folder and uses the DocumentProcessorAdapter
     to process that folder.
 
     Args:
@@ -74,32 +36,34 @@ def process_uploaded_files(uploaded_files) -> List[Dict[str, Any]]:
     start_time = time.time()
     logger.info(f"Starting processing for {len(uploaded_files)} uploaded files.")
 
-    # Save all uploaded files into the persistent temporary directory
-    logger.info(f"Saving files to persistent temporary directory: {TEMP_UPLOAD_DIR}")
-    saved_files_paths = []
-    for uploaded_file in uploaded_files:
-        # Create a safe path within the temporary directory
-        # Use uploaded_file.name directly, ensuring it's joined with the base temp dir
-        file_path = os.path.join(TEMP_UPLOAD_DIR, uploaded_file.name)
-        # Write the file content to the temporary path
-        try:
-            with open(file_path, "wb") as f:
-                f.write(uploaded_file.getvalue())
-            logger.debug(f"Saved uploaded file '{uploaded_file.name}' to '{file_path}'")
-            saved_files_paths.append(file_path)
-        except Exception as e:
-            logger.error(f"Error saving uploaded file '{uploaded_file.name}' to temporary directory: {e}", exc_info=True)
-            # Log a warning in Streamlit if a file couldn't be saved
-            st.warning(f"Could not save uploaded file '{uploaded_file.name}' temporarily. It will be skipped.")
+    # Create a temporary directory to save uploaded files
+    # This directory will be automatically cleaned up when the 'with' block exits
+    with tempfile.TemporaryDirectory() as tmpdir:
+        logger.info(f"Using temporary directory: {tmpdir}")
+        # Save all uploaded files into the temporary directory
+        for uploaded_file in uploaded_files:
+            # Create a safe path within the temporary directory
+            # Use uploaded_file.name directly, tempfile handles uniqueness if needed
+            file_path = os.path.join(tmpdir, uploaded_file.name)
+            # Write the file content to the temporary path
+            try:
+                with open(file_path, "wb") as f:
+                    f.write(uploaded_file.getvalue())
+                logger.debug(f"Saved uploaded file '{uploaded_file.name}' to '{file_path}'")
+            except Exception as e:
+                logger.error(f"Error saving uploaded file '{uploaded_file.name}' to temporary directory: {e}", exc_info=True)
+                # Log a warning in Streamlit if a file couldn't be saved
+                st.warning(f"Could not save uploaded file '{uploaded_file.name}' temporarily. It will be skipped.")
 
-    # Use the DocumentProcessorAdapter to process the entire temporary folder
-    processor = DocumentProcessorAdapter()
-    # Call process_folder with the persistent temporary directory path
-    # The process_folder method returns the list of extraction results
-    extraction_results = processor.process_folder(TEMP_UPLOAD_DIR)
+
+        # Use the DocumentProcessorAdapter to process the entire temporary folder
+        processor = DocumentProcessorAdapter() # Corrected typo here
+        # Call process_folder with the temporary directory path
+        extraction_results = processor.process_folder(tmpdir)
+        # The process_folder method returns the list of extraction results
 
     end_time = time.time()
-    logger.info(f"Finished processing uploaded files (saving and initial extraction) in {end_time - start_time:.2f} seconds.")
+    logger.info(f"Finished processing uploaded files in {end_time - start_time:.2f} seconds.")
     # The extraction_results list now contains dictionaries with 'filename', 'text', 'error', etc.
     return extraction_results
 
@@ -135,18 +99,14 @@ def setup_retrieval_system(extraction_results: List[Dict[str, Any]]) -> Tuple[Li
         raise
 
 
-def summarize_extracted_documents(extraction_results: List[Dict[str, Any]], retriever: Retriever, update_queue: queue.Queue) -> List[Dict[str, Any]]:
+def summarize_extracted_documents(extraction_results: List[Dict[str, Any]], retriever: Retriever) -> List[Dict[str, Any]]:
     """
-    Summarizes documents based on extraction results and a configured retriever,
-    reporting progress via a queue.
+    Summarizes documents based on extraction results and a configured retriever.
 
     Args:
         extraction_results: List of dictionaries from document extraction (should include chunk_size
                             populated by setup_retrieval_system).
-                            EXPECTED to be a list containing *one* document result dictionary
-                            when called from the Streamlit app for individual file processing.
         retriever: An initialized Retriever instance.
-        update_queue: A thread-safe queue.Queue object to put progress updates into.
 
     Returns:
         A list of dictionaries, each containing the summary result for a file.
@@ -156,94 +116,167 @@ def summarize_extracted_documents(extraction_results: List[Dict[str, Any]], retr
         - 'summary': The generated summary string (if successful), or None.
         - 'error': An error message string (if not successful), or None.
         - 'processing_time': Time taken for summarization of this file.
-        - 'status': Current processing status (e.g., 'completed', 'error').
     """
-    # This function is designed to handle a list of results, but the Streamlit app
-    # calls it with a list containing a single document result for parallel processing.
-    # The internal logic iterates through the provided list.
-
     start_time = time.time()
-    logger.info(f"Starting summarization for {len(extraction_results)} document(s) in background.")
+    logger.info(f"Starting summarization for {len(extraction_results)} documents.")
 
     # Initialize the summarizer with the retriever
     summarizer = DocumentSummarizer(retriever)
 
-    results = [] # List to store results for each document processed by this call
+    results = [] # List to store results for each document
 
-    # Filter out results that failed extraction or have no text/chunks from the input list
+    # Filter out results that failed extraction or have no text/chunks
     # Summarization requires extracted text and successful chunking (chunk_size > 0)
-    summarizable_input_results = [
+    summarizable_results = [
         res for res in extraction_results
         if res.get('text') and res.get('chunk_size', 0) > 0 and res.get('error') is None
     ]
-    skipped_input_results = [
+    skipped_results = [
         res for res in extraction_results
-        if res not in summarizable_input_results
+        if res not in summarizable_results
     ]
 
-    if skipped_input_results:
-        logger.warning(f"Skipping summarization for {len(skipped_input_results)} input file(s) due to extraction errors or no text/chunks.")
-        for res in skipped_input_results:
+    if skipped_results:
+        logger.warning(f"Skipping summarization for {len(skipped_results)} files due to extraction errors or no text/chunks.")
+        for res in skipped_results:
             # Add entries for skipped files to the results list
-            skipped_result = {
+            results.append({
                 'filename': res.get('filename', 'unknown'),
                 'success': False,
                 'summary': None,
                 'error': res.get('error', 'Extraction failed or no text/chunks'),
                 'processing_time': 0, # No summarization time for skipped files
-                'status': 'skipped' # Add status for skipped files
-            }
-            results.append(skipped_result)
-            # Put status update into the queue
-            update_queue.put({'filename': skipped_result['filename'], 'status': skipped_result['status'], 'result_data': skipped_result})
+            })
 
 
-    # Process only the summarizable documents from the input list
-    for result in summarizable_input_results:
+    def process_single_summary(result: Dict[str, Any]) -> Dict[str, Any]:
+        """Helper function to summarize a single document result."""
         file_start_time = time.time()
         filename = result.get('filename', 'unknown')
+        # Use detected language, default to English if detection failed
         language = result.get('language', 'en')
-        chunk_size = result.get('chunk_size', 0) # Should be > 0 for summarizable_input_results
+        chunk_size = result.get('chunk_size', 0) # Should be > 0 for summarizable_results
 
-        logger.info(f"Background: Summarizing document: {filename}")
-        # Put status update into the queue
-        update_queue.put({'filename': filename, 'status': 'summarizing'})
+        logger.info(f"Summarizing document: {filename}")
 
         try:
-            # Call the summarizer for this single document
+            # Call the summarizer for a single document
+            # The summerize_document method handles parallel processing of components internally
             summary = summarizer.summerize_document(filename, language, chunk_size)
 
             file_end_time = time.time()
-            logger.info(f"Background: Finished summarizing {filename} in {file_end_time - file_start_time:.2f} seconds.")
-            summary_result = {
+            logger.info(f"Finished summarizing {filename} in {file_end_time - file_start_time:.2f} seconds.")
+            return {
                 'filename': filename,
                 'success': True,
                 'summary': summary, # Return the summary string
                 'error': None,
                 'processing_time': file_end_time - file_start_time,
-                'status': 'completed' # Add completed status
             }
-            results.append(summary_result) # Add to the results list for this call
-            # Put status update into the queue
-            update_queue.put({'filename': filename, 'status': summary_result['status'], 'result_data': summary_result})
-
         except Exception as e:
             file_end_time = time.time()
             error_msg = str(e)
-            logger.error(f"Background: Error summarizing document {filename}: {e}", exc_info=True)
-            error_result = {
+            logger.error(f"Error summarizing document {filename}: {e}", exc_info=True)
+            return {
                 'filename': filename,
                 'success': False,
                 'summary': None,
                 'error': error_msg,
                 'processing_time': file_end_time - file_start_time,
-                'status': 'error' # Add error status
             }
-            results.append(error_result) # Add to the results list for this call
-            # Put status update into the queue
-            update_queue.put({'filename': filename, 'status': error_result['status'], 'result_data': error_result})
+
+    with ThreadPoolExecutor(max_workers=None) as executor: # Adjust max_workers as needed
+        # Submit summarizable document results to the executor
+        futures = {executor.submit(process_single_summary, res): res['filename'] for res in summarizable_results}
+
+        # Process results as they complete
+        for future in as_completed(futures):
+            filename = futures[future]
+            try:
+                summary_result = future.result()
+                results.append(summary_result)
+                logger.debug(f"Summary result received for {filename}")
+            except Exception as exc:
+                # This catches exceptions *within* the future's result retrieval
+                logger.error(f"Exception retrieving summary result for {filename}: {exc}", exc_info=True)
+                results.append({
+                    'filename': filename,
+                    'success': False,
+                    'summary': None,
+                    'error': f"Failed to retrieve result: {exc}",
+                    'processing_time': 0, # Can't determine processing time if result retrieval failed
+                })
 
     end_time = time.time()
-    logger.info(f"Background: Finished processing {len(extraction_results)} document(s) in {end_time - start_time:.2f} seconds.")
-    # Return the list of results processed by this specific call
+    logger.info(f"Finished batch summarization in {end_time - start_time:.2f} seconds.")
     return results
+
+
+# if __name__ == "__main__":
+#     start_time = time.time()
+#     logger.info("Starting document summarization process (command line).")
+#
+#     try:
+#         # Step 1: Process documents from the predefined folder
+#         logger.info(f"Processing documents from: {DOCS_FOLDER}")
+#         # DocumentProcessorAdapter().process_folder returns a list of extraction result dicts
+#         extraction_results = DocumentProcessorAdapter().process_folder(DOCS_FOLDER)
+#         logger.info(f"Document Processing Time taken: {time.time()-start_time:.2f} seconds")
+#
+#         # Step 2: Setup retrieval system
+#         setup_start_time = time.time()
+#         # setup_retrieval_system takes extraction results and returns updated results (with chunk_size) and the retriever
+#         extraction_results_with_chunks, retriever = setup_retrieval_system(extraction_results)
+#         logger.info(f"Retriever Setup Time taken: {time.time() - setup_start_time:.2f} seconds")
+#
+#         # Step 3: Summarize the documents
+#         summarization_start_time = time.time()
+#         # For command line, we might still want to save files locally
+#         output_manager = SummaryOutputManager() # Uses default output_dir from settings
+#         # summarize_extracted_documents performs the summarization and returns results
+#         summary_results = summarize_extracted_documents(extraction_results_with_chunks, retriever)
+#
+#         # Step 4: Save summaries to files (for command-line only)
+#         logger.info("Saving summaries to files.")
+#         saved_count = 0
+#         for res in summary_results:
+#             if res['success'] and res['summary']:
+#                 # Use the output_manager to save the summary string
+#                 output_manager.save_summary(res['filename'], res['summary'], formats=['markdown'])
+#                 saved_count += 1
+#         logger.info(f"Saved {saved_count} summaries.")
+#
+#
+#         logger.info(f"Summarization Time taken: {time.time() - summarization_start_time:.2f} seconds")
+#
+#
+#         # Output results summary to console
+#         logger.info("\n" + "=" * 50)
+#         logger.info("Summarization Process Complete.")
+#         logger.info("=" * 50)
+#         successful_count = sum(res.get('success', False) for res in summary_results)
+#         total_processed = len(summary_results) # Includes skipped files if they were added to results list earlier
+#         total_time = time.time() - start_time
+#
+#         logger.info(f"Total files attempted: {len(extraction_results)}") # Total files found/attempted extraction
+#         logger.info(f"Files successfully extracted and summarizable: {len(extraction_results_with_chunks)}") # Files with text and chunks
+#         logger.info(f"Files summarized: {successful_count}/{total_processed}")
+#         logger.info(f"Total process time: {total_time:.2f} seconds")
+#         logger.info("=" * 50)
+#
+#         # Print individual results status
+#         logger.info("\nIndividual File Results:")
+#         for result in summary_results:
+#             name = result.get('filename', 'unknown')
+#             status = "SUCCESS" if result['success'] else "FAILED"
+#             time_taken = result.get('processing_time', 0)
+#             error_msg = result.get('error', '')
+#             logger.info(f"- {name}: {status} ({time_taken:.2f}s) {f'Error: {error_msg}' if error_msg else ''}")
+#
+#
+#     except FileNotFoundError as fnf_error:
+#         logger.error(f"Configuration Error: {fnf_error}")
+#         print(f"Error: {fnf_error}")
+#     except Exception as main_error:
+#         logger.error(f"An unexpected error occurred during the main process: {main_error}", exc_info=True)
+#         print(f"An unexpected error occurred: {main_error}")