First commit

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Karan Verma
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,3 @@
+# Expedition-Aya-Insight
+AI-Powered Multilingual Scientific Summarization with Cohere
+Aya-Insight is a fast, multilingual AI tool that extracts structured, reasoning-driven insights from scientific research papers. 

app.py

@@ -0,0 +1,215 @@
+# app.py
+import streamlit as st
+import os
+import sys
+import logging
+
+# Add the project root to the sys.path to allow importing modules like config, document_processing, etc.
+# This assumes app.py is in the project root directory.
+# Adjust the path if your app.py is in a subdirectory.
+
+try:
+    # Import the necessary functions from your main script    print("YES1")
+    from main import process_uploaded_files, setup_retrieval_system, summarize_extracted_documents
+    # Configure Streamlit's logging to match your application's settings
+    logging.basicConfig(level='INFO', format="%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+    logger = logging.getLogger(__name__)
+    logger.info("Streamlit app started and logging configured.")
+
+    # Flag to check if modules were imported successfully
+    modules_loaded = True
+
+except ImportError as e:
+    st.error(f"Could not import application modules. Please ensure your project structure is correct and dependencies are installed.")
+    st.error(f"ImportError: {e}")
+    logger = logging.getLogger(__name__)
+    logger.error(f"Failed to import application modules: {e}", exc_info=True)
+    modules_loaded = False # Set flag to False if imports fail
+
+
+# --- Streamlit App Configuration ---
+st.set_page_config(
+    page_title="Aya Insight Document Summarizer",
+    page_icon="📄",
+    layout="wide"
+)
+
+# --- Session State Initialization ---
+# Initialize session state variables if they don't exist
+if 'api_key_entered' not in st.session_state:
+    st.session_state.api_key_entered = False
+if 'summary_results' not in st.session_state:
+    st.session_state.summary_results = None
+if 'selected_filename' not in st.session_state:
+    st.session_state.selected_filename = None
+
+
+# --- API Key Input Section ---
+if not st.session_state.api_key_entered:
+    st.title("🔒 Enter Your Cohere API Key to Unlock")
+    api_key = st.text_input("Cohere API Key", type="password", help="Enter your Cohere API key to use the summarization service.")
+
+    if st.button("Unlock"):
+        if api_key:
+            # Basic validation: Just check if it's not empty.
+            # For a real application, you might want to validate by making a small API call.
+            os.environ["COHERE_API_KEY"] = api_key # Set the environment variable
+            st.session_state.api_key_entered = True
+            st.success("API Key accepted. You can now upload documents.")
+            st.rerun() # Rerun the app to show the main content
+        else:
+            st.warning("Please enter your Cohere API key.")
+
+# --- Main Application Content (Unlocked) ---
+if st.session_state.api_key_entered and modules_loaded:
+    st.title("📄 Aya Insight Document Summarizer")
+    st.markdown("""
+    Upload one or more PDF or image files to get a structured summary for each document.
+    """)
+
+    # --- File Uploader ---
+    uploaded_files = st.file_uploader(
+        "Choose Document Files",
+        type=["pdf", "png", "jpg", "jpeg", "tiff", "bmp", "gif"], # Added image types
+        accept_multiple_files=True,
+        help="You can upload multiple PDF or image documents here."
+    )
+
+    # --- Summarize Button and Logic ---
+    if uploaded_files: # Only show button if files are uploaded
+        st.info(f"You have uploaded {len(uploaded_files)} file(s).")
+
+        if st.button("Generate Summaries", key="summarize_button"):
+            st.session_state.selected_filename = None # Reset selected file on new summary generation
+            if not uploaded_files:
+                st.warning("Please upload at least one file before generating summaries.")
+            else:
+                st.subheader("Processing Documents...")
+                all_summary_results = [] # To store results for display
+
+                # Use a spinner to indicate processing
+                with st.spinner("Processing documents and generating summaries... This may take a few minutes depending on file size and number."):
+                    try:
+                        # Step 1: Process uploaded files (Extraction)
+                        logger.info(f"Calling process_uploaded_files with {len(uploaded_files)} files.")
+                        extraction_results = process_uploaded_files(uploaded_files)
+                        logger.info(f"Finished document extraction. {len(extraction_results)} results obtained.")
+
+                        # Check if any files were successfully extracted
+                        if not any(res.get('text') for res in extraction_results):
+                            st.error("No text could be extracted from the uploaded files. Please check the file formats.")
+                            logger.error("No text extracted from any uploaded file.")
+                            st.session_state.summary_results = [] # Store empty results
+                            # st.stop() # Don't stop, allow user to try again
+
+                        # Step 2: Setup retrieval system (Vector Store and Embedding)
+                        logger.info("Calling setup_retrieval_system.")
+                        extraction_results_with_chunks, retriever = setup_retrieval_system(extraction_results)
+                        logger.info("Retriever system setup complete.")
+
+                        # Step 3: Summarize the extracted documents
+                        logger.info("Calling summarize_extracted_documents.")
+                        summary_results = summarize_extracted_documents(extraction_results_with_chunks, retriever)
+                        logger.info(f"Finished summarization. {len(summary_results)} summary results obtained.")
+
+                        st.session_state.summary_results = summary_results # Store results in session state
+
+                    except FileNotFoundError as fnf_error:
+                        st.error(f"Configuration Error: {fnf_error}. Please check your environment settings.")
+                        logger.error(f"Configuration Error during Streamlit process: {fnf_error}", exc_info=True)
+                        st.session_state.summary_results = [] # Store empty results on error
+                    except Exception as e:
+                        st.error(f"An unexpected error occurred during processing: {e}")
+                        logger.error(f"An unexpected error occurred during Streamlit process: {e}", exc_info=True)
+                        st.session_state.summary_results = [] # Store empty results on error
+
+
+    # --- Display Document Tiles and Summaries ---
+    if st.session_state.summary_results is not None:
+        st.subheader("Summaries:")
+
+        if not st.session_state.summary_results:
+            st.info("No summaries were generated. Upload files and click 'Generate Summaries'.")
+        else:
+            # Display files as a grid of clickable tiles
+            files_per_row = 3
+            rows = len(st.session_state.summary_results) // files_per_row + (len(st.session_state.summary_results) % files_per_row > 0)
+
+            # Create a list of filenames for easy access
+            filenames = [res.get('filename', f'File {i+1}') for i, res in enumerate(st.session_state.summary_results)]
+
+            for i in range(rows):
+                cols = st.columns(files_per_row)
+                for j in range(files_per_row):
+                    file_index = i * files_per_row + j
+                    if file_index < len(st.session_state.summary_results):
+                        result = st.session_state.summary_results[file_index]
+                        filename = result.get('filename', f'File {file_index+1}')
+                        is_selected = st.session_state.selected_filename == filename
+
+                        # Create a tile using a button or markdown link
+                        # Using a button inside a column for simplicity
+                        with cols[j]:
+                            # Add a border or highlight if selected
+                            tile_style = "border: 2px solid lightgrey; padding: 10px; margin: 5px; text-align: center; cursor: pointer;"
+                            if is_selected:
+                                tile_style = "border: 2px solid steelblue; padding: 10px; margin: 5px; text-align: center; cursor: pointer; background-color: #e6f3ff;" # Highlight color
+
+                            # Use markdown with HTML to create the clickable tile appearance
+                            # When clicked, set the selected filename in session state
+                            st.markdown(
+                                f"""
+                                <div style="{tile_style}" onclick="document.getElementById('hidden_button_{file_index}').click()">
+                                    📄<br>
+                                    <strong>{filename}</strong>
+                                </div>
+                                <button id="hidden_button_{file_index}" style="display: none;" onclick="document.getElementById('hidden_button_{file_index}').click()"></button>
+                                """,
+                                unsafe_allow_html=True
+                            )
+                            # Streamlit buttons don't work directly with markdown clicks like this easily.
+                            # A simpler approach is to use a standard button and handle the click.
+                            # Let's use a standard button instead of complex markdown/JS.
+
+                            # Alternative using a standard button:
+                            if st.button(f"📄 {filename}", key=f"tile_button_{file_index}"):
+                                st.session_state.selected_filename = filename
+                                logger.info(f"Selected file: {filename}")
+                                st.rerun() # Rerun to display the summary
+
+
+            # Display summary of the selected file
+            if st.session_state.selected_filename:
+                st.markdown("---") # Separator
+                st.subheader(f"Summary for: {st.session_state.selected_filename}")
+
+                # Find the summary for the selected file
+                selected_summary = None
+                selected_result = None
+                for result in st.session_state.summary_results:
+                    if result.get('filename') == st.session_state.selected_filename:
+                        selected_summary = result.get('summary')
+                        selected_result = result
+                        break
+
+                if selected_summary:
+                    if selected_result.get('success'):
+                         st.markdown(selected_summary) # Render markdown summary
+                    else:
+                         st.error(f"Could not load summary for {st.session_state.selected_filename}: {selected_result.get('error', 'Unknown error')}")
+                else:
+                    st.info(f"Summary not available for {st.session_state.selected_filename}.")
+
+            # Display overall processing status
+            successful_count = sum(res.get('success', False) for res in st.session_state.summary_results)
+            total_files = len(st.session_state.summary_results)
+            st.markdown(f"---") # Final separator
+            st.success(f"Processed {total_files} files. Successfully summarized {successful_count}.")
+            if successful_count < total_files:
+                st.warning("Some files could not be processed or summarized. See error messages above.")
+
+
+# --- Message if API Key is not entered and modules loaded ---
+if not st.session_state.api_key_entered:
+    st.info("Enter your Cohere API Key above to unlock the application functionality.")
+

app/config/__init__.py

@@ -0,0 +1,3 @@
+"""
+Configuration package.
+"""

app/config/settings.py

@@ -0,0 +1,27 @@
+import os
+from dotenv import load_dotenv
+import logging
+
+# Load environment variables from a .env file
+load_dotenv()
+
+# Configure logging for the httpx library to suppress warnings
+logging.getLogger("httpx").setLevel(logging.WARNING)
+
+# Define default paths for documents and summaries output
+DOCS_FOLDER = os.getenv("DOCS_FOLDER", "samples/pdf5")
+SUMMARIES_OUTPUT_DIR = os.getenv("SUMMARIES_OUTPUT_DIR", "summaries")
+
+# Create the output directory if it doesn't exist
+os.makedirs(SUMMARIES_OUTPUT_DIR, exist_ok=True)
+
+# Define models for embedding, reranking, and language model
+EMBEDDING_MODEL = os.getenv("EMBEDDING_MODEL", "embed-v4.0")
+COHERERANK_MODEL = os.getenv('COHERERANK_MODEL', 'rerank-v3.5')
+LLM_MODEL = os.getenv("LLM_MODEL", "command-a-03-2025")
+
+# Define settings for text splitting and retrieval
+CHUNK_SIZE = int(os.getenv("CHUNK_SIZE", "1000"))
+CHUNK_OVERLAP = int(os.getenv("CHUNK_OVERLAP", "100"))
+COHERERANK_TOPN = int(os.getenv("COHERERANK_TOPN", "100"))
+VECTOSTORE_TOPK = int(os.getenv("VECTOSTORE_TOPK", "100"))

app/document_processing/__init__.py

@@ -0,0 +1,3 @@
+"""
+Document processing package.
+"""

app/document_processing/extractors.py

@@ -0,0 +1,334 @@
+"""
+Document extraction functionality for processing documents.
+"""
+import json
+import os
+import concurrent.futures
+import time
+
+import cohere
+import logging
+from pathlib import Path
+from typing import List, Dict, Any, Optional
+from langchain.docstore.document import Document
+from ..config.settings import CHUNK_SIZE, LLM_MODEL
+
+# Configure logging with a null handler by default
+logger = logging.getLogger(__name__)
+logger.addHandler(logging.NullHandler())
+
+
+class DocumentProcessor:
+    """Base class for document processors"""
+
+    def __init__(self):
+        self.supported_extensions = []
+
+    def can_process(self, file_path: str) -> bool:
+        """Check if the processor can handle this file type"""
+        ext = Path(file_path).suffix.lower()
+        return ext in self.supported_extensions
+
+    def process(self, file_path: str, **kwargs) -> str:
+        """Process the document and extract text"""
+        raise NotImplementedError("Subclasses must implement this method")
+
+
+class PdfProcessor(DocumentProcessor):
+    """Processor for PDF documents"""
+
+    def __init__(self):
+        super().__init__()
+        self.supported_extensions = ['.pdf']
+
+    def process(self, file_path: str, **kwargs) -> str:
+        """Extract text from a PDF file"""
+        try:
+            # Import here to avoid dependency if not used
+            from pypdf import PdfReader
+
+            logger.debug(f"Processing PDF: {file_path}")
+            reader = PdfReader(file_path)
+            text = ""
+            for page in reader.pages:
+                text += page.extract_text() + "\n"
+            return text.strip()
+        except Exception as e:
+            logger.error(f"Error processing PDF {file_path}: {e}")
+            raise
+
+
+class ImageProcessor(DocumentProcessor):
+    """Processor for image files"""
+
+    def __init__(self):
+        super().__init__()
+        self.supported_extensions = ['.png', '.jpg', '.jpeg', '.tiff', '.bmp', '.gif']
+        # Default languages including multiple options
+        self.default_languages = "eng+fra+hin+spa+chi-sim"
+
+    def process(self, file_path: str, **kwargs) -> str:
+        """Extract text from an image file using OCR"""
+        try:
+            # Import here to avoid dependency if not used
+            import pytesseract
+            from PIL import Image
+
+            # Use the expanded default languages if not specified
+            lang = kwargs.get('lang', self.default_languages)
+            logger.debug(f"Processing image: {file_path} with languages: {lang}")
+            image = Image.open(file_path)
+            text = pytesseract.image_to_string(image, lang=lang)
+            return text.strip()
+        except Exception as e:
+            logger.error(f"Error processing image {file_path}: {e}")
+            raise
+
+
+class DocumentExtractor:
+    """Main class for document text extraction"""
+
+    def __init__(self):
+        """Initialize with default processors"""
+        self.processors = [
+            PdfProcessor(),
+            ImageProcessor()
+        ]
+        self.cohere_client = None
+
+    def add_processor(self, processor: DocumentProcessor) -> None:
+        """Add a custom document processor"""
+        self.processors.append(processor)
+
+    def get_processor(self, file_path: str) -> Optional[DocumentProcessor]:
+        """Get the appropriate processor for a file"""
+        for processor in self.processors:
+            if processor.can_process(file_path):
+                return processor
+        return None
+
+    def get_language(self, text: str) -> str:
+        """
+        Detect the language of the provided text using Cohere API.
+
+        Args:
+            text: Text sample to analyze
+
+        Returns:
+            String containing the detected language name
+        """
+        try:
+            # Initialize client if not already done
+            start = time.time()
+            if not self.cohere_client:
+                self.cohere_client = cohere.Client()
+
+            prompt = f"What language is this sentence written in?\n\n{text}\n\nRespond only with the language name."
+            response = self.cohere_client.chat(
+                model=LLM_MODEL,
+                message= prompt,
+                max_tokens=100,
+                temperature=0.2,
+            )
+            return response.text
+
+        except Exception as e:
+            logger.error(f"Error detecting language: {e}")
+            return "unknown"
+
+    def process_file(self, file_path: str, **kwargs) -> Dict[str, Any]:
+        """
+        Process a single file based on its extension.
+
+        Args:
+            file_path: Path to the file
+            **kwargs: Additional processing options
+
+        Returns:
+            Dictionary containing processing results and metadata
+        """
+        result = {
+            "file_path": file_path,
+            "filename": Path(file_path).name,
+            "text": "",
+            "error": None,
+            "type": None,
+            "language": None,
+            "chunk_size": 0
+        }
+
+        try:
+            processor = self.get_processor(file_path)
+
+            if processor:
+                text = processor.process(file_path, **kwargs)
+                result["text"] = text
+                result["language"] = self.get_language(text[:CHUNK_SIZE]) if text else None
+                result["type"] = processor.__class__.__name__.lower().replace('processor', '')
+            else:
+                ext = Path(file_path).suffix.lower()
+                result["error"] = f"Unsupported file type: {ext}"
+        except Exception as e:
+            result["error"] = str(e)
+
+        return result
+
+    def process_files(self, file_paths: List[str], **kwargs) -> List[Dict[str, Any]]:
+        """
+        Process multiple files in parallel.
+
+        Args:
+            file_paths: List of file paths to process
+            **kwargs: Additional processing options
+                     (max_workers: max number of processes)
+
+        Returns:
+            List of dictionaries with processing results
+        """
+        max_workers = kwargs.pop('max_workers', os.cpu_count() or 1)
+        logger.info(f"Processing {len(file_paths)} files with {max_workers} workers")
+
+        results = []
+        with concurrent.futures.ProcessPoolExecutor(max_workers=max_workers * 2) as executor:
+            futures = {
+                executor.submit(self.process_file, file_path, **kwargs): file_path
+                for file_path in file_paths
+            }
+
+            for future in concurrent.futures.as_completed(futures):
+                file_path = futures[future]
+                try:
+                    result = future.result()
+                    results.append(result)
+                except Exception as e:
+                    logger.error(f"Exception processing {file_path}: {e}")
+                    results.append({
+                        "filepath": file_path,
+                        "filename": Path(file_path).name,
+                        "text": "",
+                        "error": str(e),
+                        "type": None,
+                        "langugae": None,
+                        "chunk_size": 0
+                    })
+
+        return results
+
+    def find_supported_files(self, folder_path: str, recursive: bool = True) -> List[str]:
+        """
+        Get all supported files in a folder.
+
+        Args:
+            folder_path: Path to the folder
+            recursive: Whether to include subfolders
+
+        Returns:
+            List of file paths
+        """
+        # Get all supported extensions from processors
+        supported_extensions = []
+        for processor in self.processors:
+            supported_extensions.extend(processor.supported_extensions)
+
+        file_paths = []
+
+        if recursive:
+            for root, _, files in os.walk(folder_path):
+                for file in files:
+                    file_path = os.path.join(root, file)
+                    if Path(file).suffix.lower() in supported_extensions:
+                        file_paths.append(file_path)
+        else:
+            for file in os.listdir(folder_path):
+                file_path = os.path.join(folder_path, file)
+                if os.path.isfile(file_path) and Path(file).suffix.lower() in supported_extensions:
+                    file_paths.append(file_path)
+
+        return file_paths
+
+    def process_folder(self, folder_path: str, recursive: bool = True, **kwargs) -> List[Dict[str, Any]]:
+        """
+        Process all supported files in a folder.
+
+        Args:
+            folder_path: Path to the folder containing documents
+            recursive: Whether to process subfolders recursively
+            **kwargs: Additional processing options
+
+        Returns:
+            List of dictionaries with processing results
+        """
+        file_paths = self.find_supported_files(folder_path, recursive)
+        logger.info(f"Found {len(file_paths)} supported files in {folder_path}")
+
+        return self.process_files(file_paths, **kwargs)
+
+
+class FileOutputManager:
+    """Class for managing output of extracted text"""
+
+    def __init__(self, output_dir: str = "extracted_texts"):
+        """Initialize with output directory"""
+        self.output_dir = output_dir
+        os.makedirs(output_dir, exist_ok=True)
+
+    def save_results(self, results: List[Dict[str, Any]]) -> Dict[str, int]:
+        """
+        Save extracted text to files.
+
+        Args:
+            results: List of processing results
+
+        Returns:
+            Dictionary with counts of successful and failed saves
+        """
+        stats = {"success": 0, "skipped": 0, "failed": 0}
+
+        for result in results:
+            if not result["text"]:
+                stats["skipped"] += 1
+                continue
+
+            try:
+                # Create filename with original name + file type
+                base_name = Path(result['filename']).stem
+                file_type = result.get('type', 'unknown')
+                output_filename = f"{base_name}_{file_type}.txt"
+
+                output_path = os.path.join(self.output_dir, output_filename)
+                with open(output_path, "w", encoding="utf-8") as f:
+                    f.write(result["text"])
+                stats["success"] += 1
+            except Exception as e:
+                logger.error(f"Error saving text from {result['file_path']}: {e}")
+                stats["failed"] += 1
+
+        return stats
+
+
+# Adapter class to convert DocumentExtractor results to langchain Document objects
+class DocumentProcessorAdapter:
+    """
+    Adapter to process documents and convert them to langchain Document objects.
+    """
+    def __init__(self):
+        """Initialize document processor adapter with the extractor."""
+        self.extractor = DocumentExtractor()
+
+    def process_folder(self, folder_path):
+        """
+        Process all documents in a folder.
+
+        Args:
+            folder_path (str): Path to the folder containing documents
+
+        Returns:
+            tuple: (list of langchain Document objects, original extraction results)
+        """
+        if not os.path.exists(folder_path):
+            raise FileNotFoundError(f"Folder not found: {folder_path}")
+
+        # Extract content from documents
+        extraction_results = self.extractor.process_folder(folder_path)
+        print(f"Processed {len(extraction_results)} documents")
+        return extraction_results

app/retrieval/__init__.py

@@ -0,0 +1,3 @@
+"""
+Retrieval package.
+"""

app/retrieval/vector_store.py

@@ -0,0 +1,68 @@
+"""
+Vector database operations for document storage and retrieval.
+"""
+from langchain_chroma import Chroma
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_cohere import CohereEmbeddings
+from langchain_core.documents import Document
+
+from ..config.settings import CHUNK_SIZE, CHUNK_OVERLAP, EMBEDDING_MODEL, COHERERANK_MODEL, COHERERANK_TOPN, VECTOSTORE_TOPK
+import cohere
+
+class Retriever:
+    """
+    Wrapper for vector database operations.
+    """
+
+    def __init__(self, model=EMBEDDING_MODEL):
+        self.cohere_client = cohere.Client()
+        self.chroma_db = None
+        self.embedding_model = CohereEmbeddings(model=model)
+        self.text_splitter = RecursiveCharacterTextSplitter(
+            chunk_size=CHUNK_SIZE,
+            chunk_overlap=CHUNK_OVERLAP
+        )
+
+    def create_from_documents(self, extraction_results):
+        chunks = []
+        for result in extraction_results:
+            filename = result['filename']
+            text = result['text']
+            if text:
+                document = Document(
+                    page_content=text,
+                    metadata={"filename": filename}
+                )
+                doc_chunks = self.text_splitter.split_documents([document])
+                result['chunk_size'] = len(doc_chunks)
+                chunks.extend(doc_chunks)
+
+        self.chroma_db = Chroma.from_documents(
+            chunks,
+            embedding=self.embedding_model
+        )
+        return extraction_results
+
+    def similarity_search(self, query, k=5, filter=None):
+        if not self.chroma_db:
+            raise ValueError("Vector store has not been initialized with documents")
+
+        return self.chroma_db.similarity_search(query=query, k=k, filter=filter)
+
+    def reranking(self, query, docs, top_n=10):
+        doc_texts = [doc.page_content for doc in docs]
+        rerank_response = self.cohere_client.rerank(model=COHERERANK_MODEL, query=query, documents=doc_texts, top_n=top_n)
+        # return [docs[result.index] for result in rerank_response.results]
+        return [docs[result.index].page_content for result in rerank_response.results]
+
+
+    def get_relevant_docs(self, chromdb_query, rerank_query, filter, chunk_size):
+        dense_topk = min(chunk_size, VECTOSTORE_TOPK)
+        reranking_topk = min(chunk_size, COHERERANK_TOPN)
+        docs = self.similarity_search(chromdb_query, filter=filter, k=dense_topk)
+        if docs:
+            return self.reranking(rerank_query, docs, top_n=reranking_topk)
+        return []
+
+
+

app/summarization/__init__.py

@@ -0,0 +1,3 @@
+"""
+Summarization package.
+"""

app/summarization/output.py

@@ -0,0 +1,167 @@
+"""
+Output handling for document summaries in multiple formats.
+"""
+import os
+from ..config.settings import SUMMARIES_OUTPUT_DIR
+
+
+class SummaryOutputManager:
+    """
+    Manager for saving and retrieving document summaries in multiple formats.
+    """
+
+    def __init__(self, output_dir=SUMMARIES_OUTPUT_DIR):
+        """
+        Initialize output manager with output directory.
+
+        Args:
+            output_dir (str): Directory to save summaries
+        """
+        self.output_dir = output_dir
+        self._ensure_output_dir()
+
+    def _ensure_output_dir(self):
+        """Create output directory if it doesn't exist."""
+        if not os.path.exists(self.output_dir):
+            os.makedirs(self.output_dir)
+            print(f"Created output directory: {self.output_dir}")
+
+    def save_summary(self, filename, summary, formats=None):
+        """
+        Save a document summary to files in specified formats.
+
+        Args:
+            filename (str): Name of the original document
+            summary (str): Summary text
+            formats (list): List of formats to save. Defaults to ['markdown', 'html']
+
+        Returns:
+            dict: Paths to the saved summary files by format
+        """
+        if formats is None:
+            formats = ['markdown', 'html']
+
+        output_paths = {}
+
+        # Generate and save in each requested format
+        for fmt in formats:
+            if fmt == 'markdown':
+                output_paths['markdown'] = self._save_markdown(filename, summary)
+            elif fmt == 'html':
+                output_paths['html'] = self._save_html(filename, summary)
+            else:
+                print(f"Warning: Unsupported format '{fmt}' requested")
+
+        return output_paths
+
+    def _save_markdown(self, filename, summary):
+        """
+        Save a document summary to a markdown file.
+
+        Args:
+            filename (str): Name of the original document
+            summary (str): Summary text
+
+        Returns:
+            str: Path to the saved markdown file
+        """
+        # Create markdown output
+        markdown_content = f""
+        markdown_content += summary
+        markdown_content += "\n\n---\n"
+
+        # Save to file
+        output_path = os.path.join(self.output_dir, f"{filename}.md")
+        with open(output_path, "w") as f:
+            f.write(markdown_content)
+
+        print(f"Saved markdown summary to: {output_path}")
+        return output_path
+
+    def _save_html(self, filename, summary):
+        """
+        Save a document summary to an HTML file.
+
+        Args:
+            filename (str): Name of the original document
+            summary (str): Summary text
+
+        Returns:
+            str: Path to the saved HTML file
+        """
+        # Convert summary to HTML paragraphs
+        paragraphs = summary.split('\n\n')
+        html_paragraphs = ''.join([f"<p>{p}</p>" for p in paragraphs if p.strip()])
+
+        # Create HTML output with basic styling
+        html_content = f"""<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Summary for {filename}</title>
+    <style>
+        body {{
+            font-family: Arial, sans-serif;
+            line-height: 1.6;
+            margin: 0;
+            padding: 20px;
+            max-width: 800px;
+            margin: 0 auto;
+            color: #333;
+        }}
+        h1 {{
+            color: #2c3e50;
+            border-bottom: 1px solid #eee;
+            padding-bottom: 10px;
+        }}
+        p {{
+            margin-bottom: 16px;
+        }}
+        .footer {{
+            margin-top: 30px;
+            padding-top: 10px;
+            border-top: 1px solid #eee;
+            font-size: 0.9em;
+            color: #7f8c8d;
+        }}
+    </style>
+</head>
+<body>
+    <h1>Summary for {filename}</h1>
+    <div class="content">
+        {html_paragraphs}
+    </div>
+    <div class="footer">
+        <p>Generated summary</p>
+    </div>
+</body>
+</html>
+"""
+        # Save to file
+        output_path = os.path.join(self.output_dir, f"{filename}.html")
+        with open(output_path, "w") as f:
+            f.write(html_content)
+
+        print(f"Saved HTML summary to: {output_path}")
+        return output_path
+
+    def get_available_formats(self, filename):
+        """
+        Check which formats are available for a given file.
+
+        Args:
+            filename (str): Base filename to check
+
+        Returns:
+            list: Available formats for this file
+        """
+        available_formats = []
+        base_name = os.path.splitext(filename)[0]
+
+        if os.path.exists(os.path.join(self.output_dir, f"{base_name}.md")):
+            available_formats.append('markdown')
+        if os.path.exists(os.path.join(self.output_dir, f"{base_name}.html")):
+            available_formats.append('html')
+
+        return available_formats

app/summarization/prompt2.py

@@ -0,0 +1,627 @@
+"""
+Research Paper Summarization: Key Points Extraction for Researchers
+Note: Always return the summary in the same language as the original paper.
+"""
+
+
+# # Mini-Prompt 1: Basic Information
+# basic_info_prompt = """
+# Extract the essential identifying information from this research paper.
+#
+# | Information | Details |
+# |-------------|---------|
+# | Title | [Paper title] |
+# | Authors | [Author names] |
+# | Publication | [Journal/Conference, Year] |
+# | Field | [Research domain] |
+# | Keywords | [Key terms] |
+# | DOI/URL | [Link if available] |
+# """
+#
+# # Mini-Prompt 2: Research Objectives & Abstract
+# objectives_prompt = """
+# Extract the following:
+#
+# 1. Primary research question/objective (2-3 bullet points)
+# 2. Condensed abstract summary (2-3 sentences)
+# 3. Main contributions (3-5 bullet points)
+# 4. Theoretical foundation/frameworks
+#
+# Keep each point short and focused on a single idea.
+# """
+#
+# # Mini-Prompt 3: Methodology Details
+# methodology_prompt = """
+# Extract key methodology details:
+#
+# 1. Base model(s) used
+# 2. Architecture summary
+# 3. Dataset(s) with name, size, and characteristics
+# 4. Experimental setup (conditions, controls, parameters)
+# 5. Implementation details (hardware/software used)
+#
+# Format as a structured table with clear categories.
+# """
+#
+# # Mini-Prompt 4: Key Equations & Technical Approach
+# equations_prompt = """
+# Extract and explain key equations:
+#
+# | Equation | Purpose | Explanation |
+# |----------|---------|-------------|
+# | [Equation 1] | [What it calculates] | [Explanation] |
+# | [Equation 2] | [What it calculates] | [Explanation] |
+#
+# Include 2-3 equations central to the methodology or findings.
+# """
+#
+# # Mini-Prompt 5: Results & Performance
+# results_prompt = """
+# Summarize the performance:
+#
+# 1. 5-7 bullet points for primary findings
+# 2. Performance metrics (accuracy, F1, etc.)
+# 3. Comparison with previous work or baselines
+# 4. Highlights from ablation studies
+#
+# Present performance in a table if applicable:
+#
+# | Metric | Value | Compared to Baseline |
+# |--------|-------|-----------------------|
+# """
+#
+# # Mini-Prompt 6: Critical Analysis
+# analysis_prompt = """
+# Identify strengths and weaknesses:
+#
+# | Strengths | Limitations |
+# |-----------|-------------|
+# | [Point 1] | [Limitation 1] |
+# | [Point 2] | [Limitation 2] |
+# | [Point 3] | [Limitation 3] |
+#
+# Also note:
+# - Key assumptions made
+# - Generalizability of findings
+# - Ethical concerns, if mentioned
+# """
+#
+# # Mini-Prompt 7: Implications & Future Work
+# implications_prompt = """
+# Extract future-oriented content:
+#
+# 1. Research implications (field-level impact)
+# 2. Practical/industry applications
+# 3. Future research directions
+# 4. Unanswered or open questions
+#
+# Use 3-4 concise bullet points.
+# """
+#
+# # Mini-Prompt 8: Executive Summary
+# executive_summary_prompt = """
+# Create a concise executive summary (max 250 words):
+#
+# 1. One-paragraph overview
+# 2. Problem being addressed
+# 3. Approach and methods
+# 4. Key results
+# 5. Significance of the findings
+# """
+
+
+# basic_info_prompt = """You are extracting basic information from a research paper. Please provide:
+#
+# 1. Paper title
+# 2. Authors and affiliations
+# 3. Publication date and venue/journal
+# 4. DOI/URL if available
+# 5. Citation information
+# 6. Research domain/field
+#
+# Format as a simple table with categories and details."""
+#
+# # Mini-Prompt 2: Research Objectives and Abstract
+# objectives_prompt = """Based on the research paper, please extract:
+#
+# 1. The primary research question/objective (2-3 bullet points)
+# 2. A condensed abstract summary (2-3 sentences capturing the essence)
+# 3. The main contributions (3-5 key bullet points)
+# 4. The theoretical foundation/frameworks underlying the research
+#
+# Keep each bullet point to 1-2 sentences, focused on a single idea."""
+#
+# # Mini-Prompt 3: Methodology Details
+# methodology_prompt = """Extract the key methodological details from the paper:
+#
+# 1. Base model(s) used (if applicable)
+# 2. System/model architecture (concise description)
+# 3. Datasets used (names, sizes, characteristics)
+# 4. Experimental setup (conditions, controls, parameters)
+# 5. Implementation details (hardware, software, computational resources)
+#
+# Present this information in a structured table format."""
+#
+# # Mini-Prompt 4: Key Equations and Technical Approach
+# equations_prompt = """Identify and explain the most important equations and technical approaches:
+#
+# 1. Extract 2-3 key equations/formulations
+# 2. For each equation, explain:
+#    - What it calculates
+#    - Its purpose in the paper
+#    - How it relates to the overall methodology
+#
+# Format as a table with columns for Equation, Purpose, and Explanation."""
+#
+# # Mini-Prompt 5: Results and Performance
+# results_prompt = """Summarize the main results and performance metrics:
+#
+# 1. Primary findings (5-7 bullet points)
+# 2. Performance metrics (accuracy, F1, BLEU, etc.)
+# 3. Comparison to prior or competing approaches
+# 4. Key insights from ablation studies
+#
+# Present performance metrics in a table with columns for Metric, Value, and Comparison to Previous Work."""
+#
+# # Mini-Prompt 6: Critical Analysis
+# analysis_prompt = """Analyze the strengths and limitations of the paper:
+#
+# 1. Clearly stated limitations (3-4 bullet points)
+# 2. Key assumptions made by the authors
+# 3. Assessment of generalizability of findings
+# 4. Ethical considerations mentioned
+#
+# Present as a comparison table with Strengths and Limitations columns."""
+#
+# # Mini-Prompt 7: Implications and Future Work
+# implications_prompt = """Extract information about implications and future directions:
+#
+# 1. Research implications (how this advances the field)
+# 2. Practical/industry applications of the findings
+# 3. Future research directions identified by the authors
+# 4. Unresolved questions that emerge from this work
+#
+# Provide as 3-4 concise bullet points focusing on significance and future work."""
+#
+# # Mini-Prompt 8: Executive Summary
+# executive_summary_prompt = """Create a concise executive summary of the paper with these components:
+#
+# 1. One-paragraph overview (3-5 sentences)
+# 2. The problem being solved
+# 3. The approach taken
+# 4. The key results
+# 5. Why this matters
+#
+# Keep the entire summary under 250 words for quick reference."""
+
+
+# ================= ================= ================= ================= ================= =================
+
+
+basic_info_prompt = """# Basic Paper Information
+
+Generate a concise summary of the paper's essential metadata using the table below. Ensure all details are accurately extracted and easy for researchers to scan. Provide the output in the same language as this prompt. If a specific detail cannot be found, provide an empty string (`""`) for that item or cell, do not use placeholder text.
+
+| Information       | Details                          |
+|-------------------|----------------------------------|
+| **Title** | [Full title of the paper]        |
+| **Authors** | [Complete list of authors]       |
+| **Publication Venue** | [Journal/Conference, Year]   |
+| **Research Field**| [Primary domain or discipline]   |
+| **Keywords** | [Relevant terms and topics - use bullet points if multiple] |
+"""
+
+research_focus_prompt = """# Core Research Focus
+
+Summarize the central aim, problem, contribution, and significance of the paper. Present the information clearly and concisely using bullet points.  Provide the output in the same language as this prompt. If a specific detail cannot be found, provide an empty string (`""`) for that item or cell, do not use placeholder text.
+
+* **Research Question:** [What is being investigated? State this clearly.]
+* **Problem Statement:** [What specific gap or issue does the paper address? Be direct.]
+* **Main Contribution:** [What is the core offering, innovation, or finding? Highlight the novelty.]
+* **Significance:** [Why is this research important for the field or practice? Briefly explain the impact.]
+"""
+
+abstract_prompt = """# Abstract Summary
+
+Break down the paper's abstract into its fundamental components for quick comprehension. Present the information concisely using bullet points. Provide the output in the same language as this prompt. If a specific detail cannot be found, provide an empty string (`""`) for that item or cell, do not use placeholder text.
+
+* **Background:** [Brief context leading to the study]
+* **Problem:** [The specific issue the paper tackles]
+* **Methodology:** [Approach, methods, or techniques used]
+* **Key Findings:** [Main results or discoveries - use sub-bullets if needed]
+* **Conclusion:** [Primary takeaway or implication]
+"""
+
+methods_prompt = """# Methodology Summary
+
+Describe how the research was conducted, focusing on key aspects like study design, data, techniques, and evaluation. Present the information concisely using bullet points, with sub-bullets for details. Provide the output in the same language as this prompt. If a specific detail cannot be found, provide an empty string (`""`) for that item or cell, do not use placeholder text.
+
+* **Study Design:** [e.g., Experimental, Simulation, Case Study, etc.]
+* **Dataset(s):**
+    * Source: [Where the data came from]
+    * Size: [Amount of data]
+    * Key Characteristics: [Important features or properties]
+    * Preprocessing: [Main steps taken to prepare data]
+* **Techniques/Models:** [Specific models, algorithms, or frameworks used - list key ones]
+* **Evaluation:**
+    * Metrics: [How performance/success was measured - list key metrics]
+    * Setup: [Briefly describe evaluation setup if notable]
+* **Tools & Software:** [Libraries, platforms, hardware specifics if critical]
+"""
+
+results_prompt = """# Key Results
+
+List and explain the paper's main outcomes and their importance. Use the table for primary findings and bullet points for comparisons to prior work. Keep descriptions and insights brief and impactful. Provide the output in the same language as this prompt. If a specific detail cannot be found, provide an empty string (`""`) for that item or cell, do not use placeholder text.
+
+| Finding # | Description of Result         | Significance / Insight           |
+|-----------|-------------------------------|----------------------------------|
+| 1         | [What was observed/found?]    | [Why this result is important or novel?] |
+| 2         | [Another key result]          | [Its implication or contribution]  |
+| 3         | [Third main finding]          | [What we learn from this]        |
+| ...       | [Add more rows as needed]     | [Corresponding insight]          |
+
+**Comparison to Prior Work:**
+* [Highlight how these results differ from or improve upon previous research.]
+* [Mention specific previous work if comparison is direct.]
+* [Explain why the improvement or difference matters.]
+"""
+
+visuals_prompt = """# Important Figures & Tables
+
+Highlight the most critical visualizations and tabular data from the paper. Explain their content and why they are important for understanding the research. Use the table below.
+
+| Visual Element  | Brief Description             | Key Insight or Interpretation            |
+|-----------------|-------------------------------|--------------------------------------|
+| **Figure [Number]**| [What the figure depicts or shows] | [What key point or data trend does it illustrate?] |
+| **Table [Number]** | [Summary of data/content in the table] | [What conclusion or comparison can be drawn from this table?] |
+| **Figure [Number]**| [Another key visualization]   | [Why is this figure crucial for the results or argument?] |
+| ...             | [Add more rows as needed]     | [Corresponding insight]              |
+"""
+
+limitations_prompt = """# Limitations & Future Work
+
+Detail the limitations encountered during the research and outline suggested future directions. Use bullet points for both limitations and future work. Be concise. Provide the output in the same language as this prompt. If a specific detail cannot be found, provide an empty string (`""`) for that item or cell, do not use placeholder text.
+
+**Limitations:**
+* **Theoretical:** [Conceptual limits of the approach, with brief impact]
+* **Methodological:** [Issues with design or procedure, with brief impact]
+* **Data-Related:** [Constraints due to data quality/availability, with brief impact]
+* [Add other relevant limitations]
+
+**Future Work Suggestions:**
+* [Proposed next steps or improvements to the current work.]
+* [New areas or questions for future research based on these findings.]
+* [Potential experiments or applications to explore.]
+"""
+
+contributions_prompt = """# Main Contributions
+
+List all major contributions of the paper, categorized by type. Explain how each contribution adds value or novelty to the field. Use bullet points, with sub-bullets for novelty/advancement. Provide the output in the same language as this prompt. If a specific detail cannot be found, provide an empty string (`""`) for that item or cell, do not use placeholder text.
+
+* **Theoretical:** [New framework, concept, or insight introduced]
+    * Novelty/Advancement: [How it extends or changes existing theory]
+* **Methodological:** [New method, algorithm, or model developed]
+    * Novelty/Advancement: [What makes it different, better, or more efficient?]
+* **Empirical:** [Significant findings or results from experiments/data]
+    * Novelty/Advancement: [Why these results matter or what they demonstrate?]
+* **Practical:** [Applications, systems, or tools developed]
+    * Novelty/Advancement: [Real-world relevance or utility]
+* [Add other relevant contributions]
+
+**Most Noteworthy Contribution:** [Briefly summarize the single biggest impact or most innovative aspect of the paper.]
+"""
+
+related_work_prompt = """# Related Work
+
+Show how this research fits into the existing landscape of studies and what specific gaps it addresses. Use the table to compare this work to previous approaches and list the addressed gap using bullet points. Provide the output in the same language as this prompt. If a specific detail cannot be found, provide an empty string (`""`) for that item or cell, do not use placeholder text.
+
+| Topic/Area      | Previous Approaches             | This Paper's Innovation / Difference   |
+|-----------------|----------------------------------|----------------------------------------|
+| **[Relevant Area 1]**| [Summary of how prior work handled this] | [What new approach, technique, or finding is introduced here?] |
+| **[Relevant Area 2]**| [Other related methods or studies] | [How does this paper build upon or deviate from them?] |
+| **[Relevant Area 3]**| [Existing theories or models]    | [Enhancements, alternatives, or validations provided by this work] |
+| ...             | [Add more rows as needed]        | [Corresponding innovation]             |
+
+**Gap Addressed:**
+* [What specific problem, limitation, or missing piece in the existing literature does this paper tackle?]
+"""
+
+applications_prompt = """# Practical Applications
+
+Explore potential real-world applications of the research findings or methods. Use the table to detail potential use cases, required conditions, and feasibility. Provide the output in the same language as this prompt. If a specific detail cannot be found, provide an empty string (`""`) for that item or cell, do not use placeholder text.
+
+| Domain/Industry | Potential Use Case or Application   | Key Requirements or Dependencies        | Feasibility/Timeline (e.g., Short/Med/Long term) |
+|-----------------|--------------------------------------|-------------------------------------|-------------------|
+| **[Domain 1]** | [How can the results/methods be used here?] | [What data, technology, or infrastructure is needed?] | [Estimated time to potential deployment]  |
+| **[Domain 2]** | [Another potential application area] | [Factors affecting feasibility or adoption] | [Estimated time to potential deployment]  |
+| **[Domain 3]** | [Innovative potential application] | [Challenges or conditions for implementation] | [Estimated time to potential deployment]  |
+| ...             | [Add more rows as needed]            | [Corresponding requirements]          | [Corresponding timeline]  |
+
+**Most Promising Use Case:** [Briefly highlight the application with the highest potential impact or feasibility.]
+"""
+
+technical_prompt = """# Technical Details
+
+Provide a concise summary of the paper's specific technical aspects. Use the table for algorithms, architecture, implementation, and performance. Provide the output in the same language as this prompt. If a specific detail cannot be found, provide an empty string (`""`) for that item or cell, do not use placeholder text.
+
+| Component         | Description                        | Key Configuration or Parameters         |
+|-------------------|-------------------------------------|--------------------------------------|
+| **Algorithm(s)** | [What specific algorithm(s) are central?] | [Key hyperparameters, variations used, etc.] |
+| **Model/Architecture**| [Type or design of the model/system] | [Number of layers, components, specific structure details] |
+| **Implementation**| [Languages, key libraries, environment specifics] | [Frameworks used (TensorFlow, PyTorch, etc.), notable dependencies] |
+| **Performance** | [Key performance metrics reported]    | [Results achieved (e.g., Accuracy %, F1 score, latency ms)] |
+| ...               | [Add more rows as needed]           | [Corresponding details]              |
+
+"""
+
+quick_summary_prompt = """# Quick Summary
+
+Provide a highly concise summary of the entire paper, suitable for a quick grasp of its core message. Include both a brief paragraph and a single-sentence version. Provide the output in the same language as this prompt. If a specific detail cannot be found, provide an empty string (`""`) for that item or cell, do not use placeholder text.
+
+**Brief Summary (3–5 Sentences):**
+[Write a concise summary covering the paper's motivation, core method, main findings, and overall significance.]
+
+**One-Sentence Summary:**
+[Write a single, impactful sentence that captures the paper’s most important contribution or finding.]
+"""
+
+reading_guide_prompt = """# Reading Guide
+
+Help researchers quickly navigate the paper by highlighting the most important sections and the key information found within them. Suggest an efficient reading path. Use the table for key sections and bullet points for the reading path. Provide the output in the same language as this prompt. If a specific detail cannot be found, provide an empty string (`""`) for that item or cell, do not use placeholder text.
+
+| Section Name      | Key Information or Reason to Focus Here |
+|-------------------|------------------------------------|
+| **[Section Name 1]**| [What is the main idea or critical takeaway from this section?] |
+| **[Section Name 2]**| [Why is this section particularly insightful or important for understanding the work?] |
+| **[Section Name 3]**| [What key details or results are presented here?] |
+| ...               | [Add more rows as needed]          |
+| **[Conclusion Section]**| [Main takeaways and future implications.] |
+
+**Recommended Reading Path:**
+* [Suggest an efficient order to read the key sections for maximum understanding (e.g., Abstract -> Introduction -> Methods (key parts) -> Results (key figures/tables) -> Conclusion).]
+"""
+
+equations_prompt = """# Key Equations
+
+Highlight and explain the major equations presented in the paper. For each equation, describe its purpose, define its variables, and explain its significance to the research. Use the table below. Use LaTeX format ($$...$$ for block, $...$ for inline) for equations. Provide the output in the same language as this prompt. If a specific detail cannot be found, provide an empty string (`""`) for that item or cell, do not use placeholder text.
+
+| Equation          | Purpose or Role in the Paper      | Why It Matters to the Research     |
+|-------------------|-----------------------------------|------------------------------------|
+| $$ [Equation 1] $$ | [What the equation calculates, models, or represents] | [Its role in the method, results, or theory] |
+| $$ [Equation 2] $$ | [Purpose of this equation]       | [Its impact on the conclusions or findings] |
+| $$ [Equation 3] $$ | [Purpose of this equation]       | [How it supports the overall argument] |
+| ...               | [Add more rows as needed]         | [Corresponding significance]       |
+"""
+
+executive_summary_prompt = """# Executive Summary
+
+Provide a high-level summary of the paper, tailored for research leads, grant reviewers, or collaborators. Focus on the problem, solution, key results, and implications using concise bullet points. Provide the output in the same language as this prompt. If a specific detail cannot be found, provide an empty string (`""`) for that item or cell, do not use placeholder text.
+
+* **Research Problem:** [Clear articulation of the challenge the paper addresses]
+* **Proposed Solution:** [Brief overview of the method, model, or approach introduced]
+* **Major Results:** [Highlights of the most significant findings or achievements - use sub-bullets if needed]
+* **Implications:** [Practical, theoretical, or future impact of the work]
+* **Relevance:** [Why this paper is important and should be paid attention to]
+"""
+
+# ================= ================= ================= ================= ================= =================
+
+
+
+
+# 1. Basic Paper Information
+# basic_info_prompt = """# Basic Paper Information
+#
+# Extract all essential metadata to clearly identify and classify the research paper. Focus on accurately capturing the publication details.
+#
+# | Information       | Details                          |
+# |-------------------|----------------------------------|
+# | Title             | [Full title of the paper]        |
+# | Authors           | [Complete list of authors]       |
+# | Publication Venue | [Journal/Conference, Year]       |
+# | Research Field    | [Primary domain or discipline]   |
+# | Keywords          | [Relevant terms and topics]      |
+# """
+#
+# # 2. Core Research Focus
+# research_focus_prompt = """# Core Research Focus
+#
+# Summarize the central aim of the paper. Clearly articulate the main question, the addressed problem, and the novelty of the contribution.
+#
+# | Element              | Details                                         |
+# |----------------------|-------------------------------------------------|
+# | Research Question    | [What is being investigated?]                  |
+# | Problem Statement    | [What gap or issue does the paper address?]    |
+# | Main Contribution    | [What is the core offering or innovation?]     |
+# | Significance         | [Why is this research important?]              |
+# """
+#
+# # 3. Abstract Summary
+# abstract_prompt = """# Abstract Summary
+#
+# Break down the abstract into its fundamental components for easier comprehension.
+#
+# | Component     | Details                               |
+# |---------------|----------------------------------------|
+# | Background     | [Brief context of the study]          |
+# | Research Problem | [Specific issue the paper solves]    |
+# | Methodology    | [Approach or technique used]         |
+# | Key Findings   | [Main results or discoveries]         |
+# | Conclusion     | [Primary takeaway from the study]     |
+# """
+#
+#
+# # 4. Methodology Summary
+# methods_prompt = """# Methodology Summary
+#
+# Describe how the research was conducted, including data, tools, and procedures.
+#
+# | Component        | Details                                   |
+# |------------------|-------------------------------------------|
+# | Study Design     | [Experimental, simulation, case study etc.] |
+# | Dataset          | [Source, size, preprocessing, etc.]       |
+# | Techniques Used  | [Models, algorithms, or frameworks used]  |
+# | Evaluation Metrics| [How success or performance was measured]|
+# | Tools & Software | [Libraries, platforms, hardware specifics]|
+# """
+#
+#
+# # 5. Key Results
+# results_prompt = """# Key Results
+#
+# List and explain the main outcomes, their impact, and how they compare to past work.
+#
+# | Finding # | Description of Result         | Significance / Insight           |
+# |-----------|-------------------------------|----------------------------------|
+# | 1         | [What was observed]           | [Why it matters]                 |
+# | 2         | [What was observed]           | [Why it matters]                 |
+# | 3         | [What was observed]           | [Why it matters]                 |
+#
+# **Comparison to Prior Work:** [Highlight how these results differ or improve upon previous research.]
+# """
+#
+#
+# # 6. Important Figures & Tables
+# visuals_prompt = """# Important Figures & Tables
+#
+# Highlight the most critical visualizations and tabular data, explaining their importance.
+#
+# | Figure/Table | Description                     | Insight or Interpretation            |
+# |--------------|----------------------------------|--------------------------------------|
+# | Figure 1     | [What it shows]                  | [Why it's important]                |
+# | Table 2      | [Data/content summary]           | [What we learn from it]             |
+# | Figure 3     | [Trend or structure depicted]    | [Significance to conclusions]       |
+# """
+#
+#
+# # 7. Limitations & Future Work
+# limitations_prompt = """# Limitations & Future Work
+#
+# Detail the limitations encountered in the research and outline proposed future directions.
+#
+# | Type           | Limitation Description          | Potential Impact                     |
+# |----------------|----------------------------------|--------------------------------------|
+# | Theoretical     | [Conceptual limits]              | [Effect on validity/generalizability]|
+# | Methodological  | [Design or procedure issues]     | [Effect on robustness]               |
+# | Data-Related    | [Data quality, availability]     | [Effect on conclusions]              |
+#
+# **Future Work Suggestions:**
+# - [Proposed improvement or next step]
+# - [New areas to explore]
+# - [Potential experiments or applications]
+# """
+#
+#
+# # 8. Main Contributions
+# contributions_prompt = """# Main Contributions
+#
+# List all major contributions by type, and explain how each adds value.
+#
+# | Category       | Contribution Summary             | Novelty or Advancement                |
+# |----------------|----------------------------------|----------------------------------------|
+# | Theoretical     | [New framework or insight]       | [How it extends theory]               |
+# | Methodological  | [New method/model]               | [What makes it different or better]   |
+# | Empirical       | [Results from data/experiments]  | [Why they matter]                     |
+# | Practical       | [Applications or systems]        | [Real-world relevance]                |
+#
+# **Most Noteworthy Contribution:** [Summarize the biggest impact of the paper]
+# """
+#
+# # 9. Related Work
+# related_work_prompt = """# Related Work
+#
+# Show how this research fits into the existing landscape and what gaps it fills.
+#
+# | Topic/Area      | Previous Approaches             | This Paper's Innovation               |
+# |------------------|----------------------------------|----------------------------------------|
+# | Area 1           | [Summary of prior methods]       | [What’s new in this work]             |
+# | Area 2           | [Prior attempts or models]       | [Improvements or alternatives]        |
+# | Area 3           | [Old techniques or theories]     | [Enhancements introduced here]        |
+#
+# **Gap Addressed:** [What missing element or inefficiency this paper tackles]
+# """
+#
+# # 10. Practical Applications
+# applications_prompt = """# Practical Applications
+#
+# Explore how the research can be applied in real-world domains.
+#
+# | Domain/Industry | Use Case or Application         | Requirements or Dependencies        | Expected Timeline |
+# |------------------|----------------------------------|-------------------------------------|-------------------|
+# | Domain 1         | [What can be done]               | [Tech, data, adoption needs]        | [Short/Med/Long]  |
+# | Domain 2         | [Another use case]               | [Feasibility factors]               | [Short/Med/Long]  |
+# | Domain 3         | [Innovative potential]           | [Deployment conditions]             | [Short/Med/Long]  |
+#
+# **Most Promising Use Case:** [Brief highlight of top application potential]
+# """
+#
+#
+# # 11. Technical Details
+# technical_prompt = """# Technical Details
+#
+# Dive into the specific technical aspects, including algorithms, architecture, and implementation details.
+#
+# | Component     | Description                        | Configuration or Parameters         |
+# |---------------|-------------------------------------|--------------------------------------|
+# | Algorithm     | [What algorithm is used]            | [Hyperparameters, version etc.]     |
+# | Model/Architecture | [Type or design used]          | [Layers, connections, components]   |
+# | Implementation| [Languages, packages, environment] | [Frameworks, hardware specifics]    |
+# | Performance   | [Observed performance]              | [Accuracy, latency, etc.]           |
+#
+# **Code Repository:** [Link if available or mention if not provided]
+# """
+#
+#
+# # 12. Quick Summary
+# quick_summary_prompt = """# Quick Summary
+#
+# Provide an overview of the entire paper in both concise and single-sentence formats.
+#
+# **Brief Summary (3–5 Sentences):**
+# [Include motivation, methodology, findings, and significance.]
+#
+# **One-Sentence Summary:**
+# [A compact summary capturing the paper’s core message.]
+# """
+#
+#
+# # 13. Reading Guide
+# reading_guide_prompt = """# Reading Guide
+#
+# Help readers focus on the most insightful sections.
+#
+# | Section Name      | Key Information or Reason to Read |
+# |-------------------|------------------------------------|
+# | [Section A]       | [Main idea or takeaway]            |
+# | [Section B]       | [Core implementation detail]       |
+# | [Section C]       | [Critical results or discussion]   |
+#
+# **Recommended Reading Path:** [Suggestion for efficient reading – e.g., skip intro, read methods, then results]
+# """
+#
+#
+# # 14. Key Equations
+# equations_prompt = """# Key Equations
+#
+# Highlight and explain major equations in the paper.
+#
+# | Equation         | Purpose                          | Variable Explanation                 | Why It Matters                     |
+# |------------------|-----------------------------------|---------------------------------------|------------------------------------|
+# | [Equation 1]     | [What it calculates or models]    | [Define each term]                    | [Its role in the paper]            |
+# | [Equation 2]     | [Purpose]                         | [Define each term]                    | [Impact on method/results]         |
+# | [Equation 3]     | [Purpose]                         | [Define each term]                    | [How it supports the conclusions]  |
+# """
+#
+# # 15. Executive Summary
+# executive_summary_prompt = """# Executive Summary
+#
+# Offer a high-level summary tailored for research leads, grant reviewers, or collaborators.
+#
+# | Section       | Description                         |
+# |----------------|--------------------------------------|
+# | Research Problem | [Clear articulation of the challenge] |
+# | Proposed Solution| [Brief on method/model introduced]   |
+# | Major Results   | [Highlights of key findings]         |
+# | Implications    | [Practical, theoretical impact]      |
+# | Relevance       | [Why this paper should be read]      |
+# """

app/summarization/summarizer.py

@@ -0,0 +1,206 @@
+import time
+from typing import Dict, List
+import cohere
+from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor, as_completed
+from ..config.settings import LLM_MODEL
+
+
+class DocumentSummarizer:
+    def __init__(self, retriever, batch_size=4):
+        self.batch_size = batch_size
+        self.retriever = retriever # Store the retriever here
+
+        self.cohere_client = cohere.ClientV2()
+
+        self.components = {
+            'basic_info': "Basic Paper Information",
+            'abstract': "Abstract Summary",
+            'methods': "Methodology Summary",
+            'results': "Key Results",
+            'limitations': "Limitations & Future Work",
+            'related_work': "Related Work",
+            'applications': "Practical Applications",
+            'technical': "Technical Details",
+            'equations': "Key Equations",
+        }
+
+        self.prompts = self._initialize_prompts()
+
+        # Validate prompts dictionary matches components
+        # This helps catch missing prompts or components
+        missing_prompts = [comp for comp in self.components if comp not in self.prompts]
+        if missing_prompts:
+             print(f"Warning: No prompts found for components: {missing_prompts}")
+        missing_components = [prompt_key for prompt_key in self.prompts if prompt_key not in self.components]
+        if missing_components:
+             print(f"Warning: Prompts found for components not in self.components: {missing_components}")
+
+
+    def _initialize_prompts(self):
+        # It's better to explicitly import what you need
+        from ..summarization.prompt2 import (
+            basic_info_prompt, abstract_prompt,
+            methods_prompt, results_prompt, visuals_prompt, limitations_prompt,
+            contributions_prompt, related_work_prompt, applications_prompt,
+            technical_prompt, quick_summary_prompt, reading_guide_prompt, # quick_summary & reading_guide prompts might be needed
+            equations_prompt
+        )
+        return {
+            'basic_info': basic_info_prompt,
+            'abstract': abstract_prompt,
+            'methods': methods_prompt,
+            'results': results_prompt,
+            'limitations': limitations_prompt,
+            'related_work': related_work_prompt,
+            'applications': applications_prompt,
+            'technical': technical_prompt,
+            'equations': equations_prompt,
+        }
+
+    def summarize_text(self, documents: List[Dict], prompt: str, language: str):
+        """
+        Summarizes the provided documents using the given prompt and language
+        via the Cohere Chat API.
+        """
+        if not documents:
+            print("Warning: No documents provided for summarization.")
+            return None # Or an empty string, depending on desired behavior
+
+        # Use the initialized client
+        try:
+            response = self.cohere_client.chat(
+                model=LLM_MODEL,
+                documents=documents, # Pass the list of dicts directly
+                messages=[
+                    {"role": "system", "content": f"You are an expert summarization AI. Please respond in {language}."},
+                    {"role": "user", "content": f"{prompt}"}
+                ],
+            )
+            if response and response.message and response.message.content and response.message.content[0] and response.message.content[0].text:
+                 return response.message.content[0].text
+            else:
+                 print(f"Warning: Unexpected API response structure for prompt: {prompt[:50]}...")
+                 return None
+
+        except Exception as e:
+            print(f"Error during Cohere API call: {e}")
+            return None
+
+
+    def extract_relevant_documents(self, component: str, filename: str, chunk_size: int):
+        """
+        Extracts relevant documents for a specific component from the retriever.
+        """
+        query = f"Analyze the {self.components.get(component, component)} section from the document titled '{filename}'."
+        # Use the retriever stored in self.
+        # Pass the chunk_size parameter correctly
+        try:
+            documents = self.retriever.get_relevant_docs(
+                chromdb_query=query,
+                rerank_query=query,
+                filter={'filename': filename},
+                chunk_size=chunk_size
+            )
+            return documents
+        except Exception as e:
+            print(f"Error during document retrieval for component {component}: {e}")
+            return []
+
+
+    def summerize_document(self, filename: str, language: str, chunk_size: int):
+        """
+        Summarizes a document by processing each component in parallel.
+        """
+        start_total = time.time()
+        components = list(self.components.keys())
+        results = {}
+        errors = {} # Track errors
+
+        def process_component(comp):
+            comp_start = time.time()
+            print(f"Starting processing for component: {comp}")
+            try:
+                document_chunks = self.extract_relevant_documents(comp, filename, chunk_size)
+
+                if not document_chunks:
+                    print(f"No documents found for component: {comp} for file {filename}")
+                    return comp, None, f"No documents found"
+
+                prompt = self.prompts.get(comp)
+                if not prompt:
+                    print(f"No prompt defined for component: {comp}")
+                    return comp, None, f"No prompt defined"
+
+                # Summarize the retrieved documents
+                summary = self.summarize_text(document_chunks, prompt, language)
+
+                comp_end = time.time()
+                print(f"Finished processing for component: {comp}. Time taken: {comp_end - comp_start:.2f} seconds")
+                return comp, summary, None # Return comp, result, error
+
+            except Exception as e:
+                comp_end = time.time()
+                print(f"Error processing component {comp}: {e}. Time taken: {comp_end - comp_start:.2f} seconds")
+                return comp, None, str(e) # Return comp, result, error
+
+        # Use ThreadPoolExecutor for I/O-bound tasks (API calls)
+        # max_workers=None uses a default appropriate for the system
+        with ThreadPoolExecutor(max_workers=None) as executor:
+            # Submit all component tasks
+            future_to_component = {executor.submit(process_component, comp): comp for comp in components}
+
+            # Process results as they complete
+            for future in as_completed(future_to_component):
+                comp = future_to_component[future]
+                try:
+                    comp_name, result, error = future.result()
+                    if result is not None:
+                        results[comp_name] = result
+                    elif error:
+                        errors[comp_name] = error
+
+                except Exception as exc:
+                    # This catches exceptions *within* the future's result retrieval, less common
+                    print(f'{comp} generated an exception: {exc}')
+                    errors[comp] = str(exc)
+
+
+        end_total = time.time()
+        print(f"\n--- Total summarization time for {filename}: {end_total - start_total:.2f} seconds ---\n")
+
+        # You might want to return both results and errors
+        # For simplicity, let's just compile the available results for now
+        compiled = self.compile_summary(filename, results)
+        # Optionally, add errors to the compiled output or return them separately
+        if errors:
+             print(f"Components that failed or returned no data: {list(errors.keys())}")
+             # You could append error messages to the compiled summary
+             # compiled += "\n\n## Processing Errors\n" + "\n".join([f"- {k}: {v}" for k, v in errors.items()])
+
+
+        return compiled
+
+
+    def compile_summary(self, filename: str, results: Dict[str, str]) -> str:
+        """
+        Compiles a summary for a document by concatenating the results of all requested components.
+        Orders sections according to a predefined list.
+        """
+        # Include all components that might have results, maintaining desired order
+        sections_order = [
+            'basic_info', 'abstract',
+            'methods', 'results', 'equations', 'technical',
+            'related_work', 'applications', 'limitations'
+        ]
+
+        lines = [f"# Summary of {filename}", f"Generated on: {time.strftime('%Y-%m-%d %H:%M:%S')}\n"]
+        for section in sections_order:
+            # Only add a section if it was processed and returned a result
+            if section in results and results[section]:
+                # Use .get with a default in case a component was added to results
+                # but not self.components (though validate init helps prevent this)
+                title = self.components.get(section, section).title()
+                lines.append(f"## {title}\n") # Use ## for subheadings
+                lines.append(f"{results[section]}\n")
+
+        return "\n".join(lines)

app/utils/__init__.py

@@ -0,0 +1,3 @@
+"""
+Utility package.
+"""

app/utils/progress_tracker.py

@@ -0,0 +1,128 @@
+"""
+Progress monitoring utilities for tracking parallel processing.
+"""
+import time
+import threading
+import logging
+from typing import Dict, List, Any, Set
+
+logger = logging.getLogger(__name__)
+
+
+class ProgressTracker:
+    """
+    Tracks progress of parallel document processing tasks.
+    """
+
+    def __init__(self, total_documents, update_interval=5):
+        """
+        Initialize the progress tracker.
+
+        Args:
+            total_documents: Total number of documents to process
+            update_interval: How often to log updates (in seconds)
+        """
+        self.total = total_documents
+        self.completed = 0
+        self.failed = 0
+        self.in_progress = 0
+        self.processed_files = set()
+        self.update_interval = update_interval
+        self.lock = threading.Lock()
+        self.start_time = time.time()
+        self.monitor_thread = None
+        self.stop_monitoring = threading.Event()
+
+    def mark_started(self, filename):
+        """Mark a document as being processed."""
+        with self.lock:
+            self.in_progress += 1
+            logger.info(f"Started processing: {filename}")
+
+    def mark_completed(self, filename, success=True):
+        """Mark a document as completed."""
+        with self.lock:
+            self.in_progress -= 1
+            if filename not in self.processed_files:
+                self.processed_files.add(filename)
+                if success:
+                    self.completed += 1
+                else:
+                    self.failed += 1
+
+    def get_stats(self):
+        """Get current processing statistics."""
+        with self.lock:
+            elapsed = time.time() - self.start_time
+            remaining = self.total - (self.completed + self.failed)
+
+            # Calculate estimated time remaining
+            if self.completed > 0:
+                avg_time_per_doc = elapsed / self.completed
+                est_remaining = avg_time_per_doc * remaining
+            else:
+                est_remaining = None
+
+            return {
+                'total': self.total,
+                'completed': self.completed,
+                'failed': self.failed,
+                'in_progress': self.in_progress,
+                'remaining': remaining,
+                'elapsed_seconds': elapsed,
+                'estimated_remaining_seconds': est_remaining
+            }
+
+    def _format_time(self, seconds):
+        """Format seconds as HH:MM:SS."""
+        if seconds is None:
+            return "unknown"
+
+        hours, remainder = divmod(int(seconds), 3600)
+        minutes, seconds = divmod(remainder, 60)
+        return f"{hours:02d}:{minutes:02d}:{seconds:02d}"
+
+    def _monitor_progress(self):
+        """Monitor and log progress periodically."""
+        while not self.stop_monitoring.is_set():
+            stats = self.get_stats()
+
+            logger.info(
+                f"Progress: {stats['completed']}/{stats['total']} completed, "
+                f"{stats['failed']} failed, {stats['in_progress']} in progress | "
+                f"Elapsed: {self._format_time(stats['elapsed_seconds'])} | "
+                f"Est. remaining: {self._format_time(stats['estimated_remaining_seconds'])}"
+            )
+
+            # Check if we're done
+            if stats['completed'] + stats['failed'] >= stats['total']:
+                logger.info("All documents processed!")
+                break
+
+            # Wait for next update
+            self.stop_monitoring.wait(self.update_interval)
+
+    def start_monitoring(self):
+        """Start background monitoring thread."""
+        self.monitor_thread = threading.Thread(target=self._monitor_progress)
+        self.monitor_thread.daemon = True
+        self.monitor_thread.start()
+
+    def stop(self):
+        """Stop the monitoring thread and report final results."""
+        if self.monitor_thread and self.monitor_thread.is_alive():
+            self.stop_monitoring.set()
+            self.monitor_thread.join(timeout=2.0)
+
+        # Log final statistics
+        stats = self.get_stats()
+        logger.info(
+            f"Final results: {stats['completed']}/{stats['total']} completed, "
+            f"{stats['failed']} failed | "
+            f"Total time: {self._format_time(stats['elapsed_seconds'])}"
+        )
+
+        success_rate = (stats['completed'] / stats['total']) * 100 if stats['total'] > 0 else 0
+        logger.info(f"Success rate: {success_rate:.2f}%")
+
+        return stats

main.py

@@ -0,0 +1,282 @@
+import logging  # Import logging
+import os
+import tempfile
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from typing import List, Dict, Any, Tuple
+
+from app.config.settings import DOCS_FOLDER
+# Import classes from the renamed modules
+from app.document_processing.extractors import DocumentProcessorAdapter
+from app.retrieval.vector_store import Retriever
+from app.summarization.output import SummaryOutputManager
+from app.summarization.summarizer import DocumentSummarizer
+
+# Configure logging for the main script
+logger = logging.getLogger(__name__)
+
+
+
+def process_uploaded_files(uploaded_files) -> List[Dict[str, Any]]:
+    """
+    Processes a list of files uploaded via Streamlit.
+    Saves them temporarily into a folder and uses the DocumentProcessorAdapter
+    to process that folder.
+
+    Args:
+        uploaded_files: List of Streamlit UploadedFile objects.
+                        Type hint is omitted here to avoid needing Streamlit import at top level.
+
+    Returns:
+        List of dictionaries with original extraction results, including chunk_size.
+    """
+    # Import streamlit here, as it's used for st.warning
+    import streamlit as st
+
+    start_time = time.time()
+    logger.info(f"Starting processing for {len(uploaded_files)} uploaded files.")
+
+    # Create a temporary directory to save uploaded files
+    # This directory will be automatically cleaned up when the 'with' block exits
+    with tempfile.TemporaryDirectory() as tmpdir:
+        logger.info(f"Using temporary directory: {tmpdir}")
+        # Save all uploaded files into the temporary directory
+        for uploaded_file in uploaded_files:
+            # Create a safe path within the temporary directory
+            # Use uploaded_file.name directly, tempfile handles uniqueness if needed
+            file_path = os.path.join(tmpdir, uploaded_file.name)
+            # Write the file content to the temporary path
+            try:
+                with open(file_path, "wb") as f:
+                    f.write(uploaded_file.getvalue())
+                logger.debug(f"Saved uploaded file '{uploaded_file.name}' to '{file_path}'")
+            except Exception as e:
+                logger.error(f"Error saving uploaded file '{uploaded_file.name}' to temporary directory: {e}", exc_info=True)
+                # Log a warning in Streamlit if a file couldn't be saved
+                st.warning(f"Could not save uploaded file '{uploaded_file.name}' temporarily. It will be skipped.")
+
+
+        # Use the DocumentProcessorAdapter to process the entire temporary folder
+        processor = DocumentProcessorAdapter() # Corrected typo here
+        # Call process_folder with the temporary directory path
+        extraction_results = processor.process_folder(tmpdir)
+        # The process_folder method returns the list of extraction results
+
+    end_time = time.time()
+    logger.info(f"Finished processing uploaded files in {end_time - start_time:.2f} seconds.")
+    # The extraction_results list now contains dictionaries with 'filename', 'text', 'error', etc.
+    return extraction_results
+
+
+def setup_retrieval_system(extraction_results: List[Dict[str, Any]]) -> Tuple[List[Dict[str, Any]], Retriever]:
+    """
+    Sets up the retrieval system (vector store) from extraction results.
+
+    Args:
+        extraction_results: List of dictionaries from document extraction.
+                            Should contain 'filename' and 'text'.
+
+    Returns:
+        A tuple containing:
+        - The updated extraction_results list (with 'chunk_size' populated by Retriever).
+        - An initialized Retriever instance.
+    """
+    start_time = time.time()
+    logger.info("Setting up retrieval system.")
+    try:
+        retriever = Retriever()
+        # create_from_documents takes extraction results, chunks text, embeds, and builds the DB.
+        # It also updates the extraction_results list with the 'chunk_size' for each document.
+        updated_extraction_results = retriever.create_from_documents(extraction_results)
+        end_time = time.time()
+        logger.info(f"Retriever setup complete in {end_time - start_time:.2f} seconds.")
+        return updated_extraction_results, retriever
+    except Exception as e:
+        end_time = time.time()
+        logger.error(f"Error during retrieval system setup: {e}", exc_info=True)
+        # If retrieval setup fails, the summarization cannot proceed.
+        # Re-raise the exception so the Streamlit app can catch and display it.
+        raise
+
+
+def summarize_extracted_documents(extraction_results: List[Dict[str, Any]], retriever: Retriever) -> List[Dict[str, Any]]:
+    """
+    Summarizes documents based on extraction results and a configured retriever.
+
+    Args:
+        extraction_results: List of dictionaries from document extraction (should include chunk_size
+                            populated by setup_retrieval_system).
+        retriever: An initialized Retriever instance.
+
+    Returns:
+        A list of dictionaries, each containing the summary result for a file.
+        Each dictionary includes:
+        - 'filename': The name of the file.
+        - 'success': Boolean indicating if summarization was successful.
+        - 'summary': The generated summary string (if successful), or None.
+        - 'error': An error message string (if not successful), or None.
+        - 'processing_time': Time taken for summarization of this file.
+    """
+    start_time = time.time()
+    logger.info(f"Starting summarization for {len(extraction_results)} documents.")
+
+    # Initialize the summarizer with the retriever
+    summarizer = DocumentSummarizer(retriever)
+
+    results = [] # List to store results for each document
+
+    # Filter out results that failed extraction or have no text/chunks
+    # Summarization requires extracted text and successful chunking (chunk_size > 0)
+    summarizable_results = [
+        res for res in extraction_results
+        if res.get('text') and res.get('chunk_size', 0) > 0 and res.get('error') is None
+    ]
+    skipped_results = [
+        res for res in extraction_results
+        if res not in summarizable_results
+    ]
+
+    if skipped_results:
+        logger.warning(f"Skipping summarization for {len(skipped_results)} files due to extraction errors or no text/chunks.")
+        for res in skipped_results:
+            # Add entries for skipped files to the results list
+            results.append({
+                'filename': res.get('filename', 'unknown'),
+                'success': False,
+                'summary': None,
+                'error': res.get('error', 'Extraction failed or no text/chunks'),
+                'processing_time': 0, # No summarization time for skipped files
+            })
+
+
+    def process_single_summary(result: Dict[str, Any]) -> Dict[str, Any]:
+        """Helper function to summarize a single document result."""
+        file_start_time = time.time()
+        filename = result.get('filename', 'unknown')
+        # Use detected language, default to English if detection failed
+        language = result.get('language', 'en')
+        chunk_size = result.get('chunk_size', 0) # Should be > 0 for summarizable_results
+
+        logger.info(f"Summarizing document: {filename}")
+
+        try:
+            # Call the summarizer for a single document
+            # The summerize_document method handles parallel processing of components internally
+            summary = summarizer.summerize_document(filename, language, chunk_size)
+
+            file_end_time = time.time()
+            logger.info(f"Finished summarizing {filename} in {file_end_time - file_start_time:.2f} seconds.")
+            return {
+                'filename': filename,
+                'success': True,
+                'summary': summary, # Return the summary string
+                'error': None,
+                'processing_time': file_end_time - file_start_time,
+            }
+        except Exception as e:
+            file_end_time = time.time()
+            error_msg = str(e)
+            logger.error(f"Error summarizing document {filename}: {e}", exc_info=True)
+            return {
+                'filename': filename,
+                'success': False,
+                'summary': None,
+                'error': error_msg,
+                'processing_time': file_end_time - file_start_time,
+            }
+
+    with ThreadPoolExecutor(max_workers=None) as executor: # Adjust max_workers as needed
+        # Submit summarizable document results to the executor
+        futures = {executor.submit(process_single_summary, res): res['filename'] for res in summarizable_results}
+
+        # Process results as they complete
+        for future in as_completed(futures):
+            filename = futures[future]
+            try:
+                summary_result = future.result()
+                results.append(summary_result)
+                logger.debug(f"Summary result received for {filename}")
+            except Exception as exc:
+                # This catches exceptions *within* the future's result retrieval
+                logger.error(f"Exception retrieving summary result for {filename}: {exc}", exc_info=True)
+                results.append({
+                    'filename': filename,
+                    'success': False,
+                    'summary': None,
+                    'error': f"Failed to retrieve result: {exc}",
+                    'processing_time': 0, # Can't determine processing time if result retrieval failed
+                })
+
+    end_time = time.time()
+    logger.info(f"Finished batch summarization in {end_time - start_time:.2f} seconds.")
+    return results
+
+
+# if __name__ == "__main__":
+#     start_time = time.time()
+#     logger.info("Starting document summarization process (command line).")
+#
+#     try:
+#         # Step 1: Process documents from the predefined folder
+#         logger.info(f"Processing documents from: {DOCS_FOLDER}")
+#         # DocumentProcessorAdapter().process_folder returns a list of extraction result dicts
+#         extraction_results = DocumentProcessorAdapter().process_folder(DOCS_FOLDER)
+#         logger.info(f"Document Processing Time taken: {time.time()-start_time:.2f} seconds")
+#
+#         # Step 2: Setup retrieval system
+#         setup_start_time = time.time()
+#         # setup_retrieval_system takes extraction results and returns updated results (with chunk_size) and the retriever
+#         extraction_results_with_chunks, retriever = setup_retrieval_system(extraction_results)
+#         logger.info(f"Retriever Setup Time taken: {time.time() - setup_start_time:.2f} seconds")
+#
+#         # Step 3: Summarize the documents
+#         summarization_start_time = time.time()
+#         # For command line, we might still want to save files locally
+#         output_manager = SummaryOutputManager() # Uses default output_dir from settings
+#         # summarize_extracted_documents performs the summarization and returns results
+#         summary_results = summarize_extracted_documents(extraction_results_with_chunks, retriever)
+#
+#         # Step 4: Save summaries to files (for command-line only)
+#         logger.info("Saving summaries to files.")
+#         saved_count = 0
+#         for res in summary_results:
+#             if res['success'] and res['summary']:
+#                 # Use the output_manager to save the summary string
+#                 output_manager.save_summary(res['filename'], res['summary'], formats=['markdown'])
+#                 saved_count += 1
+#         logger.info(f"Saved {saved_count} summaries.")
+#
+#
+#         logger.info(f"Summarization Time taken: {time.time() - summarization_start_time:.2f} seconds")
+#
+#
+#         # Output results summary to console
+#         logger.info("\n" + "=" * 50)
+#         logger.info("Summarization Process Complete.")
+#         logger.info("=" * 50)
+#         successful_count = sum(res.get('success', False) for res in summary_results)
+#         total_processed = len(summary_results) # Includes skipped files if they were added to results list earlier
+#         total_time = time.time() - start_time
+#
+#         logger.info(f"Total files attempted: {len(extraction_results)}") # Total files found/attempted extraction
+#         logger.info(f"Files successfully extracted and summarizable: {len(extraction_results_with_chunks)}") # Files with text and chunks
+#         logger.info(f"Files summarized: {successful_count}/{total_processed}")
+#         logger.info(f"Total process time: {total_time:.2f} seconds")
+#         logger.info("=" * 50)
+#
+#         # Print individual results status
+#         logger.info("\nIndividual File Results:")
+#         for result in summary_results:
+#             name = result.get('filename', 'unknown')
+#             status = "SUCCESS" if result['success'] else "FAILED"
+#             time_taken = result.get('processing_time', 0)
+#             error_msg = result.get('error', '')
+#             logger.info(f"- {name}: {status} ({time_taken:.2f}s) {f'Error: {error_msg}' if error_msg else ''}")
+#
+#
+#     except FileNotFoundError as fnf_error:
+#         logger.error(f"Configuration Error: {fnf_error}")
+#         print(f"Error: {fnf_error}")
+#     except Exception as main_error:
+#         logger.error(f"An unexpected error occurred during the main process: {main_error}", exc_info=True)
+#         print(f"An unexpected error occurred: {main_error}")

packages.txt

@@ -0,0 +1,6 @@
+tesseract-ocr
+libtesseract-dev
+tesseract-ocr-hin
+tesseract-ocr-ara
+tesseract-ocr-spa
+tesseract-ocr-chi-sim

requirements.txt

@@ -0,0 +1,11 @@
+streamlit
+dotenv
+cohere
+langchain
+chromadb
+protobuf~=3.20
+langchain-chroma
+langchain-cohere
+pypdf
+pytesseract
+Pillow