Initial commit - chatbot online version

Dockerfile

@@ -0,0 +1,12 @@
+FROM python:3.10-slim
+
+WORKDIR /app
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY . .
+
+EXPOSE 7860
+
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860"]

conversation_storage.py

@@ -0,0 +1,78 @@
+import re
+from datetime import datetime
+from typing import Dict, List, Any
+
+# Global conversation storage dictionary
+SYSTEM_CHART_CONVERSATIONS: Dict[str, Dict[str, Any]] = {}
+
+def save_conversation(extraction_id: str, paper_title: str, conversation_history: List, paper_content: str, paper_type: str) -> str:
+    """
+    Save a conversation to the global conversation storage.
+    
+    Args:
+        extraction_id: Unique identifier for the extraction task
+        paper_title: Title of the paper
+        conversation_history: List of conversation messages
+        paper_content: Full content of the paper
+        paper_type: Type of paper (Experimental/Practical, Theoretical/Computational, Mixed)
+    
+    Returns:
+        str: Unique conversation key
+    """
+    # Initialize extraction_id entry if it doesn't exist
+    if extraction_id not in SYSTEM_CHART_CONVERSATIONS:
+        SYSTEM_CHART_CONVERSATIONS[extraction_id] = {}
+    
+    # Create a safe key from paper title (matching original behavior)
+    safe_title = re.sub(r'[^\w\s-]', '', paper_title)[:100]
+    safe_title = re.sub(r'[-\s]+', '_', safe_title)
+    
+    conversation_key = f"{safe_title}_{len(SYSTEM_CHART_CONVERSATIONS[extraction_id])}"
+    
+    # Save the conversation
+    SYSTEM_CHART_CONVERSATIONS[extraction_id][conversation_key] = {
+        "paper_title": paper_title,
+        "conversation_history": conversation_history.copy(),  # Make a copy to avoid modification
+        "paper_content": paper_content,
+        "paper_type": paper_type,
+        "created_at": datetime.now().isoformat(),
+        "last_updated": datetime.now().isoformat()
+    }
+    
+    print(f"💾 Saved conversation for paper: {paper_title} (key: {conversation_key})")
+    return conversation_key
+
+def get_conversations(extraction_id: str) -> Dict[str, Dict[str, Any]]:
+    """
+    Get all conversations for a specific extraction ID.
+    
+    Args:
+        extraction_id: Unique identifier for the extraction task
+    
+    Returns:
+        Dict of conversation_key -> conversation_data
+    """
+    return SYSTEM_CHART_CONVERSATIONS.get(extraction_id, {})
+
+def update_conversation(extraction_id: str, conversation_key: str, new_message, response):
+    """
+    Update an existing conversation with a new message and response.
+    
+    Args:
+        extraction_id: Unique identifier for the extraction task
+        conversation_key: Unique conversation key
+        new_message: New message to add
+        response: Response to the message
+    
+    Returns:
+        bool: True if update was successful, False otherwise
+    """
+    if (extraction_id in SYSTEM_CHART_CONVERSATIONS and 
+        conversation_key in SYSTEM_CHART_CONVERSATIONS[extraction_id]):
+        
+        conversation = SYSTEM_CHART_CONVERSATIONS[extraction_id][conversation_key]
+        conversation["conversation_history"].append(new_message)
+        conversation["conversation_history"].append(response)
+        conversation["last_updated"] = datetime.now().isoformat()
+        return True
+    return False 

hypotheses_generation.py

@@ -0,0 +1,207 @@
+import regex as re
+from typing import Dict, List, Tuple
+from langchain_core.messages import HumanMessage
+from prompts import create_hypotheses_generation_prompt
+
+from typing import Tuple
+
+PATTERN = re.compile(
+    r"PAPER:\s*(?P<paper>.*?)\s*[\r\n]+SYSTEM_CHART:\s*(?P<chart>.*?)(?=\s*={5,}|$)",
+    re.DOTALL
+)
+
+def parse_multiple_system_chart_files(content_list: List[str]) -> Tuple[str, str]:
+    """
+    Parse multiple system chart files and extract numbered tables and citations.
+    Handles new format with 100 '=' delimiters and PAPER:/SYSTEM_CHART: structure.
+    
+    Args:
+        content_list: List of raw content strings from multiple system chart files
+        
+    Returns:
+        Tuple of (numbered_tables, numbered_citations)
+    """
+    papers: List[str] = []
+    charts: List[str] = []
+
+    for blob in content_list:
+        if not blob or not blob.strip():
+            continue
+        for m in PATTERN.finditer(blob):
+            papers.append(m.group("paper").strip())
+            charts.append(m.group("chart").strip())
+
+    numbered_tables = []
+    numbered_citations = []
+    for i, (p, c) in enumerate(zip(papers, charts), 1):
+        numbered_tables.append(f"\n=== TABLE {i} ===\n{c}\n=== END OF TABLE {i} ===")
+        numbered_citations.append(f"{i}. {p}")
+        
+    return "\n".join(numbered_tables).strip(), "\n".join(numbered_citations).strip()
+
+def parse_system_charts_and_citations(content: str) -> Tuple[str, str]:
+    """
+    Parse system charts content and extract numbered tables and citations robustly.
+    Updated to handle new format with 100 '=' delimiters and PAPER:/SYSTEM_CHART: structure.
+    
+    Args:
+        content: Raw content from system_charts.txt file
+        
+    Returns:
+        Tuple of (numbered_tables, numbered_citations)
+    """
+    # Use the new multi-file parser with single content
+    return parse_multiple_system_chart_files([content])
+
+async def generate_hypotheses_from_system_charts(
+    system_charts: List[Dict[str, str]], 
+    llm, 
+    material_goal_1: str = "Property #1 for xxx material",
+    material_goal_2: str = "Property #2 for xxx material",
+    iterations_count: int = 1,
+    progress_callback=None
+) -> str:
+    """
+    Generate scientific hypotheses using the new synergistic approach.
+    
+    Args:
+        system_charts: List of parsed system chart dictionaries with 'content' or 'raw_content' fields
+        llm: Language model for hypothesis generation
+        material_goal_1: First material design goal to guide hypothesis generation
+        material_goal_2: Second material design goal to guide hypothesis generation
+        iterations_count: Number of hypothesis iterations to generate
+        progress_callback: Function to report progress updates
+        
+    Returns:
+        hypotheses_content
+    """
+    
+    if not system_charts:
+        return "No system charts provided for hypothesis generation."
+    
+    # Initial progress update
+    if progress_callback:
+        progress_callback({
+            "status": "parsing",
+            "progress": 5,
+            "message": "Parsing system charts and citations..."
+        })
+    
+    # Extract raw content from system charts
+    # Support both 'raw_content' and 'content' fields for backward compatibility
+    content_list = []
+    for chart in system_charts:
+        raw_content = chart.get('raw_content', chart.get('content', ''))
+        if raw_content and raw_content.strip():
+            content_list.append(raw_content)
+    
+    if not content_list:
+        return "No valid system chart content found for hypothesis generation."
+    
+    # Parse all system chart files together using the new multi-file parser
+    numbered_tables, numbered_citations = parse_multiple_system_chart_files(content_list)
+    
+    if progress_callback:
+        progress_callback({
+            "status": "generating",
+            "progress": 20,
+            "message": "Generating synergistic hypotheses..."
+        })
+    
+    # Create the prompt with user-specified material goals
+    hypotheses_prompt = create_hypotheses_generation_prompt(material_goal_1, material_goal_2)
+    
+    # Prepare the prompt in the specified format
+    full_prompt = f"Tables are: {numbered_tables}\n\nCitation resources are: {numbered_citations}\n\n{hypotheses_prompt}"
+    
+    # Configure LLM parameters for creativity
+    llm_kwargs = {}
+    
+    # Check if this is a reasoning model (o1 series) which doesn't support temperature
+    model_name = getattr(llm, 'model_name', getattr(llm, 'model', ''))
+    if not (model_name and model_name.lower().startswith('o')):
+        # Set higher temperature for creativity (non-reasoning models)
+        if hasattr(llm, 'temperature'):
+            original_temp = llm.temperature
+            llm.temperature = 1.0  # Higher temperature for creativity
+        elif hasattr(llm, 'model_kwargs'):
+            llm_kwargs = llm.model_kwargs.copy()
+            llm_kwargs['temperature'] = 1.0
+    
+    try:
+        # Generate hypotheses for each iteration
+        all_iterations_content = []
+        
+        for iteration in range(iterations_count):
+            # Update progress for current iteration
+            if progress_callback:
+                progress_start = 20 + (iteration * 60 // iterations_count)
+                progress_callback({
+                    "status": "llm_processing",
+                    "progress": progress_start,
+                    "message": f"LLM generating hypotheses (iteration {iteration + 1}/{iterations_count})..."
+                })
+            
+            conversation_history = [HumanMessage(content=full_prompt)]
+            
+            # Generate with appropriate parameters
+            if llm_kwargs:
+                # Use model_kwargs if available
+                original_kwargs = getattr(llm, 'model_kwargs', {})
+                llm.model_kwargs = llm_kwargs
+                response = await llm.ainvoke(conversation_history)
+                llm.model_kwargs = original_kwargs
+            else:
+                response = await llm.ainvoke(conversation_history)
+            
+            iteration_content = response.content
+            all_iterations_content.append({
+                'iteration': iteration + 1,
+                'content': iteration_content
+            })
+        
+        if progress_callback:
+            progress_callback({
+                "status": "formatting",
+                "progress": 80,
+                "message": "Formatting results..."
+            })
+        
+        # Create formatted output with all iterations
+        iterations_output = ""
+        for iteration_data in all_iterations_content:
+            iterations_output += f"\n\n"
+            iterations_output += "=" * 80 + "\n"
+            iterations_output += f"ITERATION {iteration_data['iteration']} HYPOTHESES:\n"
+            iterations_output += "=" * 80 + "\n\n"
+            iterations_output += iteration_data['content'] + "\n\n"
+            iterations_output += "=" * 80 + "\n"
+        
+        # Create formatted output
+        formatted_output = ""
+        formatted_output += "NUMBERED SYSTEM CHARTS:\n"
+        formatted_output += numbered_tables + "\n\n"
+        formatted_output += "CITATION REFERENCES:\n"
+        formatted_output += numbered_citations
+        formatted_output += iterations_output
+        
+        # Restore original temperature if modified
+        if hasattr(llm, 'temperature') and 'original_temp' in locals():
+            llm.temperature = original_temp
+            
+        if progress_callback:
+            progress_callback({
+                "status": "completed",
+                "progress": 100,
+                "message": f"Generated {iterations_count} iteration{'s' if iterations_count > 1 else ''} of synergistic hypotheses from {len(system_charts)} charts"
+            })
+        
+        return formatted_output
+        
+    except Exception as e:
+        # Restore original temperature if modified
+        if hasattr(llm, 'temperature') and 'original_temp' in locals():
+            llm.temperature = original_temp
+            
+        error_msg = f"Error generating hypotheses: {str(e)}"
+        return error_msg

hypotheses_service.py

@@ -0,0 +1,279 @@
+from io import BytesIO
+from zipfile import ZipFile
+from typing import Dict, Callable, Any, List
+from hypotheses_generation import generate_hypotheses_from_system_charts
+
+class HypothesesService:
+    """
+    Service class for Scientific Hypotheses Generation.
+    Handles the orchestration of parsing system charts, generating hypotheses,
+    and packaging results using the new synergistic approach.
+    """
+    
+    def __init__(self):
+        pass
+    
+    async def generate_hypotheses(
+        self,
+        system_charts_content_list: List[str],
+        llm,
+        material_goal_1: str = "Property #1 for xxx material",
+        material_goal_2: str = "Property #2 for xxx material",
+        iterations_count: int = 1,
+        progress_callback: Callable[[Dict[str, Any]], None] = None
+    ) -> Dict[str, Any]:
+        """
+        Main hypotheses generation pipeline using synergistic approach.
+        
+        Args:
+            system_charts_content_list: List of content strings from multiple system chart files
+            llm: Language model for hypothesis generation
+            material_goal_1: First material design goal to guide hypothesis generation
+            material_goal_2: Second material design goal to guide hypothesis generation
+            iterations_count: Number of hypothesis iterations to generate
+            progress_callback: Function to report progress updates
+            
+        Returns:
+            Dict containing generation results and metadata
+        """
+        try:
+            # Initial progress update
+            if progress_callback:
+                progress_callback({
+                    "status": "starting",
+                    "progress": 0,
+                    "message": "Preparing for synergistic hypothesis generation..."
+                })
+            
+            # Handle both single content and list of contents for backward compatibility
+            if isinstance(system_charts_content_list, str):
+                system_charts_content_list = [system_charts_content_list]
+            
+            # Create chart data structure for multiple files
+            chart_data = []
+            for i, content in enumerate(system_charts_content_list, 1):
+                if content and content.strip():
+                    chart_data.append({
+                        'content': content,
+                        'citation': f'System chart file {i}',
+                        'title': f'System Chart File {i}',
+                        'raw_content': content  # Pass the raw content
+                    })
+            
+            if not chart_data:
+                return {
+                    "success": False,
+                    "message": "No valid system chart content provided for hypothesis generation.",
+                    "zip_data": None,
+                    "filename": None
+                }
+            
+            print(f"📊 Prepared {len(chart_data)} system chart files for synergistic hypothesis generation")
+            
+            # Update progress after parsing
+            if progress_callback:
+                progress_callback({
+                    "status": "parsing_complete",
+                    "progress": 10,
+                    "total_charts": len(chart_data),
+                    "message": f"System charts prepared from {len(chart_data)} files. Starting synergistic hypothesis generation..."
+                })
+            
+            # Create progress callback function for the generation process
+            def generation_progress_callback(progress_data):
+                if progress_callback:
+                    # Adjust progress to fit within our overall range (10% to 95%)
+                    adjusted_progress = 10 + int((progress_data.get("progress", 0) / 100) * 85)
+                    progress_data["progress"] = adjusted_progress
+                    progress_callback(progress_data)
+            
+            # Generate hypotheses using the new synergistic approach
+            print(f"🧠 Starting synergistic hypothesis generation with goals: '{material_goal_1}' and '{material_goal_2}' ({iterations_count} iteration{'s' if iterations_count > 1 else ''})...")
+            hypotheses_content = await generate_hypotheses_from_system_charts(
+                chart_data, 
+                llm, 
+                material_goal_1,
+                material_goal_2,
+                iterations_count,
+                generation_progress_callback
+            )
+            
+            # Note: Cross-chart analysis is now integrated into the main generation process
+            cross_analysis_content = ""  # No longer separate
+            
+            # Create ZIP file in memory
+            if progress_callback:
+                progress_callback({
+                    "status": "packaging",
+                    "progress": 95,
+                    "message": "Creating results package..."
+                })
+            
+            zip_data = self._create_zip_file(
+                hypotheses_content, 
+                cross_analysis_content
+            )
+            
+            # Prepare result
+            return {
+                "success": True,
+                "message": f"Generated {iterations_count} iteration{'s' if iterations_count > 1 else ''} of synergistic hypotheses from {len(chart_data)} system chart files using cross-paper analysis approach!",
+                "zip_data": zip_data,
+                "filename": "hypotheses.zip",
+                "charts_processed": len(chart_data),
+                "cross_analysis_included": True,  # Now always included in main process
+                "method": "synergistic_analysis",
+                "iterations_count": iterations_count
+            }
+            
+        except Exception as e:
+            return {
+                "success": False,
+                "message": f"Error during synergistic hypothesis generation: {str(e)}",
+                "zip_data": None,
+                "filename": None
+            }
+    
+    def _create_zip_file(
+        self, 
+        hypotheses_content: str, 
+        cross_analysis_content: str = ""
+    ) -> bytes:
+        """Create ZIP file containing hypothesis generation results."""
+        zip_buffer = BytesIO()
+        
+        with ZipFile(zip_buffer, "w") as zipf:
+            # Add main hypotheses file
+            zipf.writestr("hypotheses.txt", hypotheses_content)
+
+            print(f"📦 Created synergistic hypotheses ZIP package with {len(zipf.namelist())} files")
+        
+        # Get ZIP data and close buffer
+        zip_data = zip_buffer.getvalue()
+        zip_buffer.close()
+        
+        return zip_data
+    
+    @staticmethod
+    def validate_multiple_system_charts_content(content_list: List[str]) -> Dict[str, Any]:
+        """
+        Validate that the provided content list is suitable for synergistic hypothesis generation.
+        
+        Args:
+            content_list: List of content strings to validate
+            
+        Returns:
+            Dict with validation results
+        """
+        try:
+            if not content_list:
+                return {
+                    "valid": False,
+                    "message": "No content provided for hypothesis generation.",
+                    "chart_count": 0
+                }
+            
+            valid_files = 0
+            total_data_rows = 0
+            file_details = []
+            
+            for i, content in enumerate(content_list, 1):
+                if not content or not content.strip():
+                    file_details.append(f"File {i}: Empty content")
+                    continue
+                
+                # Check for new format with 100 '=' delimiters
+                lines = content.strip().split('\n')
+                has_new_format = any(line.strip() == '=' * 100 for line in lines)
+                has_paper_keywords = any(line.strip().startswith('PAPER:') for line in lines)
+                has_chart_keywords = any(line.strip().startswith('SYSTEM_CHART:') for line in lines)
+                
+                if has_new_format and has_paper_keywords and has_chart_keywords:
+                    # Count entries in new format
+                    paper_count = len([line for line in lines if line.strip().startswith('PAPER:')])
+                    chart_count = len([line for line in lines if line.strip().startswith('SYSTEM_CHART:')])
+                    
+                    if paper_count > 0 and chart_count > 0 and paper_count == chart_count:
+                        valid_files += 1
+                        total_data_rows += paper_count
+                        file_details.append(f"File {i}: Valid format with {paper_count} entries")
+                    else:
+                        file_details.append(f"File {i}: Format issues - {paper_count} papers, {chart_count} charts")
+                else:
+                    # Fallback to table-based validation
+                    table_lines = [line for line in lines if '|' in line and not line.strip().startswith('|---')]
+                    
+                    if table_lines:
+                        data_rows = len([line for line in table_lines if line.strip() and (
+                            ('|' in line) and not any(
+                                header_word in line.lower() for header_word in ['processing', 'structure', 'property', 'mechanism']
+                            )
+                        )])
+                        
+                        if data_rows >= 2:
+                            valid_files += 1
+                            total_data_rows += data_rows
+                            file_details.append(f"File {i}: Valid table format with {data_rows} rows")
+                        else:
+                            file_details.append(f"File {i}: Insufficient table data ({data_rows} rows)")
+                    else:
+                        file_details.append(f"File {i}: No valid format found")
+            
+            if valid_files == 0:
+                return {
+                    "valid": False,
+                    "message": f"No valid system chart files found. Details: {'; '.join(file_details)}",
+                    "chart_count": 0,
+                    "valid_charts": 0,
+                    "file_details": file_details
+                }
+            
+            if total_data_rows < 2:
+                return {
+                    "valid": False,
+                    "message": f"Insufficient data across all files. Need multiple entries for synergistic analysis. Found {total_data_rows} total entries.",
+                    "chart_count": len(content_list),
+                    "valid_charts": valid_files,
+                    "file_details": file_details
+                }
+            
+            return {
+                "valid": True,
+                "message": f"Validated {valid_files}/{len(content_list)} system chart files for synergistic hypothesis generation. Found {total_data_rows} total entries.",
+                "chart_count": len(content_list),
+                "valid_charts": valid_files,
+                "data_entries": total_data_rows,
+                "file_details": file_details
+            }
+            
+        except Exception as e:
+            return {
+                "valid": False,
+                "message": f"Error validating content: {str(e)}",
+                "chart_count": 0,
+                "file_details": []
+            }
+
+    @staticmethod
+    def validate_system_charts_content(content) -> Dict[str, Any]:
+        """
+        Validate that the provided content is suitable for synergistic hypothesis generation.
+        Handles both single content string and list of content strings.
+        
+        Args:
+            content: Content to validate (str or List[str])
+            
+        Returns:
+            Dict with validation results
+        """
+        # Handle both single content and list of contents
+        if isinstance(content, str):
+            return HypothesesService.validate_multiple_system_charts_content([content])
+        elif isinstance(content, list):
+            return HypothesesService.validate_multiple_system_charts_content(content)
+        else:
+            return {
+                "valid": False,
+                "message": "Invalid content type. Expected string or list of strings.",
+                "chart_count": 0
+            } 

pdf_text_processing.py

@@ -0,0 +1,229 @@
+import os
+import PyPDF2
+import re
+from langchain_core.documents import Document
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+
+
+def detect_file_type(file_path: str) -> str:
+    """
+    Detect file type based on extension.
+    Returns 'pdf' or 'text'.
+    """
+    _, ext = os.path.splitext(file_path.lower())
+    if ext == '.pdf':
+        return 'pdf'
+    elif ext in ['.txt', '.text']:
+        return 'text'
+    else:
+        # Default to text for unknown extensions
+        return 'text'
+
+
+def get_text_title(file_path: str) -> str:
+    """
+    Extract title from text file that follows the format "Title: XXXXX Abstract: XXXXX"
+    """
+    try:
+        with open(file_path, "r", encoding="utf-8") as f:
+            content = f.read()
+        
+        # Look for "Title:" pattern (case insensitive)
+        title_match = re.search(r'title:\s*([^\n\r]+?)(?:\s+abstract:|$)', content, re.IGNORECASE)
+        if title_match:
+            title = title_match.group(1).strip()
+            # Clean up the title
+            title = re.sub(r'[^\w\s-]', '', title)[:100]
+            if title:
+                return title
+        
+        # If no "Title:" found, try to get the first non-empty line
+        lines = content.strip().split('\n')
+        for line in lines:
+            line = line.strip()
+            if line and not line.lower().startswith('abstract:'):
+                # Remove common trailing punctuation from titles
+                title = line.rstrip('.,;:!?')
+                return title[:100]
+                
+    except Exception as e:
+        print(f"❌ Error reading text file {file_path}: {e}")
+    
+    return os.path.basename(file_path)
+
+
+def get_pdf_title(file_path: str) -> str:
+    """
+    Existing code that attempts to get a PDF title.
+    """
+    try:
+        with open(file_path, "rb") as f:
+            reader = PyPDF2.PdfReader(f)
+            meta = reader.metadata
+            if meta and meta.title:
+                title = re.sub(r'[^\w\s-]', '', meta.title.strip())[:100]
+                return title
+            first_page = reader.pages[0].extract_text()
+            if first_page:
+                return first_page.strip().split('\n')[0]
+    except:
+        pass
+    return os.path.basename(file_path)
+
+
+def get_file_title(file_path: str) -> str:
+    """
+    Get title from either PDF or text file based on file type.
+    """
+    file_type = detect_file_type(file_path)
+    if file_type == 'pdf':
+        return get_pdf_title(file_path)
+    else:
+        return get_text_title(file_path)
+
+
+def remove_duplicate_chunks(docs):
+    """
+    Your existing deduplication logic.
+    """
+    seen = set()
+    unique_docs = []
+    for doc in docs:
+        key = (doc.page_content.strip(), doc.metadata.get("title", ""))
+        if key not in seen:
+            seen.add(key)
+            unique_docs.append(doc)
+    return unique_docs
+
+def extract_text_from_file(file_path: str) -> list:
+    """
+    Extract text from either PDF or text file.
+    Returns list of Document objects (typically length=1).
+    """
+    docs = []
+    try:
+        file_type = detect_file_type(file_path)
+        title = get_file_title(file_path)
+        
+        if file_type == 'pdf':
+            # Handle PDF files
+            with open(file_path, "rb") as f:
+                reader = PyPDF2.PdfReader(f)
+                text = "\n".join([p.extract_text() or "" for p in reader.pages])
+        else:
+            # Handle text files
+            with open(file_path, "r", encoding="utf-8") as f:
+                text = f.read()
+        
+        if text.strip():
+            docs = [Document(page_content=text, metadata={"source": os.path.basename(file_path), "title": title, "file_type": file_type})]
+        
+    except Exception as e:
+        print(f"❌ Error processing {file_path}: {e}")
+    
+    return docs
+
+
+def extract_text_from_pdf(file_path: str) -> list:
+    """
+    Legacy function name for backward compatibility.
+    Now handles both PDF and text files.
+    """
+    return extract_text_from_file(file_path)
+
+
+def extract_and_split(file_path: str):
+    """
+    Extract text from file (PDF or text) and split into chunks.
+    """
+    docs = []
+    try:
+        file_type = detect_file_type(file_path)
+        title = get_file_title(file_path)
+        
+        if file_type == 'pdf':
+            # Handle PDF files
+            with open(file_path, "rb") as f:
+                reader = PyPDF2.PdfReader(f)
+                text = "\n".join([p.extract_text() or "" for p in reader.pages])
+        else:
+            # Handle text files
+            with open(file_path, "r", encoding="utf-8") as f:
+                text = f.read()
+        
+        if not text.strip():
+            return []
+            
+        splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=50)
+        docs = splitter.create_documents([text], metadatas=[{"source": os.path.basename(file_path), "title": title, "file_type": file_type}])
+        docs = remove_duplicate_chunks(docs)
+        print(f"📄 {title}: {len(docs)} unique chunks after deduplication")
+        
+    except Exception as e:
+        print(f"❌ Error processing {file_path}: {e}")
+    
+    return docs
+
+def entity_match(dictionary: dict, target_phrase: str, window_size: int = 70) -> dict:
+    """
+    Given one entity (target_phrase), return a dict:
+      {paper_name: [list_of_snippets]}
+    Each snippet is a window of ±`window_size` words around the matched term.
+    """
+    output = {}
+    phrase_tokens = target_phrase.lower().split()
+    phrase_len = len(phrase_tokens)
+
+    for paper_name, text in dictionary.items():
+        words = text.split()
+        n = len(words)
+
+        intervals = []
+        
+        i = 0
+        while i < n:
+            # Enough room for entire phrase
+            if i + phrase_len <= n:
+                slice_candidate = words[i : i + phrase_len]
+                if [w.lower() for w in slice_candidate] == phrase_tokens:
+                    start_idx = max(0, i - window_size)
+                    end_idx   = min(n, i + phrase_len + window_size)
+                    intervals.append((start_idx, end_idx))
+                    # Advance i by 1 to allow overlapping matches
+                    i += 1
+                    continue
+            i += 1
+
+        # If no intervals found, just store empty list and move on
+        if not intervals:
+            output[paper_name] = []
+            continue
+
+        # Sort intervals by their start index
+        intervals.sort(key=lambda x: x[0])
+
+        # Merge overlapping intervals
+        merged_intervals = []
+        current_start, current_end = intervals[0]
+
+        for idx in range(1, len(intervals)):
+            start, end = intervals[idx]
+            if start <= current_end:  # overlap
+                current_end = max(current_end, end)
+            else:
+                merged_intervals.append((current_start, current_end))
+                current_start, current_end = start, end
+
+        # Add the final interval
+        merged_intervals.append((current_start, current_end))
+
+        # Convert merged intervals to text snippets
+        snippets = []
+        for (start_i, end_i) in merged_intervals:
+            snippet_words = words[start_i:end_i]
+            snippet_text = " ".join(snippet_words)
+            snippets.append(snippet_text)
+        
+        output[paper_name] = snippets
+
+    return output

prompts.py

@@ -0,0 +1,230 @@
+# ---- For system chart extraction ----
+
+PROMPTS_CLASSIFICATION = '''In the field of materials science research, scholarly papers typically align with one of three research paradigms:
+
+Experimental/Practical Research: These papers are grounded in experimental methodologies, detailing the use of specific material processing techniques to achieve desired material structures. The focus is on empirically demonstrating how these processing methods influence the material's mechanical, thermal, chemical, electrical, or other relevant properties. While the paper may include brief computational analysis methods, these are not the main subject or the primary emphasis. Papers in this category typically feature experimental data, analyses of structural characteristics, and discussions on how processing conditions affect the material's performance.
+
+Theoretical/Computational Research: These papers focus solely on computational models or simulations, employing methods like density functional theory (DFT), molecular dynamics (MD), finite element analysis, thermodynamic calculations, or other modeling approaches to predict and analyze the behavior and performance of materials at various scales (atomic, molecular, or macroscopic level). The focus is on theoretical predictions and simulations, excluding any actual processing of materials. Such papers discuss the theoretical framework, present simulation results, and explore the implications of these findings for potential material processing strategies.
+
+Mixed Research: These papers combine experimental and computational approaches, employing computational techniques to conceptualize potential materials or processes, followed by experimental procedures to synthesize these materials and evaluate their characteristics. The content includes substantial theoretical analysis and extensive experimental research, with both parts occupying a significant portion of the text.
+
+Please evaluate the attached paper and determine which category it best fits. Provide a succinct classification and reasoning for your decision, considering aspects like the research methodology, primary focus (experimental vs. theoretical), data analysis, and the nature of the conclusions drawn.
+
+Strictly respond in the following format:
+Category: [Experimental/Practical, Theoretical/Computational, or Mixed]
+Reasons: [Offer a concise explanation, referencing specific elements such as the research methods (experimental or computational), the primary focus of the paper, the type of data presented (experimental results or simulation data), and how the conclusions relate to practical applications or theoretical advancements in the field of materials science.]'''
+
+
+PROMPTS_EXPERIMENTAL = ['''
+Identify the material's significant properties discussed in the text. Include only those properties explicitly mentioned in the text, avoiding any not directly addressed. Prioritize information provided in the Abstract and Conclusion sections. Ensure each property description is both concise and precise. Please include the environmental condition for each property, provided that the property is valid under that specific environmental condition. Use semicolons to separate your answers. Please make sure your answer is no more than 15 words in total.''', # Get the target property. 
+                          
+'''
+Extract key structural characteristics relevant to the material's properties or behavior mechanisms as concise, noun-based phrases; avoid full sentences. Separate responses with semicolons. Please make sure your answer is no more than 20 words in total.''', # Get the structure of the material.
+
+'''
+Perorm a detailed analysis to identify the composition and primary functional components of this material. Ensure they are distinguished from the structure discussed previously; in other words, they should not overlap with answers from earlier structure questions. Provide only the results, without any additional information or explanation. Respond in this format: Composition: YOUR ANSWER; Key Components: YOUR ANSWER. If no such component is identified, state 'N/A'.''', # Get the composition and key components (if any) of the material.
+
+'''
+1. Construct a table linking structure, associated mechanisms, and resultant material properties. Consider only the structures and properties outlined in the prerequisites, without altering them when completing the table. The mechanisms described in the paper should always take precedence. Present your findings in a table format with three columns: Structure, Mechanism (Structure & Property), and Property. Include every listed structure and describe the mechanism in detail to prevent ambiguity. 
+2. Please include the effects of environmental conditions on structural characteristics within the mechanisms, if the corresponding property is influenced by environmental conditions.
+3. To ensure clarity and accuracy in indicating the source of each mechanism (except when N/A), adhere to the following guidelines in your responses: For Mechanisms Directly from the Original Text: If the explanation or mechanism you provide is directly taken or closely paraphrased from the original text, indicate this using the format: "XXX (From text)". For Mechanisms Derived from Your Own Knowledge Base: If the explanation or mechanism is not found in the original text and instead comes from your understanding or external knowledge, use the format: "XXX (From knowledge base)". Please strictly follow this format for each mechanism you explain.
+''', # Get the mechanism between the target property and the structure.
+
+'''List the key processing or manufacturing techniques explicitly stated in the text provided for this material; do not base your answers on typical experience. Avoid mentioning details about micro/macro structures. Use semicolons to separate your answers. Additional Requirements: 1. Exclude any processing methods that are hypothetical or proposed as future improvements. Only include those explicitly employed in the current context of the material's manufacturing or processing. 2. Focus solely on manufacturing/synthesis methods, excluding analytical methods like testing and characterization techniques.
+''', # Get the processing method of the material.
+
+'''From the list of processing methods of this material discussed previously, identify the method(s) most central to the paper's discussion. For instance, if process A is highlighted frequently, consider it the primary method. Exclude routine precursor preparation steps unless the paper emphasizes them as critical. Additionally, provide a brief explanation as to why this method, for example, process A, is emphasized in the paper over others, based on the frequency of its mention and its relevance to the key findings or conclusions. This will help to understand the context and significance of the chosen method(s) in the overall study. Ensure to exclude any processing methods not explicitly mentioned in the text.
+RETURN ME STRICTLY WITH THIS FORMAT: The final processing methods are: YOUR ANSWER; The explanations are: YOUR EXPLANATIONS.''', # Refine the processing method of the material.
+
+'''
+1. Construct a table that maps structures to their influential processing methods and the mechanisms responsible for their formation. Frame the relationship as follows: "The [processing method] yields [structure] via the [mechanism]." Ensure a logical and coherent connection among these three elements. The table should consist of three columns titled 'Processing', 'Mechanism (Processing & Structure)', and 'Structure'. Incorporate all relevant structures, omitting those not pertinent to the discussion. Consider only the processing methods specified in the 'processing list'. Each structure must be matched with pertinent processing methods and their mechanisms, emphasizing a clear and logical link among them. The table's content should be concise, focusing exclusively on the interplay between processing methods, mechanisms, and structures.
+2. For clarity and precision in citing the origin of each mechanism (except when not applicable), adhere to the guidelines below in your responses: For mechanisms directly quoted or closely paraphrased from the original text, use the format: "XXX (From text)". For mechanisms derived from your own knowledge or external sources, use the format: "XXX (From knowledge base)". Strictly follow this format for every mechanism explanation provided.''', # Get the mechanism between the structure and the processing.
+
+"Review the provided list that details the composition of the material, including any special component additions. Analyze and identify any components in the list that significantly influence the material's structure. If any such components are identified, add them to the 'Processing' column of the final table's corresponding row. Please do not overwrite the content in the 'Processing' column of the original table unless it is marked as N/A, and refrain from creating a new column for composition/key components.", # Try to add composition and key components into the Processing column.
+
+'''Merge the two extracted tables into a single comprehensive table, with the 'Structure' column serving as the primary criterion for merging. Here are the specific instructions:
+1. Merging Based on Structure: Combine the tables into one, aligning entries based on similarities or matches in the 'Structure' column.
+2. Handling Non-Matching Structures: For entries in the 'Structure' column that do not match, analyze their relevance and importance based on the context provided by the source papers.
+3. Decision on Exclusion: After analysis, decide whether the non-matching structure entries are essential to the merged table's overall content and purpose. If not, consider removing these entries to maintain coherence and relevance.
+4. Columns of the final table: Include Processing, Mechanism (Processing & Structure), Structure, Mechanism (Structure & Property), and Property.
+5. Ensure the final table is complete, with no gaps. Adjust content as needed for accuracy and consistency with the original texts.
+6. Mark all uncertain positions as 'N/A'. Replace all blank positions with 'N/A'.
+7. Prune the table: Examine each row; remove rows with three or more 'N/A' entries. Keep rows with fewer than three 'N/A' entries. If significant information is lost due to pruning, incorporate essential details into other rows, ensuring data coherence and rationale are preserved.
+8. After completing the above steps, review the document to assess the content's rationality and identify any extraneous content for removal.
+9. Only the final, corrected table is required; details about the process are not needed. 
+The goal is to produce a unified table that accurately reflects the combined data, focusing on maintaining relevance and coherence, especially regarding the 'Structure' column.''' # Merge the two tables.
+]
+
+
+PROMPTS_COMPUTATIONAL = [
+'''Identify the material's significant properties discussed in the text. Include only those properties explicitly mentioned in the text, avoiding any not directly addressed. Prioritize information provided in the Abstract and Conclusion sections. Ensure each property description is both concise and precise. Please include the environmental condition for each property, provided that the property is valid under that specific environmental condition. Use semicolons to separate your answers. Please make sure your answer is no more than 15 words in total.''', # Get the target property. 
+                          
+"Extract key structural characteristics relevant to the material's properties or behavior mechanisms as concise, noun-based phrases; avoid full sentences. Separate responses with semicolons. Please make sure your answer is no more than 20 words in total.", # Get the structure of the material.
+
+"Perorm a detailed analysis to identify the composition and primary functional components of this material. Ensure they are distinguished from the structure discussed previously; in other words, they should not overlap with answers from earlier structure questions. Provide only the results, without any additional information or explanation. Respond in this format: Composition: YOUR ANSWER; Key Components: YOUR ANSWER. If no such component is identified, state 'N/A'.", # Get the composition and key components (if any) of the material.
+
+'''
+1. Construct a table linking structure, associated mechanisms, and resultant material properties. Consider only the structures and properties outlined in the prerequisites, without altering them when completing the table. The mechanisms described in the paper should always take precedence. Present your findings in a table format with three columns: Structure, Mechanism (Structure & Property), and Property. Include every listed structure and describe the mechanism in detail to prevent ambiguity. 
+2. Please include the effects of environmental conditions on structural characteristics within the mechanisms, if the corresponding property is influenced by environmental conditions.
+3. To ensure clarity and accuracy in indicating the source of each mechanism (except when N/A), adhere to the following guidelines in your responses: For Mechanisms Directly from the Original Text: If the explanation or mechanism you provide is directly taken or closely paraphrased from the original text, indicate this using the format: "XXX (From text)". For Mechanisms Derived from Your Own Knowledge Base: If the explanation or mechanism is not found in the original text and instead comes from your understanding or external knowledge, use the format: "XXX (From knowledge base)". Please strictly follow this format for each mechanism you explain.
+''', # Get the mechanism between the target property and the structure.
+
+'''
+1. Construct a table that maps structures to their critical compositions/key components and the mechanisms responsible for their formation. This table should consist of three columns: 'Composition/key components', 'Mechanism (Composition/key components & Structure)', and 'Structure'. Incorporate all relevant structures provided and omit those that do not apply to the context. For each structure, if a direct causation from composition or key components is evident, list these in the 'Composition/key components' column; otherwise, enter 'N/A'. Ensure a clear logical relationship among these three categories. The table's details should be concise, focusing specifically on the interplay between the composition/key components, mechanisms, and structures.
+2. To maintain clarity and precision when referencing the origins of each mechanism (except when N/A), adhere to the following instructions in your responses: For mechanisms directly extracted or closely paraphrased from the original text, use the notation: "XXX (From text)". For mechanisms that originate from your personal knowledge or external sources, apply the notation: "XXX (From knowledge base)". Please strictly follow this notation for each mechanism described.
+''', # Get the mechanism between the structure and the composition/key components.
+
+'''Merge the two extracted tables into a single comprehensive table, focusing on the 'Structure' column as the primary criterion for merging. Follow these specific instructions:
+
+1. Merging Based on Structure: Combine the two tables into one, aligning entries by matching or similar 'Structure' column values.
+2. Handling Non-Matching Structures: For 'Structure' column entries that do not match between the two tables, analyze their relevance and importance based on the context of the source papers.
+3. Decision on Exclusion: Decide whether to include non-matching structure entries in the merged table based on their significance to its overall content and purpose. Exclude entries that are not crucial to maintain coherence and relevance.
+4. Columns of the Final Table: Include the following columns - Composition/Key Components, Mechanism (Composition/Key Components & Structure), Structure, Mechanism (Structure & Property), and Property.
+5. Ensure the table is complete, with no missing information. Modify content as necessary for accuracy and consistency with the original sources.
+6. Represent all uncertain data as 'N/A'. Replace any blank fields with 'N/A'.
+7. Perform Pruning: Review each row for instances of 'N/A'. Remove rows with three or more 'N/A' entries, but strive to incorporate essential details from such rows into others, preserving data coherence and rationale.
+8. After completing these steps, reassess the document to identify and eliminate any irrelevant content, ensuring the table's rationality and relevance.
+9. Only the final, corrected table is required without the need for detailed explanations.
+The goal is to produce a unified table that accurately represents the merged data, with an emphasis on relevance and coherence, particularly in relation to the 'Structure' column.
+
+Final modifications: If 'Processing' still shows in the table, replace it to 'Composition/key components'. The content within the column should remain unchanged. If 'Mechanism (Processing & Structure)' still shows in the table, replace it to 'Mechanism (Composition/key components & Structure)'. The content within the column should remain unchanged.''' # Final modifications and merge the two tables.
+]
+
+PROMPTS_MIXED = ['''As discussed earlier, this paper is comprehensive, combining both experimental techniques and computational models. Please divide the paper into three sections: A) Computational model and theoretical analysis of the material, B) Material synthesis and manufacturing, and C) Material's performance testing. Keep in mind not to submit the results of your division.
+Identify the material's significant properties discussed in the text. Include only those properties explicitly mentioned in the text, avoiding any not directly addressed. Prioritize information provided in the Abstract and Conclusion sections. Ensure each property description is both concise and precise. Please include the environmental condition for each property, provided that the property is valid under that specific environmental condition. Use semicolons to separate your answers. Please make sure your answer is no more than 15 words in total.''', # Get the target property. 
+                          
+"Extract key structural characteristics relevant to the material's properties or behavior mechanisms as concise, noun-based phrases; avoid full sentences. Separate responses with semicolons. Please make sure your answer is no more than 20 words in total.", # Get the structure of the material.
+
+"Perorm a detailed analysis to identify the composition and primary functional components of this material. Ensure they are distinguished from the structure discussed previously; in other words, they should not overlap with answers from earlier structure questions. Provide only the results, without any additional information or explanation. Respond in this format: Composition: YOUR ANSWER; Key Components: YOUR ANSWER. If no such component is identified, state 'N/A'.", # Get the composition and key components (if any) of the material.
+
+'''
+1. Construct a table linking structure, associated mechanisms, and resultant material properties. Consider only the structures and properties outlined in the prerequisites, without altering them when completing the table. The mechanisms described in the paper should always take precedence. Present your findings in a table format with three columns: Structure, Mechanism (Structure & Property), and Property. Include every listed structure and describe the mechanism in detail to prevent ambiguity. 
+2. Please include the effects of environmental conditions on structural characteristics within the mechanisms, if the corresponding property is influenced by environmental conditions.
+3. To ensure clarity and accuracy in indicating the source of each mechanism (except when N/A), adhere to the following guidelines in your responses: For Mechanisms Directly from the Original Text: If the explanation or mechanism you provide is directly taken or closely paraphrased from the original text, indicate this using the format: "XXX (From text)". For Mechanisms Derived from Your Own Knowledge Base: If the explanation or mechanism is not found in the original text and instead comes from your understanding or external knowledge, use the format: "XXX (From knowledge base)". Please strictly follow this format for each mechanism you explain.
+''', # Get the mechanism between the target property and the structure.
+
+'''Only focus on section B, list the key processing or manufacturing techniques explicitly stated in the text provided for this material; do not base your answers on typical experience. Avoid mentioning details about micro/macro structures. Use semicolons to separate your answers. Additional Requirements: 1. Exclude any processing methods that are hypothetical or proposed as future improvements. Only include those explicitly employed in the current context of the material's manufacturing or processing. 2. Focus solely on manufacturing/synthesis methods, excluding analytical methods like testing and characterization techniques.
+''', # Get the processing method of the material.
+
+'''From the list of processing methods of this material discussed previously, identify the method(s) most central to the paper's discussion. For instance, if process A is highlighted frequently, consider it the primary method. Exclude routine precursor preparation steps unless the paper emphasizes them as critical. Additionally, provide a brief explanation as to why this method, for example, process A, is emphasized in the paper over others, based on the frequency of its mention and its relevance to the key findings or conclusions. This will help to understand the context and significance of the chosen method(s) in the overall study. Ensure to exclude any processing methods not explicitly mentioned in the text.
+RETURN ME STRICTLY WITH THIS FORMAT: The final processing methods are: YOUR ANSWER; The explanations are: YOUR EXPLANATIONS.''', # Refine the processing method of the material.
+
+'''
+1. Construct a table that maps structures to their influential processing methods and the mechanisms responsible for their formation. Frame the relationship as follows: "The [processing method] yields [structure] via the [mechanism]." Ensure a logical and coherent connection among these three elements. The table should consist of three columns titled 'Processing', 'Mechanism (Processing & Structure)', and 'Structure'. Incorporate all relevant structures, omitting those not pertinent to the discussion. Consider only the processing methods specified in the 'processing list'. Each structure must be matched with pertinent processing methods and their mechanisms, emphasizing a clear and logical link among them. The table's content should be concise, focusing exclusively on the interplay between processing methods, mechanisms, and structures.
+2. For clarity and precision in citing the origin of each mechanism (except when not applicable), adhere to the guidelines below in your responses: For mechanisms directly quoted or closely paraphrased from the original text, use the format: "XXX (From text)". For mechanisms derived from your own knowledge or external sources, use the format: "XXX (From knowledge base)". Strictly follow this format for every mechanism explanation provided.''', # Get the mechanism between the structure and the processing.
+
+"Review the provided list that details the composition of the material, including any special component additions. Analyze and identify any components in the list that significantly influence the material's structure. If any such components are identified, add them to the 'Processing' column of the final table's corresponding row. Please do not overwrite the content in the 'Processing' column of the original table unless it is marked as N/A, and refrain from creating a new column for composition/key components.", # Try to add composition and key components into the Processing column.
+
+'''Merge the two extracted tables into a single comprehensive table, with the 'Structure' column serving as the primary criterion for merging. Here are the specific instructions:
+1. Merging Based on Structure: Combine the tables into one, aligning entries based on similarities or matches in the 'Structure' column.
+2. Handling Non-Matching Structures: For entries in the 'Structure' column that do not match, analyze their relevance and importance based on the context provided by the source papers.
+3. Decision on Exclusion: After analysis, decide whether the non-matching structure entries are essential to the merged table's overall content and purpose. If not, consider removing these entries to maintain coherence and relevance.
+4. Columns of the final table: Include Processing, Mechanism (Processing & Structure), Structure, Mechanism (Structure & Property), and Property.
+5. Ensure the final table is complete, with no gaps. Adjust content as needed for accuracy and consistency with the original texts.
+6. Mark all uncertain positions as 'N/A'. Replace all blank positions with 'N/A'.
+7. Prune the table: Examine each row; remove rows with three or more 'N/A' entries. Keep rows with fewer than three 'N/A' entries. If significant information is lost due to pruning, incorporate essential details into other rows, ensuring data coherence and rationale are preserved.
+8. After completing the above steps, review the document to assess the content's rationality and identify any extraneous content for removal.
+9. Only the final, corrected table is required; details about the process are not needed. 
+The goal is to produce a unified table that accurately reflects the combined data, focusing on maintaining relevance and coherence, especially regarding the 'Structure' column.''' # Merge the two tables.
+                  ] 
+
+# ---- For system chart normalization ----
+
+PROMPTS_SYSTEM_CHART_NORMALIZATION = [
+    '''Several structures have been extracted from a table. Categorize only those structures into groups based on their mechanistic relationships in materials science. Your categorization should be based on the table. Only provide the results and do not include any explanations.
+    structures: ''',
+
+    '''
+    Task: In the previous step, all structures were categorized. Based on these results, for each structure in the chart, add a tag to represent its category.
+    Requirements:
+    1. Do not change any part of the table except for adding the tag.
+    2. The tag should be added in parentheses after the structure name.
+    3. Your result should be a complete tagged chart.
+    Example for tag addition (For reference only):
+    Before:
+    | Processing | Mechanism (Processing & structure) | structure | Mechanism (structure & Property) | Property |
+    |------------|-----------------------------------------|----------------|--------------------------------------|----------|
+    | Processing #1 | Mechanism linking Processing #1 and structure #1 (From text)| structure #1 | Mechanism linking structure #1 and Property #1 | Property #1 |
+    After:
+    | Processing | Mechanism (Processing & structure) | structure | Mechanism (structure & Property) | Property |
+    |------------|-----------------------------------------|----------------|--------------------------------------|----------|
+    | Processing #1 | Mechanism linking Processing #1 and structure #1 (From text)| structure #1 (Category #1) | Mechanism linking structure #1 and Property #1 | Property #1 |
+    Replace 'Category #1' with the actual full category name.
+    Table to modify:
+    ''',
+
+    '''Please carefully review each row of the table.
+    Task:
+    1. If any row contains specific parameters (e.g., "Yield strength of XXX MPa"), simplify them (e.g., "Yield strength").
+    2. Do not remove any compositions or key elements.
+    3. Do not change the mechanisms.
+    4. Do not remove any environmental conditions (e.g., temperature) under 'Property'.
+    5. Unify the expression for similar properties (e.g., 'High yield strength' and 'Exceptional strength').
+    6. Ensure every property starts with a capital letter.
+    Table to modify:
+   ''',
+
+    '''Note that the mechanisms listed under "Mechanism (Processing & structure)" might have mixed mechanisms for processing methods and composition/key elements.
+    Task:
+    1. Separate the mechanisms for processing methods and for composition/key elements into individual rows. However, if they belong to the same category based on the provided paper, do not separate them.
+    2. Determine whether each mechanism is a processing method or a key element/composition using the provided paper.
+    3. Allocate the mechanisms without significant changes to their content.
+    4. Keep the rest of the table unchanged.
+    5. After splitting, re-evaluate all mechanisms under "Mechanism (Processing & structure)" to tag them as either "(From text)" or "(From knowledge base)" based on their origin.
+    Example for splitting (for reference only):
+    Before splitting:
+    | Processing | Mechanism (Processing & structure) | structure | Mechanism (structure & Property) | Property |
+    |------------|-----------------------------------------|----------------|--------------------------------------|----------|
+    | Processing #1; Key element addition #1 | Mechanism linking Processing #1 and structure #1; Mechanism linking key element addition #1 and structure #1 (From text)| structure #1 | Mechanism linking structure #1 and Property #1 | Property #1 |
+    After splitting:
+    | Processing | Mechanism (Processing & structure) | structure | Mechanism (structure & Property) | Property |
+    |------------|-----------------------------------------|----------------|--------------------------------------|----------|
+    | Processing #1 | Mechanism linking Processing #1 and structure #1 (From knowledge base)| structure #1 | Mechanism linking structure #1 and Property #1 | Property #1 |
+    | Key element addition #1 | Mechanism linking key element addition #1 and structure #1 (From text)| structure #1 | Mechanism linking structure #1 and Property #1 | Property #1 |
+    Table to modify:''',
+
+    '''Currently, a single row may contain multiple properties.
+    Task:
+    1. Separate each property into a different row while keeping the columns 'Processing', 'Mechanism (Processing & structure)', and 'structure' unchanged.
+    2. For each property, review the mechanisms in "Mechanism (structure & Property)". Determine which mechanisms are relevant to that property based on the content (ignoring degree adverbs) and include all qualified mechanisms (including those related to environmental conditions).
+    Example (for reference only):
+    Before:
+    | Processing | Mechanism (Processing & structure) | structure | Mechanism (structure & Property) | Property |
+    |------------|-----------------------------------------|----------------|--------------------------------------|----------|
+    | Processing #1| Mechanism linking Processing #1 and structure #1 (From text)| structure #1 | Mechanism #1 linking structure #1 and Property #1; Mechanism #2 linking structure #1 and Property #2 | Property #1; Property #2 |
+    After:
+    | Processing | Mechanism (Processing & structure) | structure | Mechanism (structure & Property) | Property |
+    |------------|-----------------------------------------|----------------|--------------------------------------|----------|
+    | Processing #1| Mechanism linking Processing #1 and structure #1 (From text)| structure #1 | Mechanism #1 linking structure #1 and Property #1 | Property #1 |
+    | Processing #1| Mechanism linking Processing #1 and structure #1 (From text)| structure #1 | Mechanism #2 linking structure #1 and Property #2 | Property #2 |
+    Table to modify:''',
+
+    '''Task:
+    Step 1. Check if any row has 'N/A' in the 'Structure' column.
+    Step 2. For rows with 'N/A', compare the 'Processing' and 'Mechanism (Processing & Structure)' values with other rows. If a match is found, replace 'N/A' with the corresponding value.
+    Step 3. Repeat for all 'N/A' in 'Structure'.
+    Step 4. Next, check for 'N/A' in the 'Property' column.
+    Step 5. For rows with 'N/A' in 'Property', compare the 'Structure' and 'Mechanism (Structure & Property)' values with other rows. If a match is found, replace 'N/A' accordingly.
+    Step 6. Provide the updated table. Please return only the content of the chart, removing all modifiers and comments (for example, omit phrases like 'the above table').
+    Table to modify:'''
+]
+
+# ---- For hypothesis generation ----
+
+def create_hypotheses_generation_prompt(material_goal_1, material_goal_2):
+    """Create the hypotheses generation prompt with user-specified material design goals."""
+    return f'''Background Information: The table you're referring to has been compiled from various system charts found in multiple research papers. Whenever the same paper number appears more than once in the table, it indicates that the entries originate from the same research study. In many materials science challenges, achieving {material_goal_1} requires innovative design approaches. The challenge is to design materials that can maintain {material_goal_2} across multiple demanding criteria simultaneously. This necessitates the synergistic combination of special design principles for superior material properties with relevant deformation mechanisms to find innovative design solutions.
+
+    Task Objective: The objective of this research is to design materials that exhibit {material_goal_1} while maintaining {material_goal_2}. Please strictly adhere to this theme.
+
+    Step 1: Begin by thoroughly reviewing each entry in the table. Then, identify some mechanisms that contribute to {material_goal_1}.
+
+    Step 2: Review each entry again and select some mechanisms targeting {material_goal_2}, which have synergistic effects with the ideas you selected in Step 1. Ideas selected in Step 1 and Step 2 should come from different papers. Synergistic effects occur when scientific discoveries from different papers can be combined to create a greater positive impact than each would produce on its own. These synergistic effects are not just additive but multiplicative.
+
+    Detailing Synergistic Hypotheses: For each hypothesis you formulate, provide a detailed explanation of the structural features or mechanisms that might interact synergistically to achieve both {material_goal_1} and {material_goal_2}. This explanation should go beyond simply summarizing the findings from the papers, suggesting innovative interactions not directly discussed in the original studies.
+
+    Citations: When citing specific data or findings from the table, append the source at the end of the sentence using the format '[number]', where '[number]' denotes the corresponding paper number listed in the table.
+
+    Ensure you provide me with every potential hypothesis hidden in the table, and each hypothesis should thoroughly address the task in over 400 words. Ensure that your review and hypotheses consider the full breadth of research captured in the table, without bias towards any specific range of paper numbers. Each hypothesis should be well-developed and address the task comprehensively.
+
+    Answer me in this form: Hypothesis #1: Idea selected from Step 1: XXX; Idea selected from Step 2: XXX; Synergistic mechanism: XXX.'''

requirements.txt

@@ -0,0 +1,18 @@
+fastapi
+uvicorn[standard]
+python-multipart
+aiofiles
+requests
+rapidfuzz
+chromadb
+langchain-core
+langchain-openai
+langchain-google-genai
+langchain-community
+langchain-text-splitters
+PyPDF2
+pandas
+networkx
+matplotlib
+Pillow
+regex

system_chart_extraction.py

@@ -0,0 +1,1322 @@
+import asyncio, re, textwrap, math, copy, os, uuid
+from datetime import datetime
+import pandas as pd
+import networkx as nx
+import matplotlib.pyplot as plt
+import matplotlib.patches as mpatches
+from PIL import Image
+from langchain_openai import ChatOpenAI
+from langchain_core.messages import HumanMessage, AIMessage
+from io import StringIO
+from collections import defaultdict
+
+from prompts import PROMPTS_CLASSIFICATION, PROMPTS_EXPERIMENTAL, PROMPTS_COMPUTATIONAL, PROMPTS_MIXED, PROMPTS_SYSTEM_CHART_NORMALIZATION
+
+# -----helper functions and variables-----
+
+_CATEGORY_COLORS = [
+    '#E1BEE7',  # Light Purple
+    '#F8BBD9',  # Light Pink
+    '#FFCDD2',  # Light Red
+    '#FFF9C4',  # Light Yellow
+    '#B2DFDB',  # Light Teal
+    '#D7CCC8',  # Light Brown
+    '#CFD8DC',  # Light Blue Grey
+    '#FFCCBC',  # Light Orange
+    '#DCEDC8',  # Light Green
+    '#B2EBF2'   # Light Cyan
+]
+
+def _parse_microstructures(starter_table: str) -> str:
+    """
+    Extract the 'Structure' column (or 3rd real column if no headers) from a Markdown-like table.
+    Handles both:
+      1) No outer pipes, no explicit header separator
+      2) With outer pipes and a '---|---|...' header separator
+    Returns a '; '-joined string of non-empty cells from that column.
+    """
+    # 1) Normalize lines: strip, drop blanks
+    lines = [l.strip() for l in str(starter_table).strip().splitlines() if l.strip()]
+    if not lines:
+        raise ValueError("Empty table text.")
+
+    # 2) Remove obvious Markdown header separator lines (e.g., | --- | :---: | --- |)
+    def is_md_separator(line: str) -> bool:
+        # After removing pipes, colons, dashes and spaces, nothing should remain
+        return re.sub(r'[\|\-\:\s]', '', line) == '' and ('-' in line or ':' in line) and '|' in line
+
+    lines = [ln for ln in lines if not is_md_separator(ln)]
+
+    # 3) Parse with pandas using '|' as delimiter
+    text = '\n'.join(lines)
+    df = pd.read_csv(StringIO(text), sep='|', engine='python', header=None, dtype=str, keep_default_na=False)
+
+    # 4) Trim whitespace in all cells
+    df = df.applymap(lambda x: x.strip() if isinstance(x, str) else x)
+
+    # 5) Drop fully-empty boundary columns caused by leading/trailing pipes
+    def col_empty(series):
+        s = series.astype(str).str.strip()
+        return s.eq('').all()
+
+    while df.shape[1] > 0 and col_empty(df.iloc[:, 0]):
+        df = df.iloc[:, 1:]
+    while df.shape[1] > 0 and col_empty(df.iloc[:, -1]):
+        df = df.iloc[:, :-1]
+
+    if df.shape[1] < 1:
+        raise ValueError("No columns detected after normalization.")
+
+    # 6) Detect header row (case-insensitive). If first row contains 'structure', treat it as header.
+    header_row = df.iloc[0].astype(str).tolist()
+    header_lower = [h.strip().lower() for h in header_row]
+    has_header = any(h == 'structure' for h in header_lower)
+
+    if has_header:
+        columns = [re.sub(r'^\*+(.*?)\*+$', r'\1', c or '').strip() for c in header_row]
+        df = df.iloc[1:].reset_index(drop=True)
+        df.columns = columns
+        # Prefer the 'Structure' column by name; fall back to case-insensitive match
+        if 'Structure' in df.columns:
+            target = df['Structure']
+        else:
+            # case-insensitive match
+            name_map = {c.lower(): c for c in df.columns}
+            target = df[name_map.get('structure')]
+    else:
+        # No header: need at least 3 real columns, use the 3rd
+        if df.shape[1] < 3:
+            raise ValueError("Not enough columns to extract the 3rd column.")
+        target = df.iloc[:, 2]
+
+    # 7) Clean typical markdown bullets and noise
+    def clean_cell(s: str) -> str:
+        s = s.strip()
+        s = re.sub(r'^\-\s+', '', s)          # leading "- "
+        s = re.sub(r'\s*\(from text\)\s*$', '', s, flags=re.IGNORECASE)
+        return s.strip('| ').strip()
+
+    col = target.astype(str).map(clean_cell)
+
+    # 8) Drop empty rows and join
+    col = col[col.str.strip() != '']
+    if col.empty:
+        return ''
+
+    # Deduplicate while keeping order (optional; drop if you want duplicates)
+    seen = set()
+    out = []
+    for v in col:
+        if v not in seen:
+            seen.add(v)
+            out.append(v)
+
+    return '; '.join(out)
+
+def _wrap_label(text, wrap_limit=20):
+    return textwrap.fill(text, width=wrap_limit)
+
+def _parse_composition_and_key_components(response_text: str) -> tuple:
+    """
+    Parse composition and key components from the LLM response.
+    Supports variants like:
+      - "Composition: ...; Key Components: ..."
+      - "Composition: ...; Key Component: ..."
+      - "Composition: ...; Key components addition: ..."
+    Returns (composition, key_components). Falls back to "N/A" if missing.
+    """
+    if not response_text:
+        return "N/A", "N/A"
+
+    text = str(response_text).strip()
+    composition = "N/A"
+    key_components = "N/A"
+
+    comp_label = re.search(r'Composition\s*:\s*', text, flags=re.IGNORECASE)
+    key_label = re.search(r'Key\s+Components?\s*(?:addition)?\s*:\s*', text, flags=re.IGNORECASE)
+
+    if comp_label:
+        comp_start = comp_label.end()
+        comp_end = key_label.start() if key_label and key_label.start() > comp_start else len(text)
+        comp_value = text[comp_start:comp_end].strip().strip(';')
+        composition = comp_value if comp_value else "N/A"
+
+    if key_label:
+        key_value = text[key_label.end():].strip().strip(';')
+        key_components = key_value if key_value else "N/A"
+
+    return composition, key_components
+
+def _llm_content_to_text(content) -> str:
+    """
+    Normalize heterogeneous LLM content payloads into a plain string.
+    Handles str, list (content blocks), dict payloads, and fallback objects.
+    """
+    if content is None:
+        return ""
+
+    if isinstance(content, str):
+        return content
+
+    if isinstance(content, list):
+        parts = []
+        for item in content:
+            if item is None:
+                continue
+            if isinstance(item, str):
+                parts.append(item)
+                continue
+            if isinstance(item, dict):
+                if isinstance(item.get('text'), str):
+                    parts.append(item['text'])
+                    continue
+                text_block = item.get('text')
+                if isinstance(text_block, dict):
+                    value = text_block.get('value') or text_block.get('text')
+                    if value is not None:
+                        parts.append(str(value))
+                        continue
+                if isinstance(item.get('content'), str):
+                    parts.append(item['content'])
+                    continue
+                parts.append(str(item))
+                continue
+
+            text_attr = getattr(item, 'text', None)
+            if isinstance(text_attr, str):
+                parts.append(text_attr)
+            else:
+                parts.append(str(item))
+
+        return '\n'.join([part for part in parts if part]).strip()
+
+    if isinstance(content, dict):
+        if isinstance(content.get('text'), str):
+            return content['text']
+        text_block = content.get('text')
+        if isinstance(text_block, dict):
+            value = text_block.get('value') or text_block.get('text')
+            if value is not None:
+                return str(value)
+        if isinstance(content.get('content'), str):
+            return content['content']
+        return str(content)
+
+    return str(content)
+
+class _NormalizedResponse:
+    """Proxy response object with normalized string `content`."""
+    def __init__(self, response):
+        self._response = response
+        self.content = _llm_content_to_text(getattr(response, 'content', response))
+
+    def __getattr__(self, name):
+        return getattr(self._response, name)
+
+
+class _NormalizedLLM:
+    """LLM wrapper that normalizes `.ainvoke(...).content` to plain text."""
+    def __init__(self, llm):
+        self._llm = llm
+
+    async def ainvoke(self, *args, **kwargs):
+        response = await self._llm.ainvoke(*args, **kwargs)
+        return _NormalizedResponse(response)
+
+    def __getattr__(self, name):
+        return getattr(self._llm, name)
+
+def _transform_table_into_list(total_table):
+    ps_list, sp_list = [], []
+    for i in total_table.splitlines()[2:]:  # Skip header
+        elements = [elem.strip() for elem in i.split('|') if elem]
+        ps_list.append((elements[0], elements[1], elements[2]))
+        sp_list.append((elements[2], elements[3], elements[4]))
+    return ps_list, sp_list
+
+def _initialize_category_colors(G):
+    """Extract categories and initialize colors for structure categories"""
+    structure_categories = {attr['category'] for node, attr in G.nodes(data=True) if attr['group'] == 'structure'}
+    category_colors = {}
+    for i, category in enumerate(structure_categories):
+        category_colors[category] = _CATEGORY_COLORS[i % len(_CATEGORY_COLORS)]
+    return category_colors
+
+def _get_text_width(text, fontsize=16):
+    """Calculate the width of the wrapped text"""
+    fig = plt.figure()
+    renderer = fig.canvas.get_renderer()
+    text_obj = plt.text(0, 0, text, fontsize=fontsize)
+    bbox = text_obj.get_window_extent(renderer=renderer)
+    plt.close(fig)
+    return bbox.width / fig.dpi  # Convert from pixels to inches
+
+def _create_main_diagram(G, pos, fig_width, fig_height, save_path, category_colors):
+    """Create the main network diagram and save to file"""
+    plt.figure(figsize=(fig_width, fig_height))  # Dynamic figsize
+    ax = plt.gca()
+    pos2 = copy.deepcopy(pos)
+
+    dpi = ax.figure.get_dpi()
+
+    # Draw nodes as rectangles
+    key_list = list(pos.keys())
+    for iii, nnn in enumerate(pos.items()):
+        node, (x, y) = nnn
+        group = G.nodes[node]['group']
+        header = G.nodes[node].get('header', '')
+        content = _wrap_label(G.nodes[node].get('content', node), wrap_limit=30)
+        header_width = _get_text_width(header, fontsize=16)
+        content_width = max([_get_text_width(line, fontsize=20) for line in content.split('\n')])
+        width = content_width / 2.8
+        height = 60. / dpi * (content.strip('\n').count('\n') + 1) + 0.6
+
+        # Set color for nodes
+        if group == 'structure':
+            facecolor = category_colors.get(G.nodes[node]['category'], 'white')
+        elif group == 'processing':
+            facecolor = '#FFB74D'  # Orange
+        elif group == 'property':
+            facecolor = '#ADD8E6'  # Blue
+        else:
+            facecolor = 'white'
+
+        rect = mpatches.FancyBboxPatch((x - width / 2, y - height / 2),
+                                       width, height,
+                                       boxstyle="square,pad=0.0",
+                                       edgecolor='black',
+                                       facecolor=facecolor)
+        ax.add_patch(rect)
+        plt.text(x, y + height / 2 + 36. / dpi / 2, header, fontsize=22, ha='center', va='center', fontweight='bold')
+        plt.text(x, y, content, fontsize=22, ha='center', va='center')
+
+        # Adjust node positions to avoid edges converging to the center
+        if group == 'processing':
+            pos[key_list[iii]] = (pos[key_list[iii]][0] + width / 2, pos[key_list[iii]][1])
+            pos2[key_list[iii]] = (pos2[key_list[iii]][0] + width / 2, pos2[key_list[iii]][1])
+        if group == 'property':
+            pos[key_list[iii]] = (pos[key_list[iii]][0] - width / 2, pos[key_list[iii]][1])
+            pos2[key_list[iii]] = (pos2[key_list[iii]][0] - width / 2, pos2[key_list[iii]][1])
+        elif group == 'structure':
+            pos[key_list[iii]] = (pos[key_list[iii]][0] + width / 2, pos[key_list[iii]][1])
+            pos2[key_list[iii]] = (pos2[key_list[iii]][0] - width / 2, pos2[key_list[iii]][1])
+
+    # Draw edges
+    node_list = list(G.nodes())
+    # Separate the edges based on their groups
+    processing_edges = [(u, v) for u, v in G.edges() if G.nodes[u]['group'] == 'processing']
+    property_edges = [(u, v) for u, v in G.edges() if G.nodes[v]['group'] == 'property']
+
+    # Draw processing edges using pos2
+    nx.draw_networkx_edges(G, pos2, edgelist=processing_edges, ax=ax)
+    # Draw property edges using pos
+    nx.draw_networkx_edges(G, pos, edgelist=property_edges, ax=ax)
+
+    # Draw labels for edges
+    for edge in G.edges():
+        labels = G.edges[edge]['mechanisms']
+        unique_labels = list(set(labels))
+        combined_label = ', '.join(unique_labels)
+        x, y = pos[edge[0]] if G.nodes[edge[0]]['group'] == 'processing' else pos[edge[0]]
+        x2, y2 = pos[edge[1]] if G.nodes[edge[1]]['group'] == 'property' else pos2[edge[1]]
+        xytext = ((x + x2) / 2, (y + y2) / 2)
+        dx = (x2 - x) * 0.05
+        dy = (y2 - y) * 0.05
+        rot = math.degrees(math.atan2((y2 - y) * 0.23, (x2 - x)))
+        plt.text(xytext[0] + dx, xytext[1] + dy, combined_label, size=18, ha='center', va='center', rotation=rot,
+                bbox=dict(facecolor='white', alpha=0.7, edgecolor='none', boxstyle='round,pad=0.3'))
+
+    header_y_position = max(pos.values(), key=lambda x: x[1])[1] + 1
+    plt.text(1, header_y_position + 1, 'Processing', fontsize=30, ha='center', va='center', fontweight='bold')
+    plt.text(3, header_y_position + 1, 'Microstructure', fontsize=30, ha='center', va='center', fontweight='bold')
+    plt.text(5, header_y_position + 1, 'Property', fontsize=30, ha='center', va='center', fontweight='bold')
+
+    plt.tight_layout()
+    plt.axis('off')
+    plt.savefig(save_path, format='png', bbox_inches='tight')
+    plt.close()
+    return save_path
+
+def _get_image_width(image_path):
+    """Get the width of an image file"""
+    with Image.open(image_path) as img:
+        return img.width
+
+def _add_na_nodes(G, mechanisms, pos):
+    """Add N/A nodes to the graph if they exist in edges"""
+    na_nodes = set()
+    for edge in G.edges():
+        if edge[0] == "N/A" or edge[1] == "N/A":
+            na_nodes.update(edge)
+
+    for na_node in na_nodes:
+        if na_node == "N/A":
+            continue
+        if na_node not in G.nodes:
+            G.add_node(na_node, group='na', header='N/A', content='N/A')
+            pos[na_node] = (1, -1)
+
+def _create_legend(mechanisms, category_colors, canvas_width_pixels, save_path):
+    """Create a legend for the network diagram"""
+    canvas_width_inches = canvas_width_pixels / 100
+    fig_height = len(mechanisms) * 0.3 + len(category_colors) * 0.3
+    fig, ax = plt.subplots(figsize=(canvas_width_inches, fig_height))
+    legend_patches = [mpatches.Patch(color='none', label=f'{code}: {_wrap_label(mech, wrap_limit=150)}')
+                      for mech, code in sorted(mechanisms.items(), key=lambda item: int(item[1])) if mech != "N/A"]
+    category_patches = [mpatches.Patch(color=color, label=f'{category}')
+                        for category, color in category_colors.items()]
+    ax.legend(handles=legend_patches + category_patches, loc='center', fontsize=22)
+    plt.axis('off')
+    plt.savefig(save_path, bbox_inches='tight')
+    plt.close()
+    return save_path
+
+def _combine_images(image1_path, image2_path, output_path):
+    """Combine two images vertically"""
+    img1 = Image.open(image1_path)
+    img2 = Image.open(image2_path)
+    total_width = max(img1.width, img2.width)
+    total_height = img1.height + img2.height
+    new_img = Image.new('RGB', (total_width, total_height), 'white')
+    new_img.paste(img2, ((total_width - img2.width) // 2, 0))
+    new_img.paste(img1, ((total_width - img1.width) // 2, img2.height))
+    new_img.save(output_path)
+    return output_path
+
+def _calculate_fig_dimensions(node_counts, max_label_length, min_height=10, height_per_node=2.0, width_per_char=0.2):
+    """Calculate figure dimensions based on node counts and label length"""
+    max_nodes = max(node_counts)
+    fig_height = max(min_height, max_nodes * height_per_node)
+    fig_width = max(20, max_label_length * width_per_char)
+    return fig_width, fig_height
+
+def _get_max_label_length(nodes):
+    """Get the maximum label length from a list of nodes"""
+    return max(len(label) for label in nodes)
+
+def _split_microstructure_label(label):
+    """Split microstructure label into name and category"""
+    if '(' in label and ')' in label:
+        parts = label.rsplit('(', 1)
+        microstructure = parts[0].strip()
+        category = parts[1].rstrip(')').strip()
+        return microstructure, category
+    return label, 'Other categories'
+
+def _assign_unique_identifiers(ps_mechanisms, sp_mechanisms):
+    """Assign unique identifiers to mechanisms"""
+    unique_mechanisms = {}
+    current_code = 1
+    for mech in ps_mechanisms:
+        if (mech != "N/A") and (mech not in unique_mechanisms):
+            unique_mechanisms[mech] = str(current_code)
+            current_code += 1
+    for mech in sp_mechanisms:
+        if (mech != "N/A") and (mech not in unique_mechanisms):
+            unique_mechanisms[mech] = str(current_code)
+            current_code += 1
+    return unique_mechanisms
+
+def _calculate_node_positions(nodes, x_position, vertical_gap, offset_y=0):
+    """Calculate positions for nodes in the graph"""
+    return {node: (x_position, offset_y + (len(nodes) - 1 - i) * vertical_gap) for i, node in enumerate(nodes)}
+
+def create_network_diagram(system_chart, paper_title, temp_dir):
+    """
+    Main function to create network diagram from system chart
+    
+    Args:
+        system_chart: The extracted system chart content
+        paper_title: Title of the paper for filename generation
+        temp_dir: Temporary directory to save the visualization files
+    
+    Returns:
+        str: Path to the combined diagram file
+    """
+    import os
+    import uuid
+    import re
+    
+    ps_list, sp_list = _transform_table_into_list(system_chart.strip())
+
+    ps_mechanisms = [mech for _, mech, _ in ps_list]
+    sp_mechanisms = [mech for _, mech, _ in sp_list]
+
+    unique_mechanisms = _assign_unique_identifiers(ps_mechanisms, sp_mechanisms)
+
+    G = nx.DiGraph()
+    category_to_microstructures = defaultdict(set)
+
+    for processing, mechanism, structure in ps_list:
+        if mechanism != "N/A":
+            microstructure, category = _split_microstructure_label(structure)
+            category_to_microstructures[category].add(microstructure)
+            wrapped_processing = _wrap_label(processing)
+            wrapped_microstructure = _wrap_label(microstructure)
+            G.add_node(wrapped_processing, group='processing', category='Processing')
+            G.add_node(wrapped_microstructure, group='structure', category=category)
+            G.nodes[wrapped_microstructure]['header'] = ''
+            G.nodes[wrapped_microstructure]['content'] = wrapped_microstructure
+            if (wrapped_processing, wrapped_microstructure) in G.edges:
+                G.edges[wrapped_processing, wrapped_microstructure]['mechanisms'].append(unique_mechanisms[mechanism])
+            else:
+                G.add_edge(wrapped_processing, wrapped_microstructure, mechanisms=[unique_mechanisms[mechanism]])
+
+    for structure, mechanism, property in sp_list:
+        if mechanism != "N/A":
+            microstructure, category = _split_microstructure_label(structure)
+            wrapped_property = _wrap_label(property)
+            wrapped_microstructure = _wrap_label(microstructure)
+            G.add_node(wrapped_microstructure, group='structure', category=category)
+            G.nodes[wrapped_microstructure]['header'] = ''
+            G.nodes[wrapped_microstructure]['content'] = wrapped_microstructure
+            G.add_node(wrapped_property, group='property', category='Property')
+            if (wrapped_microstructure, wrapped_property) in G.edges:
+                G.edges[wrapped_microstructure, wrapped_property]['mechanisms'].append(unique_mechanisms[mechanism])
+            else:
+                G.add_edge(wrapped_microstructure, wrapped_property, mechanisms=[unique_mechanisms[mechanism]])
+
+    # Define node positions before calling _add_na_nodes
+    processing_nodes = [node for node, attr in G.nodes(data=True) if attr['group'] == 'processing']
+    structure_nodes = [node for node, attr in G.nodes(data=True) if attr['group'] == 'structure']
+    property_nodes = [node for node, attr in G.nodes(data=True) if attr['group'] == 'property']
+
+    # Calculate the offsets to center the processing and property nodes
+    processing_height = len(processing_nodes) * 3.0
+    structure_height = len(structure_nodes) * 2.5
+    property_height = len(property_nodes) * 3.0
+
+    offset_y_processing = (structure_height - processing_height) / 2.5 if structure_height > processing_height else 0
+    offset_y_property = (structure_height - property_height) / 2.5 if structure_height > property_height else 0
+
+    processing_positions = _calculate_node_positions(processing_nodes, 1, 2.5, offset_y_processing)
+    structure_positions = _calculate_node_positions(structure_nodes, 3, 2)
+    property_positions = _calculate_node_positions(property_nodes, 5, 2.5, offset_y_property)
+
+    pos = {**processing_positions, **structure_positions, **property_positions}
+
+    max_label_length = _get_max_label_length(G.nodes)
+    fig_width, fig_height = _calculate_fig_dimensions([len(processing_nodes), len(structure_nodes), len(property_nodes)], max_label_length)
+
+    # Initialize category colors for structure nodes
+    category_colors = _initialize_category_colors(G)
+
+    # Add N/A nodes after defining pos
+    _add_na_nodes(G, unique_mechanisms, pos)
+    
+    # Create safe filename from paper title
+    safe_title = re.sub(r'[^\w\s-]', '', paper_title)[:50]  # Remove special chars, limit length
+    safe_title = re.sub(r'[-\s]+', '_', safe_title)  # Replace spaces/hyphens with underscores
+    unique_id = str(uuid.uuid4())[:8]
+    
+    # Create file paths
+    diagram_path = os.path.join(temp_dir, f"diagram_{safe_title}_{unique_id}.png")
+    legend_path = os.path.join(temp_dir, f"legend_{safe_title}_{unique_id}.png")
+    combined_path = os.path.join(temp_dir, f"system_chart_{safe_title}_{unique_id}.png")
+    
+    # Create the diagrams
+    _create_main_diagram(G, pos, fig_width, fig_height, diagram_path, category_colors)
+    diagram_img_width = _get_image_width(diagram_path)
+    _create_legend(unique_mechanisms, category_colors, diagram_img_width, legend_path)
+    _combine_images(diagram_path, legend_path, combined_path)
+    
+    # Clean up intermediate files
+    try:
+        os.remove(diagram_path)
+        os.remove(legend_path)
+    except OSError:
+        pass  # Don't fail if cleanup doesn't work
+    
+    return combined_path
+
+
+def sanitize_filename(filename: str) -> str:
+    """Sanitize filename by removing/replacing invalid characters"""
+    import re
+    # Remove or replace invalid characters
+    filename = re.sub(r'[<>:"/\\|?*]', '_', filename)
+    # Limit length
+    return filename[:100] if len(filename) > 100 else filename
+
+async def openai_api_function(input_list, llm):
+    """
+    Simple wrapper for OpenAI API calls using langchain
+    
+    Args:
+        input_list: List of previous conversation messages
+        llm: ChatOpenAI instance
+    
+    Returns:
+        tuple: (updated input_list, latest response)
+    """
+    from langchain_core.messages import HumanMessage, AIMessage
+    
+    # Convert input_list to proper message format if needed
+    messages = []
+    for i, content in enumerate(input_list):
+        if i % 2 == 0:  # Human messages (even indices)
+            messages.append(HumanMessage(content=content))
+        else:  # AI messages (odd indices)
+            messages.append(AIMessage(content=content))
+    
+    # Get response from LLM
+    response = await llm.ainvoke(messages)
+    response_text = _llm_content_to_text(getattr(response, 'content', response))
+
+    # Return updated list and response content
+    return input_list, response_text
+
+def _visualize_one_system_chart(system_chart_table_text: str, paper_text: str = "", citation: str = "", llm=None, temp_dir: str = "/tmp"):
+    """
+    Visualize system chart for a single paper, saving images in a citation-named subfolder.
+    
+    Args:
+        system_chart_table_text: The system chart table as text
+        paper_text: Full paper text for context
+        citation: Citation/title for the paper
+        llm: Language model instance
+        temp_dir: Directory to save visualizations
+    
+    Returns:
+        str: Path to the created visualization
+    """
+    import asyncio
+    
+    async def _async_visualize():
+        print("Processing system chart...")
+        
+        # Parse the table to extract microstructures
+        try:
+            df = pd.read_csv(StringIO(system_chart_table_text), delimiter='|', skipinitialspace=True).iloc[1:, 1:-1].reset_index(drop=True)
+            df.columns = df.columns.str.strip()
+            microstructures = df['Microstructure'].str.strip().tolist()
+            microstructures = '; '.join(microstructures)
+        except Exception as e:
+            print(f"Error parsing table: {e}")
+            microstructures = "N/A"
+        
+        table_string = (
+            "The table is as follows:\n"
+            f" The table: {system_chart_table_text}"
+        )
+        
+        input_list = []
+        r_last = None
+        
+        # Use the normalization prompts as our chart-to-diagram prompts
+        prompts_chart_to_diagram = PROMPTS_SYSTEM_CHART_NORMALIZATION
+        
+        for i in range(len(prompts_chart_to_diagram)):
+            print(f"Processing step {i+1}/{len(prompts_chart_to_diagram)}...")
+            
+            if i == 0:
+                prompt = prompts_chart_to_diagram[i].format(microstructures=microstructures)
+                if paper_text:
+                    prompt = (
+                        f"{prompt}\n\n"
+                        "-----\n"
+                        "Full paper context (for reference only):\n"
+                        f"{paper_text}"
+                    )
+            elif i == 1:
+                prompt = prompts_chart_to_diagram[i] + table_string
+            else:
+                prompt = prompts_chart_to_diagram[i].format(r=r_last)
+            
+            input_list.append(prompt)
+            
+            if llm:
+                input_list, r = await openai_api_function(input_list, llm)
+                input_list.append(r)
+                r_last = r
+            else:
+                print("Warning: No LLM provided, skipping API call")
+                r_last = system_chart_table_text  # Use original table as fallback
+        
+        print("Creating diagrams...")
+        
+        # Save in visualized_system_charts/citation/
+        base_folder = os.path.join(temp_dir, 'visualized_system_charts')
+        os.makedirs(base_folder, exist_ok=True)
+        
+        safe_citation = sanitize_filename(citation) if citation else "unknown_citation"
+        subfolder_path = os.path.join(base_folder, safe_citation)
+        os.makedirs(subfolder_path, exist_ok=True)
+        
+        # Create the network diagram
+        try:
+            visualization_path = create_network_diagram(r_last or system_chart_table_text, citation, subfolder_path)
+            print("Diagrams created!")
+            return visualization_path
+        except Exception as e:
+            print(f"Error creating diagram: {e}")
+            return None
+    
+    # Run the async function
+    return asyncio.run(_async_visualize())
+
+async def system_chart_extraction(paper_dict: dict, llm: ChatOpenAI, progress_callback=None, enable_visualization=False, temp_dir=None, extraction_id=None, run_normalization: bool = False) -> tuple:
+    """
+    Extract system charts from papers with detailed progress tracking.
+    
+    Args:
+        paper_dict: Dictionary of title -> paper content
+        llm: Language model for processing
+        progress_callback: Function to call with progress updates
+        enable_visualization: Whether to create visualizations
+        temp_dir: Temporary directory to save visualization files
+    """
+    # Normalize LLM output at the boundary so downstream logic always receives strings.
+    llm = _NormalizedLLM(llm)
+
+    # Initialize output with header and format information
+    output_system_charts = ''''''
+    
+    output_all_details = ''''''
+    created_visualizations = []  # Track created visualization files
+    
+    total_papers = len(paper_dict)
+    
+    # Initialize progress
+    if progress_callback:
+        progress_callback({
+            "status": "initializing",
+            "progress": 0,
+            "total_papers": total_papers,
+            "current_paper": 0,
+            "message": "Initializing..."
+        })
+    
+    # First pass: classify all papers to calculate total questions
+    paper_classifications = {}
+    questions_per_type = {
+        'Experimental/Practical': 9,
+        'Theoretical/Computational': 6,
+        'Mixed': 9
+    }
+    
+    for idx, (title, paper) in enumerate(paper_dict.items()):
+        if progress_callback:
+            progress_callback({
+                "status": "classifying paper type",
+                "progress": int((idx / total_papers) * 20),  # Classification takes 20% of total
+                "total_papers": total_papers,
+                "current_paper": idx + 1,
+                "message": f"Classifying paper {idx + 1}/{total_papers}: {title[:50]}..."
+            })
+        
+        # Add small delay to allow frontend to catch up
+        await asyncio.sleep(0.5)
+        
+        # Classify paper
+        conversation_history = [HumanMessage(content=PROMPTS_CLASSIFICATION + '\n' + paper)]
+        response = await llm.ainvoke(conversation_history)
+        classification_result = response.content
+        
+        # Yield control back to event loop to allow other requests to be processed
+        await asyncio.sleep(0)
+        
+        try:
+            type_of_paper = classification_result.split('Reasons: ')[0].strip().split('Category: ')[1].strip()
+            # Normalize the classification result
+            if type_of_paper not in questions_per_type:
+                type_of_paper = 'Mixed'  # Default fallback
+        except:
+            type_of_paper = 'Mixed'  # Default fallback if parsing fails
+            
+        paper_classifications[title] = type_of_paper
+    
+    # Calculate total questions (classification + actual questions + normalization)
+    total_questions = total_papers  # One classification per paper
+    for paper_type in paper_classifications.values():
+        total_questions += questions_per_type[paper_type]
+    
+    effective_normalization_steps = len(PROMPTS_SYSTEM_CHART_NORMALIZATION)
+    if run_normalization:
+        total_questions += effective_normalization_steps * total_papers  # Normalization steps
+    
+    completed_questions = total_papers  # We've completed all classifications
+    
+    # Update progress after classification phase is complete
+    if progress_callback:
+        progress_callback({
+            "status": "classification_complete",
+            "progress": 20,
+            "total_papers": total_papers,
+            "total_questions": total_questions,
+            "completed_questions": completed_questions,
+            "message": f"Classification complete. Starting to process {total_questions} total questions across {total_papers} papers..."
+        })
+    
+    # Second pass: process each paper
+    count_for_papers = 1
+    
+    for title, paper in paper_dict.items():
+        paper_type = paper_classifications[title]
+        
+        # Called once per paper at the start of the paper processing, high level 
+        if progress_callback:
+            progress_callback({
+                "status": "starting paper processing",
+                "progress": int((completed_questions / total_questions) * 100),
+                "total_papers": total_papers,
+                "current_paper": count_for_papers,
+                "total_questions": total_questions,
+                "completed_questions": completed_questions,
+                "current_paper_type": paper_type,
+                "phase": "starting",
+                "message": f"Starting paper {count_for_papers}/{total_papers} ({paper_type}): {title[:50]}..."
+            })
+        
+        # Reset for each paper
+        count_for_questions = 0
+        count_for_visualization = 0
+        conversation_history = []
+        conversation_history_for_visualization = []
+        five_answers_summary = {}
+        
+        # Add classification result to details
+        output_all_details += ('\n')
+        output_all_details += ('='*100 + '\n')
+        output_all_details += (title)
+        output_all_details += ('\n' + '='*100 + '\n')
+        output_all_details += ('Check the type of the paper (Experimental/Practical, Theoretical/Computational or Mixed), this paper belongs to: ')
+        output_all_details += (' '*5)
+        output_all_details += (paper_type + '\n')
+        output_all_details += ('\n')
+        
+        # Re-establish conversation history with classification
+        conversation_history.append(HumanMessage(content=PROMPTS_CLASSIFICATION + '\n' + paper))
+        classification_response = f"Category: {paper_type}\nReasons: Paper classified as {paper_type} based on content analysis."
+        conversation_history.append(AIMessage(content=classification_response))
+        
+        def update_question_progress(question_num, if_visualization=False):
+            nonlocal completed_questions
+            completed_questions += 1
+            if not if_visualization:
+                if progress_callback:
+                    progress_callback({
+                        "status": "extracting system chart",
+                        "progress": int((completed_questions / total_questions) * 100),
+                        "total_papers": total_papers,
+                        "current_paper": count_for_papers,
+                        "total_questions": total_questions,
+                        "completed_questions": completed_questions,
+                        "current_paper_type": paper_type,
+                        "current_question": question_num + 1,
+                        "questions_in_paper": questions_per_type[paper_type],
+                        "phase": "extraction",
+                        "message": f"Paper {count_for_papers}/{total_papers} - Extracting system chart (Q{question_num + 1}/{questions_per_type[paper_type]})"
+                    })
+            else:
+                if progress_callback:
+                    # Provide specific status messages for different normalization steps
+                    step_descriptions = {
+                        0: "categorizing microstructures",
+                        1: "standardizing properties", 
+                        2: "splitting mechanisms",
+                        3: "separating multi-properties",
+                        4: "filling missing values",
+                        5: "enforcing terminology consistency"
+                    }
+                    step_desc = step_descriptions.get(question_num, "normalizing system chart")
+                    effective_normalization_steps = len(PROMPTS_SYSTEM_CHART_NORMALIZATION)
+                    
+                    progress_callback({
+                        "status": step_desc,
+                        "progress": int((completed_questions / total_questions) * 100),
+                        "total_papers": total_papers,
+                        "current_paper": count_for_papers,
+                        "total_questions": total_questions,
+                        "completed_questions": completed_questions,
+                        "phase": "normalization",
+                        "current_normalization_step": question_num + 1,
+                        "total_normalization_steps": effective_normalization_steps,
+                        "message": f"Paper {count_for_papers}/{total_papers} - Normalization: {step_desc} (Step {question_num + 1}/{effective_normalization_steps})"
+                    })
+        
+        # Process questions based on paper type
+        if paper_type == 'Experimental/Practical':
+            prompts = PROMPTS_EXPERIMENTAL
+            max_questions = 9
+        elif paper_type == 'Theoretical/Computational':
+            prompts = PROMPTS_COMPUTATIONAL
+            max_questions = 6
+        else:  # Mixed
+            prompts = PROMPTS_MIXED
+            max_questions = 9
+        
+        # Process all questions for this paper
+        final_result = ""  # Initialize final_result
+        while count_for_questions < max_questions:
+            if paper_type == 'Experimental/Practical':
+                r, sp, ps, output_all_details = await process_experimental_question(count_for_questions, prompts, conversation_history, 
+                                                        llm, five_answers_summary, output_all_details, count_for_papers)
+            elif paper_type == 'Theoretical/Computational':
+                r, sp, ps, output_all_details = await process_computational_question(count_for_questions, prompts, conversation_history, 
+                                                         llm, five_answers_summary, output_all_details, count_for_papers)
+            else:  # Mixed
+                r, sp, ps, output_all_details = await process_mixed_question(count_for_questions, prompts, conversation_history, 
+                                                 llm, five_answers_summary, output_all_details, count_for_papers)
+            
+            # Store the result for the final system chart
+            final_result = r
+            
+            # Update progress after each question
+            update_question_progress(count_for_questions, if_visualization=False)
+            
+            count_for_questions += 1
+        
+        # Run normalization only when requested (visualization implies normalization).
+        normalized_chart = None
+        if final_result and run_normalization:
+            # Notify that normalization phase is starting
+            if progress_callback:
+                progress_callback({
+                    "status": "starting normalization",
+                    "progress": int((completed_questions / total_questions) * 100),
+                    "total_papers": total_papers,
+                    "current_paper": count_for_papers,
+                    "total_questions": total_questions,
+                    "completed_questions": completed_questions,
+                    "phase": "normalization_start",
+                    "message": f"Paper {count_for_papers}/{total_papers} - Starting normalization..."
+                })
+            
+            try:
+                raw_chart = final_result
+                normalized_chart = final_result
+                print(f"\n🚀 Starting normalization process for paper: {title}")
+                print(f"📊 Initial system chart length: {len(final_result)} characters")
+                print("=" * 80)
+
+                # Process all normalization steps (always execute, not just for visualization)
+                effective_steps = len(PROMPTS_SYSTEM_CHART_NORMALIZATION)
+                print(f"🔧 Total normalization steps to execute: {effective_steps}")
+                
+                while count_for_visualization < effective_steps:
+                    print(f"\n⏳ Starting normalization step {count_for_visualization + 1}/{effective_steps}")
+                    
+                    normalized_chart = await normalize_system_charts(
+                        count_for_visualization, 
+                        PROMPTS_SYSTEM_CHART_NORMALIZATION,
+                        conversation_history_for_visualization, 
+                        normalized_chart, 
+                        raw_chart,
+                        paper, 
+                        llm
+                    )
+                    
+                    print(f"📊 Chart length after step {count_for_visualization + 1}: {len(normalized_chart)} characters")
+                    
+                    # Update progress after each normalization step
+                    update_question_progress(count_for_visualization, if_visualization=True)
+                    count_for_visualization += 1
+
+                print(f"\n✅ Normalization process completed! Final chart length: {len(normalized_chart)} characters")
+                print("=" * 80)
+                print("📋 FINAL NORMALIZED SYSTEM CHART:")
+                print("-" * 60)
+                print(normalized_chart[:1000] + "..." if len(normalized_chart) > 1000 else normalized_chart)
+                print("-" * 60)
+                print("=" * 80)
+
+                # Keep the ORIGINAL final_result for system_charts.txt output
+                # (normalized_chart is only used for visualization)
+                
+                # Optional: Create visualization if requested
+                if enable_visualization and temp_dir:
+                    visualization_path = await create_visualization(
+                        normalized_chart,
+                        title,
+                        temp_dir,
+                        created_visualizations
+                    )
+                    
+                    if visualization_path:
+                        print(f"✅ Visualization completed for paper: {title}")
+                    else:
+                        print(f"⚠️ Visualization creation failed for paper: {title}")
+                    
+            except Exception as e:
+                print(f"❌ Normalization failed for paper '{title}': {str(e)}")
+                # Continue processing other papers even if normalization fails
+        
+        # Add final result to system charts with new format
+        delimiter = "=" * 100
+        output_system_charts += f'\n{delimiter}\n'
+        output_system_charts += f'PAPER: {title}\n'
+        output_system_charts += f'SYSTEM_CHART: {final_result}\n'
+        output_system_charts += f'{delimiter}\n\n'
+        
+        # Save conversation for this paper if extraction_id is provided
+        if extraction_id:
+            try:
+                from conversation_storage import save_conversation
+                conversation_key = save_conversation(
+                    extraction_id=extraction_id,
+                    paper_title=title,
+                    conversation_history=conversation_history,
+                    paper_content=paper,
+                    paper_type=paper_type
+                )
+                print(f"💾 Saved conversation for paper: {title} (key: {conversation_key})")
+            except Exception as e:
+                print(f"⚠️ Could not save conversation for paper '{title}': {str(e)}")
+        
+        count_for_papers += 1
+    
+    # Final progress update
+    if progress_callback:
+        progress_callback({
+            "status": "completed",
+            "progress": 100,
+            "total_papers": total_papers,
+            "total_questions": total_questions,
+            "completed_questions": completed_questions,
+            "message": f"System chart extraction completed! Processed {total_papers} papers with {total_questions} total questions."
+        })
+    
+    return output_system_charts, output_all_details, created_visualizations
+
+async def process_experimental_question(count_for_questions, prompts, conversation_history, llm, 
+                                five_answers_summary, output_all_details, count_for_papers):
+    """Process a single question for experimental papers"""
+    sp, ps = None, None
+    r = ""  # Initialize r
+    
+    if count_for_questions == 0:
+        conversation_history.append(HumanMessage(content=prompts[count_for_questions]))
+        response = await llm.ainvoke(conversation_history)
+        conversation_history.append(AIMessage(content=response.content))
+        await asyncio.sleep(0)
+        r = response.content
+        five_answers_summary['property'] = r.split(';')
+        output_all_details += (f'Q{count_for_questions}: ' + prompts[count_for_questions] + '\n' + f'A{count_for_questions} : Properties are: ' + r + '\n')
+    elif count_for_questions == 1:
+        output_all_details += ('~'*100 + '\n')
+        conversation_history.append(HumanMessage(content=prompts[count_for_questions]))
+        response = await llm.ainvoke(conversation_history)
+        conversation_history.append(AIMessage(content=response.content))
+        await asyncio.sleep(0)
+        r = response.content
+        five_answers_summary['microstructure'] = r.split(';')
+        output_all_details += (f'Q{count_for_questions}: ' + prompts[count_for_questions] + '\n' + f'A{count_for_questions}: Microstructures are: '+ r + '\n')
+    elif count_for_questions == 2:
+        output_all_details += ('~'*100 + '\n')
+        conversation_history.append(HumanMessage(content=prompts[count_for_questions]))
+        response = await llm.ainvoke(conversation_history)
+        conversation_history.append(AIMessage(content=response.content))
+        await asyncio.sleep(0)
+        r = response.content
+        composition, key_elements = _parse_composition_and_key_components(r)
+        five_answers_summary['composition&key_elements'] =  composition
+        if 'N/A' not in key_elements:
+            five_answers_summary['composition&key_elements'] += '; ' + key_elements
+        output_all_details += (f'Q{count_for_questions}: ' + prompts[count_for_questions] + '\n' + f'A{count_for_questions}: ' + r + '\n')
+    elif count_for_questions == 3:
+        output_all_details += ('~'*100 + '\n')
+        prerequisites = f"Prerequisites: Enhanced properties of the alloy: {five_answers_summary['property']}; Microstructure of the alloy: {five_answers_summary['microstructure']}. "
+        conversation_history.append(HumanMessage(content=prerequisites + prompts[count_for_questions]))
+        response = await llm.ainvoke(conversation_history)
+        conversation_history.append(AIMessage(content=response.content))
+        await asyncio.sleep(0)
+        r = response.content
+        sp = r
+        output_all_details += (f'Q{count_for_questions}: ' + prompts[count_for_questions] + '\n' + f'A{count_for_questions}: '+ r + '\n')
+    elif count_for_questions == 4:
+        output_all_details += ('~'*100 + '\n')
+        conversation_history.append(HumanMessage(content=prompts[count_for_questions]))
+        response = await llm.ainvoke(conversation_history)
+        conversation_history.append(AIMessage(content=response.content))
+        await asyncio.sleep(0)
+        r = response.content
+        five_answers_summary['processing'] = r.split(';')
+        output_all_details += (f'Q{count_for_questions}: ' + prompts[count_for_questions] + '\n' + f'A{count_for_questions}' + ' Processing methods are: ' + r + '\n')
+    elif count_for_questions == 5:
+        output_all_details += ('~'*100 + '\n')
+        prerequisites = f"Processing methods previously discussed: {r}. "
+        conversation_history.append(HumanMessage(content=prerequisites + prompts[count_for_questions]))
+        response = await llm.ainvoke(conversation_history)
+        conversation_history.append(AIMessage(content=response.content))
+        await asyncio.sleep(0)
+        r = response.content
+        five_answers_summary['processing'] = r.split('The explanations are:')[0].split(': ')[1]
+        output_all_details += (f'Q{count_for_questions}: ' + prompts[count_for_questions] + '\n' + f'A{count_for_questions}: '+ r + '\n')
+    elif count_for_questions == 6:
+        output_all_details += ('~'*100 + '\n')
+        prerequisites = f"Prerequisites: Processing methods of the alloy: {five_answers_summary['processing']}; Microstructure of the alloy: {five_answers_summary['microstructure']}. "
+        conversation_history.append(HumanMessage(content=prerequisites + prompts[count_for_questions]))
+        response = await llm.ainvoke(conversation_history)
+        conversation_history.append(AIMessage(content=response.content))
+        await asyncio.sleep(0)
+        r = response.content
+        output_all_details += (f'Q{count_for_questions}: ' + prerequisites + prompts[count_for_questions] + '\n' + f'A{count_for_questions}: '+ r + '\n')
+    elif count_for_questions == 7:
+        output_all_details += ('~'*100 + '\n')
+        prerequisites = f"Prerequisites: Composition & Key components (if any) of the alloy: {five_answers_summary['composition&key_elements']}; Microstructure of the alloy: {five_answers_summary['microstructure']}. "
+        conversation_history.append(HumanMessage(content=prerequisites + prompts[count_for_questions]))
+        response = await llm.ainvoke(conversation_history)
+        conversation_history.append(AIMessage(content=response.content))
+        await asyncio.sleep(0)
+        r = response.content
+        ps = r
+        output_all_details += (f'Q{count_for_questions}: ' + prerequisites + prompts[count_for_questions] + '\n' + f'A{count_for_questions}: '+ r + '\n')
+    elif count_for_questions == 8:
+        output_all_details += ('~'*100 + '\n')
+        prerequisites = f"Prerequisites: First table for relationsip between processing and microstructure: {ps}; Second table for relationsip between microstructure and properties: {sp}. "
+        conversation_history.append(HumanMessage(content=prerequisites + '\n' + prompts[count_for_questions]))
+        response = await llm.ainvoke(conversation_history)
+        conversation_history.append(AIMessage(content=response.content))
+        await asyncio.sleep(0)
+        r = response.content
+        output_all_details += ('\n' + f'Q{count_for_questions}: ' + prerequisites + prompts[count_for_questions] + '\n' + f'A{count_for_questions}: '+ r + '\n')
+    
+    return r, sp, ps, output_all_details
+
+async def process_computational_question(count_for_questions, prompts, conversation_history, llm, 
+                                 five_answers_summary, output_all_details, count_for_papers=1):
+    """Process a single question for computational papers"""
+    sp, ps = None, None
+    r = ""  # Initialize r
+    
+    if count_for_questions == 0:
+        conversation_history.append(HumanMessage(content=prompts[count_for_questions]))
+        response = await llm.ainvoke(conversation_history)
+        conversation_history.append(AIMessage(content=response.content))
+        await asyncio.sleep(0)
+        r = response.content
+        five_answers_summary['property'] = r.split(';')
+        output_all_details += (f'Q{count_for_questions}: ' + prompts[count_for_questions] + '\n' + f'A{count_for_questions} : Properties are: ' + r + '\n')
+    elif count_for_questions == 1:
+        output_all_details += ('~'*100 + '\n')
+        conversation_history.append(HumanMessage(content=prompts[count_for_questions]))
+        response = await llm.ainvoke(conversation_history)
+        conversation_history.append(AIMessage(content=response.content))
+        await asyncio.sleep(0)
+        r = response.content
+        five_answers_summary['microstructure'] = r.split(';')
+        output_all_details += (f'Q{count_for_questions}: ' + prompts[count_for_questions] + '\n' + f'A{count_for_questions}: Microstructures are: '+ r + '\n')
+    elif count_for_questions == 2:
+        output_all_details += ('~'*100 + '\n')
+        conversation_history.append(HumanMessage(content=prompts[count_for_questions]))
+        response = await llm.ainvoke(conversation_history)
+        conversation_history.append(AIMessage(content=response.content))
+        await asyncio.sleep(0)
+        r = response.content
+        composition, key_elements = _parse_composition_and_key_components(r)
+        five_answers_summary['composition&key_elements'] =  composition
+        if 'N/A' not in key_elements:
+            five_answers_summary['composition&key_elements'] += '; ' + key_elements
+        output_all_details += (f'Q{count_for_questions}: ' + prompts[count_for_questions] + '\n' + f'A{count_for_questions}: ' + r + '\n')
+    elif count_for_questions == 3:
+        output_all_details += ('~'*100 + '\n')
+        prerequisites = f"Prerequisites: Enhanced properties of the alloy: {five_answers_summary['property']}; Microstructure of the alloy: {five_answers_summary['microstructure']}."
+        conversation_history.append(HumanMessage(content=prerequisites + prompts[count_for_questions]))
+        response = await llm.ainvoke(conversation_history)
+        conversation_history.append(AIMessage(content=response.content))
+        await asyncio.sleep(0)
+        r = response.content
+        sp = r
+        output_all_details += (f'Q{count_for_questions}: ' + prompts[count_for_questions] + '\n' + f'A{count_for_questions}: '+ r + '\n')
+    elif count_for_questions == 4:
+        output_all_details += ('~'*100 + '\n')
+        prerequisites = f"Prerequisites: Composition & Key components of the alloy: {five_answers_summary['composition&key_elements']}; Microstructure of the alloy: {five_answers_summary['microstructure']}."
+        conversation_history.append(HumanMessage(content=prerequisites + prompts[count_for_questions]))
+        response = await llm.ainvoke(conversation_history)
+        conversation_history.append(AIMessage(content=response.content))
+        await asyncio.sleep(0)
+        r = response.content
+        ps = r
+        output_all_details += (f'Q{count_for_questions}: ' + prerequisites + prompts[count_for_questions] + '\n' + f'A{count_for_questions}: '+ r + '\n')
+    elif count_for_questions == 5:
+        output_all_details += ('~'*100 + '\n')
+        prerequisites = f"Prerequisites: First table for relationsip between processing and microstructure: {ps}; Second table for relationsip between microstructure and properties: {sp}. "
+        conversation_history.append(HumanMessage(content=prerequisites + '\n' + prompts[count_for_questions]))
+        response = await llm.ainvoke(conversation_history)
+        conversation_history.append(AIMessage(content=response.content))
+        await asyncio.sleep(0)
+        r = response.content
+        output_all_details += ('\n' + f'Q{count_for_questions}: ' + prerequisites + prompts[count_for_questions] + '\n' + f'A{count_for_questions}: '+ r + '\n')
+    
+    return r, sp, ps, output_all_details
+
+async def process_mixed_question(count_for_questions, prompts, conversation_history, llm, 
+                         five_answers_summary, output_all_details, count_for_papers):
+    """Process a single question for mixed papers"""
+    sp, ps = None, None
+    r = ""  # Initialize r
+    
+    if count_for_questions == 0:
+        conversation_history.append(HumanMessage(content=prompts[count_for_questions]))
+        response = await llm.ainvoke(conversation_history)
+        conversation_history.append(AIMessage(content=response.content))
+        await asyncio.sleep(0)
+        r = response.content
+        five_answers_summary['property'] = r.split(';')
+        output_all_details += (f'Q{count_for_questions}: ' + prompts[count_for_questions] + '\n' + f'A{count_for_questions} : Properties are: ' + r + '\n')
+    elif count_for_questions == 1:
+        output_all_details += ('~'*100 + '\n')
+        conversation_history.append(HumanMessage(content=prompts[count_for_questions]))
+        response = await llm.ainvoke(conversation_history)
+        conversation_history.append(AIMessage(content=response.content))
+        await asyncio.sleep(0)
+        r = response.content
+        five_answers_summary['microstructure'] = r.split(';')
+        output_all_details += (f'Q{count_for_questions}: ' + prompts[count_for_questions] + '\n' + f'A{count_for_questions}: Microstructures are: '+ r + '\n')
+    elif count_for_questions == 2:
+        output_all_details += ('~'*100 + '\n')
+        conversation_history.append(HumanMessage(content=prompts[count_for_questions]))
+        response = await llm.ainvoke(conversation_history)
+        conversation_history.append(AIMessage(content=response.content))
+        await asyncio.sleep(0)
+        r = response.content
+        composition, key_elements = _parse_composition_and_key_components(r)
+        five_answers_summary['composition&key_elements'] =  composition
+        if 'N/A' not in key_elements:
+            five_answers_summary['composition&key_elements'] += '; ' + key_elements
+        output_all_details += (f'Q{count_for_questions}: ' + prompts[count_for_questions] + '\n' + f'A{count_for_questions}: ' + r + '\n')
+    elif count_for_questions == 3:
+        output_all_details += ('~'*100 + '\n')
+        prerequisites = f"Prerequisites: Enhanced properties of the alloy: {five_answers_summary['property']}; Microstructure of the alloy: {five_answers_summary['microstructure']}."
+        conversation_history.append(HumanMessage(content=prerequisites + prompts[count_for_questions]))
+        response = await llm.ainvoke(conversation_history)
+        conversation_history.append(AIMessage(content=response.content))
+        await asyncio.sleep(0)
+        r = response.content
+        sp = r
+        output_all_details += (f'Q{count_for_questions}: ' + prompts[count_for_questions] + '\n' + f'A{count_for_questions}: '+ r + '\n')
+    elif count_for_questions == 4:
+        output_all_details += ('~'*100 + '\n')
+        conversation_history.append(HumanMessage(content=prompts[count_for_questions]))
+        response = await llm.ainvoke(conversation_history)
+        conversation_history.append(AIMessage(content=response.content))
+        await asyncio.sleep(0)
+        r = response.content
+        five_answers_summary['processing'] = r.split(';')
+        output_all_details += (f'Q{count_for_questions}: ' + prompts[count_for_questions] + '\n' + f'A{count_for_questions}' + ' Processing methods are: ' + r + '\n')
+    elif count_for_questions == 5:
+        output_all_details += ('~'*100 + '\n')
+        prerequisites = f"Processing methods previously discussed: {r}. "
+        conversation_history.append(HumanMessage(content=prerequisites + prompts[count_for_questions]))
+        response = await llm.ainvoke(conversation_history)
+        conversation_history.append(AIMessage(content=response.content))
+        await asyncio.sleep(0)
+        r = response.content
+        five_answers_summary['processing'] = r.split('The explanations are:')[0].split(': ')[1]
+        output_all_details += (f'Q{count_for_questions}: ' + prompts[count_for_questions] + '\n' + f'A{count_for_questions}: '+ r + '\n')
+    elif count_for_questions == 6:
+        output_all_details += ('~'*100 + '\n')
+        prerequisites = f"Prerequisites: Processing methods of the alloy: {five_answers_summary['processing']}; Microstructure of the alloy: {five_answers_summary['microstructure']}. "
+        conversation_history.append(HumanMessage(content=prerequisites + prompts[count_for_questions]))
+        response = await llm.ainvoke(conversation_history)
+        conversation_history.append(AIMessage(content=response.content))
+        await asyncio.sleep(0)
+        r = response.content
+        output_all_details += (f'Q{count_for_questions}: ' + prerequisites + prompts[count_for_questions] + '\n' + f'A{count_for_questions}: '+ r + '\n')
+    elif count_for_questions == 7:
+        output_all_details += ('~'*100 + '\n')
+        prerequisites = f"Prerequisites: Composition & Key components (if any) of the alloy: {five_answers_summary['composition&key_elements']}; Microstructure of the alloy: {five_answers_summary['microstructure']}. "
+        conversation_history.append(HumanMessage(content=prerequisites + prompts[count_for_questions]))
+        response = await llm.ainvoke(conversation_history)
+        conversation_history.append(AIMessage(content=response.content))
+        await asyncio.sleep(0)
+        r = response.content
+        ps = r
+        output_all_details += (f'Q{count_for_questions}: ' + prerequisites + prompts[count_for_questions] + '\n' + f'A{count_for_questions}: '+ r + '\n')
+    elif count_for_questions == 8:
+        output_all_details += ('~'*100 + '\n')
+        prerequisites = f"Prerequisites: First table for relationsip between processing and microstructure: {ps}; Second table for relationsip between microstructure and properties: {sp}. "
+        conversation_history.append(HumanMessage(content=prerequisites + '\n' + prompts[count_for_questions]))
+        response = await llm.ainvoke(conversation_history)
+        conversation_history.append(AIMessage(content=response.content))
+        await asyncio.sleep(0)
+        r = response.content
+        output_all_details += ('\n' + f'Q{count_for_questions}: ' + prerequisites + prompts[count_for_questions] + '\n' + f'A{count_for_questions}: '+ r + '\n')
+    
+    return r, sp, ps, output_all_details
+
+async def normalize_system_charts(count_for_visualization, prompts, conversation_history, system_chart, raw_chart, paper, llm):
+    """
+    Normalize system charts for visualization
+    
+    Args:
+        count_for_visualization: Current visualization step number
+        prompts: List of prompts for normalization
+        conversation_history: Conversation history with LLM
+        system_chart: The extracted system chart content
+        paper: Full content of the paper
+        llm: Language model for processing
+    
+    Returns:
+        str: Normalized system chart content
+    """
+    step_descriptions = {
+        0: "categorizing microstructures",
+        1: "standardizing properties", 
+        2: "splitting mechanisms",
+        3: "separating multi-properties",
+        4: "filling missing values",
+        5: "enforcing terminology consistency"
+    }
+    
+    step_name = step_descriptions.get(count_for_visualization, f"step {count_for_visualization}")
+    
+    print(f"\n🔄 NORMALIZATION STEP {count_for_visualization + 1}: {step_name}")
+    
+    try:
+        
+        # Handle the first step (categorization) with special processing
+        if count_for_visualization == 0:
+            microstructures = _parse_microstructures(system_chart)
+            print(f"🔍 Extracted microstructures: {microstructures}")
+            
+            # Question 1: Initial categorization
+            conversation_history.append(HumanMessage(content=prompts[count_for_visualization] + microstructures + '\nOriginal paper: ' + paper))
+            response = await llm.ainvoke(conversation_history)
+            conversation_history.append(AIMessage(content=response.content))
+            result = response.content
+        elif count_for_visualization == 1:
+            conversation_history.append(HumanMessage(content=prompts[count_for_visualization] + raw_chart))
+            response = await llm.ainvoke(conversation_history)
+            conversation_history.append(AIMessage(content=response.content))
+            result = response.content
+        else:
+            conversation_history.append(HumanMessage(content=prompts[count_for_visualization]))
+            response = await llm.ainvoke(conversation_history)
+            conversation_history.append(AIMessage(content=response.content))
+            result = response.content
+        
+        return result
+
+    except Exception as e:
+        print(f"❌ Error in normalization step {count_for_visualization}: {str(e)}")
+        print(f"📤 Returning original chart due to error\n")
+        return system_chart  # Return original chart if normalization fails
+
+async def create_visualization(normalized_chart, paper_title, temp_dir, created_visualizations):
+    """
+    Create visualization from normalized system chart
+    
+    Args:
+        normalized_chart: The normalized system chart content
+        paper_title: Title of the paper
+        temp_dir: Temporary directory to save visualization files
+        created_visualizations: List to store created visualization file paths
+    
+    Returns:
+        str: Path to the created visualization file
+    """
+    try:
+        # Create the network diagram with the normalized result
+        visualization_path = create_network_diagram(normalized_chart, paper_title, temp_dir)
+        
+        # Add to the list of created visualizations
+        created_visualizations.append(visualization_path)
+        
+        print(f"✅ Visualization created for '{paper_title}': {visualization_path}")
+        return visualization_path
+        
+    except Exception as e:
+        print(f"❌ Error creating visualization for '{paper_title}': {str(e)}")
+        return None

system_chart_service.py

@@ -0,0 +1,341 @@
+import tempfile
+import os
+import shutil
+from io import BytesIO
+from zipfile import ZipFile
+from typing import Dict, Callable, Any, List
+from datetime import datetime
+from system_chart_extraction import system_chart_extraction
+
+class SystemChartService:
+    """
+    Service class for System Chart Extraction.
+    Handles the orchestration of system chart extraction, visualization creation,
+    and result packaging.
+    """
+    
+    def __init__(self):
+        pass
+    
+    async def extract_system_charts(
+        self,
+        category: str,
+        llm,
+        pdf_dict: Dict[str, str],
+        enable_visualization: bool = False,
+        run_system_chart_normalization: bool = False,
+        progress_callback: Callable[[Dict[str, Any]], None] = None,
+        extraction_id: str = None
+    ) -> Dict[str, Any]:
+        """
+        Main system chart extraction pipeline.
+        
+        Args:
+            category: Database category for extraction
+            llm: Language model for extraction
+            pdf_dict: Dictionary mapping document titles to full text content
+            enable_visualization: Whether to create visualization diagrams
+            progress_callback: Function to report progress updates
+            
+        Returns:
+            Dict containing extraction results and metadata
+        """
+        try:
+            # Validate input
+            if not pdf_dict:
+                return {
+                    "success": False,
+                    "message": "No files found in the specified category.",
+                    "zip_data": None,
+                    "filename": None
+                }
+            
+            print(f"📄 Found {len(pdf_dict)} papers to process")
+            
+            # Initial progress update
+            if progress_callback:
+                progress_callback({
+                    "status": "starting",
+                    "progress": 0,
+                    "message": "Getting PDF files..."
+                })
+            
+            # Create progress callback function for the extraction process
+            def extraction_progress_callback(progress_data):
+                if progress_callback:
+                    print(f"📊 Progress update: {progress_data}")
+                    progress_callback(progress_data)
+                    print(f"🔄 Progress updated: {progress_data}")
+            
+            # Update progress before starting extraction
+            if progress_callback:
+                progress_callback({
+                    "status": "starting",
+                    "progress": 5,
+                    "message": f"Found {len(pdf_dict)} papers. Starting extraction..."
+                })
+            
+            print(f"🔬 Starting system chart extraction...")
+            
+            # Create temporary directory for visualization files if needed
+            temp_dir = None
+            if enable_visualization:
+                temp_dir = tempfile.mkdtemp(prefix="system_chart_viz_")
+                print(f"📁 Created temporary directory: {temp_dir}")
+            
+            # Run system chart extraction with progress tracking
+            output_system_charts, output_all_details, created_visualizations = await system_chart_extraction(
+                pdf_dict, 
+                llm, 
+                extraction_progress_callback, 
+                enable_visualization,
+                temp_dir,
+                extraction_id,  # Pass extraction_id to save conversations
+                run_normalization=run_system_chart_normalization  # ← only normalize when asked/needed
+            )
+            
+            # Create ZIP file in memory
+            zip_data = self._create_zip_file(
+                output_system_charts, 
+                output_all_details, 
+                created_visualizations
+            )
+            
+            # Clean up temporary directory
+            if temp_dir:
+                self._cleanup_temp_directory(temp_dir)
+            
+            # Prepare result
+            viz_message = f" with {len(created_visualizations)} visualizations" if created_visualizations else ""
+            
+            return {
+                "success": True,
+                "message": f"System chart extraction completed successfully{viz_message}!",
+                "zip_data": zip_data,
+                "filename": "system_chart_extraction.zip",
+                "charts_count": len(pdf_dict),
+                "visualizations_count": len(created_visualizations) if created_visualizations else 0
+            }
+            
+        except Exception as e:
+            # Clean up on error
+            if 'temp_dir' in locals() and temp_dir:
+                self._cleanup_temp_directory(temp_dir)
+            
+            return {
+                "success": False,
+                "message": f"Error during system chart extraction: {str(e)}",
+                "zip_data": None,
+                "filename": None
+            }
+    
+    def _create_zip_file(
+        self, 
+        output_system_charts: str, 
+        output_all_details: str, 
+        created_visualizations: List[str]
+    ) -> bytes:
+        """Create ZIP file containing extraction results and visualizations."""
+        zip_buffer = BytesIO()
+        
+        with ZipFile(zip_buffer, "w") as zipf:
+            # Add text results
+            zipf.writestr("system_charts.txt", output_system_charts)
+            zipf.writestr("all_details.txt", output_all_details)
+            
+            # Add visualization files if any were created
+            if created_visualizations:
+                for viz_path in created_visualizations:
+                    if os.path.exists(viz_path):
+                        # Extract just the filename for the ZIP entry
+                        viz_filename = os.path.basename(viz_path)
+                        zipf.write(viz_path, f"visualizations/{viz_filename}")
+                        print(f"📦 Added visualization to ZIP: {viz_filename}")
+        
+        # Get ZIP data and close buffer
+        zip_data = zip_buffer.getvalue()
+        zip_buffer.close()
+        
+        return zip_data
+    
+    def _cleanup_temp_directory(self, temp_dir: str) -> None:
+        """Clean up temporary directory and its contents."""
+        if temp_dir and os.path.exists(temp_dir):
+            try:
+                shutil.rmtree(temp_dir)
+                print(f"🗑️ Cleaned up temporary directory: {temp_dir}")
+            except OSError as e:
+                print(f"⚠️ Could not clean up temporary directory {temp_dir}: {e}")
+                # Don't fail the whole process if cleanup fails
+                pass
+    
+    @staticmethod
+    def parse_system_charts_file(file_content: str) -> List[Dict[str, str]]:
+        """
+        Parse a delimited system charts file into individual chart objects.
+        
+        Args:
+            file_content: Content of the system_charts.txt file
+            
+        Returns:
+            List of dictionaries, each containing:
+            - 'chart_number': The chart number (auto-generated)
+            - 'title': Paper title
+            - 'citation': Citation format (same as title for compatibility)
+            - 'content': System chart table content
+        """
+        charts = []
+        lines = file_content.split('\n')
+        
+        current_chart = None
+        chart_number = 1
+        
+        for i, line in enumerate(lines):
+            line = line.strip()
+            
+            # Check for delimiter line (100 '=' characters)
+            if line == '=' * 100:
+                # Look for PAPER: line after delimiter
+                if i + 1 < len(lines):
+                    next_line = lines[i + 1].strip()
+                    if next_line.startswith('PAPER:'):
+                        # Save previous chart if exists
+                        if current_chart:
+                            charts.append(current_chart)
+                        
+                        # Start new chart
+                        paper_title = next_line.replace('PAPER:', '').strip()
+                        current_chart = {
+                            'chart_number': str(chart_number),
+                            'title': paper_title,
+                            'citation': f'[{paper_title}]',
+                            'content': ''
+                        }
+                        chart_number += 1
+                        
+            elif line.startswith('SYSTEM_CHART:') and current_chart:
+                # Extract the system chart content
+                chart_content = line.replace('SYSTEM_CHART:', '').strip()
+                current_chart['content'] = chart_content
+        
+        # Handle last chart if file doesn't end with proper delimiter
+        if current_chart:
+            charts.append(current_chart)
+        
+        print(f"📊 Parsed {len(charts)} system charts from file")
+        return charts
+    
+    @staticmethod
+    def combine_system_charts_files(file_contents: List[str]) -> str:
+        """
+        Combine multiple system chart txt files into a single formatted string.
+        Preserves the one-to-one relationship between system charts and titles.
+        
+        Args:
+            file_contents: List of system chart file contents to combine
+            
+        Returns:
+            Combined system chart content in the new format
+        """
+        combined_charts = []
+        
+        # Parse each file and collect all charts
+        for file_content in file_contents:
+            charts = SystemChartService.parse_system_charts_file(file_content)
+            combined_charts.extend(charts)
+        
+        # Generate combined output in new format
+        combined_output = "COMBINED SYSTEM CHARTS EXTRACTION RESULTS\n"
+        combined_output += "=" * 100 + "\n"
+        
+        for chart in combined_charts:
+            delimiter = "=" * 100
+            combined_output += f'\n{delimiter}\n'
+            combined_output += f'PAPER: {chart["title"]}\n'
+            combined_output += f'SYSTEM_CHART: {chart["content"]}\n'
+            combined_output += f'{delimiter}\n\n'
+        
+        # Add summary
+        summary_delimiter = "=" * 100
+        combined_output += f'\n{summary_delimiter}\n'
+        combined_output += f'COMBINATION SUMMARY\n'
+        combined_output += f'{summary_delimiter}\n'
+        combined_output += f'Total Files Combined: {len(file_contents)}\n'
+        combined_output += f'Total System Charts: {len(combined_charts)}\n'
+        combined_output += f'Combination Date: {datetime.now().strftime("%Y-%m-%d %H:%M:%S")}\n'
+        
+        print(f"🔗 Combined {len(file_contents)} files into {len(combined_charts)} system charts")
+        return combined_output
+    
+    async def combine_multiple_system_chart_files(
+        self,
+        file_paths: List[str],
+        output_filename: str = "combined_system_charts.txt"
+    ) -> Dict[str, Any]:
+        """
+        Combine multiple system chart txt files into a single ZIP file.
+        
+        Args:
+            file_paths: List of file paths to system chart txt files
+            output_filename: Name for the output file
+            
+        Returns:
+            Dict containing success status, message, and ZIP data
+        """
+        try:
+            file_contents = []
+            
+            # Read all files
+            for file_path in file_paths:
+                if not os.path.exists(file_path):
+                    return {
+                        "success": False,
+                        "message": f"File not found: {file_path}",
+                        "zip_data": None,
+                        "filename": None
+                    }
+                
+                with open(file_path, 'r', encoding='utf-8') as f:
+                    file_contents.append(f.read())
+            
+            # Combine the files
+            combined_content = self.combine_system_charts_files(file_contents)
+            
+            # Create ZIP file
+            zip_buffer = BytesIO()
+            with ZipFile(zip_buffer, "w") as zipf:
+                zipf.writestr(output_filename, combined_content)
+            
+            zip_data = zip_buffer.getvalue()
+            zip_buffer.close()
+            
+            return {
+                "success": True,
+                "message": f"Successfully combined {len(file_paths)} system chart files!",
+                "zip_data": zip_data,
+                "filename": "combined_system_charts.zip",
+                "files_combined": len(file_paths)
+            }
+            
+        except Exception as e:
+            return {
+                "success": False,
+                "message": f"Error combining system chart files: {str(e)}",
+                "zip_data": None,
+                "filename": None
+            }
+    
+    @staticmethod
+    def get_chart_citations(file_content: str) -> List[str]:
+        """
+        Extract just the citations from a system charts file.
+        
+        Args:
+            file_content: Content of the system_charts.txt file
+            
+        Returns:
+            List of citation strings
+        """
+        charts = SystemChartService.parse_system_charts_file(file_content)
+        return [chart.get('citation', chart.get('title', f"Chart {chart['chart_number']}")) 
+                for chart in charts if chart.get('chart_number')] 

vectorstore.py

@@ -0,0 +1,24 @@
+import os
+import sys
+import hashlib
+from langchain_community.vectorstores import Chroma
+from langchain_openai import OpenAIEmbeddings
+
+def get_base_directory():
+    """Return the base directory for persistent storage (next to executable when packaged)"""
+    if hasattr(sys, '_MEIPASS'):  # Running as PyInstaller executable
+        return os.path.dirname(sys.executable)
+    else:  # Running as normal Python script
+        return os.path.dirname(os.path.abspath(__file__))
+
+# Define path relative to the base directory
+CHROMA_DB_BASE = os.path.join(get_base_directory(), "root databases")
+os.makedirs(CHROMA_DB_BASE, exist_ok=True)
+
+def compute_file_hash(content: bytes) -> str:
+    return hashlib.md5(content).hexdigest()
+
+def get_vectorstore(category: str, embedding_model):
+    path = os.path.join(CHROMA_DB_BASE, category.strip())
+    os.makedirs(path, exist_ok=True)
+    return Chroma(persist_directory=path, embedding_function=embedding_model)