Update src/model.py

src/model.py

@@ -27,7 +27,6 @@ openai.api_key = get_openai_api_key()
 # System prompts
 SYSTEM_PROMPT = """You are an advanced clinical AI assistant designed to aid healthcare professionals.
 Follow these guidelines in all responses:
-
 1.  **Clarify First**: Before providing any diagnosis or plan, if the user's query is underspecified, ALWAYS ask relevant clarifying questions to gather necessary patient information. This includes, but is not limited to, symptoms, duration, severity, medical history, age, lifestyle factors (diet, exercise), and current medications.
 2.  Professional tone: Maintain a clear, respectful, and professional tone appropriate for medical consultation.
 3.  Evidence-based practice: Base all responses on current medical evidence and guidelines.
@@ -37,20 +36,17 @@ Follow these guidelines in all responses:
 7.  Limitations: Acknowledge the limits of AI medical advice and recommend in-person consultation when appropriate.
 8.  Comprehensive approach: Consider differential diagnoses and relevant contextual factors.
 9.  Patient-centered: Focus on clinically relevant information while maintaining respect for the patient.
-
 For each consultation:
 1.  Ask clarifying questions if needed (as per guideline 1).
 2.  Provide differential diagnosis with likelihood assessment.
 3.  Suggest appropriate next steps (testing, treatment, referral).
 4.  Include reasoning for your conclusions.
 5.  Cite medical literature or guidelines supporting your assessment using [source_id].
-
 IMPORTANT: Your primary duty is to support clinical decision-making, not replace clinical judgment.
 """
 
 FOLLOW_UP_PROMPT = """Continue this medical consultation based on the previous discussion.
 Consider the information already gathered and the tentative diagnosis/plan.
-
 When responding to the follow-up:
 1.  Reference relevant details from the prior conversation.
 2.  Address the specific follow-up question with evidence-based information.
@@ -58,82 +54,13 @@ When responding to the follow-up:
 4.  Update recommendations if appropriate.
 5.  Maintain the same structured approach with transparent reasoning.
 6.  Cite additional medical literature or guidelines when relevant using [source_id].
-
 Remember that this is an ongoing consultation where continuity of care is important.
 """
 
-# Enhanced medical query preprocessing
-def preprocess_medical_query(query):
-    """
-    Use GPT-4o to extract key medical terms and concepts from the user query.
-    This improves search relevance by identifying proper medical terminology.
-    """
-    try:
-        system_prompt = """
-        You are a medical term extraction system. Extract key medical concepts from the input query.
-        Focus on:
-        1. Symptoms (e.g., fever, pain, cough)
-        2. Conditions (e.g., diabetes, hypertension)
-        3. Anatomical structures (e.g., liver, heart)
-        4. Medications (e.g., aspirin, insulin)
-        5. Procedures (e.g., MRI, surgery)
-        
-        For each term extracted, try to provide the corresponding medical terminology or MeSH term if applicable.
-        
-        Return your answer in this JSON format:
-        {
-          "extracted_terms": ["term1", "term2", ...],
-          "mesh_mappings": {"term1": "MeSH term1", "term2": "MeSH term2", ...},
-          "optimized_search_query": "term1 AND term2 AND term3..."
-        }
-        
-        Include only the JSON object, nothing else.
-        """
-        
-        response = openai.ChatCompletion.create(
-            model="gpt-4o-mini",  # Using gpt-4o-mini for efficiency, can upgrade to gpt-4o if needed
-            messages=[
-                {"role": "system", "content": system_prompt},
-                {"role": "user", "content": f"Extract medical terms from this query: {query}"}
-            ],
-            temperature=0.1,  # Low temperature for consistent outputs
-            max_tokens=500,
-        )
-        
-        # Extract json from response
-        result_text = response.choices[0].message['content']
-        try:
-            # Find JSON object
-            json_match = re.search(r'({[\s\S]*})', result_text)
-            if json_match:
-                json_str = json_match.group(1)
-                return json.loads(json_str)
-            else:
-                return json.loads(result_text)
-        except json.JSONDecodeError:
-            # Fallback to original query if parsing fails
-            return {
-                "extracted_terms": [query],
-                "mesh_mappings": {},
-                "optimized_search_query": query
-            }
-    except Exception as e:
-        print(f"Error in query preprocessing: {str(e)}")
-        # Fallback to original query
-        return {
-            "extracted_terms": [query],
-            "mesh_mappings": {},
-            "optimized_search_query": query
-        }
-
 # Function to extract source IDs and replace them with actual links
 def extract_and_link_sources(text, evidence_snippets):
-    """
-    Enhanced function to replace [source_id] placeholders with actual source information.
-    Improved to handle various citation formats and provide richer context.
-    """
-    # Expanded pattern to handle more citation formats
-    source_pattern = r'\[([\w\d:_\-\.+]+)\]'  # Basic citation format [source_id]
+    """Replace [source_id] placeholders with actual source information"""
+    source_pattern = r'\[([\w\d:_\-\.+]+)\]'  # Expanded to handle more characters including +
     matches = re.findall(source_pattern, text)
 
     source_map = {}  # Map to store source_id -> source data
@@ -146,54 +73,28 @@ def extract_and_link_sources(text, evidence_snippets):
                     "id": snippet["id"],
                     "title": snippet["title"].strip(),
                     "url": snippet["url"],
-                    "citation": snippet["citation"],
-                    "has_full_text": snippet.get("has_full_text", False),
-                    "journal": snippet.get("journal", ""),
-                    "year": snippet.get("year", "")
+                    "citation": snippet["citation"]
                 }
                 break
     
     # Next, try fuzzy matching for cases where the exact ID isn't matched
     for source_id_match in matches:
         if source_id_match not in source_map and source_id_match != "source_id":
-            # Try multiple matching strategies
-            
-            # Strategy 1: Match on ID prefix (e.g., pubmed-12345 might match pubmed-12345678)
             for snippet in evidence_snippets:
-                if snippet["id"].startswith(source_id_match) or source_id_match.startswith(snippet["id"]):
+                # Try to match on partial IDs (e.g. part before a hyphen)
+                snippet_id_parts = snippet["id"].split("-")
+                source_id_parts = source_id_match.split("-")
+                
+                # Check if the first parts match (journal name)
+                if (snippet_id_parts and source_id_parts and 
+                    snippet_id_parts[0] == source_id_parts[0]):
                     source_map[source_id_match] = {
                         "id": snippet["id"],
                         "title": snippet["title"].strip(),
                         "url": snippet["url"],
-                        "citation": snippet["citation"],
-                        "has_full_text": snippet.get("has_full_text", False),
-                        "journal": snippet.get("journal", ""),
-                        "year": snippet.get("year", "")
+                        "citation": snippet["citation"]
                     }
                     break
-            
-            # Strategy 2: Try to match on partial IDs broken by source type
-            if source_id_match not in source_map:
-                # Split on common delimiters
-                for snippet in evidence_snippets:
-                    snippet_id_parts = re.split(r'[-_:.]', snippet["id"])
-                    source_id_parts = re.split(r'[-_:.]', source_id_match)
-                    
-                    # Check if any significant parts match
-                    if (len(snippet_id_parts) > 0 and len(source_id_parts) > 0 and
-                        (snippet_id_parts[0] == source_id_parts[0] or  # First part matches (e.g., "pubmed")
-                         (len(snippet_id_parts) > 1 and len(source_id_parts) > 1 and
-                          snippet_id_parts[1] == source_id_parts[1]))):  # Second part matches (e.g., the ID number)
-                        source_map[source_id_match] = {
-                            "id": snippet["id"],
-                            "title": snippet["title"].strip(),
-                            "url": snippet["url"],
-                            "citation": snippet["citation"],
-                            "has_full_text": snippet.get("has_full_text", False),
-                            "journal": snippet.get("journal", ""),
-                            "year": snippet.get("year", "")
-                        }
-                        break
     
     # Handle generic [source_id] placeholder
     if "source_id" in matches:
@@ -205,10 +106,7 @@ def extract_and_link_sources(text, evidence_snippets):
                     "id": snippet["id"],
                     "title": snippet["title"].strip(),
                     "url": snippet["url"],
-                    "citation": snippet["citation"],
-                    "has_full_text": snippet.get("has_full_text", False),
-                    "journal": snippet.get("journal", ""),
-                    "year": snippet.get("year", "")
+                    "citation": snippet["citation"]
                 }
 
     # Replace source_id placeholders with actual links in the text
@@ -216,24 +114,13 @@ def extract_and_link_sources(text, evidence_snippets):
     for source_id_key, source_data in source_map.items():
         safe_id = re.escape(source_id_key)
         pattern = f"\\[{safe_id}\\]"
-        
-        # Create a more informative replacement that shows the title and preserves the source ID
-        title = source_data['title']
-        short_title = title[:60] + "..." if len(title) > 60 else title
-        
-        # Include year if available for better context
-        year_text = f" ({source_data['year']})" if source_data.get('year') else ""
-        
-        # Create the replacement with hover tooltip (works in many markdown renderers)
-        replacement = f"[{short_title}{year_text}]({source_data['url']} \"{title}\")"
-        
+        replacement = f"[{source_data['title']}]({source_data['url']})"
         linked_text = re.sub(pattern, replacement, linked_text)
     
     # Handle remaining [source_id] placeholders
     if "source_id" in source_map and "[source_id]" in linked_text:
         generic_data = source_map["source_id"]
-        year_text = f" ({generic_data['year']})" if generic_data.get('year') else ""
-        replacement = f"[{generic_data['title']}{year_text}]({generic_data['url']})"
+        replacement = f"[{generic_data['title']}]({generic_data['url']})"
         linked_text = re.sub(r'\[source_id\]', replacement, linked_text)
     
     # Final fallback for any [source_id] not mapped at all
@@ -243,29 +130,48 @@ def extract_and_link_sources(text, evidence_snippets):
 
 # Implement PubMed API integration for medical evidence retrieval
 def fetch_from_pubmed_api(query, max_results=3, api_key=None):
-    """
-    Enhanced PubMed API integration using E-utilities (ESearch + EFetch)
-    to retrieve more detailed article information with better abstracts.
-    """
+    """Fetch medical evidence from PubMed API using E-utilities"""
     results = []
-    base_url = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/"
     
-    # Step 1: Process the query with medical term extraction if it's complex enough
-    if len(query.split()) > 3:  # Only process complex queries
-        query_analysis = preprocess_medical_query(query)
-        search_query = query_analysis.get("optimized_search_query", query)
-        
-        # Get extracted terms for relevance scoring later
-        extracted_terms = query_analysis.get("extracted_terms", [])
+    # Clean up the query for better results
+    cleaned_query = re.sub(r'^(hi|hello|hey|greetings|good morning|good afternoon|good evening)[,\.]?\s+', '', query.lower())
+    cleaned_query = re.sub(r"(i'?m|i am)\s+a\s+\d+[-\s]year[-\s]old", '', cleaned_query)
+    cleaned_query = re.sub(r'(my name is|i am|i have been|i\'ve been|i was|i have|i\'ve had|i feel|i\'m feeling|i experienced)', '', cleaned_query)
+    
+    # Try to extract key medical symptoms
+    symptom_patterns = [
+        r'(muscle weakness)', r'(fatigue)', r'(rash)', r'(pain)', r'(swelling)',
+        r'(difficulty breathing|shortness of breath)', r'(fever)', r'(headache)',
+        r'(nausea|vomiting)', r'(dizziness)', r'(numbness)', r'(tingling)'
+    ]
+    
+    medical_terms = []
+    for pattern in symptom_patterns:
+        matches = re.findall(pattern, query.lower())
+        if matches:
+            medical_terms.extend(matches)
+    
+    # If we found medical terms, prioritize them in the search
+    if medical_terms:
+        search_query = " AND ".join(medical_terms)
+        # Add the complete cleaned query as a less weighted part
+        if cleaned_query:
+            search_query = f"({search_query}) OR ({cleaned_query})"
     else:
-        search_query = query
-        extracted_terms = [query]
+        # If no medical terms found, use the cleaned query
+        search_query = cleaned_query
     
-    # Step 2: Use ESearch to find article IDs
+    # Encode the query for the API
+    encoded_query = urllib.parse.quote(search_query)
+    
+    # Base URL for PubMed E-utilities
+    base_url = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/"
+    
+    # Search parameters
     search_params = {
         "db": "pubmed",
-        "term": search_query,
-        "retmax": max_results * 2,  # Get more results than needed for filtering
+        "term": encoded_query,
+        "retmax": max_results,
         "retmode": "json",
         "sort": "relevance"
     }
@@ -273,7 +179,7 @@ def fetch_from_pubmed_api(query, max_results=3, api_key=None):
     # Add API key if provided (increases rate limits)
     if api_key:
         search_params["api_key"] = api_key
-    
+        
     try:
         # First get article IDs
         search_response = requests.get(f"{base_url}esearch.fcgi", params=search_params)
@@ -284,132 +190,83 @@ def fetch_from_pubmed_api(query, max_results=3, api_key=None):
         search_data = search_response.json()
         
         if "esearchresult" in search_data and "idlist" in search_data["esearchresult"]:
-            pmids = search_data["esearchresult"]["idlist"]
-            
-            if not pmids:
-                return []
-            
-            # Step 3: Use EFetch to get article details
-            fetch_params = {
-                "db": "pubmed",
-                "id": ",".join(pmids),
-                "retmode": "xml",
-                "rettype": "abstract"
-            }
-            
-            if api_key:
-                fetch_params["api_key"] = api_key
-                
-            fetch_response = requests.get(f"{base_url}efetch.fcgi", params=fetch_params)
-            
-            if fetch_response.status_code != 200:
-                return []
-            
-            # Step 4: Parse XML to extract article details
-            root = ET.fromstring(fetch_response.text)
+            ids = search_data["esearchresult"]["idlist"]
             
-            # Process each article
-            for i, article in enumerate(root.findall(".//PubmedArticle")):
-                try:
-                    # Extract PMID
-                    pmid = article.findtext(".//PMID")
-                    
-                    # Extract title
-                    title = article.findtext(".//ArticleTitle") or "No title available"
-                    
-                    # Extract authors
-                    authors = []
-                    for author in article.findall(".//Author"):
-                        last_name = author.findtext(".//LastName") or ""
-                        initials = author.findtext(".//Initials") or ""
-                        if last_name or initials:
-                            authors.append(f"{last_name} {initials}".strip())
-                    
-                    # Extract journal and publication date
-                    journal = article.findtext(".//Journal/Title") or "Unknown Journal"
-                    year = article.findtext(".//PubDate/Year") or "Unknown Year"
-                    
-                    # Extract abstract with sections
-                    abstract_texts = []
-                    
-                    # Get structured abstract if available
-                    abstract_sections = article.findall(".//AbstractText")
-                    if abstract_sections:
-                        for section in abstract_sections:
-                            label = section.get("Label", "")
-                            text = section.text or ""
-                            if label and text:
-                                abstract_texts.append(f"{label}: {text}")
-                            elif text:
-                                abstract_texts.append(text)
-                    else:
-                        # Try single abstract text
-                        main_abstract = article.findtext(".//Abstract/AbstractText")
-                        if main_abstract:
-                            abstract_texts.append(main_abstract)
-                    
-                    abstract = " ".join(abstract_texts) if abstract_texts else "Abstract not available"
-                    
-                    # Check if full text is available in PMC
-                    pmc_id = article.findtext(".//ArticleId[@IdType='pmc']")
-                    pmc_link = f"https://www.ncbi.nlm.nih.gov/pmc/articles/{pmc_id}/" if pmc_id else None
-                    has_full_text = bool(pmc_id)
-                    
-                    # Build citation
-                    author_citation = ""
-                    if authors:
-                        if len(authors) == 1:
-                            author_citation = authors[0]
-                        elif len(authors) == 2:
-                            author_citation = f"{authors[0]} & {authors[1]}"
-                        else:
-                            author_citation = f"{authors[0]} et al."
-                    
-                    citation = f"{author_citation} ({year}). {title}. {journal}."
+            if ids:
+                # Fetch article details
+                fetch_params = {
+                    "db": "pubmed",
+                    "id": ",".join(ids),
+                    "retmode": "xml"
+                }
+                if api_key:
+                    fetch_params["api_key"] = api_key
                     
-                    # Build result object
-                    result = {
-                        "id": f"pubmed-{pmid}",
-                        "title": title,
-                        "authors": authors,
-                        "journal": journal,
-                        "year": year,
-                        "text": abstract,
-                        "url": f"https://pubmed.ncbi.nlm.nih.gov/{pmid}/",
-                        "pmc_url": pmc_link,
-                        "has_full_text": has_full_text,
-                        "source_type": "PubMed" + (" (Full Text Available)" if has_full_text else ""),
-                        "citation": citation
-                    }
+                fetch_response = requests.get(f"{base_url}efetch.fcgi", params=fetch_params)
+                
+                if fetch_response.status_code != 200:
+                    return []
                     
-                    # Calculate relevance score
-                    relevance_score = 0
-                    for term in extracted_terms:
-                        if term.lower() in title.lower():
-                            relevance_score += 2  # Higher weight for terms in title
-                        if term.lower() in abstract.lower():
-                            relevance_score += 1  # Lower weight for terms in abstract
+                try:
+                    # Parse XML response
+                    root = ET.fromstring(fetch_response.text)
                     
-                    result["relevance_score"] = relevance_score
-                    results.append(result)
+                    for article in root.findall(".//PubmedArticle"):
+                        try:
+                            pmid = article.findtext(".//PMID")
+                            title = article.findtext(".//ArticleTitle") or "No title available"
+                            
+                            # Extract abstract
+                            abstract_elements = article.findall(".//AbstractText")
+                            abstract = " ".join([(elem.text or "") for elem in abstract_elements])
+                            
+                            # Extract authors
+                            authors = []
+                            for author in article.findall(".//Author"):
+                                last_name = author.findtext(".//LastName") or ""
+                                initials = author.findtext(".//Initials") or ""
+                                if last_name and initials:
+                                    authors.append(f"{last_name} {initials}")
+                            
+                            author_str = ", ".join(authors[:3])
+                            if len(authors) > 3:
+                                author_str += " et al."
+                                
+                            # Extract journal and date
+                            journal = article.findtext(".//Journal/Title") or "Journal not specified"
+                            year = article.findtext(".//PubDate/Year") or "N/A"
+                            
+                            # Create citation
+                            citation = f"{author_str}. ({year}). {title}. {journal}. PMID: {pmid}"
+                            
+                            # Create direct access URL
+                            url = f"https://pubmed.ncbi.nlm.nih.gov/{pmid}/"
+                            
+                            # Check if free full text is available via PMC
+                            pmc_id = article.findtext(".//ArticleId[@IdType='pmc']")
+                            has_free_text = bool(pmc_id) or article.findtext(".//PublicationStatus") == "epublish"
+                            
+                            # If PMC ID is available, use that URL instead as it provides full text
+                            if pmc_id:
+                                url = f"https://www.ncbi.nlm.nih.gov/pmc/articles/{pmc_id}/"
+                                
+                            results.append({
+                                "id": f"pubmed:{pmid}",
+                                "title": title,
+                                "text": abstract[:800] + "..." if len(abstract) > 800 else abstract,
+                                "citation": citation,
+                                "url": url,
+                                "source_type": "PubMed" + (" (Free Full Text)" if has_free_text else ""),
+                                "is_open_access": has_free_text
+                            })
+                        except Exception:
+                            continue
+                except ET.ParseError:
+                    return []
                     
-                    # If we have enough results, break
-                    if len(results) >= max_results:
-                        break
-                        
-                except Exception as e:
-                    print(f"Error parsing article {pmid}: {str(e)}")
-                    continue
-            
-            # Sort by relevance score
-            results.sort(key=lambda x: x.get("relevance_score", 0), reverse=True)
-            return results[:max_results]
-    
-    except Exception as e:
-        print(f"Error in PubMed API fetch: {str(e)}")
+        return results
+    except Exception:
         return []
-        
-    return results
 
 def fetch_from_pmc_api(query, max_results=2, api_key=None):
     """Fetch free full text articles from PubMed Central (PMC)"""
@@ -749,346 +606,50 @@ def fetch_from_core_api(query, max_results=2, api_key=None):
     except Exception:
         return []
 
-# Europe PMC API integration for open access full-text articles
-def fetch_from_europe_pmc(query, max_results=2):
-    """
-    Fetch research articles from Europe PMC's API, which provides better
-    access to full-text content than regular PubMed.
-    """
-    results = []
-    
-    # Process the query with medical term extraction if it's complex enough
-    if len(query.split()) > 3:
-        query_analysis = preprocess_medical_query(query)
-        search_query = query_analysis.get("optimized_search_query", query)
-        extracted_terms = query_analysis.get("extracted_terms", [])
-    else:
-        search_query = query
-        extracted_terms = [query]
-    
-    try:
-        # Europe PMC REST API URL
-        api_url = "https://www.ebi.ac.uk/europepmc/webservices/rest/search"
-        
-        # Request parameters
-        params = {
-            "query": search_query,
-            "format": "json",
-            "resultType": "core",  # Core result fields
-            "pageSize": max_results * 2,  # Get more results for filtering
-            "cursorMark": "*",  # Starting point for pagination
-            "sort": "relevance",  # Sort by relevance
-            "synonym": "TRUE",  # Include MeSH term synonyms for better matching
-            "hasTextMinedTerms": "TRUE",  # Filter for text-mined terms
-            "hasLabsLinks": "TRUE"  # Include links to full text
-        }
-        
-        # Make the request
-        response = requests.get(api_url, params=params)
-        
-        if response.status_code != 200:
-            return []
-        
-        data = response.json()
-        
-        # Check if results exist
-        if "resultList" not in data or "result" not in data["resultList"]:
-            return []
-        
-        # Process results
-        for article in data["resultList"]["result"]:
-            try:
-                # Extract basic article info
-                pmid = article.get("pmid", "")
-                title = article.get("title", "No title available")
-                
-                # Extract abstract - Europe PMC sometimes provides better abstracts
-                abstract = article.get("abstractText", "Abstract not available")
-                
-                # Get author information
-                authors = []
-                if "authorList" in article and "author" in article["authorList"]:
-                    for author in article["authorList"]["author"]:
-                        author_name = []
-                        if "lastName" in author:
-                            author_name.append(author["lastName"])
-                        if "initials" in author:
-                            author_name.append(author["initials"])
-                        if author_name:
-                            authors.append(" ".join(author_name))
-                
-                # Get journal info
-                journal = article.get("journalTitle", "Unknown Journal")
-                year = article.get("pubYear", "Unknown Year")
-                
-                # Check if full text is available
-                has_full_text = False
-                full_text_url = None
-                
-                # Europe PMC provides several indicators for full text
-                if "isOpenAccess" in article and article["isOpenAccess"] == "Y":
-                    has_full_text = True
-                
-                # Get PMC ID if available
-                pmc_id = article.get("pmcid", "")
-                if pmc_id:
-                    has_full_text = True
-                    full_text_url = f"https://www.ncbi.nlm.nih.gov/pmc/articles/{pmc_id}/"
-                
-                # Build citation
-                author_citation = ""
-                if authors:
-                    if len(authors) == 1:
-                        author_citation = authors[0]
-                    elif len(authors) == 2:
-                        author_citation = f"{authors[0]} & {authors[1]}"
-                    else:
-                        author_citation = f"{authors[0]} et al."
-                
-                citation = f"{author_citation} ({year}). {title}. {journal}."
-                
-                # Building the URL - prefer Europe PMC links as they often have better HTML rendering
-                url = f"https://europepmc.org/article/MED/{pmid}" if pmid else ""
-                
-                # If no PMID but has DOI, use DOI link
-                if not url and "doi" in article:
-                    url = f"https://doi.org/{article['doi']}"
-                
-                # Use PMC link if available
-                if full_text_url:
-                    url = full_text_url
-                
-                # Create result object
-                result = {
-                    "id": f"epmc-{pmid if pmid else article.get('id', uuid.uuid4().hex[:8])}",
-                    "title": title,
-                    "authors": authors,
-                    "journal": journal,
-                    "year": year,
-                    "text": abstract,
-                    "url": url,
-                    "has_full_text": has_full_text,
-                    "source_type": "Europe PMC" + (" (Full Text Available)" if has_full_text else ""),
-                    "citation": citation
-                }
-                
-                # Calculate relevance score
-                relevance_score = 0
-                for term in extracted_terms:
-                    if term.lower() in title.lower():
-                        relevance_score += 2  # Higher weight for terms in title
-                    if term.lower() in abstract.lower():
-                        relevance_score += 1  # Lower weight for terms in abstract
-                
-                result["relevance_score"] = relevance_score
-                results.append(result)
-                
-            except Exception as e:
-                print(f"Error processing Europe PMC article: {str(e)}")
-                continue
-        
-        # Sort by relevance score
-        results.sort(key=lambda x: x.get("relevance_score", 0), reverse=True)
-        return results[:max_results]
-        
-    except Exception as e:
-        print(f"Error in Europe PMC API fetch: {str(e)}")
-        return []
-        
-    return results
-
-# Relevance filtering for medical evidence
-def assess_evidence_relevance(query, evidence_snippets, max_results=5):
-    """
-    Use GPT-4o to assess the relevance of evidence snippets to the user query
-    and select the most applicable ones for response generation.
-    """
-    if not evidence_snippets:
-        return []
-    
-    try:
-        # If we already have ranking from the API calls, use that first
-        if "relevance_score" in evidence_snippets[0]:
-            # Sort by the existing relevance score
-            evidence_snippets.sort(key=lambda x: x.get("relevance_score", 0), reverse=True)
-            
-            # If we have less than or equal to max_results, return all
-            if len(evidence_snippets) <= max_results:
-                return evidence_snippets
-            
-            # If more than 10 snippets, only run GPT filtering on the top 10
-            snippets_to_assess = evidence_snippets[:min(10, len(evidence_snippets))]
-        else:
-            snippets_to_assess = evidence_snippets
-        
-        # Prepare snippets for assessment
-        snippets_text = ""
-        for i, snippet in enumerate(snippets_to_assess):
-            snippets_text += f"---ARTICLE {i+1}---\n"
-            snippets_text += f"Title: {snippet['title']}\n"
-            snippets_text += f"Source: {snippet['source_type']}\n"
-            
-            # Limit text length to avoid token limits
-            text = snippet['text']
-            if len(text) > 800:
-                text = text[:800] + "..."
-                
-            snippets_text += f"Content: {text}\n\n"
-        
-        # Create system prompt for assessment
-        system_prompt = """
-        You are an expert medical research assistant helping to select the most relevant medical evidence for a patient query.
-        
-        Review the provided medical articles and evaluate their relevance to the patient's query.
-        Focus on these factors:
-        1. Direct relevance to the medical condition or symptoms described
-        2. Quality and comprehensiveness of the information
-        3. Whether it covers diagnosis, treatment, or management aspects needed
-        4. Recency and reliability of the source
-        5. Presence of actionable information that would help answer the query
-        
-        For each article, assign a relevance score from 1-10 (10 being most relevant).
-        
-        Return your assessment as a JSON object:
-        {
-          "article_rankings": [
-            {"article_index": 1, "relevance_score": 8, "reason": "Directly addresses the primary symptom with treatment options"},
-            {"article_index": 2, "relevance_score": 4, "reason": "Tangentially related but not focused on the main condition"},
-            ...
-          ],
-          "recommended_indices": [1, 3, 5]  // Indices of the most relevant articles to use, in order of relevance
-        }
-        
-        Include only the JSON in your response.
-        """
-        
-        # Call GPT-4o
-        response = openai.ChatCompletion.create(
-            model="gpt-4o-mini",
-            messages=[
-                {"role": "system", "content": system_prompt},
-                {"role": "user", "content": f"Patient query: {query}\n\nArticles to assess:\n{snippets_text}"}
-            ],
-            temperature=0.1,
-            max_tokens=1000,
-        )
-        
-        result_text = response.choices[0].message['content']
-        
-        # Extract the JSON
-        try:
-            # Find JSON object
-            json_match = re.search(r'({[\s\S]*})', result_text)
-            if json_match:
-                json_str = json_match.group(1)
-                assessment = json.loads(json_str)
-            else:
-                assessment = json.loads(result_text)
-            
-            # Get recommended indices
-            recommended_indices = assessment.get("recommended_indices", [])
-            
-            # If no recommendations, fall back to top 5 from original list
-            if not recommended_indices and len(evidence_snippets) > max_results:
-                return evidence_snippets[:max_results]
-            
-            # Filter evidence snippets based on recommended indices
-            filtered_snippets = []
-            for idx in recommended_indices:
-                if 0 <= idx-1 < len(snippets_to_assess):
-                    filtered_snippets.append(snippets_to_assess[idx-1])
-            
-            # If we have less than max_results, add more from original sorted list
-            if len(filtered_snippets) < max_results and len(evidence_snippets) > len(filtered_snippets):
-                # Get indices of snippets already included
-                included_indices = set()
-                for snippet in filtered_snippets:
-                    for i, original in enumerate(evidence_snippets):
-                        if snippet["id"] == original["id"]:
-                            included_indices.add(i)
-                            break
-                
-                # Add more snippets that weren't already included
-                for i, snippet in enumerate(evidence_snippets):
-                    if i not in included_indices and len(filtered_snippets) < max_results:
-                        filtered_snippets.append(snippet)
-            
-            return filtered_snippets[:max_results]
-            
-        except json.JSONDecodeError:
-            # Fallback to sorted snippets if parsing fails
-            return evidence_snippets[:max_results]
-    
-    except Exception as e:
-        print(f"Error in evidence relevance assessment: {str(e)}")
-        # Fallback to the original sorting
-        return evidence_snippets[:max_results]
-
 # Enhanced RAG System with real medical sources
 def fetch_medical_evidence(query, max_results=5):
-    """Fetch medical evidence from multiple sources using real APIs with improved relevance assessment"""
-    all_results = []
+    """Fetch medical evidence from multiple sources using real APIs"""
+    results = []
     
     # Define API keys
     pubmed_api_key = os.environ.get("PUBMED_API_KEY")
     core_api_key = os.environ.get("CORE_API_KEY")
     
-    # Step 1: Query preprocessing with GPT-4o to extract medical terms
-    query_analysis = preprocess_medical_query(query)
-    processed_query = query_analysis.get("optimized_search_query", query)
-    
-    # Step 2: Gather evidence from multiple sources
-    
     # Source 1: PubMed API - prioritize for relevant medical research
-    pubmed_results = fetch_from_pubmed_api(query, max_results=max(3, max_results//2), api_key=pubmed_api_key)
+    pubmed_results = fetch_from_pubmed_api(query, max_results=max(2, max_results//2), api_key=pubmed_api_key)
     if pubmed_results:
-        all_results.extend(pubmed_results)
+        results.extend(pubmed_results)
     
-    # Source 2: Europe PMC - for better full text access
-    epmc_results = fetch_from_europe_pmc(query, max_results=max(2, max_results//3))
-    if epmc_results:
-        all_results.extend(epmc_results)
-    
-    # Source 3: PubMed Central - free full text articles
-    if len(all_results) < max_results * 2:  # Get more than needed for filtering
-        remaining = (max_results * 2) - len(all_results)
-        pmc_results = fetch_from_pmc_api(processed_query, max_results=remaining, api_key=pubmed_api_key)
+    # Source 2: PubMed Central - free full text articles
+    if len(results) < max_results:
+        remaining = max_results - len(results)
+        pmc_results = fetch_from_pmc_api(query, max_results=remaining, api_key=pubmed_api_key)
         if pmc_results:
-            all_results.extend(pmc_results)
+            results.extend(pmc_results)
     
-    # Source 4: CORE API - open access research papers
-    if len(all_results) < max_results * 2:
-        remaining = (max_results * 2) - len(all_results)
-        core_results = fetch_from_core_api(processed_query, max_results=remaining, api_key=core_api_key)
+    # Source 3: CORE API - open access research papers
+    if len(results) < max_results:
+        remaining = max_results - len(results)
+        core_results = fetch_from_core_api(query, max_results=remaining, api_key=core_api_key)
         if core_results:
-            all_results.extend(core_results)
+            results.extend(core_results)
     
-    # Source 5: WHO Guidelines - if still need more results
-    if len(all_results) < max_results:
-        remaining = max_results - len(all_results)
-        who_results = fetch_from_who_api(processed_query, max_results=remaining)
+    # Source 4: WHO Guidelines - if still need more results
+    if len(results) < max_results:
+        remaining = max_results - len(results)
+        who_results = fetch_from_who_api(query, max_results=remaining)
         if who_results:
-            all_results.extend(who_results)
+            results.extend(who_results)
     
-    # Step 3: Initial sorting by source quality and full text availability
     # Prioritize sources with full text for better diagnosis
-    all_results.sort(key=lambda x: (
-        x.get("relevance_score", 0),  # First by relevance score if available
-        "Full Text" in x.get("source_type", ""),  # Then by full text availability
-        "Europe PMC" in x.get("source_type", ""),  # Europe PMC often has better full text access
-        "CORE" in x.get("source_type", ""),  # CORE for open access
-        "PMC" in x.get("source_type", ""),  # PMC for free full text
-        "PubMed" in x.get("source_type", "")  # Regular PubMed last
+    results.sort(key=lambda x: (
+        "Full Text" in x.get("source_type", ""), 
+        "CORE" in x.get("source_type", ""),
+        "PMC" in x.get("source_type", ""),
+        "PubMed" in x.get("source_type", "")
     ), reverse=True)
     
-    # Step 4: Relevance assessment with GPT-4o
-    # Only run this if we have more results than needed
-    if len(all_results) > max_results:
-        filtered_results = assess_evidence_relevance(query, all_results, max_results)
-    else:
-        filtered_results = all_results
-    
-    return filtered_results  # Return the filtered and ranked results
+    return results[:max_results]  # Limit to requested number after sorting
 
 # Function to parse doctor agent responses
 def parse_doctor_response(response_text):
@@ -1158,7 +719,7 @@ def doctor_agent(messages):
 
 # Single orchestrator turn with enhanced reasoning and citation tracking
 def orchestrator_chat(history, query, use_rag, is_follow_up=False):
-    """Handle a single turn of conversation with the doctor agent with improved evidence utilization"""
+    """Handle a single turn of conversation with the doctor agent"""
     # Select appropriate system prompt based on whether this is a follow-up
     if is_follow_up:
         system = {"role": "system", "content": FOLLOW_UP_PROMPT}
@@ -1167,97 +728,30 @@ def orchestrator_chat(history, query, use_rag, is_follow_up=False):
 
     msgs = [system] + history
 
-    # Evidence gathering with improved relevance
+    # Evidence gathering
     evidence_snippets = []
     if use_rag:
         # Only fetch and format evidence if RAG is enabled
         evidence_snippets = fetch_medical_evidence(query)
 
-        # Format evidence for the model with enhanced structure
+        # Format evidence for the model
         if evidence_snippets:
             evidence_text = "MEDICAL EVIDENCE FROM AUTHORITATIVE SOURCES:\n\n"
 
             for i, snippet in enumerate(evidence_snippets):
-                # Create a more readable format with clear section separation
-                evidence_text += f"SOURCE {i+1}: [{snippet['id']}]\n"
-                evidence_text += f"TITLE: {snippet['title']}\n"
-                
-                # Add author information if available
-                if "authors" in snippet and snippet["authors"]:
-                    authors_text = ", ".join(snippet["authors"][:3])
-                    if len(snippet["authors"]) > 3:
-                        authors_text += " et al."
-                    evidence_text += f"AUTHORS: {authors_text}\n"
-                
-                # Add journal and year if available
-                journal_info = []
-                if "journal" in snippet and snippet["journal"]:
-                    journal_info.append(snippet["journal"])
-                if "year" in snippet and snippet["year"]:
-                    journal_info.append(snippet["year"])
-                if journal_info:
-                    evidence_text += f"PUBLICATION: {', '.join(journal_info)}\n"
-                
-                # Format the source type with emphasis on full text availability
-                source_type = snippet.get("source_type", "Unknown Source")
-                evidence_text += f"SOURCE TYPE: {source_type}\n"
-                
-                # Format the text with section labels if available
-                text = snippet.get("text", "").strip()
-                # Split by section labels if they exist (e.g., "METHODS:", "RESULTS:")
-                sections = re.split(r'([A-Z][A-Z\s]+:)', text)
-                
-                if len(sections) > 1:
-                    formatted_text = ""
-                    current_section = None
-                    for section in sections:
-                        if re.match(r'[A-Z][A-Z\s]+:', section):
-                            current_section = section
-                            formatted_text += f"\n{current_section}\n"
-                        elif current_section is not None:
-                            formatted_text += section.strip() + "\n"
-                    evidence_text += f"CONTENT:\n{formatted_text}\n"
-                else:
-                    evidence_text += f"CONTENT:\n{text}\n"
-                
-                # Add URL for verification
-                evidence_text += f"URL: {snippet.get('url', 'No URL available')}\n"
-                
-                # Additional link to full text if available
-                if snippet.get("has_full_text", False) and snippet.get("pmc_url"):
-                    evidence_text += f"FULL TEXT: {snippet.get('pmc_url')}\n"
-                
-                # Add citation
-                evidence_text += f"CITATION: {snippet.get('citation', 'Citation not available')}\n\n"
-                
-                # Add a separator between articles
-                evidence_text += "----------------------------------------------\n\n"
+                evidence_text += f"[{snippet['id']}] {snippet['title']}\n"
+                evidence_text += f"Source: {snippet['source_type']}\n"
+                evidence_text += f"Content: {snippet['text']}\n"
+                evidence_text += f"Citation: {snippet['citation']}\n"
+                evidence_text += f"URL: {snippet['url']}\n\n"
 
             # Enhanced instructions for better source utilization
-            evidence_text += """CITATION AND EVIDENCE USE INSTRUCTIONS: 
-
-1. IMPORTANT: When referencing these sources in your response, use the format [source_id] to cite them.
-   Example: "Recent studies have shown improved outcomes with early intervention [pubmed-12345678]."
-
-2. Focus on articles marked "Full Text Available" as they provide more comprehensive information.
-
-3. When multiple sources support a claim, cite all of them for stronger evidence.
-   Example: "This treatment approach is supported by multiple studies [pubmed-12345678][epmc-87654321]."
-
-4. For each diagnostic or treatment recommendation, provide at least one citation.
-
-5. Read the CONTENT sections carefully and extract specific details - don't just cite generally.
-
-6. If sources have conflicting information, acknowledge this and present both perspectives with citations.
-
-7. Use the most recent sources when available, especially for treatment recommendations.
-
-8. For each recommendation, try to provide evidence on:
-   - Efficacy (how well it works)
-   - Safety (potential side effects)
-   - Appropriateness for this specific patient scenario
-
-9. If full text is available, prioritize information from those sources as they contain more complete data.
+            evidence_text += """CITATION INSTRUCTIONS: 
+1. When referencing these sources in your response, use the format [source_id] to cite them.
+2. Prioritize information from sources marked with "Full Text Available" as they provide more comprehensive data.
+3. CORE API sources provide open access full text articles that are particularly valuable for diagnosis.
+4. Use the most relevant medical evidence to support your diagnostic reasoning.
+5. Try to cite multiple sources to provide a well-rounded assessment.
 """
 
             msgs.append({"role": "system", "content": evidence_text})
@@ -1272,38 +766,31 @@ def orchestrator_chat(history, query, use_rag, is_follow_up=False):
     if use_rag:
         output_instructions = """
         Please structure your response clearly.
-
         **Priority 1: Ask Clarifying Questions**
         If the user's query lacks detail for a proper assessment (e.g., age, specific symptoms, medical history, duration, severity), your HIGHEST priority is to ask these questions first. Do not provide a diagnosis or plan until sufficient information is gathered.
-
         **Priority 2: Main Response (After Clarification)**
         Once sufficient information is available (either initially or after asking questions), provide:
         1.  A direct answer to the patient's concerns.
         2.  If appropriate, a clear diagnosis or differential diagnosis.
         3.  Recommendations for a treatment plan or next steps.
         4.  Ensure you cite medical evidence using the [source_id] format for any claims or information taken from the provided MEDICAL EVIDENCE snippets.
-
         **After your main response, ALWAYS include these sections:**
         -   **Reasoning**: Bullet points detailing your clinical reasoning.
-        -   **Sources**: A list of all references cited in your main response, with their full titles and publication information.
+        -   **Sources**: A list of all references cited in your main response, using their full titles and corresponding URLs if they were linked (e.g., [Title of Source](URL)). If a source was just an ID without a direct link in the text, list its ID or citation.
         """
     else:
         # Different instructions when RAG is disabled - no mention of sources or citations
         output_instructions = """
         Please structure your response clearly.
-
         **Priority 1: Ask Clarifying Questions**
         If the user's query lacks detail for a proper assessment (e.g., age, specific symptoms, medical history, duration, severity), your HIGHEST priority is to ask these questions first. Do not provide a diagnosis or plan until sufficient information is gathered.
-
         **Priority 2: Main Response (After Clarification)**
         Once sufficient information is available (either initially or after asking questions), provide:
         1.  A direct answer to the patient's concerns.
         2.  If appropriate, a clear diagnosis or differential diagnosis.
         3.  Recommendations for a treatment plan or next steps.
-
         **After your main response, ALWAYS include this section:**
         -   **Reasoning**: Bullet points detailing your clinical reasoning.
-
         IMPORTANT: Since database search is disabled, do not include citations or sources in your response.
         """
 
@@ -1313,42 +800,261 @@ def orchestrator_chat(history, query, use_rag, is_follow_up=False):
     # Get response from doctor agent
     response = doctor_agent(msgs)
 
-    # Extract and process sources
-    explanation = None
-    evidence = None
-    
+    # Process the response based on whether RAG is enabled
     if use_rag:
-        # Parse the doctor's response
+        # Process the response to replace source placeholders with actual links
+        linked_response, source_map = extract_and_link_sources(response, evidence_snippets)
+        
+        # Parse the response
+        parsed_response = parse_doctor_response(linked_response)
+        
+        # Enhance source information with evidence snippets data
+        enhanced_sources = []
+        # Use the source_map from extract_and_link_sources as the primary guide for cited sources
+        for source_id_key, mapped_data in source_map.items():
+            enhanced_sources.append({
+                "id": mapped_data["id"], # This is the original ID from the snippet
+                "title": mapped_data["title"],
+                "citation": mapped_data["citation"],
+                "url": mapped_data["url"],
+                "source_type": "Referenced Source" # Or derive from snippet if available
+            })
+
+        # Get source types and open access status from original snippets
+        for es in enhanced_sources:
+            for snippet in evidence_snippets:
+                if es["id"] == snippet["id"]:
+                    es["source_type"] = snippet.get("source_type", "Referenced Source")
+                    es["is_open_access"] = snippet.get("is_open_access", False)
+                    break
+        
+        # If there are sources in parsed_response["sources"] that are not in source_map
+        # (e.g., LLM hallucinated an ID or cited something not in snippets), add them.
+        current_enhanced_ids = {es['id'] for es in enhanced_sources}
+        
+        for source_text in parsed_response["sources"]: # source_text could be "[id]", "title (url)", or just "citation"
+            source_id_candidate = source_text.strip("[]") # Basic extraction
+            
+            # Check if this source_id_candidate was part of the original evidence
+            found_in_evidence = False
+            for snippet in evidence_snippets:
+                if source_id_candidate == snippet["id"]:
+                    if source_id_candidate not in current_enhanced_ids:
+                        enhanced_sources.append({
+                            "id": snippet["id"],
+                            "title": snippet["title"],
+                            "citation": snippet["citation"],
+                            "url": snippet["url"],
+                            "source_type": snippet["source_type"],
+                            "is_open_access": snippet.get("is_open_access", False)
+                        })
+                        current_enhanced_ids.add(snippet["id"]) # Add to set to avoid re-adding
+                    found_in_evidence = True
+                    break
+            
+            if not found_in_evidence:
+                # If it's not in source_map and not directly in evidence_snippets by a simple ID match,
+                # it might be a raw citation or a URL. Add it with available info.
+                is_duplicate = False
+                for es_item in enhanced_sources:
+                    if es_item["title"] == source_text or es_item["url"] == source_text or es_item["citation"] == source_text:
+                        is_duplicate = True
+                        break
+                if not is_duplicate and source_text not in current_enhanced_ids:
+                     # Try to extract a URL if present in markdown format
+                    url_match = re.search(r'\[(.*?)\]\((https?://[^)]+)\)', source_text)
+                    if url_match:
+                        title = url_match.group(1)
+                        url = url_match.group(2)
+                    else:
+                        title = source_text # Could be a citation string or a plain title
+                        url = "" # No URL found directly
+
+                    enhanced_sources.append({
+                        "id": source_id_candidate, # Use the candidate, might be a simple title or part of citation
+                        "title": title,
+                        "citation": source_text, # The original text from LLM's source list
+                        "url": url,
+                        "source_type": "Referenced Source (uncategorized)"
+                    })
+                    current_enhanced_ids.add(source_id_candidate)
+
+        # Add the enhanced sources back to the parsed response
+        parsed_response["enhanced_sources"] = enhanced_sources
+        main_response = linked_response
+    else:
+        # If RAG is disabled, just parse the response without source processing
         parsed_response = parse_doctor_response(response)
+        parsed_response["enhanced_sources"] = []
+        main_response = response
+
+    # Create detailed explanation with reasoning and sources
+    explanation = []
+
+    # Add reasoning section
+    if parsed_response["reasoning"]:
+        explanation.append("## REASONING")
+        for i, reason in enumerate(parsed_response["reasoning"]):
+            explanation.append(f"{i+1}. {reason}")
+        explanation.append("")
+
+    # Only add sources section if RAG is enabled
+    if use_rag and parsed_response["enhanced_sources"]:
+        explanation.append("## SOURCES USED")
+
+        # Add enhanced sources first (these are the ones actually cited in the response)
+        source_added_count = 0
         
-        # Main response with embedded citations
-        response = parsed_response.get("main_response", response)
+        unique_sources_for_display = {} # id: {title, url, citation, source_type}
+        for source in parsed_response["enhanced_sources"]:
+            # Prefer using the mapped title and URL from extract_and_link_sources if available
+            display_id = source.get('id', source.get('title', 'Unknown Source'))
+
+            if display_id not in unique_sources_for_display:
+                 unique_sources_for_display[display_id] = {
+                    "title": source.get('title', 'N/A'),
+                    "url": source.get('url', ''),
+                    "citation": source.get('citation', ''),
+                    "source_type": source.get('source_type', 'Referenced Source'),
+                    "is_open_access": source.get('is_open_access', False)
+                 }
+
+        # Create a categorized display of sources
+        source_categories = {
+            "CORE": [],   # CORE API full text
+            "PMC": [],    # PubMed Central full text
+            "PubMed": [], # PubMed abstracts
+            "WHO": [],    # WHO guidelines
+            "Other": []   # Uncategorized
+        }
         
-        # Extract reasoning for separate display
-        reasoning = parsed_response.get("reasoning", [])
-        if reasoning:
-            if isinstance(reasoning, list):
-                explanation = "\n".join([f"- {r}" for r in reasoning])
+        # Categorize sources
+        for key, src_data in unique_sources_for_display.items():
+            source_type = src_data['source_type']
+            
+            if "CORE" in source_type:
+                source_categories["CORE"].append((key, src_data))
+            elif "PMC" in source_type:
+                source_categories["PMC"].append((key, src_data))
+            elif "PubMed" in source_type:
+                source_categories["PubMed"].append((key, src_data))
+            elif "WHO" in source_type:
+                source_categories["WHO"].append((key, src_data))
             else:
-                explanation = reasoning
+                source_categories["Other"].append((key, src_data))
         
-        # Process source citations
-        if evidence_snippets:
-            # Replace source IDs with actual links
-            linked_response, source_map = extract_and_link_sources(response, evidence_snippets)
-            response = linked_response
+        # Display sources by category
+        for category, sources in source_categories.items():
+            if sources:
+                if category != "Other":  # Skip category header for Other
+                    explanation.append(f"### {category} Sources:")
+                
+                for key, src_data in sources:
+                    title = src_data['title']
+                    url = src_data['url']
+                    is_open_access = src_data.get('is_open_access', False)
+                    
+                    if url: # If URL exists, make it a markdown link
+                        explanation.append(f"- [{title}]({url}) {' 🔓' if is_open_access else ''}")
+                    else: # Otherwise, just list the title or ID
+                        explanation.append(f"- {title}")
+
+                    if src_data['source_type']:
+                        explanation.append(f"  Source Type: {src_data['source_type']}")
+                    if src_data['citation']: # Always show citation if available
+                        explanation.append(f"  Citation: {src_data['citation']}")
+                    explanation.append("") # Add a blank line for spacing
+                    source_added_count += 1
+
+        if source_added_count == 0 and parsed_response["sources"]: # Fallback to raw sources if enhanced list is empty but LLM listed some
+            explanation.append("## SOURCES MENTIONED (Raw)") # Indicate these are less processed
+            for source_text in parsed_response["sources"]:
+                explanation.append(f"- {source_text.strip()}")
+            explanation.append("")
+            source_added_count +=1
+
+        # If we still have no sources, remove the header
+        if source_added_count == 0: # Check if any sources were actually added to explanation
+            # Remove "## SOURCES USED" header if it was added but no sources followed
+            if explanation and explanation[-1] == "## SOURCES USED":
+                explanation.pop()
+
+        # Enhanced version to display clickable article links
+        # Check if we have evidence snippets but no sources in the explanation
+        if evidence_snippets and "## SOURCES USED" not in "\n".join(explanation):
+            # If AI didn't explicitly cite sources, show available evidence anyway
+            additional_explanation = ["\n## AVAILABLE MEDICAL SOURCES"]
             
-            # Collect evidence for display
-            evidence = []
-            for source_id, source_data in source_map.items():
-                evidence.append({
-                    "id": source_data["id"],
-                    "title": source_data["title"],
-                    "url": source_data["url"],
-                    "citation": source_data["citation"]
-                })
-    
-    return response, explanation, evidence
+            # Create categorized display of all available sources
+            categorized_snippets = {
+                "CORE Open Access": [],   # CORE API full text
+                "PubMed Central": [],     # PMC full text
+                "PubMed": [],             # PubMed abstracts
+                "WHO Guidelines": [],     # WHO guidelines
+                "Other": []               # Uncategorized
+            }
+            
+            # Categorize snippets
+            for snippet in evidence_snippets:
+                source_type = snippet.get("source_type", "")
+                
+                if "CORE" in source_type:
+                    categorized_snippets["CORE Open Access"].append(snippet)
+                elif "PMC" in source_type:
+                    categorized_snippets["PubMed Central"].append(snippet)
+                elif "PubMed" in source_type and "PMC" not in source_type:
+                    categorized_snippets["PubMed"].append(snippet)
+                elif "WHO" in source_type:
+                    categorized_snippets["WHO Guidelines"].append(snippet)
+                else:
+                    categorized_snippets["Other"].append(snippet)
+            
+            # Display snippets by category
+            for category, snippets in categorized_snippets.items():
+                if snippets:
+                    if category != "Other":  # Skip category header for Other
+                        additional_explanation.append(f"### {category}:")
+                    
+                    for snippet in snippets:
+                        title = snippet.get("title", "Unknown Title")
+                        url = snippet.get("url", "")
+                        source_type = snippet.get("source_type", "Medical Source")
+                        is_open_access = snippet.get("is_open_access", False)
+                        
+                        if url:
+                            # Format as clickable markdown link with open access indicator
+                            additional_explanation.append(f"- [{title}]({url}) {' 🔓' if is_open_access else ''}")
+                        else:
+                            additional_explanation.append(f"- {title} {' 🔓' if is_open_access else ''}")
+                            
+                        if "source_type" in snippet:
+                            additional_explanation.append(f"  Source Type: {snippet['source_type']}")
+                        if "citation" in snippet:
+                            additional_explanation.append(f"  Citation: {snippet['citation']}")
+                        additional_explanation.append("")
+                
+            # Add to the main explanation
+            explanation.extend(additional_explanation)
+
+        # Add a note about data availability
+        data_availability_note = [
+            "\n## DATA AVAILABILITY NOTE",
+            "- PubMed sources typically provide abstracts only, unless marked as free full text",
+            "- PubMed Central (PMC) sources provide complete free full text articles",
+            "- CORE Open Access sources provide full text content from research repositories",
+            "- WHO Guidelines provide official medical recommendations and protocols",
+            "- Sources marked with 🔓 indicate open access content with full text available"
+        ]
+        explanation.extend(data_availability_note)
+
+    # Format explanation as string
+    explanation_text = "\n".join(explanation)
+
+    # Update conversation history
+    history.append({"role": "user", "content": query})
+    history.append({"role": "assistant", "content": main_response})
+
+    return main_response, explanation_text, evidence_snippets
 
 # Enhanced interactive loop with better handling of consultations
 def run_consultation(use_rag=True):