Upload 8 files

Medcleave.iml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager" inherit-compiler-output="true">
+    <exclude-output />
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="jdk" jdkName="Python 3.12 (Medcleave)" jdkType="Python SDK" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

app.py

@@ -0,0 +1,176 @@
+import faiss
+import numpy as np
+import torch
+from transformers import AutoModel, AutoTokenizer, pipeline
+import requests
+from bs4 import BeautifulSoup
+import os
+import gradio as gr
+
+# Step 1: Define PromptTemplate class using LangChain's format
+class PromptTemplate:
+    def __init__(self, template):
+        self.template = template
+
+    def format(self, **kwargs):
+        formatted_text = self.template
+        for key, value in kwargs.items():
+            formatted_text = formatted_text.replace("{" + key + "}", str(value))
+        return formatted_text
+
+# Step 2: Load embedding model and tokenizer
+embedding_model_name = "ls-da3m0ns/bge_large_medical"
+embedding_tokenizer = AutoTokenizer.from_pretrained(embedding_model_name)
+embedding_model = AutoModel.from_pretrained(embedding_model_name)
+embedding_model.eval()  # Set model to evaluation mode
+
+# Move the embedding model to GPU
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+embedding_model.to(device)
+
+# Step 3: Load Faiss index
+index_file = "faiss_index.index"
+if os.path.exists(index_file):
+    index = faiss.read_index(index_file)
+    assert isinstance(index, faiss.IndexFlat), "Expected Faiss IndexFlat type"
+    assert index.d == 1024, f"Expected index dimension 1024, but got {index.d}"
+else:
+    raise ValueError(f"Faiss index file '{index_file}' not found.")
+
+# Step 4: Prepare URLs
+urls_file = "crawled_urls.txt"
+if os.path.exists(urls_file):
+    with open(urls_file, "r") as f:
+        urls = [line.strip() for line in f]
+else:
+    raise ValueError(f"URLs file '{urls_file}' not found.")
+
+# Step 5: Check if sample embeddings file exists, if not create it
+sample_embeddings_file = "sample_embeddings.npy"
+if not os.path.exists(sample_embeddings_file):
+    print("Sample embeddings file not found, creating new sample embeddings...")
+    # Generate sample data to fit PCA
+    sample_texts = [
+        "medical diagnosis",
+        "healthcare treatment",
+        "patient care",
+        "clinical research",
+        "disease prevention"
+    ]
+
+    sample_embeddings = []
+    for text in sample_texts:
+        inputs = embedding_tokenizer(text, return_tensors="pt").to(device)
+        with torch.no_grad():
+            outputs = embedding_model(**inputs)
+            embedding = outputs.last_hidden_state.mean(dim=1).cpu().numpy()
+            sample_embeddings.append(embedding)
+
+    sample_embeddings = np.vstack(sample_embeddings)
+    np.save(sample_embeddings_file, sample_embeddings)
+else:
+    sample_embeddings = np.load(sample_embeddings_file)
+
+# Step 6: Define function for similarity search
+def search_similar(query_text, top_k=3):
+    inputs = embedding_tokenizer(query_text, return_tensors="pt").to(device)
+    with torch.no_grad():
+        outputs = embedding_model(**inputs)
+        query_embedding = outputs.last_hidden_state.mean(dim=1).cpu().numpy()
+
+    query_embedding = query_embedding / np.linalg.norm(query_embedding)
+    query_embedding = query_embedding.reshape(1, -1).astype(np.float32)
+    _, idx = index.search(query_embedding, top_k)
+
+    results = []
+    for i in range(top_k):
+        key = int(idx[0][i])
+        results.append(urls[key])  # Return URLs only for simplicity
+
+    return results
+
+# Step 7: Function to extract content from URLs
+def extract_content(url):
+    try:
+        response = requests.get(url)
+        response.raise_for_status()
+        soup = BeautifulSoup(response.content, 'html.parser')
+
+        # Example: Extracting relevant content based on query
+        paragraphs = soup.find_all('p')
+        relevant_content = ""
+        for para in paragraphs:
+            relevant_content += para.get_text().strip()
+
+        return relevant_content.strip()  # Return relevant content as a single string
+    except requests.RequestException as e:
+        print(f"Error fetching content from {url}: {e}")
+        return ""
+
+# Step 8: Use the LangChain text generation pipeline for generating answers
+generation_model_name = "microsoft/Phi-3-mini-4k-instruct"
+text_generator = pipeline("text-generation", model=generation_model_name, device=0)
+
+# Step 9: Function to generate answer based on query and content
+def generate_answer(query, contents):
+    answers = []
+    prompt_template = PromptTemplate("""
+    
+    ### Medical Assistant Context ###
+As a helpful medical assistant, I'm here to assist you with your query.
+
+### Medical Query ###
+Query: {query}
+
+### Explanation ###
+{generated_text}
+
+### Revised Response ###
+Response: {generated_text}
+""")
+
+    for content in contents:
+        if content:
+            prompt = prompt_template.format(query=query, content=content, generated_text="")
+            # Ensure prompt is wrapped in a list for text generation
+            generated_texts = text_generator([prompt], max_new_tokens=200, num_return_sequences=1, truncation=True)
+            # Debugging: print the generated_texts object
+            #print(f"DEBUG: generated_texts: {generated_texts}")
+            # Ensure generated_texts is a list and not None
+            if generated_texts and isinstance(generated_texts, list) and len(generated_texts) > 0:
+                # Extract the response text only from the generated result
+                response = generated_texts[0][0]["generated_text"]
+                response_start = response.find("Response:") + len("Response:")
+                answers.append(response[response_start:].strip())
+            else:
+                answers.append("No AI-generated text found.")
+        else:
+            answers.append("No content available to generate an answer.")
+    return answers
+
+# Gradio interface
+def process_query(query):
+    top_results = search_similar(query, top_k=3)
+    if top_results:
+        content = extract_content(top_results[0])
+        answer = generate_answer(query, [content])[0]
+
+        response = f"Rank 1: URL - {top_results[0]}\n"
+        response += f"Generated Answer:\n{answer}\n"
+
+        similar_urls = "\n".join(top_results[1:])  # The second and third URLs as similar URLs
+        return response, similar_urls
+    else:
+        return "No results found.", "No similar URLs found."
+
+demo = gr.Interface(
+    fn=process_query,
+    inputs=gr.Textbox(label="Enter your query"),
+    outputs=[
+        gr.Textbox(label="Generated Answer"),
+        gr.Textbox(label="Similar URLs")
+    ]
+)
+
+if __name__ == "__main__":
+    demo.launch(share=True)

crawled_contents.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7d4abd213fc62a689e50e351eb69762b2c6a38e074832321fc4f5e498f59a4f
+size 2373777

crawled_urls.txt

@@ -0,0 +1,98 @@
+https://go.drugbank.com/drugs/DB00001
+https://go.drugbank.com/drugs/DB00002
+https://go.drugbank.com/drugs/DB00003
+https://go.drugbank.com/drugs/DB00004
+https://go.drugbank.com/drugs/DB00005
+https://go.drugbank.com/drugs/DB00006
+https://go.drugbank.com/drugs/DB00007
+https://go.drugbank.com/drugs/DB00008
+https://go.drugbank.com/drugs/DB00009
+https://go.drugbank.com/drugs/DB00010
+https://go.drugbank.com/drugs/DB00011
+https://go.drugbank.com/drugs/DB00012
+https://go.drugbank.com/drugs/DB00013
+https://go.drugbank.com/drugs/DB00014
+https://go.drugbank.com/drugs/DB00015
+https://go.drugbank.com/drugs/DB00016
+https://go.drugbank.com/drugs/DB00017
+https://go.drugbank.com/drugs/DB00018
+https://go.drugbank.com/drugs/DB00019
+https://go.drugbank.com/drugs/DB00020
+https://go.drugbank.com/drugs/DB00021
+https://go.drugbank.com/drugs/DB00022
+https://go.drugbank.com/drugs/DB00023
+https://go.drugbank.com/drugs/DB00024
+https://go.drugbank.com/drugs/DB00025
+https://go.drugbank.com/drugs/DB00026
+https://go.drugbank.com/drugs/DB00027
+https://go.drugbank.com/drugs/DB00028
+https://go.drugbank.com/drugs/DB00029
+https://go.drugbank.com/drugs/DB00030
+https://go.drugbank.com/drugs/DB00031
+https://go.drugbank.com/drugs/DB00032
+https://go.drugbank.com/drugs/DB00033
+https://go.drugbank.com/drugs/DB00034
+https://go.drugbank.com/drugs/DB00035
+https://go.drugbank.com/drugs/DB00036
+https://go.drugbank.com/drugs/DB00037
+https://go.drugbank.com/drugs/DB00038
+https://go.drugbank.com/drugs/DB00039
+https://go.drugbank.com/drugs/DB00040
+https://go.drugbank.com/drugs/DB00041
+https://go.drugbank.com/drugs/DB00042
+https://go.drugbank.com/drugs/DB00043
+https://go.drugbank.com/drugs/DB00044
+https://go.drugbank.com/drugs/DB00045
+https://go.drugbank.com/drugs/DB00046
+https://go.drugbank.com/drugs/DB00047
+https://go.drugbank.com/drugs/DB00048
+https://go.drugbank.com/drugs/DB00049
+https://go.drugbank.com/drugs/DB00050
+https://go.drugbank.com/drugs/DB00051
+https://go.drugbank.com/drugs/DB00052
+https://go.drugbank.com/drugs/DB00053
+https://go.drugbank.com/drugs/DB00054
+https://go.drugbank.com/drugs/DB00055
+https://go.drugbank.com/drugs/DB00056
+https://go.drugbank.com/drugs/DB00057
+https://go.drugbank.com/drugs/DB00058
+https://go.drugbank.com/drugs/DB00059
+https://go.drugbank.com/drugs/DB00060
+https://go.drugbank.com/drugs/DB00061
+https://go.drugbank.com/drugs/DB00062
+https://go.drugbank.com/drugs/DB00063
+https://go.drugbank.com/drugs/DB00064
+https://go.drugbank.com/drugs/DB00065
+https://go.drugbank.com/drugs/DB00066
+https://go.drugbank.com/drugs/DB00067
+https://go.drugbank.com/drugs/DB00068
+https://go.drugbank.com/drugs/DB00069
+https://go.drugbank.com/drugs/DB00070
+https://go.drugbank.com/drugs/DB00071
+https://go.drugbank.com/drugs/DB00072
+https://go.drugbank.com/drugs/DB00073
+https://go.drugbank.com/drugs/DB00074
+https://go.drugbank.com/drugs/DB00075
+https://go.drugbank.com/drugs/DB00076
+https://go.drugbank.com/drugs/DB00078
+https://go.drugbank.com/drugs/DB00080
+https://go.drugbank.com/drugs/DB00081
+https://go.drugbank.com/drugs/DB00082
+https://go.drugbank.com/drugs/DB00083
+https://go.drugbank.com/drugs/DB00084
+https://go.drugbank.com/drugs/DB00085
+https://go.drugbank.com/drugs/DB00086
+https://go.drugbank.com/drugs/DB00087
+https://go.drugbank.com/drugs/DB00088
+https://go.drugbank.com/drugs/DB00089
+https://go.drugbank.com/drugs/DB00090
+https://go.drugbank.com/drugs/DB00091
+https://go.drugbank.com/drugs/DB00092
+https://go.drugbank.com/drugs/DB00093
+https://go.drugbank.com/drugs/DB00094
+https://go.drugbank.com/drugs/DB00095
+https://go.drugbank.com/drugs/DB00096
+https://go.drugbank.com/drugs/DB00097
+https://go.drugbank.com/drugs/DB00098
+https://go.drugbank.com/drugs/DB00099
+https://go.drugbank.com/drugs/DB00100

crawler.py

@@ -0,0 +1,190 @@
+import requests
+from bs4 import BeautifulSoup
+from urllib.parse import urljoin, urlparse
+import os
+from transformers import BertModel, BertTokenizer
+import torch
+import numpy as np
+import faiss
+from concurrent.futures import ThreadPoolExecutor
+from retrying import retry
+import time
+from ratelimit import limits, sleep_and_retry
+import threading
+
+# Global counters for URLs and FAISS index initialization
+total_urls_crawled = 0
+index_file = 'faiss_index.bin'  # FAISS index file path
+
+# Set of visited URLs to prevent duplicates
+visited_urls = set()
+
+# Directory to save crawled URLs
+urls_dir = 'crawled_urls'
+os.makedirs(urls_dir, exist_ok=True)
+urls_file = os.path.join(urls_dir, 'crawled_urls.txt')
+
+# Initialize FAISS index
+def initialize_faiss_index(dimension):
+    if os.path.exists(index_file):
+        os.remove(index_file)
+        print("Deleted previous FAISS index file.")
+    index = faiss.IndexFlatL2(dimension)
+    return index
+
+# Initialize or load FAISS index
+dimension = 768  # Dimension of BERT embeddings
+index = initialize_faiss_index(dimension)
+
+# Initialize tokenizer and model
+tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
+model = BertModel.from_pretrained('bert-base-uncased')
+
+# Lock for thread-safe update of total_urls_crawled
+lock = threading.Lock()
+
+# Function to update and print live count of crawled URLs
+def update_live_count():
+    global total_urls_crawled
+    while True:
+        with lock:
+            print(f"\rURLs crawled: {total_urls_crawled}", end='')
+        time.sleep(1)  # Update every second
+
+# Start live count update thread
+live_count_thread = threading.Thread(target=update_live_count, daemon=True)
+live_count_thread.start()
+
+# Function to save crawled URLs to a file
+def save_crawled_urls(url):
+    with open(urls_file, 'a') as f:
+        f.write(f"{url}\n")
+        f.flush()  # Flush buffer to ensure immediate write
+        os.fsync(f.fileno())  # Ensure write is flushed to disk
+
+# Function to get all links from a webpage with retry mechanism and rate limiting
+@retry(stop_max_attempt_number=3, wait_fixed=2000)
+@sleep_and_retry
+@limits(calls=10, period=1)  # Adjust calls and period based on website's rate limits
+def get_links(url, domain):
+    global total_urls_crawled
+    links = []
+    try:
+        headers = {
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
+        }
+        response = requests.get(url, headers=headers, timeout=50)
+        response.raise_for_status()
+        soup = BeautifulSoup(response.content, 'html.parser')
+        for link in soup.find_all('a', href=True):
+            href = link['href']
+            normalized_url = normalize_url(href, domain)
+            if normalized_url and normalized_url not in visited_urls:
+                links.append(normalized_url)
+                visited_urls.add(normalized_url)
+                with lock:
+                    total_urls_crawled += 1
+                save_crawled_urls(normalized_url)  # Save crawled URL to file
+
+                # Convert text to BERT embeddings and add to FAISS index
+                try:
+                    text = soup.get_text()
+                    if text:
+                        embeddings = convert_text_to_bert_embeddings(text, tokenizer, model)
+                        index.add(np.array([embeddings]))
+                except Exception as e:
+                    print(f"Error adding embeddings to FAISS index: {e}")
+
+    except requests.HTTPError as e:
+        if e.response.status_code == 404:
+            print(f"HTTP 404 Error: {e}")
+        else:
+            print(f"HTTP error occurred: {e}")
+    except requests.RequestException as e:
+        print(f"Error accessing {url}: {e}")
+    return links
+
+# Function to normalize and validate URLs
+def normalize_url(url, domain):
+    parsed_url = urlparse(url)
+    if not parsed_url.scheme:
+        url = urljoin(domain, url)
+    if url.startswith(domain):
+        return url
+    return None
+
+# Function to recursively get all pages and collect links with retry mechanism and rate limiting
+@retry(stop_max_attempt_number=3, wait_fixed=2000)
+@sleep_and_retry
+@limits(calls=10, period=1)  # Adjust calls and period based on website's rate limits
+def crawl_site(base_url, domain, depth=0, max_depth=10):  # Increased max_depth to 10
+    if depth > max_depth or base_url in visited_urls:
+        return []
+    visited_urls.add(base_url)
+
+    links = get_links(base_url, domain)
+    print(f"Crawled {len(links)} links from {base_url} at depth {depth}.")  # Debugging info
+
+    try:
+        headers = {
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
+        }
+        response = requests.get(base_url, headers=headers, timeout=30)
+        response.raise_for_status()
+        soup = BeautifulSoup(response.content, 'html.parser')
+        links_to_crawl = []
+        for link in soup.find_all('a', href=True):
+            href = link['href']
+            normalized_url = normalize_url(href, domain)
+            if normalized_url and normalized_url not in visited_urls:
+                links_to_crawl.append(normalized_url)
+
+        with ThreadPoolExecutor(max_workers=500) as executor:
+            results = executor.map(lambda url: crawl_site(url, domain, depth + 1, max_depth), links_to_crawl)
+            for result in results:
+                links.extend(result)
+
+    except requests.HTTPError as e:
+        if e.response.status_code == 404:
+            print(f"HTTP 404 Error: {e}")
+        else:
+            print(f"HTTP error occurred: {e}")
+    except requests.RequestException as e:
+        print(f"Error accessing {base_url}: {e}")
+
+    return links
+
+# Function to convert text to BERT embeddings
+def convert_text_to_bert_embeddings(text, tokenizer, model):
+    inputs = tokenizer(text, return_tensors='pt', max_length=512, truncation=True, padding=True)
+
+    with torch.no_grad():
+        outputs = model(**inputs)
+        embeddings = outputs.last_hidden_state.mean(dim=1).squeeze().numpy()  # Average pool last layer's output
+
+    return embeddings
+
+# Main process
+def main():
+    global total_urls_crawled
+    domain = 'https://go.drugbank.com/'  # Replace with your new domain
+    start_url = 'https://go.drugbank.com/drugs/DB00001'  # Replace with your starting URL
+
+
+    try:
+        # Save the FAISS index at the beginning of the execution
+        faiss.write_index(index, index_file)
+        print("Initial FAISS index saved.")
+
+        urls = crawl_site(start_url, domain)
+        print(f"\n\nFound {total_urls_crawled} URLs.")
+
+        # Save the FAISS index at the end of execution
+        faiss.write_index(index, index_file)
+        print("Final FAISS index saved.")
+
+    except Exception as e:
+        print(f"Exception encountered: {e}")
+
+if __name__ == "__main__":
+    main()

requirements.txt

@@ -0,0 +1,7 @@
+faiss-cpu
+numpy
+torch
+transformers
+requests
+beautifulsoup4
+gradio

sample_embeddings.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29c28c327a9952d067087d04c9550baf1b41db8028e4aee5a2d46c4f6ac91983
+size 20608