Adding Files

.dvc/.gitignore

@@ -0,0 +1,3 @@
+/config.local
+/tmp
+/cache

.dvc/config

@@ -0,0 +1,4 @@
+[core]
+    autostage = true
+['remote "origin"']
+    url = https://dagshub.com/Aryan-coder-student/Med-VQA.git

.dvcignore

@@ -0,0 +1,3 @@
+# Add patterns of files dvc should ignore, which could improve
+# the performance. Learn more at
+# https://dvc.org/doc/user-guide/dvcignore

.gitignore

@@ -0,0 +1,3 @@
+models/best-saved-model
+models/last-saved-model
+Deployment/.env

Deployment/.gitignore

@@ -0,0 +1,2 @@
+.env
+Model/14-last-blip-saved-model/

Deployment/app.py

@@ -0,0 +1,98 @@
+import torch
+from flask import Flask, request, jsonify
+from transformers import BlipProcessor, BlipForQuestionAnswering
+from langchain.agents import initialize_agent, AgentType
+from langchain.tools import Tool
+from langchain_community.utilities import SerpAPIWrapper, PubMedAPIWrapper
+from langchain_groq import ChatGroq
+from langchain.memory import ConversationBufferMemory
+from dotenv import load_dotenv
+import os
+from PIL import Image
+from flask_cors import CORS
+
+ 
+load_dotenv()
+os.getenv("SERPAPI_API_KEY")
+os.getenv("GROQ_API_KEY")
+
+ 
+app = Flask(__name__)
+CORS(app)
+
+ 
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+ 
+fine_tuned_model = BlipForQuestionAnswering.from_pretrained(os.path.join("Deployment/Model/14-last-blip-saved-model")).to(device)
+processor = BlipProcessor.from_pretrained("Salesforce/blip-vqa-base")
+ 
+memory = ConversationBufferMemory()
+def chat_bot_query(query: str):
+    search_tool = Tool(
+        name="Medical_Web_Search",
+        func=SerpAPIWrapper().run,
+        description="Searches the web for medical information related to brain, CT, and MRI scans."
+    )
+
+    pubmed_tool = Tool(
+        name="PubMed_Search",
+        func=PubMedAPIWrapper().run,
+        description="Searches PubMed for research papers related to brain, CT, and MRI scans."
+    )
+
+    agent = initialize_agent(
+        agent=AgentType.ZERO_SHOT_REACT_DESCRIPTION,
+        tools=[search_tool, pubmed_tool],
+        llm=ChatGroq(model="gemma2-9b-it"),
+        memory=memory,
+        verbose=False,
+        max_iterations=10,
+        handle_parsing_errors=True
+    )
+
+    response = agent.run(query)
+    return response
+
+
+ 
+def predict_answer(image, question):
+    try:
+        image = image.convert('RGB')
+        inputs = processor(image, question, return_tensors="pt").to(device)
+        fine_tuned_output = fine_tuned_model.generate(**inputs)
+        fine_tuned_answer = processor.tokenizer.decode(fine_tuned_output[0], skip_special_tokens=True)
+        return fine_tuned_answer
+    except Exception as e:
+        return f"Error in prediction: {e}"
+
+ 
+@app.route('/predict/', methods=['GET', 'POST'])
+def predict():
+    try:
+        file = request.files['file']
+        question = request.form['question']
+        if not file or not question:
+            return jsonify({'error': 'No file or question provided'}), 400
+        image = Image.open(file)
+        answer = predict_answer(image, question)
+        return jsonify({'answer': answer})
+    except Exception as e:
+        return jsonify({'error': str(e)}), 500
+ 
+@app.route('/chat/', methods=['POST'])
+def chat():
+    try:
+        data = request.get_json()
+        query = data.get('query')
+        if not query:
+            return jsonify({'error': 'No query provided'}), 400
+
+        response = chat_bot_query(query)
+        return jsonify({'response': response})
+    except Exception as e:
+        print(e)
+        return jsonify({'error': str(e)}), 500
+
+ 
+if __name__ == '__main__':
+    app.run(debug=True, host='0.0.0.0', port='5000')

Deployment/streamlit/.gitignore

@@ -0,0 +1 @@
+.env

Deployment/streamlit/main.py

@@ -0,0 +1,27 @@
+import streamlit as st
+import requests
+from PIL import Image
+import io
+
+st.set_page_config(page_title="VQA with BLIP", layout="centered", page_icon="🤖")
+st.title("Visual Question Answering (VQA) with BLIP")
+
+uploaded_file = st.file_uploader("Upload an Image", type=["jpg", "png", "jpeg"])
+question = st.text_input("Ask a question about the image:")
+
+if uploaded_file and question:
+    image = Image.open(uploaded_file)
+    st.image(image, caption="Uploaded Image", use_column_width=True)
+    if st.button("Get Answer"):
+        img_bytes = io.BytesIO()
+        image.save(img_bytes, format="JPEG")
+        img_bytes = img_bytes.getvalue()
+        files = {"file": ("image.jpg", img_bytes, "image/jpeg")}
+        data = {"question": question}
+        response = requests.post("http://127.0.0.1:5000//predict/", files=files, data=data)
+        
+        if response.status_code == 200:
+            answer = response.json().get("answer", "No answer received")
+            st.success(f"Answer: {answer}")
+        else:
+            st.error("Error in fetching the answer. Please try again.")

Deployment/test_api.py

@@ -0,0 +1,31 @@
+import requests
+
+# Define the URL of the Flask server
+url = "http://127.0.0.1:5000/chat/"
+
+# Define the query you want to send
+query = "Provide recent papers links or researches on brain tumors."
+
+# Create the JSON payload
+payload = {
+    "query": query
+}
+
+# Set the headers to specify JSON content
+headers = {
+    "Content-Type": "application/json"
+}
+
+# Send the POST request
+try:
+    response = requests.post(url, json=payload, headers=headers)
+    
+    # Check if the request was successful
+    if response.status_code == 200:
+        print("Response from server:")
+        print(response.json())
+    else:
+        print(f"Error: {response.status_code}")
+        print(response.json())
+except Exception as e:
+    print(f"An error occurred: {e}")

Dockerfile

@@ -0,0 +1,14 @@
+# FROM python:3.10 
+# WORKDIR /app
+# COPY requirements.txt . 
+# RUN pip install -r requirements.txt
+# COPY . .
+# EXPOSE 5000 8501
+# CMD ["sh", "-c", "python Deployment/app.py & while ! nc -z localhost 5000; do sleep 1; done; streamlit run Deployment/streamlit/main.py --server.port=8501 --server.address=0.0.0.0"]
+FROM python:3.10
+WORKDIR /app
+COPY requirements.txt .
+RUN pip install -r requirements.txt
+COPY . .
+EXPOSE 5000
+CMD ["python", "Deployment/app.py"]

README.md

@@ -1 +1 @@
-# NeuroVision-BHPC-VQA
+# NeuroVision-VQA

config.yaml

@@ -0,0 +1,10 @@
+finetune_model:
+  best: models/best-saved-model
+  last: models/last-saved-model
+  orignal_model_id: Salesforce/blip-vqa-base
+data_location:
+  data: data/bronze/flaviagiammarino___vqa-rad
+  train_processed_data: data/silver/train_dataset.pkl
+  test_processed_data: data/silver/test_dataset.pkl
+
+result: results

data/.gitignore

@@ -0,0 +1,2 @@
+/bronze
+/silver

data/bronze.dvc

@@ -0,0 +1,6 @@
+outs:
+- md5: 32db974481a7ea051c9c852355c7b221.dir
+  size: 153579281
+  nfiles: 7
+  hash: md5
+  path: bronze

dvc.lock

@@ -0,0 +1,20 @@
+schema: '2.0'
+stages:
+  preprocess:
+    cmd: python src/preprocess_data.py
+    deps:
+    - path: data/bronze/downloads
+      hash: md5
+      md5: 1454b6162c5ef2c24c037ba883797c3b.dir
+      size: 34497027
+      nfiles: 4
+    - path: src/preprocess_data.py
+      hash: md5
+      md5: b1141ff5738a52bee364a048006112a2
+      size: 2333
+    outs:
+    - path: data/silver
+      hash: md5
+      md5: 1b44b8c6bbc048557b5285a27b9cc1f5.dir
+      size: 941687
+      nfiles: 2

dvc.yaml

@@ -0,0 +1,27 @@
+stages:
+  preprocess:
+    cmd: python src/preprocess_data.py
+    deps:
+      - src/preprocess_data.py
+      - data/bronze/downloads  
+    outs:
+      - data/silver  
+
+  train:
+    cmd: python src/train.py
+    deps:
+      - src/train.py
+      - data/silver 
+      - param.yaml
+    outs:
+      - models 
+
+
+  evaluate:
+    cmd: python src/evaluate.py
+    deps:
+      - src/evaluate.py
+      - models  
+      - data/silver
+    outs:
+      - results  

mlruns/0/meta.yaml

@@ -0,0 +1,6 @@
+artifact_location: file:///D:/My%20File/Project/MedHelp-Brain-VQA/End-To-End-Pipline/VQA/mlruns/0
+creation_time: 1738400123102
+experiment_id: '0'
+last_update_time: 1738400123102
+lifecycle_stage: active
+name: Default

mlruns/271729933174040292/0ddd5ed748f54c8883bf1830e5bf029c/meta.yaml

@@ -0,0 +1,15 @@
+artifact_uri: file:///D:/My%20File/Project/MedHelp-Brain-VQA/End-To-End-Pipline/VQA/mlruns/271729933174040292/0ddd5ed748f54c8883bf1830e5bf029c/artifacts
+end_time: 1742659076026
+entry_point_name: ''
+experiment_id: '271729933174040292'
+lifecycle_stage: active
+run_id: 0ddd5ed748f54c8883bf1830e5bf029c
+run_name: clean-bee-82
+run_uuid: 0ddd5ed748f54c8883bf1830e5bf029c
+source_name: ''
+source_type: 4
+source_version: ''
+start_time: 1742659058810
+status: 4
+tags: []
+user_id: ASUS

mlruns/271729933174040292/0ddd5ed748f54c8883bf1830e5bf029c/tags/mlflow.runName

@@ -0,0 +1 @@
+clean-bee-82

mlruns/271729933174040292/0ddd5ed748f54c8883bf1830e5bf029c/tags/mlflow.source.git.commit

@@ -0,0 +1 @@
+df4034786b530485650893743b7ee5889c863b0a

mlruns/271729933174040292/0ddd5ed748f54c8883bf1830e5bf029c/tags/mlflow.source.name

@@ -0,0 +1 @@
+src/train.py

mlruns/271729933174040292/0ddd5ed748f54c8883bf1830e5bf029c/tags/mlflow.source.type

@@ -0,0 +1 @@
+LOCAL

mlruns/271729933174040292/0ddd5ed748f54c8883bf1830e5bf029c/tags/mlflow.user

@@ -0,0 +1 @@
+ASUS

mlruns/271729933174040292/257978a9195e4891a4b58f979bd21da7/meta.yaml

@@ -0,0 +1,15 @@
+artifact_uri: file:///D:/My%20File/Project/MedHelp-Brain-VQA/End-To-End-Pipline/VQA/mlruns/271729933174040292/257978a9195e4891a4b58f979bd21da7/artifacts
+end_time: 1738904546941
+entry_point_name: ''
+experiment_id: '271729933174040292'
+lifecycle_stage: active
+run_id: 257978a9195e4891a4b58f979bd21da7
+run_name: tasteful-wren-640
+run_uuid: 257978a9195e4891a4b58f979bd21da7
+source_name: ''
+source_type: 4
+source_version: ''
+start_time: 1738904472965
+status: 4
+tags: []
+user_id: VIT

mlruns/271729933174040292/257978a9195e4891a4b58f979bd21da7/tags/mlflow.runName

@@ -0,0 +1 @@
+tasteful-wren-640

mlruns/271729933174040292/257978a9195e4891a4b58f979bd21da7/tags/mlflow.source.name

@@ -0,0 +1 @@
+src/train.py

mlruns/271729933174040292/257978a9195e4891a4b58f979bd21da7/tags/mlflow.source.type

@@ -0,0 +1 @@
+LOCAL

mlruns/271729933174040292/257978a9195e4891a4b58f979bd21da7/tags/mlflow.user

@@ -0,0 +1 @@
+VIT

mlruns/271729933174040292/581fe86b792242ec88ae6c7793f69919/meta.yaml

@@ -0,0 +1,15 @@
+artifact_uri: file:///D:/My%20File/Project/MedHelp-Brain-VQA/End-To-End-Pipline/VQA/mlruns/271729933174040292/581fe86b792242ec88ae6c7793f69919/artifacts
+end_time: 1738508905313
+entry_point_name: ''
+experiment_id: '271729933174040292'
+lifecycle_stage: active
+run_id: 581fe86b792242ec88ae6c7793f69919
+run_name: peaceful-snipe-124
+run_uuid: 581fe86b792242ec88ae6c7793f69919
+source_name: ''
+source_type: 4
+source_version: ''
+start_time: 1738508889425
+status: 4
+tags: []
+user_id: ASUS

mlruns/271729933174040292/581fe86b792242ec88ae6c7793f69919/tags/mlflow.runName

@@ -0,0 +1 @@
+peaceful-snipe-124

mlruns/271729933174040292/581fe86b792242ec88ae6c7793f69919/tags/mlflow.source.git.commit

@@ -0,0 +1 @@
+f1692ec7d45a18ee2c8f0fe487553d6a184bfc3a

mlruns/271729933174040292/581fe86b792242ec88ae6c7793f69919/tags/mlflow.source.name

@@ -0,0 +1 @@
+src/train.py

mlruns/271729933174040292/581fe86b792242ec88ae6c7793f69919/tags/mlflow.source.type

@@ -0,0 +1 @@
+LOCAL

mlruns/271729933174040292/581fe86b792242ec88ae6c7793f69919/tags/mlflow.user

@@ -0,0 +1 @@
+ASUS

mlruns/271729933174040292/67e7980b68664f9385c14de927d46167/meta.yaml

@@ -0,0 +1,15 @@
+artifact_uri: file:///D:/My%20File/Project/MedHelp-Brain-VQA/End-To-End-Pipline/VQA/mlruns/271729933174040292/67e7980b68664f9385c14de927d46167/artifacts
+end_time: 1738400286679
+entry_point_name: ''
+experiment_id: '271729933174040292'
+lifecycle_stage: active
+run_id: 67e7980b68664f9385c14de927d46167
+run_name: rebellious-asp-756
+run_uuid: 67e7980b68664f9385c14de927d46167
+source_name: ''
+source_type: 4
+source_version: ''
+start_time: 1738400274297
+status: 4
+tags: []
+user_id: ASUS

mlruns/271729933174040292/67e7980b68664f9385c14de927d46167/tags/mlflow.runName

@@ -0,0 +1 @@
+rebellious-asp-756

mlruns/271729933174040292/67e7980b68664f9385c14de927d46167/tags/mlflow.source.name

@@ -0,0 +1 @@
+d:\My File\Project\MedHelp-Brain-VQA\End-To-End-Pipline\VQA\src\train.py

mlruns/271729933174040292/67e7980b68664f9385c14de927d46167/tags/mlflow.source.type

@@ -0,0 +1 @@
+LOCAL

mlruns/271729933174040292/67e7980b68664f9385c14de927d46167/tags/mlflow.user

@@ -0,0 +1 @@
+ASUS

mlruns/271729933174040292/7c6de0cfdddd47c299c5235291b99bf5/meta.yaml

@@ -0,0 +1,15 @@
+artifact_uri: file:///D:/My%20File/Project/MedHelp-Brain-VQA/End-To-End-Pipline/VQA/mlruns/271729933174040292/7c6de0cfdddd47c299c5235291b99bf5/artifacts
+end_time: 1739270329295
+entry_point_name: ''
+experiment_id: '271729933174040292'
+lifecycle_stage: active
+run_id: 7c6de0cfdddd47c299c5235291b99bf5
+run_name: clumsy-robin-616
+run_uuid: 7c6de0cfdddd47c299c5235291b99bf5
+source_name: ''
+source_type: 4
+source_version: ''
+start_time: 1739259742693
+status: 4
+tags: []
+user_id: VIT

mlruns/271729933174040292/7c6de0cfdddd47c299c5235291b99bf5/metrics/bleu_score

@@ -0,0 +1,5 @@
+1739261831601 0.05723518642092382 0
+1739263932368 0.0565093265115535 1
+1739266009599 0.05611502952484864 2
+1739268078542 0.05611502952484864 3
+1739270173935 0.056453157920654924 4

mlruns/271729933174040292/7c6de0cfdddd47c299c5235291b99bf5/metrics/eval_loss

@@ -0,0 +1,5 @@
+1739261831601 11.475591375116716 0
+1739263932368 10.817912503292685 1
+1739266009599 10.372373430352463 2
+1739268078542 10.242192536069636 3
+1739270173935 10.166932239867092 4

mlruns/271729933174040292/7c6de0cfdddd47c299c5235291b99bf5/metrics/train_loss

@@ -0,0 +1,5 @@
+1739261831601 11.472894626193577 0
+1739263932368 11.154893341064453 1
+1739266009599 10.531031091478136 2
+1739268078542 10.308383038838704 3
+1739270173935 10.192635604010688 4

mlruns/271729933174040292/7c6de0cfdddd47c299c5235291b99bf5/params/learning_rate_0

@@ -0,0 +1 @@
+0.0

mlruns/271729933174040292/7c6de0cfdddd47c299c5235291b99bf5/params/learning_rate_1

@@ -0,0 +1 @@
+4.444444444444445e-08

mlruns/271729933174040292/7c6de0cfdddd47c299c5235291b99bf5/params/learning_rate_2

@@ -0,0 +1 @@
+8.88888888888889e-08

mlruns/271729933174040292/7c6de0cfdddd47c299c5235291b99bf5/params/learning_rate_3

@@ -0,0 +1 @@
+1.3333333333333334e-07

mlruns/271729933174040292/7c6de0cfdddd47c299c5235291b99bf5/params/learning_rate_4

@@ -0,0 +1 @@
+1.777777777777778e-07