yash's hf bipolar demo code with github action set

.gitattributes

@@ -0,0 +1,36 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.faiss filter=lfs diff=lfs merge=lfs -text

.github/workflows/sync-to-hf.yml

@@ -0,0 +1,36 @@
+name: Sync hf-bipolar to HF Space main
+
+on:
+  push:
+    branches: [ demo_bipolar ]
+  workflow_dispatch: {}
+
+jobs:
+  sync:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout demo_bipolar (with LFS)
+        uses: actions/checkout@v4
+        with:
+          ref: demo_bipolar
+          lfs: true
+          fetch-depth: 0
+
+      - name: Setup Git LFS
+        run: |
+          git lfs install
+          git lfs fetch --all
+
+      - name: Push hf-demo_bipolar to HF Space main
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+        run: |
+          set -e
+          git config user.email "actions@github.com"
+          git config user.name "github-actions[bot]"
+          SPACE_ID="ymali/bipolar"
+          TARGET_BRANCH="main"
+          AUTHED_URL="https://user:${HF_TOKEN}@huggingface.co/spaces/${SPACE_ID}"
+          git remote add hf "$AUTHED_URL" || git remote set-url hf "$AUTHED_URL"
+          git lfs push hf --all
+          git push hf "HEAD:refs/heads/${TARGET_BRANCH}" --force

.gitignore

@@ -0,0 +1 @@
+.env

.idea/.gitignore

@@ -0,0 +1,8 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Editor-based HTTP Client requests
+/httpRequests/
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml

.idea/bipolar.iml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="jdk" jdkName="Python 3.13" jdkType="Python SDK" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+  <component name="PyDocumentationSettings">
+    <option name="format" value="PLAIN" />
+    <option name="myDocStringFormat" value="Plain" />
+  </component>
+</module>

.idea/inspectionProfiles/Project_Default.xml

@@ -0,0 +1,7 @@
+<component name="InspectionProjectProfileManager">
+  <profile version="1.0">
+    <option name="myName" value="Project Default" />
+    <inspection_tool class="Eslint" enabled="true" level="WARNING" enabled_by_default="true" />
+    <inspection_tool class="PyTypeCheckerInspection" enabled="false" level="WARNING" enabled_by_default="false" />
+  </profile>
+</component>

.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

.idea/misc.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="Black">
+    <option name="sdkName" value="Python 3.13" />
+  </component>
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.13" project-jdk-type="Python SDK" />
+</project>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/bipolar.iml" filepath="$PROJECT_DIR$/.idea/bipolar.iml" />
+    </modules>
+  </component>
+</project>

.idea/vcs.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="" vcs="Git" />
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>

.streamlit/config.toml

@@ -0,0 +1,55 @@
+[server]
+enableStaticServing = false
+#
+#[[theme.fontFaces]]
+#family = "SpaceGrotesk"
+#url = "app/static/SpaceGrotesk-VariableFont_wght.ttf"
+#
+#[[theme.fontFaces]]
+#family = "SpaceMono"
+#url = "app/static/SpaceMono-Bold.ttf"
+#style = "normal"
+#weight = 700
+#
+#[[theme.fontFaces]]
+#family = "SpaceMono"
+#url = "app/static/SpaceMono-BoldItalic.ttf"
+#style = "italic"
+#weight = 700
+#
+#[[theme.fontFaces]]
+#family = "SpaceMono"
+#url = "app/static/SpaceMono-Italic.ttf"
+#style = "italic"
+#weight = 400
+#
+#[[theme.fontFaces]]
+#family = "SpaceMono"
+#url = "app/static/SpaceMono-Regular.ttf"
+#style = "normal"
+#weight = 400
+
+[theme]
+primaryColor = "#cb785c"
+backgroundColor = "#fdfdf8"
+secondaryBackgroundColor = "#ecebe3"
+textColor = "#3d3a2a"
+linkColor = "#3d3a2a"
+borderColor = "#d3d2ca"
+showWidgetBorder = true
+baseRadius = "0.75rem"
+buttonRadius = "full"
+#font = "SpaceGrotesk"
+headingFontWeights = [600,500,500,500,500,500]
+headingFontSizes = ["3rem", "2rem"]
+#codeFont = "SpaceMono"
+codeFontSize = ".75rem"
+codeBackgroundColor = "#ecebe4"
+showSidebarBorder = true
+chartCategoricalColors = ["#0ea5e9", "#059669", "#fbbf24"]
+
+[theme.sidebar]
+backgroundColor = "#f0f0ec"
+secondaryBackgroundColor = "#ecebe3"
+headingFontSizes = ["1.6rem", "1.4rem", "1.2rem"]
+dataframeHeaderBackgroundColor = "#e4e4e0"

Dockerfile

@@ -0,0 +1,21 @@
+FROM python:3.9-slim
+
+WORKDIR /app
+
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    curl \
+    software-properties-common \
+    git \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt ./
+COPY src/ ./src/
+
+RUN pip3 install -r requirements.txt
+
+EXPOSE 8501
+
+HEALTHCHECK CMD curl --fail http://localhost:8501/_stcore/health
+
+ENTRYPOINT ["streamlit", "run", "src/app.py", "--server.port=8501", "--server.address=0.0.0.0"]

README.md

@@ -1,2 +1,19 @@
-# chatbot_nlp
-NLP experimentation, deployment and testing for the depression and bipolar Chatbot. 
+---
+title: Bipolar Chatbot
+emoji: 🚀
+colorFrom: red
+colorTo: red
+sdk: streamlit
+app_port: 8501
+sdk_version: "1.45.1"
+app_file: src/app.py
+pinned: true
+short_description: Streamlit template space
+---
+
+# Welcome to Streamlit!
+
+Edit `/src/app.py` to customize this app to your heart's desire. :heart:
+
+If you have any questions, checkout our [documentation](https://docs.streamlit.io) and [community
+forums](https://discuss.streamlit.io).

data/embeddings/Qwen_Qwen3-Embedding-0.6B.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dc3ceea12f082aff546c36423f436922abb5600de3c77d494a5d8bd87f9edf62
+size 1495168

data/readme.md

@@ -0,0 +1,7 @@
+- The `embeddings/` contains the embedded data by different embedders.
+
+- The `faiss_index/` contains the index built by faiss on different embedding files
+
+- The `processed/` contains the chuncked data that's been preprocessed and used in the RAG system
+
+- The `raw/` contains the original data from the clinical guideline

requirements.txt

@@ -0,0 +1,13 @@
+streamlit
+torch>=2.1.0      # >=2.1 preferred for compatibility on Apple Silicon
+faiss-cpu>=1.7.4
+numpy>=1.24.0
+python-dotenv>=1.0.0
+
+# Required by Qwen Embedding model
+transformers>=4.51.0
+sentence-transformers>=2.7.0
+
+# LLM providers
+openai
+together>=0.2.8

src/Rag.py

@@ -0,0 +1,275 @@
+import os
+import json
+import time
+import requests
+import numpy as np
+
+from dotenv import load_dotenv
+from sentence_transformers import SentenceTransformer
+from together import Together
+
+global db, referenced_tables_db, embedder, index, llm_client
+
+
+def load_json_to_db(file_path):
+    with open(file_path) as f:
+        db = json.load(f)
+    return db
+
+
+# -------- Embedding Functions --------
+def make_embeddings(embedder, embedder_name, db):
+    texts = [chunk['text'] for chunk in db]
+    embeddings = embedder.encode(texts, convert_to_numpy=True, batch_size=1, show_progress_bar=True)
+    return embeddings
+
+
+def get_project_root():
+    return os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+
+
+def save_embeddings(embedder_name, embeddings):
+    root = get_project_root()
+    file_path = os.path.join(root, "data", "embeddings", f"{embedder_name.replace('/', '_')}.npy")
+    os.makedirs(os.path.dirname(file_path), exist_ok=True)
+    np.save(file_path, embeddings)
+    print(f"Saved embeddings to: {file_path}")
+
+
+def load_embeddings(embedder_name):
+    root = get_project_root()
+    file_path = os.path.join(root, "data", "embeddings", f"{embedder_name.replace('/', '_')}.npy")
+
+    try:
+        embeddings = np.load(file_path, allow_pickle=True)
+        print(f"Loaded embeddings from: {file_path}")
+    except FileNotFoundError:
+        print(f"Embeddings not found. Recomputing for: {embedder_name}")
+        embeddings = make_embeddings(embedder, embedder_name, db)
+        save_embeddings(embedder_name, embeddings)
+
+    return embeddings
+
+
+def load_embedder_with_fallbacks(embedder_name):
+    print(f"Loading embedder {embedder_name}")
+    model = SentenceTransformer(
+        embedder_name,
+        trust_remote_code=True,
+        tokenizer_kwargs={"padding_side": "left"},
+        device='cpu'
+    )
+    return model
+
+
+# -------- Cosine Similarity Index (no FAISS) --------
+def build_cosine_index(embeddings):
+    norms = np.linalg.norm(embeddings, axis=1, keepdims=True)
+    return embeddings / norms
+
+
+def load_cosine_index(embedder_name):
+    embeddings = load_embeddings(embedder_name)
+    normalized_embeddings = build_cosine_index(embeddings)
+    return normalized_embeddings
+
+
+# -------- Cosine Similarity Search (Brute Force) --------
+def vector_search(query, embedder, db, index, referenced_table_db, k=6):
+    def get_detailed_instruct(task_description: str, query: str) -> str:
+        return f'Instruct: {task_description}\nQuery:{query}'
+
+    task = 'Given a search query, retrieve relevant passages that answer the query'
+    query_embedding = embedder.encode([get_detailed_instruct(task, query)], convert_to_numpy=True)
+    query_vec = query_embedding / np.linalg.norm(query_embedding)
+
+    cosine_similarities = np.dot(index, query_vec.T).flatten()
+    top_k_indices = np.argsort(-cosine_similarities)[:k]
+
+    results = []
+    referenced_tables = set()
+    existed_tables = set()
+
+    for i in top_k_indices:
+        results.append({
+            "text": db[i]['text'],
+            "section": db[i]['metadata']['section'],
+            "chunk_id": db[i]['metadata']['chunk_id'],
+            "similarity": float(cosine_similarities[i]),
+        })
+        if db[i]['metadata']['referee_id']:
+            existed_tables.add(db[i]['metadata']['referee_id'])
+        try:
+            if db[i]['metadata']['referenced_tables']:
+                referenced_tables.update(db[i]['metadata']['referenced_tables'])
+        except KeyError:
+            continue
+
+    table_to_add = [table for table in referenced_tables if table not in existed_tables]
+
+    for chunk in referenced_table_db:
+        if chunk['metadata']['referee_id'] in table_to_add:
+            results.append({
+                "text": chunk['text'],
+                "section": chunk['metadata']['section'],
+                "chunk_id": chunk['metadata']['chunk_id'],
+            })
+    return results
+
+
+def load_together_llm_client():
+    load_dotenv()
+    return Together(api_key=os.getenv("TOGETHER_API_KEY"))
+
+
+# -------- Prompt Construction --------
+def construct_prompt(query, faiss_results):
+    with open("src/system_prompt.txt", "r") as f:
+        system_prompt = f.read().strip()
+
+    prompt = f"""
+### System Prompt
+{system_prompt}
+
+### User Query
+{query}
+
+### Clinical Guidelines Context
+"""
+    for res in faiss_results:
+        prompt += f"- reference: {res['section']}\n- This paragraph is from section: {res['text']}\n"
+    return prompt
+
+
+def construct_prompt_with_memory(query, faiss_results, chat_history=None, history_limit=4):
+    with open("src/system_prompt.txt", "r") as f:
+        system_prompt = f.read().strip()
+
+    prompt = f"### System Prompt\n{system_prompt}\n\n"
+
+    if chat_history:
+        prompt += "### Chat History\n"
+        for m in chat_history[-history_limit:]:
+            prompt += f"{m['role'].title()}: {m['content']}\n"
+        prompt += "\n"
+
+    prompt += f"### User Query\n{query}\n\n"
+    prompt += "### Clinical Guidelines Context\n"
+    for res in faiss_results:
+        prompt += f"- reference: {res['section']}\n- This paragraph is from section: {res['text']}\n"
+    return prompt
+
+
+def call_llm(llm_client, prompt, stream_flag=False, max_tokens=500, temperature=0.05, top_p=0.9, model_name="meta-llama/Llama-3.3-70B-Instruct-Turbo-Free"):
+    print(f"Calling LLM with model: {model_name}")
+    try:
+        if stream_flag:
+            def stream_generator():
+                response = llm_client.chat.completions.create(
+                    model=model_name,
+                    messages=[{"role": "user", "content": prompt}],
+                    max_tokens=max_tokens,
+                    temperature=temperature,
+                    top_p=top_p,
+                    stream=True,
+                )
+                for chunk in response:
+                    if chunk.choices and chunk.choices[0].delta.content:
+                        yield chunk.choices[0].delta.content
+            return stream_generator()
+        else:
+            response = llm_client.chat.completions.create(
+                model=model_name,
+                messages=[{"role": "user", "content": prompt}],
+                max_tokens=max_tokens,
+                temperature=temperature,
+                top_p=top_p,
+                stream=False,
+            )
+            return response.choices[0].message.content
+    except Exception as e:
+        print("Error in call_llm:", str(e))
+        import traceback
+        traceback.print_exc()
+        raise
+
+
+def call_ollama(prompt, model="mistral", stream_flag=False, max_tokens=500, temperature=0.05, top_p=0.9):
+    url = "http://localhost:11434/api/generate"
+    payload = {
+        "model": model,
+        "prompt": prompt,
+        "temperature": temperature,
+        "top_p": top_p,
+        "max_tokens": max_tokens,
+        "stream": True
+    }
+
+    with requests.post(url, json=payload, stream=True) as response:
+        for line in response.iter_lines():
+            if line:
+                try:
+                    data = json.loads(line.decode("utf-8"))
+                    yield data["response"]
+                except Exception:
+                    continue
+
+
+# -------- Main Assistant Entry Points --------
+def launch_depression_assistant(embedder_name, designated_client=None):
+    global db, referenced_tables_db, embedder, index, llm_client
+
+    db = load_json_to_db("data/processed/guideline_db.json")
+    referenced_tables_db = load_json_to_db("data/processed/referenced_table_chunks.json")
+
+    embedder = load_embedder_with_fallbacks(embedder_name)
+    index = load_cosine_index(embedder_name)
+
+    if designated_client is None:
+        print("No LLM client provided. Loading Together LLM client...")
+        try:
+            llm_client = load_together_llm_client()
+        except Exception:
+            print("Failed to load Together LLM client. Please check your API key.")
+    else:
+        llm_client = designated_client
+
+    print("---------Depression Assistant is ready to use!--------------\n\n")
+
+
+def depression_assistant(query, model_name="meta-llama/Llama-3.3-70B-Instruct-Turbo-Free", max_tokens=500, temperature=0.05, top_p=0.9, stream_flag=False, chat_history=None):
+    results = vector_search(query, embedder, db, index, referenced_tables_db, k=3)
+    prompt = construct_prompt_with_memory(query, results, chat_history=chat_history)
+
+    if llm_client == "Run Ollama Locally":
+        return results, call_ollama(prompt, model_name, stream_flag, max_tokens, temperature, top_p)
+    else:
+        return results, call_llm(llm_client, prompt, stream_flag, max_tokens, temperature, top_p, model_name)
+
+
+def load_queries_and_answers(query_file, answers_file):
+    with open(query_file, 'r') as f:
+        queries = f.readlines()
+    with open(answers_file, 'r') as f:
+        answers = f.readlines()
+    return queries, answers
+
+
+def write_batched_results(embedder_name, result_path):
+    launch_depression_assistant(embedder_name)
+    queries, answers = load_queries_and_answers("data/raw/queries.txt", "data/raw/answers.txt")
+    embedder_filename = embedder_name.replace('/', '_')
+
+    with open(f"{result_path}Retrieved_Results_by_{embedder_filename}.md", "w") as f1, \
+         open(f"{result_path}Response_by_{embedder_filename}.md", "w") as f2:
+
+        for i, query in enumerate(queries):
+            result, response = depression_assistant(query)
+
+            f1.write(f"## Query {i+1}\n{query.strip()}\n\n## Answer\n{answers[i].strip()}\n\n## Retrieved Results\n")
+            for res in result:
+                f1.write(f"\n\n#### {res['section']}\n\n{res['text']}\n")
+            f1.write("\n\n---\n\n")
+
+            f2.write(f"## Query {i+1}\n{query.strip()}\n\n## Answer\n{answers[i].strip()}\n\n## Response\n{response}\n\n---\n\n")
+            break  # remove this `break` if you want to process all queries

src/app.py

@@ -0,0 +1,137 @@
+import streamlit as st
+from Rag import launch_depression_assistant, depression_assistant
+from openai import OpenAI
+from together import Together
+import time
+import os
+from dotenv import load_dotenv
+
+load_dotenv()
+
+if "embedder_loaded" not in st.session_state:
+    st.session_state.embedder_loaded = False
+if "current_embedder_name" not in st.session_state:
+    st.session_state.current_embedder_name = None
+if "last_sources" not in st.session_state:
+    st.session_state.last_sources = []
+
+st.set_page_config(
+    page_title="Bipolar Assistant Chatbot",
+    page_icon=":robot_face:",
+    layout="wide",
+    initial_sidebar_state="collapsed"
+)
+
+model_options = [
+    "Qwen/Qwen3-Embedding-0.6B",
+    "jinaai/jina-embeddings-v3",
+    "BAAI/bge-large-en-v1.5",
+    "BAAI/bge-small-en-v1.5",
+    "BAAI/bge-base-en-v1.5",
+    "sentence-transformers/all-mpnet-base-v2",
+    "Other"
+]
+
+st.sidebar.title("Settings")
+with st.sidebar:
+    st.subheader("Model Selection")
+    embedder_name = st.selectbox("Select embedder model", model_options, index=0)
+
+    if embedder_name == "Other":
+        embedder_name = st.text_input('Enter the embedder model name')
+
+    llm_client = Together(api_key=os.getenv("TOGETHER_API_KEY"))
+
+    if (not st.session_state.embedder_loaded or
+            st.session_state.current_embedder_name != embedder_name):
+
+        with st.spinner(f"Loading embedding model: {embedder_name}..."):
+            launch_depression_assistant(embedder_name=embedder_name, designated_client=llm_client)
+            st.session_state.embedder_loaded = True
+            st.session_state.current_embedder_name = embedder_name
+            st.success(f"✅ Embedding model {embedder_name} loaded successfully!")
+    else:
+        st.info(f"📋 Current embedding model: {st.session_state.current_embedder_name}")
+
+    selected_model = st.selectbox('Choose a model for generation',
+                                  ["meta-llama/Llama-3.3-70B-Instruct-Turbo-Free",
+                                   "deepseek-ai/deepseek-r1",
+                                   "meta/llama-3.3-70b-instruct"],
+                                  key='selected_model')
+
+    if selected_model in ["deepseek-ai/deepseek-r1", "meta/llama-3.3-70b-instruct"]:
+        max_length = 1000
+        llm_client = OpenAI(
+            base_url="https://integrate.api.nvidia.com/v1",
+            api_key=os.getenv("NVIDIA_API_KEY", None),
+        )
+
+    temperature = st.slider('temperature', min_value=0.01, max_value=1.0, value=0.05, step=0.01)
+    top_p = st.slider('top_p', min_value=0.01, max_value=1.0, value=0.9, step=0.01)
+    max_length = st.slider('max_length', min_value=100, max_value=1000, value=500, step=10)
+
+st.title("💬 Bipolar Assistant Chatbot")
+
+if "messages" not in st.session_state:
+    st.session_state.messages = [{
+        "role": "assistant",
+        "content": "This is a simple Bipolar assistant bot that uses **RAG (Retrieval-Augmented Generation)** to answer questions related to bipolar. You can ask questions related to bipolar and get responses based on [CANMAT clinical guidelines](https://pmc.ncbi.nlm.nih.gov/articles/PMC11351064/)."
+    }]
+
+chat_col, sources_col = st.columns([1, 1])
+
+with sources_col:
+    st.markdown("### Sources")
+    sources_placeholder = st.empty()
+
+    with sources_placeholder.container():
+        if st.session_state.last_sources:
+            for i, result in enumerate(st.session_state.last_sources):
+                st.markdown(f"**Source {i + 1}** | Similarity: {result.get('similarity', 'N/A')}")
+                st.markdown(f"- **Section:** {result['section']}")
+                st.markdown(f"> {result['text']}")
+                st.markdown("---")
+        else:
+            st.markdown("*Sources will appear here after you ask a question.*")
+
+with chat_col:
+    for message in st.session_state.messages:
+        with st.chat_message(message["role"]):
+            st.markdown(message["content"])
+
+    if user_input := st.chat_input("Ask me questions about the CANMAT bipolar guideline!"):
+        st.chat_message("user").markdown(user_input)
+        st.session_state.messages.append({"role": "user", "content": user_input})
+
+        history = st.session_state.messages[:-1][-4:]
+
+        collected = ""
+        t0 = time.perf_counter()
+        results, response = depression_assistant(user_input, model_name=selected_model, max_tokens=max_length,
+                                                 temperature=temperature, top_p=top_p, stream_flag=True,
+                                                 chat_history=history)
+
+        st.session_state.last_sources = results if results else []
+
+        with sources_placeholder.container():
+            if st.session_state.last_sources:
+                for i, result in enumerate(st.session_state.last_sources):
+                    st.markdown(f"**Source {i + 1}** | Similarity: {result.get('similarity', 'N/A')}")
+                    st.markdown(f"- **Section:** {result['section']}")
+                    st.markdown(f"> {result['text']}")
+                    st.markdown("---")
+            else:
+                st.markdown("*Sources will appear here after you ask a question.*")
+
+        placeholder = st.empty()
+        for chunk in response:
+            collected += chunk
+            placeholder.markdown(collected)
+
+        t1 = time.perf_counter()
+        print(f"[Time] Retriever + Generator takes: {t1 - t0:.2f} seconds in total.")
+        print(f"============== Finish R-A-Generation for Current Query {user_input} ==============")
+
+        st.session_state.messages.append({"role": "assistant", "content": collected})
+
+        st.rerun()

src/data_processing/image_processing.py

@@ -0,0 +1,91 @@
+from bs4 import BeautifulSoup
+import json
+import re
+
+def get_graph_metadata(graph, url="https://pmc.ncbi.nlm.nih.gov/articles/PMC11351064/#"):
+
+    
+
+    figure = graph.find_parent("figure")
+    figure_flag = False
+
+    section = graph.find_parent(id=re.compile(r'^section\d+-\d+$'))
+    section_id = section.get("id")
+    section_url = url + section_id
+
+    section_heading = section.find("h2").get_text()
+    section_subheading = section.find("h3").get_text()
+    headings = section_heading + " > " + section_subheading
+
+    attribution = ""
+
+    
+    if figure:
+
+        figure_flag = True
+
+        image_url = graph.get("src")
+
+        name = figure.select_one(".obj_head").get_text()
+        all_p = [p.get_text() for p in figure.find_all("p") if not p.attrs]
+        caption = all_p[0]
+        label = name + " " + caption
+
+        attribution = "(" + figure.select_one('[aria-label="Attribution"]').get_text() + ")"
+        number = "_".join(re.findall(r"(.{1})\.", name)).lower()
+        referee_id = f"figure_{number}"
+        
+          
+    else:
+
+        image_url = graph.get("src")
+
+        table_section = graph.find_parent("section")
+
+        name = table_section.select_one(".obj_head").get_text()
+        caption = table_section.select_one(".caption p").get_text()
+        label = name + " " + caption
+
+        number = "_".join(re.findall(r"(.{1})\.", name)).lower()
+        referee_id = f"table_{number}"
+    
+
+    return attribution, caption, figure_flag, headings, image_url, label, name, referee_id, section_url
+
+def to_chunk(text_block, section_url, referee_id, headings):
+
+    d = {
+    "text": text_block,
+    "metadata": {
+        "section": section_url,
+        "type": "table image",
+        "referee_id": referee_id,
+        "headings": headings,
+        }
+    }
+    return d
+
+def main():
+
+    with open('../data/raw/source.html', encoding="utf-8") as f:
+        html = f.read()
+        soup = BeautifulSoup(html)
+
+    with open("../data/processed/parsed_images.txt",  encoding="utf-8") as f:
+        text = f.read()
+        text_blocks = text.split("------")
+
+    docs = []
+    for graph, text_block in zip(soup.select(".graphic"), text_blocks):
+        attribution, caption, figure_flag, headings, image_url, label, name, referee_id, section_url = get_graph_metadata(graph)
+        text_block = text_block.strip()
+     
+        if text_block.startswith(name):
+            chunk = to_chunk(text_block, section_url, referee_id, headings)
+            docs.append(chunk)
+    
+    with open("../data/processed/graphs.json", "w", encoding="utf-8") as f:
+        json.dump(docs, f, indent=4)
+
+if __name__ == "__main__":
+    main()

src/data_processing/main.py

@@ -0,0 +1,294 @@
+import re
+import json
+from tables import get_table_metadata, to_text, get_table_data
+from bs4 import BeautifulSoup, Tag, NavigableString
+
+abbr_map = {
+    "ACT":   "Acceptance and commitment therapy",
+    "ADHD":  "Attention-deficit hyperactivity disorder",
+    "AI":    "Artificial intelligence",
+    "BA":    "Behavioural activation",
+    "CAM":   "Complementary and alternative medicine",
+    # "CANMAT":"Canadian Network for Mood and Anxiety Treatments",
+    "CBASP": "Cognitive behavioural analysis system of psychotherapy",
+    "CBT":   "Cognitive-behavioural therapy",
+    "CPD":   "Continuing professional development",
+    "CYP":   "Cytochrome P450",
+    "DBS":   "Deep brain stimulation",
+    "DHI":   "Digital health intervention",
+    "DLPFC": "Dorsolateral prefrontal cortex",
+    "DSM-5": "Diagnostic and Statistical Manual",
+    "DSM-5-TR": "Diagnostic and Statistical Manual, 5th edition, Text Revision",
+    "DSM-IV-TR":"Diagnostic and Statistical Manual, 4th edition, Text Revision",
+    "DTD":   "Difficult-to-treat depression",
+    "ECG":   "Electrocardiography",
+    "ECT":   "Electroconvulsive therapy",
+    "EEG":   "Electroencephalography",
+    "GRADE": "Grading of Recommendations Assessment, Development, and Evaluation",
+    "ICD":   "International Classification of Diseases",
+    "IPT":   "Interpersonal therapy",
+    "MAOI":  "Monoamine oxidase inhibitor",
+    "MBC":   "Measurement-based care",
+    "MBCT":  "Mindfulness-based cognitive therapy",
+    "MCT":   "Metacognitive therapy",
+    "MDD":   "Major depressive disorder",
+    "MDE":   "Major depressive episode",
+    "MI":    "Motivational interviewing",
+    "MST":   "Magnetic seizure therapy",
+    "NbN":   "Neuroscience-based nomenclature",
+    "NDRI":  "Norepinephrine-dopamine reuptake inhibitor",
+    "NMDA":  "N-methyl-D-aspartate",
+    "NSAID": "Nonsteroidal anti-inflammatory drug",
+    "PDD":   "Persistent depressive disorder",
+    "PDT":   "Psychodynamic psychotherapy",
+    "PHQ":   "Patient health questionnaire",
+    "PST":   "Problem-solving therapy",
+    "RCT":   "Randomized controlled trial",
+    "rTMS":  "Repetitive transcranial magnetic stimulation",
+    "SDM":   "Shared decision-making",
+    "SNRI":  "Serotonin-norepinephrine reuptake inhibitor",
+    "SSRI":  "Selective serotonin reuptake inhibitor",
+    "STPP":  "Short-term psychodynamic psychotherapy",
+    "TBS":   "Theta burst stimulation",
+    "TCA":   "Tricyclic antidepressants",
+    "tDCS":  "Transcranial direct current stimulation",
+    "TMS":   "Transcranial magnetic stimulation",
+    "TRD":   "Treatment-resistant depression",
+    "VNS":   "Vagus nerve stimulation",
+    "WHO":   "World Health Organization",
+    }
+
+def append_definition(guideline):
+    pattern = re.compile(r'\b([A-Z]{2,})\b')
+
+    for i in range(len(guideline)):
+        if guideline[i]['metadata']['referee_id'] == 'table_c':
+            continue
+        
+        text = guideline[i]['text']
+        
+        # Find all abbreviations in the text
+        found_abbrs = set()
+        matches = pattern.findall(text)
+        for abbr in matches:
+            if abbr in abbr_map:
+                found_abbrs.add(abbr)
+        
+        # Create definitions section if abbreviations found
+        if found_abbrs:
+            definitions = []
+            for abbr in sorted(found_abbrs):
+                definitions.append(f"{abbr}: {abbr_map[abbr]}")
+            
+            definitions_text = "Abbreviations: " + "; ".join(definitions) + "\n\n"
+            guideline[i]['text'] =  text + "\n" + definitions_text
+
+    return guideline
+
+def parse_title(soup):
+    title = soup.find("h1")
+    if title:
+        title = title.decode_contents().replace('\n', '')
+    return {
+        "text": title,
+        "metadata": {
+            "section": "title",
+            "type": "title",
+            "headings": "Title of the guideline document",
+            "referenced_tables": [],
+            "referee_id": ""
+        }
+    }
+
+def prepend_headings_to_text(guideline):
+    for i in range(len(guideline)):
+        guideline[i]['metadata']['chunk_id'] = i
+        guideline[i]['text'] = guideline[i]['metadata']['headings'] + " > paragraph id: " + str(i) + "\n\n" + guideline[i]['text']
+
+
+def build_headings_trail(p):
+    # build headings trail
+    heading = p.find_previous_sibling(lambda tag: bool(re.match(r'^h[2-6]$', tag.name)))
+    headings = heading.get_text(strip=True) if heading else 'No heading'
+    
+    parent_sec = p.find_parent(["section",'figure'], id=True)
+    while parent_sec:
+        heading = parent_sec.find_previous_sibling(lambda tag: bool(re.match(r'^h[2-6]$', tag.name)))
+        if heading:
+            headings = heading.get_text(strip=True) + ' > ' + headings
+        parent_sec = parent_sec.find_parent("section", id=True)
+    headings = headings.strip().replace('\n', ' ')
+    return headings
+
+
+def delete_bib_links(soup):
+    for a in soup.find_all("a", href=True):
+        if a["href"].startswith("#bdi12609-bib-"):
+            a.decompose()
+    return soup
+
+def delete_fig_and_tbl_sections(soup):
+    # Remove all <figure> and <table> sections
+    for fig in soup.find_all('figure'):
+        fig.decompose()
+    # find section that has class "tw xbox font-sm" and remove it
+    for section in soup.find_all('section', class_="tw xbox font-sm"):
+        section.decompose()
+    return soup
+    
+
+def scan_links_and_tables(p):
+    referenced_tables = set()
+    # scan for numeric links and reconstruct table IDs
+    for link in p.find_all('a', href=re.compile(r'-(fig|tbl)-')):
+        href = link['href']
+        tables = re.findall(r'(fig|tbl)-(\d+)', href)
+        for table_id in tables:
+            print(f"Found table links: {table_id[0]}-{table_id[1]}")
+            referenced_tables.add(f"{table_id[0]}-{table_id[1]}")
+    if referenced_tables:
+        print("--------")
+
+    return referenced_tables
+
+def parse_paragraph(soup, output):
+    paragraphs = soup.find_all('p')
+    for p in paragraphs:
+        parent = p.find_parent("section")
+        sec_id = parent["id"] if parent else "unknown"
+        
+        output.append({
+            "text": p.get_text(strip=False),
+            "metadata": {
+                "section": f"https://pmc.ncbi.nlm.nih.gov/articles/PMC5947163/#{sec_id}",
+                "type": "paragraph",
+                "headings": build_headings_trail(p),
+                "referenced_tables": list(scan_links_and_tables(p)),
+                "referee_id": "",
+            }
+        })            
+
+    
+def parse_figures(soup, output):
+    figures = soup.find_all('figure')
+    for fig in figures:
+        sec_id = fig["id"] 
+        fig_caption = fig.find('figcaption').find('p').get_text()
+        img_src_link = fig.find('img')['src'] if fig.find('img') else "No image link found"
+        referee_id = re.search(r'(fig)-(\d+)', sec_id)
+        
+        p = fig.find('p')
+        output.append({
+            "text": f" > Figure: Image link: {img_src_link}-----\nFigure Caption: {fig_caption}\n",
+            "metadata": {
+                "section": f"https://pmc.ncbi.nlm.nih.gov/articles/PMC5947163/#{sec_id}",
+                "type": "figure",
+                "headings": build_headings_trail(p),
+                "referenced_tables": list(scan_links_and_tables(p)),
+                "referee_id": referee_id.group(0) if referee_id else "fig_unknown",
+            }
+        })
+        
+def parse_tables(soup, output):
+    tables = soup.find_all("section", class_="tw xbox font-sm")
+    for table in tables:
+        sec_id = table["id"]
+        referee_id = re.search(r'(tbl)-(\d+)', sec_id)
+        
+        img = table.find('img')
+        if img: # then this is a img table
+            img_src_link = img['src'] if img else "No image link found"
+            p = table.find('p')
+            captions = table.find_all('div', class_='caption p')
+            caption = ""
+            for cap in captions:
+                caption += cap.get_text(strip=True) + " "
+            output.append({
+                "text": f" > Table: Image link: {img_src_link}-----\nTable Caption: {caption}\n",
+                "metadata": {
+                    "section": f"https://pmc.ncbi.nlm.nih.gov/articles/PMC5947163/#{sec_id}",
+                    "type": "table",
+                    "headings": build_headings_trail(p),
+                    "referenced_tables": [],
+                    "referee_id": referee_id.group(0) if referee_id else "tbl_unknown",
+                    }
+                })
+        else:  # else it is a html table
+            name, caption, footnotes, headings, label, ref_id, section_url = get_table_metadata(table, base_url="")
+            table_data = get_table_data(table, footnotes)
+            text = to_text(table_data, label, caption)
+            p = table.find('div')
+            
+            output.append({
+                "text": text,
+                "metadata": {
+                    "section": f"https://pmc.ncbi.nlm.nih.gov/articles/PMC5947163/#{sec_id}",
+                    "type": "table",
+                    "headings": build_headings_trail(p),
+                    "referenced_tables": [],
+                    "referee_id": referee_id.group(0) if referee_id else "tbl_unknown",
+                }
+            })
+        
+        
+
+def parse_main_article(soup, output):
+    
+    soup = delete_bib_links(soup)
+    parse_figures(soup, output)
+    parse_tables(soup, output)
+    
+    paragraph_only_soup = delete_fig_and_tbl_sections(soup)
+
+    parse_paragraph(paragraph_only_soup, output)
+    
+    # parse_tables(soup, output)
+
+    return output
+
+
+def main():
+    with open('bipolar.html', 'r', encoding='utf-8') as f:
+        html = f.read()
+    soup = BeautifulSoup(html, "html.parser")
+    output = []
+
+    output.append(parse_title(soup))
+    output = parse_main_article(soup, output)
+    print(f"Parsed {len(output)} paragraphs from the main article.")
+
+    combined = output
+    prepend_headings_to_text(combined)
+    append_definition(combined)
+
+    with open("guideline_db.json", "w", encoding="utf-8") as f:
+        json.dump(combined, f, ensure_ascii=False, indent=4)
+        print(f"guideline_db.json for bipolar created with {len(combined)} chunks.")
+
+
+def write_referenced_tables():
+    with open("guideline_db.json", "r", encoding="utf-8") as f:
+        guideline_db = json.load(f)
+        with open("table12_textual.txt", "r", encoding="utf-8") as f:
+            guideline_db[21]['text'] += f.read().strip()
+        with open("table14_textual.txt", "r", encoding="utf-8") as f:
+            guideline_db[23]['text'] += f.read().strip()
+        with open("table17_textual.txt", "r", encoding="utf-8") as f:
+            guideline_db[26]['text'] += f.read().strip()
+            
+        figures_and_tables = guideline_db[1:34]  # Assuming these are the table chunks
+        #write back to the original file
+        with open ("guideline_db.json", "w", encoding="utf-8") as f:
+            json.dump(guideline_db, f, ensure_ascii=False, indent=4)
+            print(f"guideline_db.json updated with table 12 14 17 chunks.")
+        with open("referenced_table_chunks.json", "w", encoding="utf-8") as f:
+            json.dump(figures_and_tables, f, ensure_ascii=False, indent=4)
+            print(f"referenced_table_chunks.json created with {len(figures_and_tables)} chunks.")
+            
+
+if __name__ == "__main__":
+    # main()
+    # read in the file guideline_db.json
+
+    # make the referenced_tables.json
+    write_referenced_tables()

src/data_processing/table12_textual.txt

@@ -0,0 +1,223 @@
+
+#### Table 12. Hierarchical rankings of first and second‐line treatments recommended for management of acute mania
+
+
+##### First-line treatments: Monotherapies
+
+
+###### Lithium
+
+
+####### Level of evidence by phase of treatment
+
+Acute mania: level 1 evidence. Prevention of any mood episode during maintenance: level 1 evidence. Prevention of mania during maintenance: level 1 evidence. Prevention of depression during maintenance: level 1 evidence. Acute depression: level 2 evidence.
+
+####### Considerations for treatment selection
+
+Acute phase safety concerns: minor impact on treatment selection. Acute phase tolerability concerns: minor impact on treatment selection. Maintenance phase safety concerns: moderate impact on treatment selection. Maintenance phase tolerability concerns: moderate impact on treatment selection. Risk of depressive switch: limited impact on treatment selection.
+
+###### Quetiapine
+
+
+####### Level of evidence by phase of treatment
+
+Acute mania: level 1 evidence. Prevention of any mood episode during maintenance: level 1 evidence. Prevention of mania during maintenance: level 1 evidence. Prevention of depression during maintenance: level 1 evidence. Acute depression: level 1 evidence.
+
+####### Considerations for treatment selection
+
+Acute phase safety concerns: minor impact on treatment selection. Acute phase tolerability concerns: moderate impact on treatment selection. Maintenance phase safety concerns: moderate impact on treatment selection. Maintenance phase tolerability concerns: moderate impact on treatment selection. Risk of depressive switch: limited impact on treatment selection.
+
+###### Divalproex
+
+
+####### Level of evidence by phase of treatment
+
+Acute mania: level 1 evidence. Prevention of any mood episode during maintenance: level 1 evidence. Prevention of mania during maintenance: level 3 evidence. Prevention of depression during maintenance: level 2 evidence. Acute depression: level 2 evidence.
+
+####### Considerations for treatment selection
+
+Acute phase safety concerns: limited impact on treatment selection. Acute phase tolerability concerns: minor impact on treatment selection. Maintenance phase safety concerns: moderate impact on treatment selection. Maintenance phase tolerability concerns: minor impact on treatment selection. Risk of depressive switch: limited impact on treatment selection.
+
+###### Asenapine
+
+
+####### Level of evidence by phase of treatment
+
+Acute mania: level 1 evidence. Prevention of any mood episode during maintenance: level 2 evidence. Prevention of mania during maintenance: level 2 evidence. Prevention of depression during maintenance: level 2 evidence. Acute depression: no data.
+
+####### Considerations for treatment selection
+
+Acute phase safety concerns: limited impact on treatment selection. Acute phase tolerability concerns: minor impact on treatment selection. Maintenance phase safety concerns: limited impact on treatment selection. Maintenance phase tolerability concerns: minor impact on treatment selection. Risk of depressive switch: limited impact on treatment selection.
+
+###### Aripiprazole
+
+
+####### Level of evidence by phase of treatment
+
+Acute mania: level 1 evidence. Prevention of any mood episode during maintenance: level 2 evidence. Prevention of mania during maintenance: level 2 evidence. Prevention of depression during maintenance: no data; although monotherapies are listed above combination therapies in the hierarchy, combination therapies may be indicated as the preferred choice in patients with previous history of partial response to monotherapy and in those with psychotic mania or in situations where rapid response is desirable. Acute depression: level 1 negative evidence.
+
+####### Considerations for treatment selection
+
+Acute phase safety concerns: limited impact on treatment selection. Acute phase tolerability concerns: minor impact on treatment selection. Maintenance phase safety concerns: limited impact on treatment selection. Maintenance phase tolerability concerns: minor impact on treatment selection. Risk of depressive switch: limited impact on treatment selection.
+
+###### Paliperidone(>6mg)
+
+
+####### Level of evidence by phase of treatment
+
+Acute mania: level 1 evidence. Prevention of any mood episode during maintenance: level 2 evidence. Prevention of mania during maintenance: level 2 evidence. Prevention of depression during maintenance: no data; although monotherapies are listed above combination therapies in the hierarchy, combination therapies may be indicated as the preferred choice in patients with previous history of partial response to monotherapy and in those with psychotic mania or in situations where rapid response is desirable. Acute depression: no data.
+
+####### Considerations for treatment selection
+
+Acute phase safety concerns: limited impact on treatment selection. Acute phase tolerability concerns: minor impact on treatment selection. Maintenance phase safety concerns: minor impact on treatment selection. Maintenance phase tolerability concerns: moderate impact on treatment selection. Risk of depressive switch: limited impact on treatment selection.
+
+###### Risperidone
+
+
+####### Level of evidence by phase of treatment
+
+Acute mania: level 1 evidence. Prevention of any mood episode during maintenance: level 3 evidence. Prevention of mania during maintenance: level 3 evidence. Prevention of depression during maintenance: no data. Acute depression: no data.
+
+####### Considerations for treatment selection
+
+Acute phase safety concerns: limited impact on treatment selection. Acute phase tolerability concerns: minor impact on treatment selection. Maintenance phase safety concerns: minor impact on treatment selection. Maintenance phase tolerability concerns: moderate impact on treatment selection. Risk of depressive switch: limited impact on treatment selection.
+
+###### Cariprazine
+
+
+####### Level of evidence by phase of treatment
+
+Acute mania: level 1 evidence. Prevention of any mood episode during maintenance: no data. Prevention of mania during maintenance: no data. Prevention of depression during maintenance: no data. Acute depression: level 1 evidence.
+
+####### Considerations for treatment selection
+
+Acute phase safety concerns: limited impact on treatment selection. Acute phase tolerability concerns: minor impact on treatment selection. Maintenance phase safety concerns: limited impact on treatment selection. Maintenance phase tolerability concerns: limited impact on treatment selection. Risk of depressive switch: limited impact on treatment selection.
+
+##### First-line treatments: Combination therapies
+
+
+###### Quetiapine and Lithium/divalproex
+
+
+####### Level of evidence by phase of treatment
+
+Acute mania: level 1 evidence. Prevention of any mood episode during maintenance: level 1 evidence. Prevention of mania during maintenance: level 1 evidence. Prevention of depression during maintenance: level 1 evidence. Acute depression: level 4 evidence; no controlled trials; however, clinical experience suggests that it is a useful strategy.
+
+####### Considerations for treatment selection
+
+Acute phase safety concerns: minor impact on treatment selection. Acute phase tolerability concerns: moderate impact on treatment selection. Maintenance phase safety concerns: significant impact on treatment selection; divalproex and carbamazepine should be used with caution in women of childbearing age. Maintenance phase tolerability concerns: moderate impact on treatment selection. Risk of depressive switch: limited impact on treatment selection.
+
+###### Aripiprazole and Lithium/divalproex
+
+
+####### Level of evidence by phase of treatment
+
+Acute mania: level 2 evidence. Prevention of any mood episode during maintenance: level 2 evidence. Prevention of mania during maintenance: level 2 evidence. Prevention of depression during maintenance: no data; did not separate from placebo in those with index mania; no studies available in index depression. Acute depression: level 4 evidence.
+
+####### Considerations for treatment selection
+
+Acute phase safety concerns: minor impact on treatment selection. Acute phase tolerability concerns: minor impact on treatment selection. Maintenance phase safety concerns: moderate impact on treatment selection; divalproex and carbamazepine should be used with caution in women of childbearing age. Maintenance phase tolerability concerns: moderate impact on treatment selection. Risk of depressive switch: limited impact on treatment selection.
+
+###### Risperidone and Lithium/divalproex
+
+
+####### Level of evidence by phase of treatment
+
+Acute mania: level 1 evidence. Prevention of any mood episode during maintenance: level 4 evidence. Prevention of mania during maintenance: level 4 evidence. Prevention of depression during maintenance: no data. Acute depression: level 4 evidence.
+
+####### Considerations for treatment selection
+
+Acute phase safety concerns: minor impact on treatment selection. Acute phase tolerability concerns: moderate impact on treatment selection. Maintenance phase safety concerns: significant impact on treatment selection; divalproex and carbamazepine should be used with caution in women of childbearing age. Maintenance phase tolerability concerns: moderate impact on treatment selection. Risk of depressive switch: limited impact on treatment selection.
+
+###### Asenapine and Lithium/divalproex
+
+
+####### Level of evidence by phase of treatment
+
+Acute mania: level 2 evidence. Prevention of any mood episode during maintenance: level 4 evidence. Prevention of mania during maintenance: level 4 evidence. Prevention of depression during maintenance: no data. Acute depression: level 4 evidence.
+
+####### Considerations for treatment selection
+
+Acute phase safety concerns: minor impact on treatment selection. Acute phase tolerability concerns: minor impact on treatment selection. Maintenance phase safety concerns: moderate impact on treatment selection; divalproex and carbamazepine should be used with caution in women of childbearing age. Maintenance phase tolerability concerns: minor impact on treatment selection. Risk of depressive switch: limited impact on treatment selection.
+
+##### Second-line treatments: Combination therapies
+
+
+###### Olanzapine
+
+
+####### Level of evidence by phase of treatment
+
+Acute mania: level 1 evidence. Prevention of any mood episode during maintenance: level 1 evidence. Prevention of mania during maintenance: level 1 evidence. Prevention of depression during maintenance: level 1 evidence. Acute depression: level 1 evidence; did not separate from placebo on core symptoms of depression.
+
+####### Considerations for treatment selection
+
+Acute phase safety concerns: minor impact on treatment selection. Acute phase tolerability concerns: moderate impact on treatment selection. Maintenance phase safety concerns: significant impact on treatment selection. Maintenance phase tolerability concerns: moderate impact on treatment selection. Risk of depressive switch: limited impact on treatment selection.
+
+###### Carbamazepine
+
+
+####### Level of evidence by phase of treatment
+
+Acute mania: level 1 evidence. Prevention of any mood episode during maintenance: level 2 evidence. Prevention of mania during maintenance: level 2 evidence. Prevention of depression during maintenance: level 2 evidence. Acute depression: level 3 evidence.
+
+####### Considerations for treatment selection
+
+Acute phase safety concerns: moderate impact on treatment selection. Acute phase tolerability concerns: minor impact on treatment selection. Maintenance phase safety concerns: moderate impact on treatment selection; divalproex and carbamazepine should be used with caution in women of childbearing age. Maintenance phase tolerability concerns: moderate impact on treatment selection. Risk of depressive switch: limited impact on treatment selection.
+
+###### Olanzapine and Lithium/divalproex
+
+
+####### Level of evidence by phase of treatment
+
+Acute mania: level 1 evidence. Prevention of any mood episode during maintenance: level 4 evidence. Prevention of mania during maintenance: level 4 evidence. Prevention of depression during maintenance: level 4 evidence. Acute depression: no data.
+
+####### Considerations for treatment selection
+
+Acute phase safety concerns: minor impact on treatment selection. Acute phase tolerability concerns: moderate impact on treatment selection. Maintenance phase safety concerns: moderate impact on treatment selection; divalproex and carbamazepine should be used with caution in women of childbearing age. Maintenance phase tolerability concerns: moderate impact on treatment selection. Risk of depressive switch: limited impact on treatment selection.
+
+###### Lithium and divalproex
+
+
+####### Level of evidence by phase of treatment
+
+Acute mania: level 2 evidence. Prevention of any mood episode during maintenance: level 2 evidence. Prevention of mania during maintenance: level 2 evidence. Prevention of depression during maintenance: no data. Acute depression: no data.
+
+####### Considerations for treatment selection
+
+Acute phase safety concerns: minor impact on treatment selection. Acute phase tolerability concerns: moderate impact on treatment selection. Maintenance phase safety concerns: moderate impact on treatment selection. Maintenance phase tolerability concerns: moderate impact on treatment selection. Risk of depressive switch: limited impact on treatment selection.
+
+###### Ziprasidone
+
+
+####### Level of evidence by phase of treatment
+
+Acute mania: level 1 evidence. Prevention of any mood episode during maintenance: level 3 evidence. Prevention of mania during maintenance: level 3 evidence. Prevention of depression during maintenance: no data. Acute depression: level 1 negative evidence.
+
+####### Considerations for treatment selection
+
+Acute phase safety concerns: moderate impact on treatment selection. Acute phase tolerability concerns: moderate impact on treatment selection. Maintenance phase safety concerns: moderate impact on treatment selection. Maintenance phase tolerability concerns: minor impact on treatment selection. Risk of depressive switch: limited impact on treatment selection.
+
+###### Haloperidol
+
+
+####### Level of evidence by phase of treatment
+
+Acute mania: level 1 evidence. Prevention of any mood episode during maintenance: no data. Prevention of mania during maintenance: level 4 evidence. Prevention of depression during maintenance: level 4 negative evidence. Acute depression: no data.
+
+####### Considerations for treatment selection
+
+Acute phase safety concerns: minor impact on treatment selection. Acute phase tolerability concerns: moderate impact on treatment selection. Maintenance phase safety concerns: significant impact on treatment selection. Maintenance phase tolerability concerns: moderate impact on treatment selection. Risk of depressive switch: moderate impact on treatment selection.
+
+###### ECT
+
+
+####### Level of evidence by phase of treatment
+
+Acute mania: level 3 evidence. Prevention of any mood episode during maintenance: level 4 evidence. Prevention of mania during maintenance: level 4 evidence. Prevention of depression during maintenance: level 4 evidence. Acute depression: level 4 evidence.
+
+####### Considerations for treatment selection
+
+Acute phase safety concerns: minor impact on treatment selection. Acute phase tolerability concerns: moderate impact on treatment selection. Maintenance phase safety concerns: minor impact on treatment selection. Maintenance phase tolerability concerns: moderate impact on treatment selection. Risk of depressive switch: limited impact on treatment selection.
+
+

src/data_processing/table14_textual.txt

@@ -0,0 +1,131 @@
+
+
+#### Table 14. Hierarchical rankings of first and second‐line treatments recommended for management of acute bipolar I depression
+
+
+##### First-line treatments
+
+###### Quetiapine
+
+
+####### Level of evidence by phase of treatment
+
+Acute depression: level 1 evidence. Prevention of any mood episode during maintenance: level 1 evidence. Prevention of depression during maintenance: level 1 evidence. Prevention of mania during maintenance: level 1 evidence. Acute mania: level 1 evidence.
+
+####### Considerations for treatment selection
+
+Acute phase safety concerns: minor impact on treatment selection. Acute phase tolerability concerns: moderate impact on treatment selection. Maintenance phase safety concerns: moderate impact on treatment selection. Maintenance phase tolerability concerns: moderate impact on treatment selection. Risk of manic / hypomanic switch: limited impact on treatment selection.
+
+###### Lurasidone and Lithium/divalproex
+
+
+####### Level of evidence by phase of treatment
+
+Acute depression: level 1 evidence. Prevention of any mood episode during maintenance: level 3 evidence; trend for superiority on the primary efficacy measure, hence the lower rating. Prevention of depression during maintenance: level 3 evidence; effective in those with an index episode of depression. Prevention of mania during maintenance: level 4 evidence; negative data from the trial are probably due to methodological issues; rating based on expert opinion. Acute mania: no data.
+
+####### Considerations for treatment selection
+
+Acute phase safety concerns: minor impact on treatment selection. Acute phase tolerability concerns: moderate impact on treatment selection. Maintenance phase safety concerns: moderate impact on treatment selection; divalproex and carbamazepine should be used with caution in women of childbearing age. Maintenance phase tolerability concerns: moderate / limited impact on treatment selection. Risk of manic / hypomanic switch: limited impact on treatment selection.
+
+###### Lithium
+
+
+####### Level of evidence by phase of treatment
+
+Acute depression: level 2 evidence. Prevention of any mood episode during maintenance: level 1 evidence. Prevention of depression during maintenance: level 1 evidence. Prevention of mania during maintenance: level 1 evidence. Acute mania: no data.
+
+####### Considerations for treatment selection
+
+Acute phase safety concerns: minor impact on treatment selection. Acute phase tolerability concerns: minor impact on treatment selection. Maintenance phase safety concerns: moderate impact on treatment selection. Maintenance phase tolerability concerns: moderate impact on treatment selection. Risk of manic / hypomanic switch: limited impact on treatment selection.
+
+###### Lamotrigine
+
+
+####### Level of evidence by phase of treatment
+
+Acute depression: level 2 evidence. Prevention of any mood episode during maintenance: level 1 evidence. Prevention of depression during maintenance: level 1 evidence. Prevention of mania during maintenance: level 2 evidence. Acute mania: level 1 negative evidence.
+
+####### Considerations for treatment selection
+
+Acute phase safety concerns: moderate impact on treatment selection. Acute phase tolerability concerns: limited impact on treatment selection. Maintenance phase safety concerns: limited impact on treatment selection. Maintenance phase tolerability concerns: limited impact on treatment selection. Risk of manic / hypomanic switch: limited impact on treatment selection.
+
+###### Lurasidone
+
+
+####### Level of evidence by phase of treatment
+
+Acute depression: level 2 evidence. Prevention of any mood episode during maintenance: level 4 evidence. Prevention of depression during maintenance: level 4 evidence. Prevention of mania during maintenance: level 4 evidence. Acute mania: no data.
+
+####### Considerations for treatment selection
+
+Acute phase safety concerns: limited impact on treatment selection. Acute phase tolerability concerns: minor impact on treatment selection. Maintenance phase safety concerns: limited impact on treatment selection. Maintenance phase tolerability concerns: minor impact on treatment selection. Risk of manic / hypomanic switch: limited impact on treatment selection.
+
+###### Lamotrigine(adj)
+
+
+####### Level of evidence by phase of treatment
+
+Acute depression: level 2 evidence. Prevention of any mood episode during maintenance: level 4 evidence. Prevention of depression during maintenance: level 4 evidence. Prevention of mania during maintenance: level 4 evidence. Acute mania: level 4 negative evidence.
+
+####### Considerations for treatment selection
+
+Acute phase safety concerns: moderate impact on treatment selection. Acute phase tolerability concerns: minor impact on treatment selection. Maintenance phase safety concerns: moderate impact on treatment selection. Maintenance phase tolerability concerns: moderate impact on treatment selection. Risk of manic / hypomanic switch: limited impact on treatment selection.
+
+##### Second-line treatments
+
+###### Divalproex
+
+
+####### Level of evidence by phase of treatment
+
+Acute depression: level 2 evidence. Prevention of any mood episode during maintenance: level 1 evidence. Prevention of depression during maintenance: level 2 evidence. Prevention of mania during maintenance: level 3 evidence. Acute mania: level 1 evidence.
+
+####### Considerations for treatment selection
+
+Acute phase safety concerns: limited impact on treatment selection. Acute phase tolerability concerns: minor impact on treatment selection. Maintenance phase safety concerns: moderate impact on treatment selection; divalproex and carbamazepine should be used with caution in women of childbearing age. Maintenance phase tolerability concerns: minor impact on treatment selection. Risk of manic / hypomanic switch: limited impact on treatment selection.
+
+###### SSRIs/bupropion(adj)
+
+
+####### Level of evidence by phase of treatment
+
+Acute depression: level 1 evidence. Prevention of any mood episode during maintenance: no data. Prevention of depression during maintenance: level 4 evidence. Prevention of mania during maintenance: no data. Acute mania: no data.
+
+####### Considerations for treatment selection
+
+Acute phase safety concerns: limited impact on treatment selection. Acute phase tolerability concerns: minor impact on treatment selection. Maintenance phase safety concerns: limited impact on treatment selection. Maintenance phase tolerability concerns: minor impact on treatment selection. Risk of manic / hypomanic switch: minor impact on treatment selection.
+
+###### ECT
+
+
+####### Level of evidence by phase of treatment
+
+Acute depression: level 4 evidence. Prevention of any mood episode during maintenance: level 4 evidence. Prevention of depression during maintenance: level 4 evidence. Prevention of mania during maintenance: level 4 evidence. Acute mania: level 3 evidence.
+
+####### Considerations for treatment selection
+
+Acute phase safety concerns: minor impact on treatment selection. Acute phase tolerability concerns: moderate impact on treatment selection. Maintenance phase safety concerns: minor impact on treatment selection. Maintenance phase tolerability concerns: moderate impact on treatment selection. Risk of manic / hypomanic switch: limited impact on treatment selection.
+
+###### Cariprazine
+
+
+####### Level of evidence by phase of treatment
+
+Acute depression: level 1 evidence. Prevention of any mood episode during maintenance: no data. Prevention of depression during maintenance: no data. Prevention of mania during maintenance: no data. Acute mania: level 1 evidence.
+
+####### Considerations for treatment selection
+
+Acute phase safety concerns: limited impact on treatment selection. Acute phase tolerability concerns: minor impact on treatment selection. Maintenance phase safety concerns: limited impact on treatment selection. Maintenance phase tolerability concerns: limited impact on treatment selection. Risk of manic / hypomanic switch: limited impact on treatment selection.
+
+###### Olanzapine-fluoxetine
+
+
+####### Level of evidence by phase of treatment
+
+Acute depression: level 2 evidence. Prevention of any mood episode during maintenance: no data. Prevention of depression during maintenance: no data. Prevention of mania during maintenance: no data. Acute mania: no data.
+
+####### Considerations for treatment selection
+
+Acute phase safety concerns: minor impact on treatment selection. Acute phase tolerability concerns: moderate impact on treatment selection. Maintenance phase safety concerns: significant impact on treatment selection. Maintenance phase tolerability concerns: minor impact on treatment selection. Risk of manic / hypomanic switch: minor impact on treatment selection.
+
+

src/data_processing/table17_textual.txt

@@ -0,0 +1,178 @@
+
+
+
+#### Table 17. Hierarchical rankings of first‐ and second‐line treatments recommended for maintenance treatment in bipolar disorder
+
+
+##### First-line treatment
+
+
+###### Lithium 
+
+
+####### Level of evidence by phase of treatment
+
+Prevention of any mood episode during maintenance: level 1 evidence. Prevention of depression during maintenance: level 1 evidence. Prevention of mania during maintenance: level 1 evidence. Acute Depression: level 2 evidence. Acute Mania: level 1 evidence. 
+
+####### Considerations for treatment selection
+
+Acute phase safety concerns: minor impact on treatment selection. Acute phase tolerability concerns: minor impact on treatment selection. Maintenance phase safety concerns: moderate impact on treatment selection. Maintenance phase tolerability concerns: moderate impact on treatment selection.
+
+###### Quetiapine 
+
+
+####### Level of evidence by phase of treatment
+
+Prevention of any mood episode during maintenance: level 1 evidence. Prevention of depression during maintenance: level 1 evidence. Prevention of mania during maintenance: level 1 evidence. Acute Depression: level 1 evidence. Acute Mania: level 1 evidence. 
+
+####### Considerations for treatment selection
+
+Acute phase safety concerns: minor impact on treatment selection. Acute phase tolerability concerns: moderate impact on treatment selection. Maintenance phase safety concerns: moderate impact on treatment selection. Maintenance phase tolerability concerns: moderate impact on treatment selection.
+
+###### Lamotrigine 
+
+
+####### Level of evidence by phase of treatment
+
+Prevention of any mood episode during maintenance: level 1 evidence. Prevention of depression during maintenance: level 1 evidence. Prevention of mania during maintenance: level 2 evidence. Acute Depression: level 1 evidence. Acute Mania: level 1 negative evidence. 
+
+####### Considerations for treatment selection
+
+Acute phase safety concerns: moderate impact on treatment selection. Acute phase tolerability concerns: limited impact on treatment selection. Maintenance phase safety concerns: limited impact on treatment selection. Maintenance phase tolerability concerns: limited impact on treatment selection.
+
+###### Asenapine 
+
+
+####### Level of evidence by phase of treatment
+
+Prevention of any mood episode during maintenance: level 2 evidence. Prevention of depression during maintenance: level 2 evidence. Prevention of mania during maintenance: level 2 evidence. Acute Depression: no data. Acute Mania: level 1 evidence. 
+
+####### Considerations for treatment selection
+
+Acute phase safety concerns: limited impact on treatment selection. Acute phase tolerability concerns: minor impact on treatment selection. Maintenance phase safety concerns: limited impact on treatment selection. Maintenance phase tolerability concerns: minor impact on treatment selection.
+
+###### Quetiapine + Lithium/divalproex 
+
+
+####### Level of evidence by phase of treatment
+
+Prevention of any mood episode during maintenance: level 1 evidence. Prevention of depression during maintenance: level 1 evidence. Prevention of mania during maintenance: level 1 evidence. Acute Depression: level 4 evidence. Acute Mania: level 1 evidence. 
+
+####### Considerations for treatment selection
+
+Acute phase safety concerns: minor impact on treatment selection. Acute phase tolerability concerns: moderate impact on treatment selection. Maintenance phase safety concerns: significant impact on treatment selection; divalproex and carbamazepine should be used with caution in women of childbearing age. Maintenance phase tolerability concerns: moderate impact on treatment selection.
+
+###### Aripiprazole + Lithium/divalproex 
+
+
+####### Level of evidence by phase of treatment
+
+Prevention of any mood episode during maintenance: level 2 evidence. Prevention of depression during maintenance: no data; did not separate from placebo in those with index mania; no studies available in index depression. Prevention of mania during maintenance: level 2 evidence. Acute Depression: level 4 evidence. Acute Mania: level 2 evidence. 
+
+####### Considerations for treatment selection
+
+Acute phase safety concerns: minor impact on treatment selection. Acute phase tolerability concerns: minor impact on treatment selection. Maintenance phase safety concerns: moderate impact on treatment selection; divalproex and carbamazepine should be used with caution in women of childbearing age. Maintenance phase tolerability concerns: moderate impact on treatment selection.
+
+###### Aripiprazole 
+
+
+####### Level of evidence by phase of treatment
+
+Prevention of any mood episode during maintenance: level 2 evidence. Prevention of depression during maintenance: no data; did not separate from placebo in those with index mania; no studies available in index depression. Prevention of mania during maintenance: level 2 evidence. Acute Depression: level 1 negative evidence. Acute Mania: level 1 evidence. 
+
+####### Considerations for treatment selection
+
+Acute phase safety concerns: limited impact on treatment selection. Acute phase tolerability concerns: minor impact on treatment selection. Maintenance phase safety concerns: limited impact on treatment selection. Maintenance phase tolerability concerns: minor impact on treatment selection.
+
+###### Aripiprazole OM 
+
+
+####### Level of evidence by phase of treatment
+
+Prevention of any mood episode during maintenance: level 2 evidence. Prevention of depression during maintenance: no data; did not separate from placebo in those with index mania; no studies available in index depression. Prevention of mania during maintenance: level 2 evidence. Acute Depression: no data. Acute Mania: no data. 
+
+####### Considerations for treatment selection
+
+Acute phase safety concerns: limited impact on treatment selection. Acute phase tolerability concerns: minor impact on treatment selection. Maintenance phase safety concerns: limited impact on treatment selection. Maintenance phase tolerability concerns: minor impact on treatment selection.
+
+##### Second-line treatments
+
+
+###### Olanzapine 
+
+
+####### Level of evidence by phase of treatment
+
+Prevention of any mood episode during maintenance: level 1 evidence. Prevention of depression during maintenance: level 1 evidence. Prevention of mania during maintenance: level 1 evidence. Acute Depression: level 1 evidence; did not separate from placebo on core symptoms of depression. Acute Mania: level 1 evidence. 
+
+####### Considerations for treatment selection
+
+Acute phase safety concerns: minor impact on treatment selection. Acute phase tolerability concerns: moderate impact on treatment selection. Maintenance phase safety concerns: significant impact on treatment selection. Maintenance phase tolerability concerns: moderate impact on treatment selection.
+
+###### Risperidone LAI 
+
+
+####### Level of evidence by phase of treatment
+
+Prevention of any mood episode during maintenance: level 1 evidence. Prevention of depression during maintenance: no data; did not separate from placebo in those with index mania; no studies available in index depression. Prevention of mania during maintenance: level 1 evidence. Acute Depression: no data. Acute Mania: no data. 
+
+####### Considerations for treatment selection
+
+Acute phase safety concerns: limited impact on treatment selection. Acute phase tolerability concerns: minor impact on treatment selection. Maintenance phase safety concerns: minor impact on treatment selection. Maintenance phase tolerability concerns: moderate impact on treatment selection.
+
+###### Risperidone LAI (adj) 
+
+
+####### Level of evidence by phase of treatment
+
+Prevention of any mood episode during maintenance: level 2 evidence. Prevention of depression during maintenance: level 4 evidence. Prevention of mania during maintenance: level 2 evidence. Acute Depression: no data. Acute Mania: no data. 
+
+####### Considerations for treatment selection
+
+Acute phase safety concerns: minor impact on treatment selection. Acute phase tolerability concerns: moderate impact on treatment selection. Maintenance phase safety concerns: significant impact on treatment selection. Maintenance phase tolerability concerns: moderate impact on treatment selection.
+
+###### Carbamazepine 
+
+
+####### Level of evidence by phase of treatment
+
+Prevention of any mood episode during maintenance: level 2 evidence. Prevention of depression during maintenance: level 2 evidence. Prevention of mania during maintenance: level 2 evidence. Acute Depression: level 3 evidence. Acute Mania: level 1 evidence. 
+
+####### Considerations for treatment selection
+
+Acute phase safety concerns: moderate impact on treatment selection. Acute phase tolerability concerns: moderate impact on treatment selection. Maintenance phase safety concerns: minor impact on treatment selection; divalproex and carbamazepine should be used with caution in women of childbearing age. Maintenance phase tolerability concerns: moderate impact on treatment selection.
+
+###### Paliperidone (>6 mg) 
+
+
+####### Level of evidence by phase of treatment
+
+Prevention of any mood episode during maintenance: level 2 evidence. Prevention of depression during maintenance: no data; did not separate from placebo in those with index mania; no studies available in index depression. Prevention of mania during maintenance: level 2 evidence. Acute Depression: no data. Acute Mania: level 1 evidence. 
+
+####### Considerations for treatment selection
+
+Acute phase safety concerns: limited impact on treatment selection. Acute phase tolerability concerns: minor impact on treatment selection. Maintenance phase safety concerns: minor impact on treatment selection. Maintenance phase tolerability concerns: moderate impact on treatment selection.
+
+###### Lurasidone + Lithium/divalproex 
+
+
+####### Level of evidence by phase of treatment
+
+Prevention of any mood episode during maintenance: level 3 evidence; trend for superiority on the primary efficacy measure, hence the lower rating. Prevention of depression during maintenance: level 3 evidence; effective in those with an index episode of depression. Prevention of mania during maintenance: level 4 evidence. Acute Depression: level 2 evidence. Acute Mania: no data. 
+
+####### Considerations for treatment selection
+
+Acute phase safety concerns: minor impact on treatment selection. Acute phase tolerability concerns: moderate impact on treatment selection. Maintenance phase safety concerns: moderate impact on treatment selection; divalproex and carbamazepine should be used with caution in women of childbearing age. Maintenance phase tolerability concerns: moderate / limited impact on treatment selection.
+
+###### Ziprasidone + Lithium/divalproex 
+
+
+####### Level of evidence by phase of treatment
+
+Prevention of any mood episode during maintenance: level 2 evidence. Prevention of depression during maintenance: no data; did not separate from placebo in those with index mania; no studies available in index depression. Prevention of mania during maintenance: level 2 evidence. Acute Depression: level 3 negative evidence. Acute Mania: level 2 negative evidence. 
+
+####### Considerations for treatment selection
+
+Acute phase safety concerns: moderate impact on treatment selection. Acute phase tolerability concerns: moderate impact on treatment selection. Maintenance phase safety concerns: moderate impact on treatment selection; divalproex and carbamazepine should be used with caution in women of childbearing age. Maintenance phase tolerability concerns: minor impact on treatment selection.
+
+

src/data_processing/tables.py

@@ -0,0 +1,172 @@
+from bs4 import BeautifulSoup
+import json
+import pandas as pd
+import re
+
+
+def get_table_metadata(table, base_url):
+    # Find the nearest ancestor <section> that has an id
+    section = table.find_parent('section')
+    while section and not section.get('id'):
+        section = section.find_parent('section')
+
+    section_id = section.get("id") if section else None
+    if section_id:
+        section_url = base_url + section_id
+        # Try to get the main heading
+        heading_el = section.select_one(f"[data-anchor-id={section_id}]")
+        section_heading = heading_el.get_text(strip=True) if heading_el else ""
+        # Try to get the subheading from its parent section
+        parent_sec = section.find_parent('section')
+        subheading_el = parent_sec.select_one(".pmc_sec_title") if parent_sec else None
+        section_subheading = subheading_el.get_text(strip=True) if subheading_el else ""
+        headings = " > ".join(filter(None, [section_heading, section_subheading]))
+    else:
+        # Fallback if no section id is found
+        section_url = base_url
+        headings = ""
+
+    # Table name and caption
+    name_el = section.find("h4") if section else table.find("caption")
+    name = name_el.get_text(strip=True) if name_el else "Table"
+    caption_el = section.select_one('.caption p') if section else table.find("caption")
+    caption = caption_el.get_text(strip=True) if caption_el else ""
+
+    # Generate a referee_id from the table name
+    # e.g., "Table 1." → number = "1" → referee_id = "table_1"
+    # Look for parent <section> with an id containing 'tbl-'
+    section = table.find_parent("section", id=True)
+    table_id = section["id"] if section and "tbl-" in section["id"] else None
+    print(table_id)
+    match = re.search(r"tbl-(\d+)", table_id or "")
+    number = match.group(1).lstrip("0") if match else ""
+    referee_id = f"table_{number}" if number else "table_unknown"
+    label = f"Table {number}. " + caption if number else "Table"
+
+    # Collect footnotes
+    footnotes = {}
+    # case 1: <sup> outside <p>
+    for sup in section.select('.fn sup') if section else []:
+        sibling = sup.find_next_sibling("p")
+        if sibling:
+            key = sup.get_text(strip=True)
+            footnotes[key] = sibling.get_text(strip=True)
+
+    # case 2: <sup> inside <p>
+    for p in section.select('.fn p') if section else []:
+        # matches like "* text" or "# text"
+        matches = re.findall(r"(?<=(\*|#))\s*(.*?)(?=\s\*|\s#|$)", p.get_text())
+        for key, text in matches:
+            footnotes[key] = text.strip()
+    print(name)
+    return name, caption, footnotes, headings, label, referee_id, section_url
+
+
+def get_table_data(table, footnotes):
+    table_data = []
+    rowspan_tracker = {}
+    subsec = ""
+
+    for tr in table.find_all("tr"):
+        row = []
+        col_index = 0
+
+        # Pre-fill cells carried over by rowspan
+        while col_index in rowspan_tracker:
+            value, remaining = rowspan_tracker[col_index]
+            row.append(value)
+            remaining -= 1
+            if remaining:
+                rowspan_tracker[col_index] = (value, remaining)
+            else:
+                del rowspan_tracker[col_index]
+            col_index += 1
+
+        for cell in tr.find_all(["th", "td"]):
+            cell_text = cell.get_text(separator="\n", strip=True)
+            cell_sups = [sup.get_text() for sup in cell.find_all("sup")]
+
+            # Normalize text if superscripts are inside
+            if cell_sups:
+                # remove short tokens
+                lines = [t for t in cell_text.split("\n") if len(t) > 1]
+                cell_text = " ".join(lines)
+
+            # Append footnote text if any
+            for sup in cell_sups:
+                if sup in footnotes:
+                    cell_text += f" ({footnotes[sup]})"
+
+            # Handle colspan as a subsection marker
+            colspan = int(cell.get("colspan", 1))
+            if colspan > 1:
+                subsec = cell_text
+                continue
+
+            row.append(cell_text)
+
+            # Track rowspan for this column
+            rowspan = int(cell.get("rowspan", 1))
+            if rowspan > 1:
+                rowspan_tracker[col_index] = (cell_text, rowspan - 1)
+
+            col_index += 1
+
+        if row:
+            if subsec:
+                row.insert(0, subsec)
+            table_data.append(row)
+
+    return table_data
+
+
+def to_text(table_data, label, caption):
+    lines = []
+    lines.append(f"**{label}**")
+
+    # Skip header row when enumerating data rows
+    headers = table_data[0] if table_data else []
+    for i, row in enumerate(table_data[1:], start=1):
+        row_text = ", ".join(f"{h}: {v}" for h, v in zip(headers, row) if v)
+        lines.append(f"{{Row {i} - {row_text}}}")
+
+    return "[" + "\n".join(lines) + "]"
+
+
+def to_chunk(text_block, section_url, referee_id, headings):
+    return {
+        "text": text_block,
+        "metadata": {
+            "section": section_url,
+            "type": "HTML table",
+            "referee_id": referee_id,
+            # "headings": headings,
+        }
+    }
+
+
+def tables_to_json(input_path="bipolar.html", base_url="https://pmc.ncbi.nlm.nih.gov/articles/PMC5947163/#"):
+    doc = []
+    with open(input_path, encoding="utf-8") as f:
+        html = f.read()
+        soup = BeautifulSoup(html, features="html.parser")
+        tables = soup.find_all("table")
+
+    print(f"Found {len(tables)} tables in document.")
+
+    for idx, tbl in enumerate(tables, start=1):
+        name, caption, footnotes, headings, label, referee_id, section_url = \
+            get_table_metadata(tbl, base_url)
+        table_data = get_table_data(tbl, footnotes)
+        text_block = to_text(table_data, label, caption)
+        chunk = to_chunk(text_block, section_url, referee_id, headings)
+        doc.append(chunk)
+
+    return doc
+
+
+if __name__ == "__main__":
+    # doc = tables_to_json()
+    # with open("tables.json", "w", encoding="utf-8") as f:
+    #     json.dump(doc, f, indent=4)
+    pass

src/precompute.py

@@ -0,0 +1,23 @@
+
+from sentence_transformers import SentenceTransformer
+from Rag import load_json_to_db, make_embeddings, save_embeddings  # Adjust import
+
+def precompute_and_save(embedder_name, db_path):
+    print("Loading database...")
+    db = load_json_to_db(db_path)
+
+    print(f"Loading embedder: {embedder_name}")
+    model = SentenceTransformer(embedder_name, trust_remote_code=True)
+
+    print("Computing embeddings...")
+    embeddings = make_embeddings(model, embedder_name, db)
+
+    print("Saving embeddings...")
+    save_embeddings(embedder_name, embeddings)
+
+    print("Done.")
+
+if __name__ == "__main__":
+    embedder_name = "Qwen/Qwen3-Embedding-0.6B"  # Example embedder name
+    db_path = "../data/processed/guideline_db.json"
+    precompute_and_save(embedder_name, db_path)

src/readme.md

@@ -0,0 +1,9 @@
+- `Rag.py` is the Core RAG pipeline, backend
+
+- `app.py` is the frontend code implemented with streamlit
+
+- `data_processing.py` contains code to process the original guideline knowledge base for the rag system
+
+- `system_prompt.txt` is the system prompt we give to the LLM
+
+- `run_batched_queries/` contains code to run multiple queries using the system and write results to a markdown file

src/system_prompt.txt

@@ -0,0 +1,15 @@
+You are a clinical decision support assistant. Use provided Clinical Guidelines Context to answer the user's question.
+
+- Carefully review the retrieved text and find any relevant medication recommendations, treatment considerations, or patient-specific factors.
+- If multiple options exist, summarize the top 1–2 and explain briefly why they are preferred.
+- Quote the reference link provided if you used the info in that context to answer the question.
+- If there is no explicit answer, you may reasonably infer from related sections (e.g. similar symptoms, comorbidities, or past treatments), but make that clear.
+- If absolutely no relevant information is available in the manual, respond with: "No clear recommendation found in the clinical guideline."
+- Use EXACT medication names from the context, Use markdown bold fonts on all the medication.
+- Specify treatment line (first-line, second-line, etc.). If not mentioned, provide first-line option will be sufficient.
+- Include relevant clinical details (dosing, monitoring, contraindications)
+- If multiple options exist, list them clearly
+- Base recommendations STRICTLY on the provided context
+
+- Output in structural format with bullet points and bold fonts if necessary, and quote the important reference links that you used like this:
+    - **Reference:** [Table 3.1](https://pmc.ncbi.nlm.nih.gov/articles/PMC11351064/#section4F-07067437241245384)