✨ Working version of RAG with HYDE

app.py

@@ -1,18 +1,468 @@
+import re
+import math
+import yaml
+import json
+import torch
+import faiss
+import string
+import asyncio
+import pymupdf
 import gradio as gr
 
-def greet(files, query):
-    names = [file.name for file in files]
-    return f"{len(files)}, {names}, {query}"
+from time import time
+from pathlib import Path
+from functools import lru_cache
+from ast import literal_eval
+from collections import defaultdict
+from concurrent.futures import ThreadPoolExecutor
+from sentence_transformers import SentenceTransformer
+from langchain.text_splitter import SentenceTransformersTokenTextSplitter
+from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, AutoModelForCausalLM
 
-with gr.Blocks() as demo:
-    pdf_input = gr.File(label='upload PDF(s)', file_types=['.pdf'], file_count='multiple')
-    query = gr.Textbox(label='question')
-    answer = gr.Textbox(label='answer')
+from src.config import PROMPTS_FILEPATH, log
+
+
+async def load_pdfs(files, max_concurrence=5):
+    """
+    Load multiple PDF files async.
+
+    Args:
+        folder (str): Path to folder containing PDF files
+        max_concurrence (int): Maximum number of concurrent PDF processing tasks
+
+    Returns:
+        list: List of tuples containing (filename, extracted_text)
+    """
+
+    def _load_pdf_sync(file):
+        """Synchronous PDF loading function for thread pool execution"""
+        text = ""
+        try:
+            with pymupdf.open(file, filetype="pdf") as doc:
+                text = "\n".join(page.get_text() for page in doc)
+        except Exception:
+            log.exception(f"Error reading {file.name}")
+            pass
+
+        return (file.name, text)
+
+    loop = asyncio.get_event_loop()
+    with ThreadPoolExecutor(max_workers=max_concurrence) as executor:
+        futures = [
+            loop.run_in_executor(executor, _load_pdf_sync, file)
+            for file in files
+            if file is not None
+        ]
+
+        results = await asyncio.gather(*futures, return_exceptions=True)
+
+    valid_results = [result for result in results if not isinstance(result, Exception)]
+
+    log.info(f"successfully processed {len(valid_results)} out of {len(files)} PDFs ")
+    return valid_results
+
+
+async def build_corpus(pdfs, text_splitter, **load_kwargs):
+    texts = await load_pdfs(pdfs, **load_kwargs)
+    corpus, meta = [], []
+    for file_name, raw_text in texts:
+        chunks = text_splitter.split_text(raw_text)
+        for i, chunk in enumerate(chunks):
+            corpus.append(chunk)
+            meta.append({"file": file_name, "chunk_id": i})
+    return corpus, meta
+
+
+def generate_text(
+    tokenizer, model, user_prompts, system_prompt=None, **llm_kwargs
+):  # max_new_tokens=512, temperature=.4):
+    if system_prompt is None or "":
+        system_prompt = "You are a helpful assistant."
+
+    if isinstance(user_prompts, str):
+        user_prompts = [user_prompts]
+
+    messages = [
+        [
+            {"role": "system", "content": system_prompt},
+            {"role": "user", "content": user_prompt},
+        ]
+        for user_prompt in user_prompts
+    ]
+
+    texts = tokenizer.apply_chat_template(
+        messages, tokenize=False, add_generation_prompt=True
+    )
+
+    model_inputs = tokenizer(
+        texts, return_tensors="pt", truncation=True, padding=True
+    ).to(model.device)
+    generated_ids = model.generate(
+        **model_inputs,
+        max_new_tokens=llm_kwargs.pop("max_new_tokens", 512),
+        temperature=llm_kwargs.pop("temperature", 0.4),
+    )
+    generated_ids = [
+        output_ids[len(input_ids) :]
+        for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)
+    ]
+    response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)
+    return response if len(user_prompts) > 1 else response[0]
+
+
+def load_models(
+    embed_model_name: str,
+    gen_model_name: str,
+    causal_lm: bool = False,
+    device=None,
+    bitsandbytesconfig=None,
+):
+    # This will take some time to run for the first time if the model(s) don't exist locally.
+    if not device:
+        device = "cuda" if torch.cuda.is_available() else "cpu"
+    embedder = SentenceTransformer(
+        embed_model_name,
+        device=device,
+        model_kwargs={"dtype": "float16"} if device == "cuda" else {},
+    )
+
+    if not causal_lm:
+        tok = AutoTokenizer.from_pretrained(gen_model_name)
+        gen = AutoModelForSeq2SeqLM.from_pretrained(
+            gen_model_name,  # device_map='auto',
+            quantization_config=bitsandbytesconfig if bitsandbytesconfig else None,
+        )
+    else:
+        tok = AutoTokenizer.from_pretrained(gen_model_name, padding_side="left")
+        gen = AutoModelForCausalLM.from_pretrained(
+            gen_model_name,
+            dtype="float16",  # device_map='auto',
+            quantization_config=bitsandbytesconfig if bitsandbytesconfig else None,
+        )
+    gen.to(device)
+    return embedder, tok, gen
+
+
+def make_query_variants(
+    tokenizer, model, query: str, prompt: str, n: int = 3, **llm_kwargs
+):
+    instructions = f"Now give me at least {n} variations."
+    resp = generate_text(tokenizer, model, query + instructions, prompt, **llm_kwargs)
+
+    clean_resp = re.sub(r"^\d+\.\s*", "", resp, flags=re.MULTILINE).split("\n")
+    return [query] + [q for q in clean_resp if q.strip()]
+
+
+def clean_rewrite_resp(resp):
+    try:
+        resp = json.loads(resp)  # Parse JSON
+    except json.JSONDecodeError:
+        try:
+            resp = literal_eval(resp)  # Fallback parse
+        except Exception:
+            pass  # Keep resp as-is if both fail
+
+    # Ensure resp is a string before strip and slicing
+    if isinstance(resp, str):
+        resp = resp.strip()
+        if resp:
+            start = resp.find("{")
+            if start != -1:
+                end = resp[::-1].find("}")
+                if end != -1:
+                    resp = resp[start : len(resp) - end]
+                    return clean_rewrite_resp(resp)
+    return resp
+
+
+def transform_query(
+    tokenizer, model, query: str, rewrite_prompt: str, **llm_kwargs
+) -> dict:
+    """split the query into things to search and actions to take"""
+    resp = generate_text(tokenizer, model, query, rewrite_prompt, **llm_kwargs)
+    try:
+        resp = clean_rewrite_resp(resp)
+    except:
+        pass
+    return resp
+
+
+def aggregate_queries_and_tasks(
+    tokenizer,
+    model,
+    orig_query,
+    rewrite_prompt,
+    variants_prompt,
+    n_variations=3,
+    **llm_kwargs,
+):
+    # make variations for the original query as is
+    queries = make_query_variants(
+        tokenizer,
+        model,
+        orig_query.strip(),
+        variants_prompt,
+        n_variations,
+        **llm_kwargs,
+    )
+
+    start = time()
+    tr_q = transform_query(tokenizer, model, orig_query.strip(), rewrite_prompt)
+    end = time()
+    log.debug(f"\t\t transforming query task took {(end - start):.1f} seconds...")
+
+    # transformed query might have multiple things to search and tasks to perform depending on user query
+    # recursively get variations for each of the search queries but keep the tasks as is.
+    tasks = []
+    if isinstance(tr_q, dict):
+        search_results, tasks = tr_q.get("search", []), tr_q.get("tasks", [])
+        for search_result in search_results:
+            queries.extend(
+                make_query_variants(
+                    tokenizer,
+                    model,
+                    search_result,
+                    variants_prompt,
+                    n_variations,
+                    **llm_kwargs,
+                )
+            )
+
+    queries = [q.strip(string.punctuation) for q in queries]
+    tasks = [t.strip(string.punctuation) for t in tasks]
+
+    # keep the original user query as is (if in case LLM messes up the original query) and pick some after shuffling the rest
+    # This is disabled as we don't do loops and instead take advantage of batches.
+    # Since it's efficient, we can take many query variations at once without worrying about performance.
+    # q, queries = queries[:1], queries[1:]
+    # shuffle(queries)
+    # q += queries[:n_variations-1]
+
+    return queries, tasks
+
+
+def build_index(corpus_emb, n_cells=5, n_probe=2):
+    log.debug(f"building index with {n_cells=}, {n_probe=}")
+    d = corpus_emb.shape[1]
+    quantizer = faiss.IndexFlatIP(d)
+    index = faiss.IndexIVFFlat(quantizer, d, n_cells)
+    index.n_probe = n_probe
+    index.train(corpus_emb)
+    index.add(corpus_emb)
+    # index.make_direct_map()
+    return index
+
+
+def reciprocal_rank_fusion(indices, top_k=3, denom=50):
+    ii = indices.tolist()
+    scores = defaultdict(int)
+    for row in ii:
+        for rank, chunk_id in enumerate(row):
+            scores[chunk_id] += 1 / (rank + denom)
+    results = sorted(scores.items(), key=lambda x: x[1], reverse=True)[:top_k]
+    return [chunk_id for chunk_id, _ in results]
+
+
+class HyDeRAGFusion:
+    def __init__(
+        self,
+        embed_model: str,
+        generator_llm_model: str,
+        causal_lm: bool = True,
+        chunk_overlap: int = 50,
+        tokens_per_chunk: int = 256,
+        embed_batch_size: int = 64,
+        bitsandbytesconfig=None,
+    ):
+        self.embed_batch_size = embed_batch_size
+        self.text_splitter = SentenceTransformersTokenTextSplitter(
+            chunk_overlap, embed_model, tokens_per_chunk
+        )
+        self.embedder, self.tok, self.gen = load_models(
+            embed_model, generator_llm_model, causal_lm, bitsandbytesconfig
+        )
+        with open(PROMPTS_FILEPATH) as fl:
+            self.prompts = yaml.safe_load(fl)
+
+    @lru_cache(maxsize=8)
+    def preprocess_pdfs(self, pdfs, data_load_kwargs={}, faiss_index_kwargs={}):
+        self.corpus, self.meta = asyncio.run(
+            build_corpus(pdfs, self.text_splitter, **data_load_kwargs)
+        )
+        log.debug(f"{len(self.corpus)}, {len(self.meta)}")
+        self.corpus_emb = self.embedder.encode(
+            self.corpus,
+            batch_size=self.embed_batch_size,
+            show_progress_bar=True,
+            normalize_embeddings=True,
+        )
+        log.debug(f'{self.corpus_emb.shape}')
+
+        # https://github.com/facebookresearch/faiss/issues/112
+        # n_cells = int(round(4 * (self.corpus_emb.shape[0])**.5))
+
+        # one centroid for every 100 or so vectors and 20% of them as n_probe
+        n_cells = faiss_index_kwargs.pop("n_cells", self.corpus_emb.shape[0] // 100 + 1)
+        n_probe = faiss_index_kwargs.pop("n_probe", math.ceil(0.2 * n_cells))
+
+        self.index = build_index(self.corpus_emb, n_cells, n_probe)
+
+    def retrieve(
+        self, query, n_variants=3, top_k_per_variant=10, top_k_retrieve=3, **llm_kwargs
+    ):
+        start = time()
+
+        queries, tasks = aggregate_queries_and_tasks(
+            self.tok,
+            self.gen,
+            query.strip(),
+            self.prompts["rewrite"],
+            self.prompts["variants"],
+            n_variants,
+            **llm_kwargs,
+        )
+
+        end = time()
+        log.debug(f"aggregate task took {(end - start):.1f} seconds...")
+
+        start = time()
+        hyde_docs = generate_text(
+            self.tok, self.gen, queries, self.prompts["hyde"], **llm_kwargs
+        )
+        end = time()
+        log.debug(f"generating hyde docs took {(end - start):.1f} seconds...")
+
+        start = time()
+        chunks = []
+        for hyde_doc in hyde_docs:
+            chunks.extend(self.text_splitter.split_text(hyde_doc))
+        q_emb = self.embedder.encode(
+            chunks, batch_size=self.embed_batch_size, normalize_embeddings=True
+        )
+        end = time()
+        log.debug(f"embedding hyde docs took {(end - start):.1f} seconds...")
+
+        _, I = self.index.search(q_emb, top_k_per_variant)
+        chunk_ids = reciprocal_rank_fusion(I, top_k_retrieve)
+        return chunk_ids, tasks
+
+    def answer(self, query, doc_ids, tasks, max_ctx_chars=128000):
+        total, text, prompt_length = 0, "", 10000
+        sep = "\n\n-----\n\n"
+        tasks = ", ".join(tasks)
+
+        for doc_id in doc_ids:
+            # adding tags in the context caused more hallucinations.
+            # Instead, we list them as sources beneath the model response.
+            # _meta = self.meta[doc_id]
+            # tag = f"(source: {_meta['file_name']}:{_meta['chunk_id']})"
+            chunk = self.corpus[doc_id].strip()
+            tag = ""
+
+            ctx = f"{sep}{tag}\n\n{chunk}"
+            if total + len(ctx) + len(tasks) + len(sep) + prompt_length > max_ctx_chars:
+                break
+
+            text += ctx
+            total = len(text)
+
+        text += f"{sep}{tasks}"
+
+        # instruction = "Answer concisely and also cite file names & chunk ids inline like (pdf_file_name:chunk_id)."
+        instruction = "go ahead and answer!"
+        user_query = f"\nq: {query}\n\nctx:{text}" + f"\n\n{instruction}\n\n"
+
+        start = time()
+        resp = generate_text(
+            self.tok,
+            self.gen,
+            user_query,
+            self.prompts["final_answer"],
+            temperature=0.3,
+        )
+        end = time()
+        log.debug(f"final resp took {(end - start):.1f} seconds...")
+
+        return resp
+
+
+def initial_setup(embed_model, generator_model, bitsandbytesconfig=None):
+    return HyDeRAGFusion(
+        embed_model, generator_model, bitsandbytesconfig=bitsandbytesconfig
+    )
+
+start = time()
+HRF = initial_setup("sentence-transformers/LaBSE", "Qwen/Qwen2.5-0.5B-Instruct")
+end = time()
+msg = f"init took {(end - start):.1f} seconds"
+log.debug(msg)
+
+def main(
+    pdfs, query,
+    n_variants=3,
+    top_k_per_variant=5,
+    top_k_retrieve=3,
+    temperature=0.4,
+    max_new_tokens=512,
+):
+    start = time()
+    if pdfs:
+        HRF.preprocess_pdfs(tuple(sorted(pdfs)))
+
+    if query:
+        llm_kwargs = {
+            "temperature": temperature,
+            "max_new_tokens": max_new_tokens,
+        }
+        doc_ids, tasks = HRF.retrieve(
+            query,
+            int(n_variants),
+            int(top_k_per_variant),
+            int(top_k_retrieve),
+            **llm_kwargs,
+        )
+        docs = [HRF.corpus[doc_id] for doc_id in doc_ids]
+        reply = HRF.answer(query, doc_ids, tasks)
+        sources = [
+            {
+                "source": f"{Path(HRF.meta[doc_id]['file']).stem}:{HRF.meta[doc_id]['chunk_id']}",
+                "content": doc,
+            }
+            for doc_id, doc in zip(doc_ids, docs)
+        ]
+
+        resp = f"{reply}\n\n{'-'*25}\n\n"
+        resp += "Top 3 sources:"
+        resp += f"\n\n{'-'*25}\n\n"
+        for source in sources:
+            resp += f'source: {source["source"]}\n\n'
+            resp += source['content']
+            resp += f"\n\n{'-'*25}\n\n"
+
+        end = time()
+        log.debug(f"final resp took {(end - start):.1f} seconds")
+        return resp
+
+def reset_text_on_file_change(pdfs):
+    """
+    Reset text input when input docs change
+    """
+    return ""
+
+
+with gr.Blocks(title='RAG with HYDE') as demo:
+    gr.Markdown('# RAG with HYDE')
+    with gr.Row():
+        pdf_input = gr.File(label='upload PDF(s)', file_types=['.pdf'], file_count='multiple')
+        query = gr.Textbox(label='question')
 
     btn = gr.Button('Submit')
-    btn.click(greet, inputs=[pdf_input, query], outputs=answer)
+    answer = gr.Markdown(label='### Answer')
+
+    btn.click(main, inputs=[pdf_input, query], outputs=answer)
+    pdf_input.change(reset_text_on_file_change, inputs=pdf_input, outputs=query)
 
 
 if __name__ == '__main__':
-    demo.launch()
+    demo.launch(debug=True)
 

src/config.py

@@ -0,0 +1,23 @@
+import logging
+from pathlib import Path
+
+PROMPTS_FILEPATH = Path("src/prompts.yaml")
+
+
+def get_logger(LOG_LEVEL="INFO"):
+    LOG_PATH = Path("logs.log")
+    formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
+
+    log = logging.Logger("hyde_rag")
+    log.setLevel(LOG_LEVEL)
+
+    file_handler = logging.FileHandler(LOG_PATH)
+    file_handler.setLevel(LOG_LEVEL)
+    file_handler.setFormatter(formatter)
+
+    log.addHandler(file_handler)
+
+    return log
+
+
+log = get_logger("DEBUG")

src/hyde_rag.py

@@ -0,0 +1,206 @@
+# hyde_ragfusion.py
+# Minimal HyDE + RAG-Fusion over local PDFs.
+# Dependencies: transformers, sentence-transformers, scikit-learn, pymupdf, numpy
+
+import os
+import re
+import heapq
+import fitz  # PyMuPDF
+from sklearn.neighbors import NearestNeighbors
+from sentence_transformers import SentenceTransformer
+from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
+
+
+# -----------------------------
+# Ingestion & Chunking
+# -----------------------------
+def load_pdfs(folder):
+    docs = []
+    for fn in os.listdir(folder):
+        if fn.lower().endswith(".pdf"):
+            path = os.path.join(folder, fn)
+            with fitz.open(path) as doc:
+                text = "\n".join(page.get_text("text") for page in doc)
+            text = re.sub(r"\s+\n", "\n", text).strip()
+            docs.append((fn, text))
+    return docs
+
+
+def chunk_text(text, chunk_size=300, overlap=50):
+    words = text.split()
+    chunks, i = [], 0
+    while i < len(words):
+        chunk = " ".join(words[i : i + chunk_size])
+        chunks.append(chunk)
+        i += chunk_size - overlap
+    return chunks
+
+
+def build_corpus(pdf_folder):
+    raw = load_pdfs(pdf_folder)
+    corpus, meta = [], []
+    for fn, txt in raw:
+        for i, ch in enumerate(chunk_text(txt)):
+            corpus.append(ch)
+            meta.append({"file": fn, "chunk_id": i})
+    return corpus, meta
+
+
+# -----------------------------
+# Models (local)
+# -----------------------------
+def load_models():
+    # Small, fast encoder for embeddings
+    embedder = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")
+    # Lightweight local generator for HyDE + answers
+    gen_name = "google/flan-t5-base"
+    tok = AutoTokenizer.from_pretrained(gen_name)
+    gen = AutoModelForSeq2SeqLM.from_pretrained(gen_name)
+    return embedder, tok, gen
+
+
+# -----------------------------
+# Index (cosine)
+# -----------------------------
+def fit_index(embeddings, n_neighbors=12):
+    nn = NearestNeighbors(metric="cosine", algorithm="auto")
+    nn.fit(embeddings)
+    return nn
+
+
+# -----------------------------
+# RAG-Fusion (query variants) + HyDE
+# -----------------------------
+Q_VARIANTS_PROMPT = """You rewrite the user query into {n} diverse, specific search queries (short).
+User query: "{q}"
+Return each on a new line, no numbering, no extra text."""
+
+HYDE_PROMPT = """Write a factual, neutral, self-contained paragraph that could answer:
+"{q}"
+Avoid fluff. Include likely key terms and entities. 120-180 words."""
+
+ANSWER_PROMPT = """You are a helpful assistant. Use ONLY the provided context.
+Question: {q}
+
+Context:
+{ctx}
+
+Answer concisely and cite file names & chunk ids inline like (file:chunk).
+"""
+
+
+def generate_text(gen, tok, prompt, max_new_tokens=160, temperature=0.3):
+    inputs = tok(prompt, return_tensors="pt")
+    out = gen.generate(
+        **inputs,
+        max_new_tokens=max_new_tokens,
+        do_sample=False,
+        temperature=temperature,
+    )
+    return tok.decode(out[0], skip_special_tokens=True).strip()
+
+
+def make_query_variants(gen, tok, q, n=4):
+    txt = generate_text(
+        gen, tok, Q_VARIANTS_PROMPT.format(q=q, n=n), max_new_tokens=120
+    )
+    # Split cleanly into lines (drop empties/dups; include original)
+    lines = [l.strip(" -•\t") for l in txt.split("\n") if l.strip()]
+    uniq = []
+    seen = set()
+    for l in lines + [q]:
+        if l not in seen:
+            seen.add(l)
+            uniq.append(l)
+    return uniq[:n]
+
+
+def hyde_doc(gen, tok, q):
+    return generate_text(gen, tok, HYDE_PROMPT.format(q=q), max_new_tokens=220)
+
+
+# -----------------------------
+# Retrieval + RRF
+# -----------------------------
+def cosine_search(nn, corpus_embeddings, query_vec, top_k=8):
+    dists, idxs = nn.kneighbors(query_vec.reshape(1, -1), n_neighbors=top_k)
+    # Convert cosine distance to similarity
+    sims = 1 - dists[0]
+    return list(zip(idxs[0].tolist(), sims.tolist()))
+
+
+def reciprocal_rank_fusion(rank_lists, k=60, top_k=8):
+    # rank_lists: list of [doc_id, ...] ordered best→worst
+    scores = {}
+    for ranks in rank_lists:
+        for rank, doc_id in enumerate(ranks, start=1):
+            scores[doc_id] = scores.get(doc_id, 0.0) + 1.0 / (k + rank)
+    # top by fused score
+    best = heapq.nlargest(top_k, scores.items(), key=lambda x: x[1])
+    return [doc_id for doc_id, _ in best]
+
+
+# -----------------------------
+# Pipeline
+# -----------------------------
+class HyDeRAGFusion:
+    def __init__(self, pdf_folder):
+        self.corpus, self.meta = build_corpus(pdf_folder)
+        self.embedder, self.tok, self.gen = load_models()
+        self.corpus_emb = self.embedder.encode(
+            self.corpus,
+            batch_size=64,
+            show_progress_bar=True,
+            normalize_embeddings=True,
+        )
+        self.nn = fit_index(self.corpus_emb)
+
+    def retrieve(self, query, n_variants=4, per_variant_k=8, final_top_k=6, rrf_k=60):
+        variants = make_query_variants(self.gen, self.tok, query, n=n_variants)
+        rank_lists = []
+        for v in variants:
+            hypo = hyde_doc(self.gen, self.tok, v)  # HyDE
+            q_vec = self.embedder.encode([hypo], normalize_embeddings=True)[0]
+            hits = cosine_search(self.nn, self.corpus_emb, q_vec, top_k=per_variant_k)
+            rank_lists.append([doc_id for doc_id, _ in hits])
+        fused = reciprocal_rank_fusion(rank_lists, k=rrf_k, top_k=final_top_k)
+        return fused
+
+    def answer(self, query, doc_ids, max_ctx_chars=4000):
+        # Build compact context with inline provenance
+        ctx_parts = []
+        total = 0
+        for i in doc_ids:
+            piece = self.corpus[i]
+            tag = f"(source: {self.meta[i]['file']}:{self.meta[i]['chunk_id']})"
+            chunk = piece.strip()
+            if total + len(chunk) + len(tag) + 5 > max_ctx_chars:
+                break
+            ctx_parts.append(f"{chunk}\n{tag}")
+            total += len(chunk) + len(tag) + 5
+        ctx = "\n\n---\n\n".join(ctx_parts)
+        prompt = ANSWER_PROMPT.format(q=query, ctx=ctx)
+        return generate_text(self.gen, self.tok, prompt, max_new_tokens=300)
+
+
+# -----------------------------
+# Example usage
+# -----------------------------
+if __name__ == "__main__":
+    import argparse
+
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--pdf_folder", required=True, help="Folder with PDFs to index")
+    ap.add_argument("--query", required=True, help="Your user question")
+    ap.add_argument("--show_sources", action="store_true")
+    args = ap.parse_args()
+
+    rag = HyDeRAGFusion(args.pdf_folder)
+    doc_ids = rag.retrieve(args.query)
+    answer = rag.answer(args.query, doc_ids)
+    print("\n=== ANSWER ===\n")
+    print(answer)
+    if args.show_sources:
+        print("\n=== TOP SOURCES ===")
+        for i in doc_ids:
+            print(f"- {rag.meta[i]['file']}:{rag.meta[i]['chunk_id']}")

src/main.py

@@ -0,0 +1,504 @@
+import pymupdf
+import math
+import faiss
+import string
+import yaml
+import re
+import json
+import asyncio
+import torch
+import streamlit as st
+import click
+
+from collections import defaultdict
+from ast import literal_eval
+from time import time
+from sentence_transformers import SentenceTransformer
+from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, AutoModelForCausalLM
+from concurrent.futures import ThreadPoolExecutor
+from langchain.text_splitter import SentenceTransformersTokenTextSplitter
+from src.config import PROMPTS_FILEPATH, log
+
+
+async def load_pdfs(files, max_concurrence=5):
+    """
+    Load multiple PDF files async.
+
+    Args:
+        folder (str): Path to folder containing PDF files
+        max_concurrence (int): Maximum number of concurrent PDF processing tasks
+
+    Returns:
+        list: List of tuples containing (filename, extracted_text)
+    """
+
+    def _load_pdf_sync(file):
+        """Synchronous PDF loading function for thread pool execution"""
+        text = ""
+        try:
+            with pymupdf.open(stream=file.getvalue(), filetype="pdf") as doc:
+                text = "\n".join(page.get_text() for page in doc)
+        except Exception:
+            log.exception(f"Error reading {file.name}")
+            pass
+
+        return (file.name, text)
+
+    loop = asyncio.get_event_loop()
+    with ThreadPoolExecutor(max_workers=max_concurrence) as executor:
+        futures = [
+            loop.run_in_executor(executor, _load_pdf_sync, file)
+            for file in files
+            if file is not None
+        ]
+
+        results = await asyncio.gather(*futures, return_exceptions=True)
+
+    valid_results = [result for result in results if not isinstance(result, Exception)]
+
+    log.info(f"successfully processed {len(valid_results)} out of {len(files)} PDFs ")
+    return valid_results
+
+
+async def build_corpus(pdfs, text_splitter, **load_kwargs):
+    texts = await load_pdfs(pdfs, **load_kwargs)
+    corpus, meta = [], []
+    for file_name, raw_text in texts:
+        chunks = text_splitter.split_text(raw_text)
+        for i, chunk in enumerate(chunks):
+            corpus.append(chunk)
+            meta.append({"file": file_name, "chunk_id": i})
+    return corpus, meta
+
+
+def generate_text(
+    tokenizer, model, user_prompts, system_prompt=None, **llm_kwargs
+):  # max_new_tokens=512, temperature=.4):
+    if system_prompt is None or "":
+        system_prompt = "You are a helpful assistant."
+
+    if isinstance(user_prompts, str):
+        user_prompts = [user_prompts]
+
+    messages = [
+        [
+            {"role": "system", "content": system_prompt},
+            {"role": "user", "content": user_prompt},
+        ]
+        for user_prompt in user_prompts
+    ]
+
+    texts = tokenizer.apply_chat_template(
+        messages, tokenize=False, add_generation_prompt=True
+    )
+
+    model_inputs = tokenizer(
+        texts, return_tensors="pt", truncation=True, padding=True
+    ).to(model.device)
+    generated_ids = model.generate(
+        **model_inputs,
+        max_new_tokens=llm_kwargs.pop("max_new_tokens", 512),
+        temperature=llm_kwargs.pop("temperature", 0.4),
+    )
+    generated_ids = [
+        output_ids[len(input_ids) :]
+        for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)
+    ]
+    response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)
+    return response if len(user_prompts) > 1 else response[0]
+
+
+def load_models(
+    embed_model_name: str,
+    gen_model_name: str,
+    causal_lm: bool = False,
+    device=None,
+    bitsandbytesconfig=None,
+):
+    # This will take some time to run for the first time if the model(s) don't exist locally.
+    if not device:
+        device = "cuda" if torch.cuda.is_available() else "cpu"
+    embedder = SentenceTransformer(
+        embed_model_name,
+        device=device,
+        model_kwargs={"dtype": "float16"} if device == "cuda" else {},
+    )
+
+    if not causal_lm:
+        tok = AutoTokenizer.from_pretrained(gen_model_name)
+        gen = AutoModelForSeq2SeqLM.from_pretrained(
+            gen_model_name,  # device_map='auto',
+            quantization_config=bitsandbytesconfig if bitsandbytesconfig else None,
+        )
+    else:
+        tok = AutoTokenizer.from_pretrained(gen_model_name, padding_side="left")
+        gen = AutoModelForCausalLM.from_pretrained(
+            gen_model_name,
+            dtype="float16",  # device_map='auto',
+            quantization_config=bitsandbytesconfig if bitsandbytesconfig else None,
+        )
+    gen.to(device)
+    return embedder, tok, gen
+
+
+def make_query_variants(
+    tokenizer, model, query: str, prompt: str, n: int = 3, **llm_kwargs
+):
+    instructions = f"Now give me at least {n} variations."
+    resp = generate_text(tokenizer, model, query + instructions, prompt, **llm_kwargs)
+
+    clean_resp = re.sub(r"^\d+\.\s*", "", resp, flags=re.MULTILINE).split("\n")
+    return [query] + [q for q in clean_resp if q.strip()]
+
+
+def clean_rewrite_resp(resp):
+    try:
+        resp = json.loads(resp)  # Parse JSON
+    except json.JSONDecodeError:
+        try:
+            resp = literal_eval(resp)  # Fallback parse
+        except Exception:
+            pass  # Keep resp as-is if both fail
+
+    # Ensure resp is a string before strip and slicing
+    if isinstance(resp, str):
+        resp = resp.strip()
+        if resp:
+            start = resp.find("{")
+            if start != -1:
+                end = resp[::-1].find("}")
+                if end != -1:
+                    resp = resp[start : len(resp) - end]
+                    return clean_rewrite_resp(resp)
+    return resp
+
+
+def transform_query(
+    tokenizer, model, query: str, rewrite_prompt: str, **llm_kwargs
+) -> dict:
+    """split the query into things to search and actions to take"""
+    resp = generate_text(tokenizer, model, query, rewrite_prompt, **llm_kwargs)
+    try:
+        resp = clean_rewrite_resp(resp)
+    except:
+        pass
+    return resp
+
+
+def aggregate_queries_and_tasks(
+    tokenizer,
+    model,
+    orig_query,
+    rewrite_prompt,
+    variants_prompt,
+    n_variations=3,
+    **llm_kwargs,
+):
+    # make variations for the original query as is
+    queries = make_query_variants(
+        tokenizer,
+        model,
+        orig_query.strip(),
+        variants_prompt,
+        n_variations,
+        **llm_kwargs,
+    )
+
+    start = time()
+    tr_q = transform_query(tokenizer, model, orig_query.strip(), rewrite_prompt)
+    end = time()
+    log.debug(f"\t\t transforming query task took {(end - start):.1f} seconds...")
+
+    # transformed query might have multiple things to search and tasks to perform depending on user query
+    # recursively get variations for each of the search queries but keep the tasks as is.
+    tasks = []
+    if isinstance(tr_q, dict):
+        search_results, tasks = tr_q.get("search", []), tr_q.get("tasks", [])
+        for search_result in search_results:
+            queries.extend(
+                make_query_variants(
+                    tokenizer,
+                    model,
+                    search_result,
+                    variants_prompt,
+                    n_variations,
+                    **llm_kwargs,
+                )
+            )
+
+    queries = [q.strip(string.punctuation) for q in queries]
+    tasks = [t.strip(string.punctuation) for t in tasks]
+
+    # keep the original user query as is (if in case LLM messes up the original query) and pick some after shuffling the rest
+    # This is disabled as we don't do loops and instead take advantage of batches.
+    # Since it's efficient, we can take many query variations at once without worrying about performance.
+    # q, queries = queries[:1], queries[1:]
+    # shuffle(queries)
+    # q += queries[:n_variations-1]
+
+    return queries, tasks
+
+
+def build_index(corpus_emb, n_cells=5, n_probe=2):
+    log.debug(f"building index with {n_cells=}, {n_probe=}")
+    d = corpus_emb.shape[1]
+    quantizer = faiss.IndexFlatIP(d)
+    index = faiss.IndexIVFFlat(quantizer, d, n_cells)
+    index.n_probe = n_probe
+    index.train(corpus_emb)
+    index.add(corpus_emb)
+    # index.make_direct_map()
+    return index
+
+
+def reciprocal_rank_fusion(indices, top_k=3, denom=50):
+    ii = indices.tolist()
+    scores = defaultdict(int)
+    for row in ii:
+        for rank, chunk_id in enumerate(row):
+            scores[chunk_id] += 1 / (rank + denom)
+    results = sorted(scores.items(), key=lambda x: x[1], reverse=True)[:top_k]
+    return [chunk_id for chunk_id, _ in results]
+
+
+class HyDeRAGFusion:
+    def __init__(
+        self,
+        embed_model: str,
+        generator_llm_model: str,
+        causal_lm: bool = True,
+        chunk_overlap: int = 50,
+        tokens_per_chunk: int = 256,
+        embed_batch_size: int = 64,
+        bitsandbytesconfig=None,
+    ):
+        self.embed_batch_size = embed_batch_size
+        self.text_splitter = SentenceTransformersTokenTextSplitter(
+            chunk_overlap, embed_model, tokens_per_chunk
+        )
+        self.embedder, self.tok, self.gen = load_models(
+            embed_model, generator_llm_model, causal_lm, bitsandbytesconfig
+        )
+        with open(PROMPTS_FILEPATH) as fl:
+            self.prompts = yaml.safe_load(fl)
+
+    def preprocess_pdfs(self, pdfs, data_load_kwargs={}, faiss_index_kwargs={}):
+        self.corpus, self.meta = asyncio.run(
+            build_corpus(pdfs, self.text_splitter, **data_load_kwargs)
+        )
+        self.corpus_emb = self.embedder.encode(
+            self.corpus,
+            batch_size=self.embed_batch_size,
+            show_progress_bar=True,
+            normalize_embeddings=True,
+        )
+
+        # https://github.com/facebookresearch/faiss/issues/112
+        # n_cells = int(round(4 * (self.corpus_emb.shape[0])**.5))
+
+        # one centroid for every 100 or so vectors and 20% of them as n_probe
+        n_cells = faiss_index_kwargs.pop("n_cells", self.corpus_emb.shape[0] // 100 + 1)
+        n_probe = faiss_index_kwargs.pop("n_probe", math.ceil(0.2 * n_cells))
+
+        self.index = build_index(self.corpus_emb, n_cells, n_probe)
+
+    def retrieve(
+        self, query, n_variants=3, top_k_per_variant=10, top_k_retrieve=3, **llm_kwargs
+    ):
+        start = time()
+
+        queries, tasks = aggregate_queries_and_tasks(
+            self.tok,
+            self.gen,
+            query.strip(),
+            self.prompts["rewrite"],
+            self.prompts["variants"],
+            n_variants,
+            **llm_kwargs,
+        )
+
+        end = time()
+        log.debug(f"aggregate task took {(end - start):.1f} seconds...")
+
+        start = time()
+        hyde_docs = generate_text(
+            self.tok, self.gen, queries, self.prompts["hyde"], **llm_kwargs
+        )
+        end = time()
+        log.debug(f"generating hyde docs took {(end - start):.1f} seconds...")
+
+        start = time()
+        chunks = []
+        for hyde_doc in hyde_docs:
+            chunks.extend(self.text_splitter.split_text(hyde_doc))
+        q_emb = self.embedder.encode(
+            chunks, batch_size=self.embed_batch_size, normalize_embeddings=True
+        )
+        end = time()
+        log.debug(f"embedding hyde docs took {(end - start):.1f} seconds...")
+
+        _, I = self.index.search(q_emb, top_k_per_variant)
+        chunk_ids = reciprocal_rank_fusion(I, top_k_retrieve)
+        return chunk_ids, tasks
+
+    def answer(self, query, doc_ids, tasks, max_ctx_chars=128000):
+        total, text, prompt_length = 0, "", 10000
+        sep = "\n\n-----\n\n"
+        tasks = ", ".join(tasks)
+
+        for doc_id in doc_ids:
+            # adding tags in the context caused more hallucinations.
+            # Instead, we list them as sources beneath the model response.
+            # _meta = self.meta[doc_id]
+            # tag = f"(source: {_meta['file_name']}:{_meta['chunk_id']})"
+            chunk = self.corpus[doc_id].strip()
+            tag = ""
+
+            ctx = f"{sep}{tag}\n\n{chunk}"
+            if total + len(ctx) + len(tasks) + len(sep) + prompt_length > max_ctx_chars:
+                break
+
+            text += ctx
+            total = len(text)
+
+        text += f"{sep}{tasks}"
+
+        # instruction = "Answer concisely and also cite file names & chunk ids inline like (pdf_file_name:chunk_id)."
+        instruction = "go ahead and answer!"
+        user_query = f"\nq: {query}\n\nctx:{text}" + f"\n\n{instruction}\n\n"
+
+        start = time()
+        resp = generate_text(
+            self.tok,
+            self.gen,
+            user_query,
+            self.prompts["final_answer"],
+            temperature=0.3,
+        )
+        end = time()
+        log.debug(f"final resp took {(end - start):.1f} seconds...")
+
+        return resp
+
+
+@st.cache_resource
+def initial_setup(embed_model, generator_model, bitsandbytesconfig=None):
+    return HyDeRAGFusion(
+        embed_model, generator_model, bitsandbytesconfig=bitsandbytesconfig
+    )
+
+
+@click.command(context_settings=dict(show_default=True))
+@click.option(
+    "--embed-model",
+    default="sentence-transformers/LaBSE",
+    help="sentence transformers embedding model",
+)
+@click.option(
+    "--generator-llm-model",
+    default="Qwen/Qwen2.5-0.5B-Instruct",
+    help="Seq2Seq or CausalLM model (preferably multi-lingual)",
+)
+@click.option("--n-variants", default=3, help="no. of query variants")
+@click.option(
+    "--top-k-per-variant",
+    default=5,
+    help="top `k` hits per each query variant to consider for RRF",
+)
+@click.option(
+    "--top-k-retrieve", default=3, help="top `k` chunks to retrieve after RRF"
+)
+@click.option("--temperature", default=0.4, help="LLM Model Temperature")
+@click.option("--max-new-tokens", default=512, help="LLM max tokens")
+@click.option(
+    "--faiss-index-kwargs",
+    default=dict(),
+    help="kwargs to pass to FAISS Index such as `n_cells, n_probe`",
+)
+def main(
+    embed_model,
+    generator_llm_model,
+    n_variants,
+    top_k_per_variant,
+    top_k_retrieve,
+    temperature,
+    max_new_tokens,
+    faiss_index_kwargs,
+):
+    # bits_and_bytes_cfg = BitsAndBytesConfig(
+    #     load_in_8bit=True
+    # )
+    start = time()
+    hrf = initial_setup(embed_model, generator_llm_model)
+    end = time()
+    msg = f"init took {(end - start):.1f} seconds"
+    log.debug(msg)
+    st.write(msg)
+
+    st.set_page_config(page_title="RAG HYDE")
+    st.header("Ask Questions")
+
+    state = st.session_state
+    if "uploaded_names" not in state:
+        state.uploaded_names = []
+
+    pdfs = st.file_uploader(
+        "Upload your PDF(s)", type="pdf", accept_multiple_files=True, key="upload"
+    )
+    if pdfs:
+        current_names = sorted([pdf.name for pdf in pdfs])
+        # reinitialize if uploaded files are changed
+        if current_names != state.uploaded_names:
+            start = time()
+
+            hrf = initial_setup(embed_model, generator_llm_model)
+            hrf.preprocess_pdfs(
+                pdfs, faiss_index_kwargs=literal_eval(faiss_index_kwargs)
+            )
+
+            end = time()
+            st.write(
+                f"corpus embeddings shape: {hrf.corpus_emb.shape}, computed in {end - start:.1f} seconds"
+            )
+
+            state.uploaded_names = current_names
+    else:
+        state.uploaded_names = []
+        st.write("upload data to query")
+
+    query = st.text_input("ask question").strip()
+    if query and state.uploaded_names:
+        start = time()
+        llm_kwargs = {
+            "temperature": temperature,
+            "max_new_tokens": max_new_tokens,
+        }
+        doc_ids, tasks = hrf.retrieve(
+            query,
+            int(n_variants),
+            int(top_k_per_variant),
+            int(top_k_retrieve),
+            **llm_kwargs,
+        )
+        docs = [hrf.corpus[doc_id] for doc_id in doc_ids]
+        end = time()
+        reply = hrf.answer(query, doc_ids, tasks)
+        st.write(f"search took {(end - start):.1f} seconds")
+        st.write(f"\n\nFinal Answer: \n{reply}\n\n")
+        st.write("Top 3 sources:")
+        sources = [
+            {
+                "source": f"{hrf.meta[doc_id]['file']}:{hrf.meta[doc_id]['chunk_id']}",
+                "content": doc,
+            }
+            for doc_id, doc in zip(doc_ids, docs)
+        ]
+        st.json(sources[:3])
+
+
+if __name__ == "__main__":
+    # faiss_index_kwargs = {
+    #     'n_cells': 20,
+    #     'n_probe': 8
+    # }
+    main()

src/prompts.yaml

@@ -0,0 +1,307 @@
+rewrite: >
+    You are a professional content writer and editor who deeply pays attention to user's query & intention. You strictly reply ONLY in JSON.
+
+    Your mission is to analyze user input and intention and transform it in the same language as the input query to make it search engine optimised by determining the appropriate context.
+
+    The user input can be a query or a statement. There can be multiple of them. And sometimes the input also contains actions to be taken depending on the query/statement.
+
+
+    Split & Transform the user input to JSON in the following format: 
+
+    {{ "search": Optional[List[str]], "tasks": Optional[List[str]] }}
+
+
+    Description:
+
+    - search: 
+        - list of questions or statements in the input that is to be searched, if any.
+            - Rewrite them so as to paste as is on the search engine (e.g., Google) to get better search results by stripping of extraneous text.
+        - Feel free to paraphrase them in full without losing any meaning or context.
+        - Ignore any instructions/actions/tasks to be taken based upon this search.
+        - Strip any extraneous text.
+
+    - tasks:
+        - list of things to be done AFTER a search is performed, if any.
+            - Normally you wouldn't search for this (on Google) rather you perform it (with other tools). 
+        - Ignore any questions or statements and focus only on the final instructions/actions/tasks to be performed. 
+
+    Ensure your final answer contains only the content in the format: 
+    
+    {{ "search": Optional[List[str]], "tasks": Optional[List[str]] }}
+
+    Ensure the final output does not include any code block markers like ```json or ```python.
+
+
+    Examples: 
+
+    -----------
+
+
+    user: "explore sales of EVs in Munich and plot their evolution in the last decade"
+
+    assistant: {'search': ['EV sales in Munich', 'Automobile sales in Munich', 'History in numbers of EV sales'], 'tasks': ['plot chart of sales history']}
+
+
+    user: "what's the EBITDA for this quarter? and how does it compare to last two?"
+
+    assistant: {'search': ['EBITDA current quarter', 'compare current EBITDA with the last two quarters'], 'tasks': ['']}
+
+
+    user: "Name 5 planets in the solar system"
+
+    assistant: {'search': ['planets in solar system', '5 smallest planets in solar system'], 'tasks': ['']}
+
+
+    user: "summarize earth's history in a few words"
+
+    assistant: {'search': ["earth's history"], 'tasks': ['']}
+
+
+    user: "write a report in markdown and send an email to NGO on effects of global warming on north africa esp. the malnourished regions in Congo"
+
+    assistant: {
+        'search': [
+                    'global warming affects on north africa',
+                    'effect of global warming on congo', 
+                    'malnourishment in congo'
+        ], 
+        'tasks': [
+            'write a markdown report of the analysis performed',
+            'send email to NGO',
+        ]
+    }
+
+
+    user: "Untersuche den Verkauf von Elektrofahrzeugen in München und stelle ihre Entwicklung in den letzten zehn Jahren grafisch dar."
+
+    assistant: {'search': ['Verkauf von Elektrofahrzeugen in München', 'Automobilverkauf in München', 'Entwicklung des Verkaufs von Elektrofahrzeugen in Zahlen'], 'tasks': ['Verkaufsentwicklung grafisch darstellen']}
+
+
+    user: "Wie hoch ist das EBITDA für dieses Quartal? Und wie sieht es im Vergleich zu den letzten beiden Quartalen aus?"
+
+    assistant: {'search': ['EBITDA aktuelles Quartal', 'aktuelles EBITDA mit den letzten beiden Quartalen vergleichen'], 'tasks': ['']}
+
+
+    user: "Nenne 5 Planeten im Sonnensystem"
+
+    assistant: {'search': ['Planeten im Sonnensystem', '5 kleinste Planeten im Sonnensystem'], 'tasks': ['']}
+
+
+    user: "Fasse die Geschichte der Erde in wenigen Worten zusammen."
+
+    assistant: {'search': ["Geschichte der Erde"], 'tasks': ['']}
+
+
+    user: "Schreibe einen Bericht in Markdown und sende eine E-Mail an eine NGO über die Auswirkungen der globalen Erwärmung auf Nordafrika, insbesondere auf die unterernährten Regionen im Kongo."
+
+    assistant: {
+        'search': [
+                    'Auswirkungen der globalen Erwärmung auf Nordafrika',
+                    'Auswirkungen der globalen Erwärmung auf den Kongo', 
+                    'Unterernährung im Kongo'
+        ], 
+        'tasks': [
+            'Erstellen Sie einen Markdown-Bericht über die durchgeführte Analyse',
+            'Senden Sie eine E-Mail an die NGO',
+        ]
+    }
+
+    -----------
+
+
+    Remember to not answer the user's question but only transform it and in the same language as given.
+
+
+    user:
+
+
+variants: >
+    You are a multilingual professional content writer and editor who deeply pays attention to user's query & intention.
+    
+    Your goal is to transform the given query into diverse search queries keeping the user's context & intention in mind.
+
+    You MUST respond in the same language as the user query which need not always be English.
+
+    You MUST respond with only what's asked. Avoid explanations or verbose information of your actions.
+
+
+    Examples:
+
+    --------------
+
+
+    user: "EBITDA last quarter"
+    
+    assistant:
+        "What was the EBITDA for the quarter ending March?",
+        "How has the company's EBITDA performance changed from the previous quarter?",
+        "What is the current trend of EBITDA growth over the past few quarters?",
+        "Which companies have had similar EBITDA performance recently?",
+        "What factors might be influencing the changes in EBITDA?",
+
+
+    user: "what are the growing concerns of the middle class?"
+
+    assistant:
+        "How are the economic challenges impacting the middle class?",
+        "What are the social and political pressures on the middle class?",
+        "What are the long-term implications for the middle class's well-being?",
+        "What are the current trends and future prospects for the middle class"
+
+
+    user: "Capital of France"
+    
+    assistant:
+        "What is the capital city of France?",
+        "How is Paris known internationally?",
+        "Where is Paris located on the map?",
+
+    
+    user: "EBITDA letztes Quartal"
+
+    assistant:
+        "Wie hoch war das EBITDA für das im März endende Quartal?",
+        "Wie hat sich die EBITDA-Performance des Unternehmens gegenüber dem Vorquartal verändert?",
+        "Wie ist der aktuelle Trend des EBITDA-Wachstums in den letzten Quartalen?",
+        "Welche Unternehmen hatten in letzter Zeit eine ähnliche EBITDA-Entwicklung?",
+        "Welche Faktoren könnten die Veränderungen des EBITDA beeinflussen?",
+
+
+    user: "Was sind die wachsenden Sorgen der Mittelschicht?"
+
+    assistant:
+        "Wie wirken sich die wirtschaftlichen Herausforderungen auf die Mittelschicht aus?",
+        "Welchen sozialen und politischen Druck erlebt die Mittelschicht?",
+        "Was sind die langfristigen Auswirkungen auf das Wohlergehen der Mittelschicht?",
+        "Was sind die aktuellen Trends und Zukunftsaussichten für die Mittelschicht?"
+
+
+    user: "Hauptstadt von Frankreich"
+
+    assistant:
+        "Was ist die Hauptstadt von Frankreich?",
+        "Wie ist Paris international bekannt?",
+        "Wo liegt Paris auf der Karte?",
+
+
+    --------------
+
+
+    user:
+
+
+hyde: >
+    You are a professional editor at a prestigious international media organization.
+    Given user's query, write a neutral, self-contained paragraph ABSOLUTELY GROUNDED IN FACTS and established sources. Avoid fluff. Include likely key terms and entities. 120-180 words.
+    You write content in the same language as the user query which need not always be in English.
+
+    Examples:
+    --------
+
+    user: Quelle est le niveau actuel de l'engagement de Deutsche Telekom avec le développement durable
+    
+    assistant: Deutsche Telekom, la plus grande entreprise d'électricité et d'information au monde, a lancé un programme d'engagement durable en 2015 pour atteindre ses objectifs de développement durable. Ce programme visait à réduire son émission de gaz à effet de serre (CO2) par rapport aux niveaux de 1990, ainsi que les émissions de déchets et de produits chimiques. Le groupe a mis en place des politiques d'énergie basées sur la transition verte, comme la production de biogaz, l'utilisation de technologies solaires et l'innovation dans la gestion des ressources naturelles. L'objectif principal était de se démarquer du marché mondial en termes de performance énergétique et environnementale.
+
+
+    user: BMW Group expansion into southern Asia
+
+    assistant: The BMW Group has expanded its presence in southern Asia through acquisitions of several companies, including Honda Motor Company, which was acquired by BMW in 2019. This acquisition aims to strengthen the company's presence in the region, where it will be able to offer a wider range of products and services to consumers in China, India, and other countries in the South Asian market.
+
+
+    user: Arten von Unternehmen am anfälligsten für Cyber-Bedrohungen
+
+    assistant: Die wichtigsten Arten von Unternehmen, die am anfälligsten für Cyber-Bedrohungen sind, sind:\n\n1. Unternehmen mit einem großen Datennetz: Diese Unternehmen haben eine hohe Anzahl von Computern und Server, die ihre Daten und Informationen verstecken.\n\n2. Unternehmen mit einer vielfältigen Software: Diese Unternehmen haben eine Vielzahl von Softwareprogrammen, die sie nutzen, um ihre Produkte oder Dienstleistungen zu verbessern.\n\n3. Unternehmen mit einem großen Netzwerk von Internetreichen: Diese Unternehmen haben einen großen Netzwerk von Internetreichen, die ihre Kunden und Kundengruppen verfolgen.\n\n4. Unternehmen mit einem großen Netzwerk von E-Mail-Reichen: Diese Unternehmen haben einen großen Netzwerk von E-Mail-Reichen, die ihre Kunden und Kundengruppen verfolgen.\n\n5. Unternehmen mit einem großen Netzwerk von Webseiten: Diese Unternehmen haben einen großen Netzwerk von Webseiten, die ihre Kunden und Kundengruppen verfolgen.
+    
+    ----------
+
+    Remember to only write factual content.
+
+    user: 
+
+
+final_answer: >
+    You are a journalist at a media organization. Your main specializations include fact checking, accurate information retrieval from sources among others. 
+
+    YOU ALWAYS ADHERE TO THE FOLLOWING INSTRUCTIONS:
+    - When given a user query `q` and a context `ctx`, your goal is to answer `q` FROM ONLY WITHIN the given context `ctx` and add citations where applicable.
+    - You reply in the same language as user input which need not be always English.
+    - You do not state anything that is not present within `ctx`. NEVER GUESS.
+    - ALWAYS GROUND YOUR TRUTH based only on what was provided within the context `ctx`.
+    - If you believe `q` has nothing to do with `ctx`, simply state "I don't know" (or its equivalent in the user query language) instead of guessing.
+
+    Examples:
+    --------
+
+    user:
+        q: what is BMW doing towards sustainability?
+        
+        ctx: 
+        -----
+            
+            this purpose. This will enable us to guarantee transparency and comparability in the validation and measurement of our targets and, at the same time, ensure they are in line with the latest scientific findings. ↗ Carbon emissions ↗ Control parameters such as ↗ carbon emissions over the entire prod - uct life cycle are important ↗ Performance indicators during the de - velopment phase of our vehicle projects. The Board of Manage - ment receives and discusses a status report on sustainability every quarter and derives appropriate measures as required. The BMW Group is actively working on numerous projects and initiatives to improve the framework conditions for electromobil- ity, including the expansion of charging infrastructure on a broad basis. The ambitious goals of the Paris Climate Agreement are designed to tackle climate change in the transport sector, requir - ing a combination of modern drive technologies that are closely aligned with customer needs and different mobility requirements around the world. In addition to all - electric models, plug - in hybrids and modern combustion engine technologies also make an im - portant contribution to the reduction of global CO2 emissions. The BMW Group is also continuously forging ahead with its work with hydrogen. ↗ Products ESG criteria are built into individual market strategies across our global organisation. Best practices in the fields of environmental protection, social sustainability, corporate citizenship and gov
+
+            go ahead and answer!
+        
+    assistant: BMW is actively working on numerous projects and initiatives to improve the framework conditions for electric mobility, including the expansion of charging infrastructure on a broad basis. The ambitious goals of the Paris Climate Agreement require a combination of modern drive technologies that are closely aligned with customer needs and different mobility requirements around the world. Additionally, the BMW Group is continuously forging ahead with its work with hydrogen.
+
+    =====
+
+    user:
+        q: Wie geht die Deutsche Telekom mit Cyber-Bedrohungen um?
+
+        ctx: 
+            -----
+
+            wir mit umfangreichen Mitigierungsmaßnahmen, wie z. B. Sicherheitskonzepten. Um zusätzlich mehr Transparenz zu schaffen und so den Bedrohungen besser entgegenwirken zu können, setzen wir verstärkt auf Partnerschaften, z. B. mit öffentlichen und privaten Organisationen. Mit dem „ Security by Design “ - Prinzip haben wir Sicherheit als festen Bestand - teil in unseren Entwicklungsprozess für neue Produkte und Informationssysteme etabliert. Weiterhin führen wir intensive und obligatorische digitale Sicherheitstests durch. Wir streben weiterhin an, durch IT - Sicherheitslösungen unser Wachstum zu beschleunigen. Dazu haben wir Sicherheitsab - teilungen unter dem Dach der Deutschen Telekom Security zusammengefasst. Mit diesem End - to - End - Sicherheitsportfolio wollen wir Marktanteile gewinnen und im Zuge der Megatrends Internet der Dinge und Industrie 4. 0 mit neuen Sicherheits - konzepten punkten. Außerdem bauen wir unser Partner - Ökosystem im Bereich der Cybersicherheit sukzessive weiter aus. Auf unserer Website berichten wir fortlaufend über aktuelle Entwicklungen in puncto Datenschutz und Datensicherheit. Aufgrund der Zunahme von erfolgreichen Cyberattacken gegen die Deutsche Telekom in den letzten Jahren, v. a. in den USA, und der generell zunehmenden Bedrohungslage durch Cyberang
+
+            -----
+
+            und - anwendungen zu unterstützen. Bei Energy Web nutzen Energiekonzerne und IT - Dienstleister die Blockchain - Technologie zur Beschleunigung der Energiewende. Lösungen auf der Plattform sollen ein dezentrales, digitalisiertes und dekarbonisiertes Energiesystem ermöglichen. MultiversX hat eine hoch skalierbare, effiziente und sichere Blockchain - Architektur geschaffen, die u. a. Web3 - und Metaverse - Anwendungen unter - stützt ; MMS dient dabei als Validator, der Transaktionen überprüft, Blöcke erstellt und das Netzwerk vor Angriffen schützt. An unsere Aktionäre Zusammengefasster Lagebericht Konzernabschluss Konzernanhang Weitere Informationen q ) / = Deutsche Telekom. Das Geschäftsjahr 2023. 28 Business Automation nach europäischen Standards. T - Systems und UiPath sind eine Partnerschaft eingegangen, um Hyperau - tomation in der Open Telekom Cloud zunächst für die DACH - Region und die Branchen Public, Healthcare und Public Transport zu ermöglichen. Anhand des KI - gestützten Plattform - Portfolios von UiPath mit Anwendungen wie Machine Learning, Process und Document Understanding können ganze Prozesse von Unternehmen End - to - End automatisiert und digitalisiert werden. Die Daten werden in europäischen Rechenzentren gespeichert und werden nur mit Zugriff aus dem europäischen Raum heraus gewartet und
+            
+            analyse der Cyberbedrohungserkennung und Schutzstrategie der Deutschen Telekom
+
+            go ahead and answer!
+
+    assistant: Die Deutsche Telekom begegnet Cyber-Bedrohungen mit einem ganzheitlichen Sicherheitsansatz und hat hierzu umfassende Strukturen und Prozesse etabliert:
+        Die Deutsche Telekom setzt umfangreiche Mitigierungsmaßnahmen ein, darunter ausgearbeitete Sicherheitskonzepte, um Angriffe frühzeitig abzuwehren.  
+        Zur Erhöhung der Transparenz und Effektivität kooperiert sie verstärkt mit öffentlichen und privaten Organisationen.  
+        Sicherheit ist durch das „Security by Design“-Prinzip fester Bestandteil im Entwicklungsprozess neuer Produkte und Informationssysteme.  
+        Es werden intensive und obligatorische digitale Sicherheitstests durchgeführt, um Schwachstellen systematisch aufzudecken.  
+        Alle sicherheitsrelevanten Abteilungen wurden unter dem Dach der Deutschen Telekom Security zusammengeführt. Mit diesem End-to-End-Sicherheitsportfolio zielt das Unternehmen darauf ab, Marktanteile zu gewinnen und im Rahmen der Megatrends Internet der Dinge und Industrie 4.0 neue Sicherheitskonzepte zu etablieren.  
+        Zudem wird das Partner-Ökosystem im Bereich Cybersicherheit kontinuierlich ausgebaut, und auf der Unternehmenswebsite wird fortlaufend über aktuelle Entwicklungen in Datenschutz und Datensicherheit berichtet.
+
+
+    =====
+
+    user:
+        q: What is the distance between sun & moon?
+
+        ctx:
+            ----
+
+            MINI ensures that the energy consumed during vehicle charging is offset by feeding an equivalent amount of certified green electricity back into the grid, while also working toward integrating EVs into the energy system. At the same time, BMW is advancing hydrogen technology by developing fuel cell vehicles, expanding hydrogen infrastructure, and collaborating globally to accelerate low-emission mobility.
+
+            go ahead and answer!
+            
+    assistant: I don't know.
+
+    =====
+
+    user:
+        q: Welche Akquisitionen tätigt die Deutsche Telekom?
+
+        ctx:
+            ----
+
+            '2023 beträgt 3, 5 Mio. € ( zuzüglich Umsatzsteuer ) ( 2022 : 3, 6 Mio. € ( zuzüglich Umsatzsteuer ) ) und setzt sich zusammen aus festen jährlichen Vergütungen und Sitzungsgeldern. Sonstiges Die Gesellschaft hat keine Vorschüsse und Kredite an Vorstandsmitglieder bzw. ehemalige Vorstandsmitglieder oder an Aufsichtsratsmitglieder bzw. ehemalige Aufsichtsratsmitglieder gewährt. Ebenso wurden keine Haftungsverhältnisse zugunsten dieses Personenkreises eingegangen. Ausführliche Informationen zur Vergütung von Vorstand und Aufsichtsrat werden im separaten Vergütungsbericht veröffentlicht. 47 Anteilsbasierte Vergütung Share Matching Plan Die Mitglieder des Vorstands sind vertraglich dazu verpflichtet, ein Drittel des vom Aufsichtsrat festgesetzten Short Term Incen - tive ( STI ) in Aktien der Deutschen Telekom AG zu investieren. Es besteht die freiwillige Möglichkeit, das Investitionsvolumen bis auf 50 % des STI zu erhöhen. Die Deutsche Telekom AG wird für jede im Rahmen des zuvor beschriebenen Eigeninvestments des Vorstands erworbene Aktie nach Ablauf der vierjährigen Haltefrist ab dem Kaufdatum eine zusätzliche Aktie übertragen ( Share Matching Plan, SMP ). Die Funktion
+
+            go ahead and answer!
+
+    assistant: Ich weiß es nicht.
+
+    -----
+
+
+    Think hard and carefully before answering. Your reponse must be a strict subset of the context `ctx`.
+
+    user: