rag/splitter.py

from langchain_core.documents import Document
from .loader import ITEMS

documents = [
    Document(
        page_content=f"Q: {item['question']}\nA: {item['answer']}",
        metadata={
            "id": item["id"],
            "url": item["url"],
            **({"section_no": item["section_no"]} if item.get("section_no") else {}),
            **({"section_title": item["section_title"]} if item.get("section_title") else {}),
            **({"question_no": item["question_no"]} if item.get("question_no") else {}),
        },
    )
    for item in ITEMS
    if item.get("id") and item.get("question") and item.get("answer") and item.get("url")
]