utils/load_data.py

import json 
from langchain_core.documents import Document

class Article():
  def __init__(self, id, title, url, content, customer_types):
    self.id = id
    self.title = title
    self.url = url
    self.content = content
    self.customer_types = customer_types

def load_data(data_path:str) -> list[Document]:
    data = []

    with open(data_path, "r") as f:
        scraps = json.load(f)
        for article in scraps["articles"]:
            data.append(
                Article(
                    id=article["id"],
                    title=article["title"],
                    url=article["url"],
                    content=article["content"],
                    customer_types=article["customer_types"]
                )
            )

    docs = [
        Document(
            page_content=chunk.content,
            metadata={"id": chunk.id, "url": chunk.url, "title": chunk.title, "customer_types":chunk.customer_types})
            for chunk in data
    ]

    return docs

# if __name__ == "__main__":
    # data = load_data("vrbo_articles.json")