Spaces:
Sleeping
Sleeping
File size: 1,071 Bytes
5ba9e5a bec36c3 5ba9e5a bec36c3 5ba9e5a 4acb607 bec36c3 4acb607 5ba9e5a bec36c3 5ba9e5a bec36c3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 | import json
from langchain_core.documents import Document
class Article():
def __init__(self, id, title, url, content, customer_types):
self.id = id
self.title = title
self.url = url
self.content = content
self.customer_types = customer_types
def load_data(data_path:str) -> list[Document]:
data = []
with open(data_path, "r") as f:
scraps = json.load(f)
for article in scraps["articles"]:
data.append(
Article(
id=article["id"],
title=article["title"],
url=article["url"],
content=article["content"],
customer_types=article["customer_types"]
)
)
docs = [
Document(
page_content=chunk.content,
metadata={"id": chunk.id, "url": chunk.url, "title": chunk.title, "customer_types":chunk.customer_types})
for chunk in data
]
return docs
# if __name__ == "__main__":
# data = load_data("vrbo_articles.json")
|