File size: 683 Bytes
b22c324 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 | import pandas as pd
import json
import ollama
import re
from tqdm import tqdm
tqdm.pandas()
from pathlib import Path
data_dir = Path("../datasets")
# faq qa pair
train_faq_df = pd.read_csv(data_dir / "processed" / "bank_faq" / "train_faq.csv")
train_faq_df["QA_pair"] = "Q: " + train_faq_df["Question"] + "\nA: " + train_faq_df["Answer"]
train_faq_df.to_csv(data_dir / "processed" / "bank_faq" / "train_faq.csv", index=False)
# train_faq_df = train_faq_df[["Question", "Answer", "QA_pair", "Class"]].rename(columns={
# "Question": "question",
# "Answer": "answer",
# "Class": "domain"
# })
# train_faq_df["source"] = "faq"
# train_faq_df["industry"] = "banking"
|