| import json | |
| import bz2 | |
| import os | |
| import csv | |
| import pandas as pd | |
| path = '/home/icml01/multi_rag/RAG/Search-in-the-Chain/data/strategyqa_corpus.csv' | |
| def load_data(path=None): | |
| assert path | |
| corpus = [] | |
| count = 0 | |
| with open( | |
| '/home/icml01/multi_rag/RAG/Search-in-the-Chain/data/strategyqa.tsv', | |
| 'w', encoding='utf-8-sig') as w: | |
| writer = csv.writer(w, delimiter="\t") | |
| df = pd.read_csv(path) | |
| for index, row in df.iterrows(): | |
| writer.writerow([count, 'Title: '+row['title']+' Text: '+ row['text']]) | |
| count += 1 | |
| print(count) | |
| return corpus | |
| load_data(path) | |