File size: 1,274 Bytes
cc37925 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 |
import json
def import_question(qa_dir):
questions = []
options = []
ids = []
with open(qa_dir, 'r', encoding='utf-8') as f:
for line in f:
data = json.loads(line)
questions.append(data['question'])
options.append([data['A'], data['B'], data['C'], data['D'], data['E']])
ids.append(data['uuid'])
return questions, options, ids
qa_dir = r"C:\Users\vuvan\Desktop\An_Plaza\ViMedLLM\Vietnamese-Medical-LLM\dataset\QA Data\MedAB\MedABv2.jsonl"
questions, options, ids = import_question(qa_dir)
from chatbot import Chatbot
cb = Chatbot("llama3.1:8b", max_token=10000)
from prompt import request_retrieve_prompt
from concurrent.futures import ThreadPoolExecutor
from tqdm import tqdm
def process_question(i):
response = cb.chat(request_retrieve_prompt.format(question=questions[i], options=options[i]))
result = {
'id': ids[i],
'query': response
}
with open('query.jsonl', 'a', encoding='utf-8') as f:
f.write(json.dumps(result, ensure_ascii=False) + '\n')
print(result)
with ThreadPoolExecutor(max_workers=2) as executor:
list(tqdm(executor.map(process_question, range(len(questions))), total=len(questions)))
|