File size: 6,599 Bytes
650cfb0 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 |
import re
import json
from vllm import LLM, SamplingParams
if __name__ == "__main__":
model_name = 'guide-r2'
model_path = f"************************************/reasoner-{model_name}"
vllm_model = LLM(model=model_path, gpu_memory_utilization=0.90, max_model_len=4000, max_num_seqs=64)
sampling_params = SamplingParams(temperature=0., top_p=1., max_tokens=4000)
for file_name in ['FEVEROUS','HOVER','Open_FEVEROUS','Open_HOVER','LLM-AggreFact_test']:
file_path = f'testset/{file_name}.json'
data = []
# Open the file and read line by line
with open(file_path, 'r', encoding='utf-8') as file:
raw_data = json.load(file)
for item in raw_data:
data.append(item)
prompt_list = []
for now in range(len(data)):
# Now `data` contains all the JSON objects from the file
context = data[now]['evidence']
prompt_judge1 = f'''<|begin_of_text|><|start_header_id|>user<|end_header_id|>
Task: Validate the following claim using the provided context.
Your goal is to determine whether the claim can be supported by the context. Choose between "support" or "refute".
Instructions:
1. Analyze the claim step by step, verifying each crucial component in the claim as they appear.
2. Structure your reasoning on crucial components in the claim in detailed steps, from 1 to a maximum of 10. Make sure each step is the smallest possible logical unit necessary for validation.
3. Ensure that your reasoning correlates consistently with your conclusion. Use "##" to format each step clearly, e.g., "## Reasoning Step 1".
4. Finally, conclude with either "support" or "refute" enclosed in a pair of curly braces, noting the overall judgment regarding the claim.
Context: {context}
Claim: {data[now]['claim']}
<|eot_id|><|start_header_id|>assistant<|end_header_id|>
'''
prompt_list.append(prompt_judge1)
outputs = vllm_model.generate(prompt_list, sampling_params)
for i, output in enumerate(outputs):
prompt = output.prompt
generated_text = output.outputs[0].text
match = re.findall(r'\{([^{}]*)\}', generated_text)
with open(f'{file_name}-llama3-8b-nei-{model_name}.txt', 'a') as file:
# Append new content to the file
file.write(f'{match}\n')
for file_name in ['Healthver_test','Open_Healthver_test', 'Scifact_dev','Scifact_train', 'VitaminC_dev','VitaminC_test']:
file_path = f'testset/{file_name}.json'
data = []
# Open the file and read line by line
with open(file_path, 'r', encoding='utf-8') as file:
raw_data = json.load(file)
for item in raw_data:
data.append(item)
prompt_list = []
for now in range(len(data)):
# Now `data` contains all the JSON objects from the file
context = data[now]['evidence']
prompt_judge1 = f'''<|begin_of_text|><|start_header_id|>user<|end_header_id|>
Task: Validate the following claim using the provided context.
Your goal is to determine whether the claim can be supported with the context. Choose between "support", "refute", or "not enough information".
Instructions:
1. Analyze the claim step by step, verifying each crucial component in the claim as they appear.
2. Structure your reasoning on crucial components in the claim in detailed steps, from 1 to a maximum of 10. Make sure each step is the smallest possible logical unit necessary for validation.
3. Ensure that your reasoning correlates consistently with your conclusion. Use "##" to format each step clearly, e.g., "## Reasoning Step 1".
4. Finally, conclude with "support", "refute", or "not enough information" enclosed in a pair of curly braces, noting the overall judgment regarding the claim.
Context: {context}
Claim: {data[now]['claim']}
<|eot_id|><|start_header_id|>assistant<|end_header_id|>
'''
prompt_list.append(prompt_judge1)
outputs = vllm_model.generate(prompt_list, sampling_params)
for i, output in enumerate(outputs):
prompt = output.prompt
generated_text = output.outputs[0].text
match = re.findall(r'\{([^{}]*)\}', generated_text)
with open(f'{file_name}-llama3-8b-nei-{model_name}.txt', 'a') as file:
# Append new content to the file
file.write(f'{match}\n')
for file_name in ['Healthver_test','Open_Healthver_test', 'Scifact_dev','Scifact_train', 'VitaminC_dev','VitaminC_test']:
file_path = f'testset/{file_name}.json'
data = []
# Open the file and read line by line
with open(file_path, 'r', encoding='utf-8') as file:
raw_data = json.load(file)
for item in raw_data:
data.append(item)
prompt_list = []
for now in range(len(data)):
# Now `data` contains all the JSON objects from the file
context = data[now]['evidence']
prompt_judge1 = f'''<|begin_of_text|><|start_header_id|>user<|end_header_id|>
Task: Validate the following claim using the provided context.
Your goal is to determine whether the claim can be supported by the context. Choose between "support" or "refute".
Instructions:
1. Analyze the claim step by step, verifying each crucial component in the claim as they appear.
2. Structure your reasoning on crucial components in the claim in detailed steps, from 1 to a maximum of 10. Make sure each step is the smallest possible logical unit necessary for validation.
3. Ensure that your reasoning correlates consistently with your conclusion. Use "##" to format each step clearly, e.g., "## Reasoning Step 1".
4. Finally, conclude with either "support" or "refute" enclosed in a pair of curly braces, noting the overall judgment regarding the claim.
Context: {context}
Claim: {data[now]['claim']}
<|eot_id|><|start_header_id|>assistant<|end_header_id|>
'''
prompt_list.append(prompt_judge1)
outputs = vllm_model.generate(prompt_list, sampling_params)
for i, output in enumerate(outputs):
prompt = output.prompt
generated_text = output.outputs[0].text
match = re.findall(r'\{([^{}]*)\}', generated_text)
with open(f'{file_name}-llama3-8b-wo-nei-{model_name}.txt', 'a') as file:
# Append new content to the file
file.write(f'{match}\n') |