Spaces:
Runtime error
Runtime error
Commit ·
36cd40f
1
Parent(s): ac97428
more paths fixing later
Browse files- src/gen/gen_judgment.py +9 -8
src/gen/gen_judgment.py
CHANGED
|
@@ -6,6 +6,7 @@ import os
|
|
| 6 |
import re
|
| 7 |
|
| 8 |
import huggingface_hub
|
|
|
|
| 9 |
from tqdm import tqdm
|
| 10 |
from utils import (
|
| 11 |
chat_completion_anthropic,
|
|
@@ -126,8 +127,8 @@ def judgment(**args):
|
|
| 126 |
|
| 127 |
if __name__ == "__main__":
|
| 128 |
parser = argparse.ArgumentParser()
|
| 129 |
-
parser.add_argument("--setting-file", type=str, default="
|
| 130 |
-
parser.add_argument("--endpoint-file", type=str, default="
|
| 131 |
args = parser.parse_args()
|
| 132 |
print(args)
|
| 133 |
|
|
@@ -142,10 +143,10 @@ if __name__ == "__main__":
|
|
| 142 |
if configs["regex_pattern"]:
|
| 143 |
pattern = re.compile(configs["regex_pattern"])
|
| 144 |
|
| 145 |
-
question_file = os.path.join("
|
| 146 |
-
|
| 147 |
-
|
| 148 |
-
ref_answer_dir = os.path.join("data", configs["bench_name"], "reference_answer")
|
| 149 |
|
| 150 |
questions = load_questions(question_file)
|
| 151 |
model_answers_external = load_model_answers(external_dir)
|
|
@@ -157,7 +158,7 @@ if __name__ == "__main__":
|
|
| 157 |
# if user choose a set of models, only judge those models
|
| 158 |
models = [
|
| 159 |
model.split("/")[-1].split(".")[0]
|
| 160 |
-
for model in glob.glob("
|
| 161 |
]
|
| 162 |
|
| 163 |
ref_answers = None
|
|
@@ -166,7 +167,7 @@ if __name__ == "__main__":
|
|
| 166 |
ref_answers = [ref_answers[model] for model in configs["ref_model"]]
|
| 167 |
|
| 168 |
output_files = {}
|
| 169 |
-
output_dir = f"data/{configs['bench_name']}/model_judgment/{configs['judge_model']}"
|
| 170 |
for model in models:
|
| 171 |
output_files[model] = os.path.join(
|
| 172 |
output_dir,
|
|
|
|
| 6 |
import re
|
| 7 |
|
| 8 |
import huggingface_hub
|
| 9 |
+
from envs import HF_HOME
|
| 10 |
from tqdm import tqdm
|
| 11 |
from utils import (
|
| 12 |
chat_completion_anthropic,
|
|
|
|
| 127 |
|
| 128 |
if __name__ == "__main__":
|
| 129 |
parser = argparse.ArgumentParser()
|
| 130 |
+
parser.add_argument("--setting-file", type=str, default=f"{HF_HOME}/src/gen/config/judge_config.yaml")
|
| 131 |
+
parser.add_argument("--endpoint-file", type=str, default="{HF_HOME}/src/gen/config/api_config.yaml")
|
| 132 |
args = parser.parse_args()
|
| 133 |
print(args)
|
| 134 |
|
|
|
|
| 143 |
if configs["regex_pattern"]:
|
| 144 |
pattern = re.compile(configs["regex_pattern"])
|
| 145 |
|
| 146 |
+
question_file = os.path.join(f"{HF_HOME}/data", configs["bench_name"], "question.jsonl")
|
| 147 |
+
internal_dir = os.path.join(f"{HF_HOME}/data", configs["bench_name"], "model_answers/internal")
|
| 148 |
+
external_dir = os.path.join(f"{HF_HOME}/data", configs["bench_name"], "model_answers/external")
|
| 149 |
+
ref_answer_dir = os.path.join(f"{HF_HOME}/data", configs["bench_name"], "reference_answer")
|
| 150 |
|
| 151 |
questions = load_questions(question_file)
|
| 152 |
model_answers_external = load_model_answers(external_dir)
|
|
|
|
| 158 |
# if user choose a set of models, only judge those models
|
| 159 |
models = [
|
| 160 |
model.split("/")[-1].split(".")[0]
|
| 161 |
+
for model in glob.glob(f"{HF_HOME}/data/arena-hard-v0.1/model_answers/external/*.jsonl")
|
| 162 |
]
|
| 163 |
|
| 164 |
ref_answers = None
|
|
|
|
| 167 |
ref_answers = [ref_answers[model] for model in configs["ref_model"]]
|
| 168 |
|
| 169 |
output_files = {}
|
| 170 |
+
output_dir = f"{HF_HOME}/data/{configs['bench_name']}/model_judgment/{configs['judge_model']}"
|
| 171 |
for model in models:
|
| 172 |
output_files[model] = os.path.join(
|
| 173 |
output_dir,
|