File size: 3,036 Bytes
01f199c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 | import sys
import os
from datetime import datetime
from constants.paths import *
from models.Gemini import Gemini
from models.OpenAI import OpenAIModel
from results.Results import Results
from promptings.PromptingFactory import PromptingFactory
from datasets.DatasetFactory import DatasetFactory
from models.ModelFactory import ModelFactory
import argparse
parser = argparse.ArgumentParser()
parser.add_argument(
"--dataset",
type=str,
default="HumanEval",
choices=[
"HumanEval",
"MBPP",
"APPS",
"xCodeEval",
"CC",
]
)
parser.add_argument(
"--strategy",
type=str,
default="MapCoder",
choices=[
"Direct",
"CoT",
"SelfPlanning",
"Analogical",
"MapCoder",
"DebateCoder",
]
)
parser.add_argument(
"--model",
type=str,
default="ChatGPT",
choices=[
"ChatGPT",
"GPT4",
"Gemini",
"DeepSeek",
"Pangu",
"Qwen",
"Pangu72B",
]
)
parser.add_argument(
"--temperature",
type=float,
default=0
)
parser.add_argument(
"--pass_at_k",
type=int,
default=1
)
parser.add_argument(
"--language",
type=str,
default="Python3",
choices=[
"C",
"C#",
"C++",
"Go",
"PHP",
"Python3",
"Ruby",
"Rust",
]
)
args = parser.parse_args()
DATASET = args.dataset
STRATEGY = args.strategy
MODEL_NAME = args.model
TEMPERATURE = args.temperature
PASS_AT_K = args.pass_at_k
LANGUAGE = args.language
RUN_NAME = f"{MODEL_NAME}-{STRATEGY}-{DATASET}-{LANGUAGE}-{TEMPERATURE}-{PASS_AT_K}"
RESULTS_PATH = f"./outputs/{RUN_NAME}.jsonl"
print(f"#########################\nRunning start {RUN_NAME}, Time: {datetime.now()}\n##########################\n")
# Instantiate model separately so we can log its info
model_class = ModelFactory.get_model_class(MODEL_NAME)
model_instance = model_class(temperature=TEMPERATURE)
# Build a compact model info string
def _format_model_info(model_obj):
info_lines = []
info_lines.append(f"model_class={model_obj.__class__.__name__}")
# include simple attributes
for k, v in getattr(model_obj, "__dict__", {}).items():
try:
if isinstance(v, (str, int, float, bool)):
info_lines.append(f"{k}={v}")
else:
info_lines.append(f"{k}=<{type(v).__name__}>")
except Exception:
info_lines.append(f"{k}=<unrepr>")
return "; ".join(info_lines)
model_info_str = _format_model_info(model_instance)
# also print to console
print(f"[MODEL INFO] {model_info_str}")
strategy = PromptingFactory.get_prompting_class(STRATEGY)(
model=model_instance,
data=DatasetFactory.get_dataset_class(DATASET)(),
language=LANGUAGE,
pass_at_k=PASS_AT_K,
results=Results(RESULTS_PATH),
)
strategy.run()
print(f"#########################\nRunning end {RUN_NAME}, Time: {datetime.now()}\n##########################\n")
|