Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+evalplus/gallary/overview.png filter=lfs diff=lfs merge=lfs -text
+evalplus/gallary/render.gif filter=lfs diff=lfs merge=lfs -text
+nohup.out filter=lfs diff=lfs merge=lfs -text

ans.sh

@@ -0,0 +1,26 @@
+#python vllmans.py --path /home/aiscuser/fhw/model_weights/llama --datapath /home/aiscuser/fhw/data/athene_python_7w.json
+#python vllmans.py --path /home/aiscuser/fhw/model_weights/llama --datapath /home/aiscuser/fhw/data/deepseekcoder_python_7w.json
+#python vllmans.py --path /home/aiscuser/fhw/model_weights/llama --datapath /home/aiscuser/fhw/data/llama_python_7w.json
+#python vllmans.py --path /home/aiscuser/fhw/model_weights/llama --datapath /home/aiscuser/fhw/data/qwen_python_7w.json
+#python vllmans.py --path /home/aiscuser/fhw/model_weights/llama --datapath /home/aiscuser/fhw/data/qwq_python_7w.json
+#python vllmans.py --path /home/aiscuser/fhw/model_weights/deepseekcoder --datapath /home/aiscuser/fhw/data/athene_python_7w.json
+#python vllmans.py --path /home/aiscuser/fhw/model_weights/deepseekcoder --datapath /home/aiscuser/fhw/data/deepseekcoder_python_7w.json
+#python vllmans.py --path /home/aiscuser/fhw/model_weights/deepseekcoder --datapath /home/aiscuser/fhw/data/llama_python_7w.json
+#python vllmans.py --path /home/aiscuser/fhw/model_weights/deepseekcoder --datapath /home/aiscuser/fhw/data/qwen_python_7w.json
+#python vllmans.py --path /home/aiscuser/fhw/model_weights/deepseekcoder --datapath /home/aiscuser/fhw/data/qwq_python_7w.json
+#python vllmans.py --path /home/aiscuser/fhw/model_weights/athene --datapath /home/aiscuser/fhw/data/athene_python_7w.json
+#python vllmans.py --path /home/aiscuser/fhw/model_weights/athene --datapath /home/aiscuser/fhw/data/deepseekcoder_python_7w.json
+#python vllmans.py --path /home/aiscuser/fhw/model_weights/athene --datapath /home/aiscuser/fhw/data/llama_python_7w.json
+#python vllmans.py --path /home/aiscuser/fhw/model_weights/athene --datapath /home/aiscuser/fhw/data/qwen_python_7w.json
+#python vllmans.py --path /home/aiscuser/fhw/model_weights/athene --datapath /home/aiscuser/fhw/data/qwq_python_7w.json
+python vllmans.py --path /home/aiscuser/fhw/model_weights/qwen --datapath /home/aiscuser/fhw/data/athene_python_7w.json
+#python vllmans.py --path /home/aiscuser/fhw/model_weights/qwen --datapath /home/aiscuser/fhw/data/deepseekcoder_python_7w.json
+#python vllmans.py --path /home/aiscuser/fhw/model_weights/qwen --datapath /home/aiscuser/fhw/data/llama_python_7w.json
+#python vllmans.py --path /home/aiscuser/fhw/model_weights/qwen --datapath /home/aiscuser/fhw/data/qwen_python_7w.json
+#python vllmans.py --path /home/aiscuser/fhw/model_weights/qwen --datapath /home/aiscuser/fhw/data/qwq_python_7w.json
+#python vllmans.py --path /home/aiscuser/fhw/model_weights/qwq --datapath /home/aiscuser/fhw/data/athene_python_7w.json
+#python vllmans.py --path /home/aiscuser/fhw/model_weights/qwq --datapath /home/aiscuser/fhw/data/deepseekcoder_python_7w.json
+#python vllmans.py --path /home/aiscuser/fhw/model_weights/qwq --datapath /home/aiscuser/fhw/data/llama_python_7w.json
+#python vllmans.py --path /home/aiscuser/fhw/model_weights/qwq --datapath /home/aiscuser/fhw/data/qwen_python_7w.json
+#python vllmans.py --path /home/aiscuser/fhw/model_weights/qwq --datapath /home/aiscuser/fhw/data/qwq_python_7w.json
+python /data/local/zhangdi/DPO/DPO_train.py

arenaans.sh

@@ -0,0 +1,6 @@
+python vllmarenaans.py --model target --judge athene --split 0
+python vllmarenaans.py --model target --judge deepseekcoder --split 0
+python vllmarenaans.py --model target --judge llama --split 0
+python vllmarenaans.py --model target --judge qwen --split 0
+python vllmarenaans.py --model target --judge qwq --split 0
+python /data/local/zhangdi/DPO/DPO_train.py

battle.py

@@ -0,0 +1,55 @@
+from transformers import AutoTokenizer
+from vllm import LLM, SamplingParams
+import argparse
+import json
+from tqdm import tqdm
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--path', type=str,help='模型路径')
+parser.add_argument('--start', type=int,help='开始')
+parser.add_argument('--end', type=int,help='终止')
+args = parser.parse_args()
+
+name = args.path[args.path.rfind('/')+1:]
+
+fw = open(f"alignment-handbook/data/llama_battle_mistral_qwen_{args.start}_{args.end}.json", 'w+')
+
+prompts = []
+
+
+# Initialize the tokenizer
+tokenizer = AutoTokenizer.from_pretrained(args.path, trust_remote_code=True)
+f1 = open("alignment-handbook/data/llama_python_mistral_answer_0_70000_sft.json", "r+")
+f2 = open("alignment-handbook/data/llama_python_qwen_answer_0_70000_sft.json", "r+")
+lines1 = f1.readlines()[args.start:args.end]
+lines2 = f2.readlines()[args.start:args.end]
+t = 0
+for line1, line2 in zip(lines1, lines2):
+    d1 = json.loads(line1)
+    d2 = json.loads(line2)
+    instruction = d1["messages"][0]["content"]
+    answer1 = d1["messages"][1]["content"]
+    answer2 = d2["messages"][1]["content"]
+    #print(answer1)
+    #print(answer2)
+    if t%2 == 0:
+        prompt = f"This is a chatbot arena. You will be given assistant A’s answer, and assistant B’s answer. Please act as an impartial judge and evaluate the capability of two AI assistants. You should choose the assistant that follows instructions and answers questions better. Your evaluation should consider factors such as helpfulness, relevance, and accuracy. Begin your evaluation by comparing the responses of the two assistants and provide a short explanation. Avoid any position biases and ensure that the order in which the responses were presented does not influence your decision. DO NOT allow the LENGTH of the responses to influence your evaluation, choose the one that is straight-to-the-point instead of unnecessarily verbose. When the two candidates perform equally well, choose the SHORTER answer. Do not favor certain names of the assistants. Be as objective as possible. After providing your explanation concisely within 200 words, output your final verdict by strictly following this format: “[[A]]” if assistant A is better, “[[B]]” if assistant B is better, and “[[Tie]]” for a tie. Finish your judgement within 300 words.\n\n[User Question]\n{instruction}\n\n[The Start of Assistant A’s Answer]\n{answer1}\n[The End of Assistant A’s Answer]\n\n[The Start of Assistant B’s Answer]\n{answer2}\n[The End of Assistant B’s Answer]"
+    else:
+        prompt = f"This is a chatbot arena. You will be given assistant A’s answer, and assistant B’s answer. Please act as an impartial judge and evaluate the capability of two AI assistants. You should choose the assistant that follows instructions and answers questions better. Your evaluation should consider factors such as helpfulness, relevance, and accuracy. Begin your evaluation by comparing the responses of the two assistants and provide a short explanation. Avoid any position biases and ensure that the order in which the responses were presented does not influence your decision. DO NOT allow the LENGTH of the responses to influence your evaluation, choose the one that is straight-to-the-point instead of unnecessarily verbose. When the two candidates perform equally well, choose the SHORTER answer. Do not favor certain names of the assistants. Be as objective as possible. After providing your explanation concisely within 200 words, output your final verdict by strictly following this format: “[[A]]” if assistant A is better, “[[B]]” if assistant B is better, and “[[Tie]]” for a tie. Finish your judgement within 300 words.\n\n[User Question]\n{instruction}\n\n[The Start of Assistant A’s Answer]\n{answer2}\n[The End of Assistant A’s Answer]\n\n[The Start of Assistant B’s Answer]\n{answer1}\n[The End of Assistant B’s Answer]"
+    messages = [{"role": "user", "content": prompt}]
+    text = tokenizer.apply_chat_template(
+        messages,
+        tokenize=False
+    )
+    prompts.append(text)
+    t = t + 1
+    
+# Input the model name or path. Can be GPTQ or AWQ models.
+llm = LLM(args.path, dtype="float16", tensor_parallel_size=8, trust_remote_code=True, max_model_len=8192, enforce_eager=True)
+sampling_params = SamplingParams(temperature=1.0, top_p=0.995, max_tokens=8192)
+outputs = llm.generate(prompts=prompts, sampling_params=sampling_params)
+t = 0
+for output in outputs:
+    d = {"arena": output.outputs[0].text, "t": t}
+    t = t + 1
+    fw.write(json.dumps(d)+"\n")

battle.sh

@@ -0,0 +1,21 @@
+#python vllmbattle.py --judge /home/aiscuser/fhw/model_weights/athene --model /home/aiscuser/fhw/model_weights/deepseekcoder
+#python vllmbattle.py --judge /home/aiscuser/fhw/model_weights/athene --model /home/aiscuser/fhw/model_weights/llama
+#python vllmbattle.py --judge /home/aiscuser/fhw/model_weights/athene --model /home/aiscuser/fhw/model_weights/qwen
+#python vllmbattle.py --judge /home/aiscuser/fhw/model_weights/athene --model /home/aiscuser/fhw/model_weights/qwq
+#python vllmbattle.py --judge /home/aiscuser/fhw/model_weights/deepseekcoder --model /home/aiscuser/fhw/model_weights/athene
+python vllmbattle.py --judge /home/aiscuser/fhw/model_weights/deepseekcoder --model /home/aiscuser/fhw/model_weights/llama
+python vllmbattle.py --judge /home/aiscuser/fhw/model_weights/deepseekcoder --model /home/aiscuser/fhw/model_weights/qwen
+python vllmbattle.py --judge /home/aiscuser/fhw/model_weights/deepseekcoder --model /home/aiscuser/fhw/model_weights/qwq
+python vllmbattle.py --judge /home/aiscuser/fhw/model_weights/llama --model /home/aiscuser/fhw/model_weights/athene
+python vllmbattle.py --judge /home/aiscuser/fhw/model_weights/llama --model /home/aiscuser/fhw/model_weights/deepseekcoder
+#python vllmbattle.py --judge /home/aiscuser/fhw/model_weights/llama --model /home/aiscuser/fhw/model_weights/qwen
+#python vllmbattle.py --judge /home/aiscuser/fhw/model_weights/llama --model /home/aiscuser/fhw/model_weights/qwq
+#python vllmbattle.py --judge /home/aiscuser/fhw/model_weights/qwen --model /home/aiscuser/fhw/model_weights/athene
+#python vllmbattle.py --judge /home/aiscuser/fhw/model_weights/qwen --model /home/aiscuser/fhw/model_weights/deepseekcoder
+#python vllmbattle.py --judge /home/aiscuser/fhw/model_weights/qwen --model /home/aiscuser/fhw/model_weights/llama
+#python vllmbattle.py --judge /home/aiscuser/fhw/model_weights/qwen --model /home/aiscuser/fhw/model_weights/qwq
+#python vllmbattle.py --judge /home/aiscuser/fhw/model_weights/qwq --model /home/aiscuser/fhw/model_weights/athene
+#python vllmbattle.py --judge /home/aiscuser/fhw/model_weights/qwq --model /home/aiscuser/fhw/model_weights/deepseekcoder
+#python vllmbattle.py --judge /home/aiscuser/fhw/model_weights/qwq --model /home/aiscuser/fhw/model_weights/llama
+#python vllmbattle.py --judge /home/aiscuser/fhw/model_weights/qwq --model /home/aiscuser/fhw/model_weights/qwen
+python /data/local/zhangdi/DPO/DPO_train.py

battlescore.py

@@ -0,0 +1,105 @@
+from transformers import AutoTokenizer
+from vllm import LLM, SamplingParams
+import argparse
+import json
+from tqdm import tqdm
+import re
+def extract_score(judgement):
+    d = {}
+    extracted = re.findall(r"\[\[(\d*\.\d+|\d+)/10\]\]", judgement, re.S)
+    if len(extracted) > 0:
+        d["score"] = float(extracted[-1])
+        return int(d["score"])
+    extracted = re.findall(r"\[\[(\d*\.\d+|\d+)\]\]", judgement, re.S)
+    if len(extracted) > 0:
+        d["score"] = float(extracted[-1])
+        return int(d["score"])
+    extracted = re.findall(r"\*\*Score: \[(\d*\.\d+|\d+)/10\]\*\*", judgement, re.S)
+    if len(extracted) > 0:
+        d["score"] = float(extracted[-1])
+        return int(d["score"])
+    extracted = re.findall(r"\*\*Score: \[(\d*\.\d+|\d+)\]\*\*", judgement, re.S)
+    if len(extracted) > 0:
+        d["score"] = float(extracted[-1])
+        return int(d["score"])
+    extracted = re.findall(r"\*\*Score: (\d*\.\d+|\d+)/10\*\*", judgement, re.S)
+    if len(extracted) > 0:
+        d["score"] = float(extracted[-1])
+        return int(d["score"])
+    extracted = re.findall(r"\*\*Score: (\d*\.\d+|\d+)\*\*", judgement, re.S)
+    if len(extracted) > 0:
+        d["score"] = float(extracted[-1])
+        return int(d["score"])
+    extracted = re.findall(r"\*\*Score:\*\* (\d*\.\d+|\d+)/10", judgement, re.S)
+    if len(extracted) > 0:
+        d["score"] = float(extracted[-1])
+        return int(d["score"])
+    extracted = re.findall(r"\*\*Score:\*\* (\d*\.\d+|\d+)", judgement, re.S)
+    if len(extracted) > 0:
+        d["score"] = float(extracted[-1])
+        return int(d["score"])
+    extracted = re.findall(r"Score(.*?)", judgement, re.S)
+    if len(extracted) > 0:
+        judgement = extracted[-1]
+        extracted = re.findall(r"\d*\.\d+|\d+", judgement, re.S)
+        if len(extracted) > 0:
+            d["score"] = float(extracted[-1])
+            return int(d["score"])
+    return -1
+parser = argparse.ArgumentParser()
+parser.add_argument('--judgename', type=str,help='模型路径')
+parser.add_argument('--modelnames', nargs='+')
+args = parser.parse_args()
+
+f = open(f"/home/aiscuser/fhw/data/{args.judgename}_filtered_by_answer.json", "r+")
+ddd = json.loads(f.readlines()[0])
+
+fr = open(f"/home/aiscuser/fhw/data/{args.judgename}_answerby_{args.judgename}.json", 'r+')
+linesr = fr.readlines()
+
+all_lines = []
+for modelname in args.modelnames:
+    f = open(f"/home/aiscuser/fhw/data/{args.judgename}_judge_{modelname}.json", 'r+')
+    all_lines.append(f.readlines())
+
+
+a, b, c, d = 0, 0, 0, 0
+
+fw = open(f"/home/aiscuser/fhw/data/{args.judgename}_with_best_answer.json", "w+")
+
+for i in tqdm(ddd[args.judgename]):
+    reference = json.loads(linesr[i])
+    da = json.loads(all_lines[0][a]) if a<len(all_lines[0]) else json.loads(all_lines[0][0])
+    db = json.loads(all_lines[1][b]) if b<len(all_lines[1]) else json.loads(all_lines[1][0])
+    dc = json.loads(all_lines[2][c]) if c<len(all_lines[2]) else json.loads(all_lines[2][0])
+    dd = json.loads(all_lines[3][d]) if d<len(all_lines[3]) else json.loads(all_lines[3][0])
+
+    da["battlescore"], db["battlescore"], dc["battlescore"], dd["battlescore"] = -1, -1, -1, -1
+
+    if da["index"] == i:
+        da["battlescore"] = extract_score(da["battle"])
+        a = a + 1
+    if db["index"] == i:
+        db["battlescore"] = extract_score(db["battle"])
+        b = b + 1
+    if dc["index"] == i:
+        dc["battlescore"] = extract_score(dc["battle"])
+        c = c + 1
+    if dd["index"] == i:
+        dd["battlescore"] = extract_score(dd["battle"])
+        d = d + 1
+
+    instruction = reference["instruction"]
+    scorelist = [da["battlescore"], db["battlescore"], dc["battlescore"], dd["battlescore"]]
+    maxscore = max(scorelist)
+    maxindex = scorelist.index(maxscore)
+
+    if maxscore>6:
+        bestname = args.modelnames[maxindex]
+        bestanswer = [da, db, dc, dd][maxindex]["response"]
+    else:
+        bestname = args.judgename
+        bestanswer = reference["response"]
+    fw.write(json.dumps({"instruction": instruction, "scorelist": scorelist, "bestname": bestname, "bestanswer": bestanswer, "modelnames": args.modelnames, "judgename": args.judgename})+"\n")
+
+

bertencode.py

@@ -0,0 +1,40 @@
+"""
+This example starts multiple processes (1 per GPU), which encode
+sentences in parallel. This gives a near linear speed-up
+when encoding large text collections.
+"""
+from tqdm import tqdm
+import logging
+import json
+import torch
+from sentence_transformers import LoggingHandler, SentenceTransformer
+
+logging.basicConfig(
+    format="%(asctime)s - %(message)s", datefmt="%Y-%m-%d %H:%M:%S", level=logging.INFO, handlers=[LoggingHandler()]
+)
+
+# Important, you need to shield your code with if __name__. Otherwise, CUDA runs into issues when spawning new processes.
+if __name__ == "__main__":
+    # Create a large list of 100k sentences
+    f = open("/home/aiscuser/fhw/data/qwq_python_selected.json","r+")
+    lines = f.readlines()
+    sentences = []
+    for line in tqdm(lines):
+        d= json.loads(line)
+        sentences.append(d["instruction"])
+
+
+    # Define the model
+    model = SentenceTransformer("/home/aiscuser/fhw/model_weights/all-roberta-large-v1")
+
+    # Start the multi-process pool on all available CUDA devices
+    pool = model.start_multi_process_pool(["cuda:0", "cuda:0", "cuda:0", "cuda:0", "cuda:0", "cuda:0", "cuda:0", "cuda:0", "cuda:0", "cuda:0", "cuda:1", "cuda:1", "cuda:1", "cuda:1", "cuda:1", "cuda:1", "cuda:1", "cuda:1", "cuda:1", "cuda:1", "cuda:2", "cuda:2", "cuda:2", "cuda:2", "cuda:2", "cuda:2", "cuda:2", "cuda:2", "cuda:2", "cuda:2", "cuda:3", "cuda:3", "cuda:3", "cuda:3", "cuda:3", "cuda:3", "cuda:3", "cuda:3", "cuda:3", "cuda:3", "cuda:4", "cuda:4", "cuda:4", "cuda:4", "cuda:4", "cuda:4", "cuda:4", "cuda:4", "cuda:4", "cuda:4", "cuda:5", "cuda:5", "cuda:5", "cuda:5", "cuda:5", "cuda:5", "cuda:5", "cuda:5", "cuda:5", "cuda:5", "cuda:6", "cuda:6", "cuda:6", "cuda:6", "cuda:6", "cuda:6", "cuda:6", "cuda:6", "cuda:6", "cuda:6", "cuda:7", "cuda:7", "cuda:7", "cuda:7", "cuda:7", "cuda:7", "cuda:7", "cuda:7", "cuda:7", "cuda:7", ])
+
+    # Compute the embeddings using the multi-process pool
+    emb = model.encode_multi_process(sentences, pool)
+    
+    print("Embeddings computed. Shape:", emb.shape)
+
+    # Optional: Stop the processes in the pool
+    model.stop_multi_process_pool(pool)
+    torch.save(emb, "/home/aiscuser/fhw/embeddings/qwq_ins_embeddings.pt", pickle_protocol=4)

cleanans.py

@@ -0,0 +1,10 @@
+from tqdm import tqdm
+import json
+f = open("/home/aiscuser/fhw/data/all_instruct_with_answers.json", "r+")
+fw = open("/home/aiscuser/fhw/data/all_instruct_with_answers_cleaned.json", "w+")
+
+lines = f.readlines()
+for line in lines:
+    d = json.loads(line)
+    d["bestanswer"] = d["bestanswer"].strip("<|start_header_id|>assistant<|end_header_id|>").strip("\n")
+    fw.write(json.dumps(d)+"\n")

cloudgpt-apim-token-cache.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc1dedc3209b111ca9ac7676ffad159ce9bff625b0980c4897653414b796f3aa
+size 300

cloudgpt_aoai.py

@@ -0,0 +1,619 @@
+from __future__ import annotations
+from typing import (
+    Any,
+    AsyncGenerator,
+    Callable,
+    Coroutine,
+    Literal,
+    Optional,
+    ParamSpec,
+    TypeVar,
+    cast,
+    Dict,
+    TYPE_CHECKING,
+)
+import sys, os
+import contextlib
+import functools
+
+__all__ = [
+    "get_openai_token_provider",
+    "get_openai_token",
+    "get_openai_client",
+    "get_chat_completion",
+    "encode_image",
+    "cloudgpt_available_models",
+]
+
+TokenProvider = Callable[[], str]
+AsyncTokenProvider = Callable[[], Coroutine[Any, Any, str]]
+
+
+def check_module():
+    try:
+        import openai, azure.identity.broker  # type: ignore
+
+        del openai, azure.identity.broker
+    except ImportError:
+        print("Please install the required packages by running the following command:")
+        print("pip install openai azure-identity-broker --upgrade")
+        exit(1)
+
+
+check_module()
+
+import openai
+from openai import OpenAI
+
+_depRt = TypeVar("_depRt")
+_depParam = ParamSpec("_depParam")
+
+
+def _deprecated(message: str):
+    def deprecated_decorator(
+        func: Callable[_depParam, _depRt]
+    ) -> Callable[_depParam, _depRt]:
+        def deprecated_func(
+            *args: _depParam.args, **kwargs: _depParam.kwargs
+        ) -> _depRt:
+            import traceback
+
+            print(
+                "\n ⚠️  \x1b[31m{} is a deprecated function. {}".format(
+                    func.__name__, message
+                )
+            )
+            traceback.print_stack()
+            print("\x1b[0m")
+            return func(*args, **kwargs)
+
+        return deprecated_func
+
+    return deprecated_decorator
+
+
+def _validate_token(token: str) -> bool:
+    import requests
+
+    url = "https://cloudgpt-openai.azure-api.net/openai/ping"
+
+    headers = {
+        "Authorization": f"Bearer {token}",
+    }
+    try:
+        response = requests.get(url, headers=headers)
+        assert response.status_code == 200 and response.text == "OK", response.text
+        return True
+    except Exception as e:
+        print("Failed to validate token", e)
+        return False
+
+
+@functools.lru_cache(maxsize=3)
+def get_openai_token_provider(
+    token_cache_file: str = "cloudgpt-apim-token-cache.bin",
+    client_id: Optional[str] = None,
+    client_secret: Optional[str] = None,
+    use_azure_cli: Optional[bool] = None,
+    use_broker_login: Optional[bool] = None,
+    use_managed_identity: Optional[bool] = None,
+    use_device_code: Optional[bool] = None,
+    skip_access_validation: Optional[bool] = False,
+    **kwargs: Any,
+) -> TokenProvider:
+    """
+    Get a token provider function that could return a valid access token for CloudGPT OpenAI.
+
+    The return value is a function that should be used with AzureOpenAIClient constructor as azure_ad_token_provider parameter.
+    The following code snippet shows how to use it with AzureOpenAIClient:
+
+    ```python
+    token_provider = get_openai_token_provider()
+    client = openai.AzureOpenAI(
+        api_version="2024-06-01",
+        azure_endpoint="https://cloudgpt-openai.azure-api.net/",
+        azure_ad_token_provider=token_provider,
+    )
+    ```
+
+    Parameters
+    ----------
+    token_cache_file : str, optional
+        path to the token cache file, by default 'cloudgpt-apim-token-cache.bin' in the current directory
+    client_id : Optional[str], optional
+        client id for AAD app, by default None
+    client_secret : Optional[str], optional
+        client secret for AAD app, by default None
+    use_azure_cli : Optional[bool], optional
+        use Azure CLI for authentication, by default None. If AzCli has been installed and logged in,
+        it will be used for authentication. This is recommended for headless environments and AzCLI takes
+        care of token cache and token refresh.
+    use_broker_login : Optional[bool], optional
+        use broker login for authentication, by default None.
+        If not specified, it will be enabled for known supported environments (e.g. Windows, macOS, WSL, VSCode),
+        but sometimes it may not always could cache the token for long-term usage.
+        In such cases, you can disable it by setting it to False.
+    use_managed_identity : Optional[bool], optional
+        use managed identity for authentication, by default None.
+        If not specified, it will use user assigned managed identity if client_id is specified,
+        For use system assigned managed identity, client_id could be None but need to set use_managed_identity to True.
+    use_device_code : Optional[bool], optional
+        use device code for authentication, by default None. If not specified, it will use interactive login on supported platform.
+    skip_access_validation : Optional[bool], optional
+        skip access token validation, by default False.
+
+    Returns
+    -------
+    TokenProvider
+        the token provider function that could return a valid access token for CloudGPT OpenAI
+    """
+    import shutil
+    from azure.identity.broker import InteractiveBrowserBrokerCredential
+    from azure.identity import (
+        ManagedIdentityCredential,
+        ClientSecretCredential,
+        DeviceCodeCredential,
+        AuthenticationRecord,
+        AzureCliCredential,
+    )
+    from azure.identity import TokenCachePersistenceOptions
+    import msal  # type: ignore
+
+    api_scope_base = "api://feb7b661-cac7-44a8-8dc1-163b63c23df2"
+    tenant_id = "72f988bf-86f1-41af-91ab-2d7cd011db47"
+    scope = api_scope_base + "/.default"
+
+    token_cache_option = TokenCachePersistenceOptions(
+        name=token_cache_file,
+        enable_persistence=True,
+        allow_unencrypted_storage=True,
+    )
+
+    def save_auth_record(auth_record: AuthenticationRecord):
+        try:
+            with open(token_cache_file, "w") as cache_file:
+                cache_file.write(auth_record.serialize())
+        except Exception as e:
+            print("failed to save auth record", e)
+
+    def load_auth_record() -> Optional[AuthenticationRecord]:
+        try:
+            if not os.path.exists(token_cache_file):
+                return None
+            with open(token_cache_file, "r") as cache_file:
+                return AuthenticationRecord.deserialize(cache_file.read())
+        except Exception as e:
+            print("failed to load auth record", e)
+            return None
+
+    auth_record: Optional[AuthenticationRecord] = load_auth_record()
+
+    current_auth_mode: Literal[
+        "client_secret",
+        "managed_identity",
+        "az_cli",
+        "interactive",
+        "device_code",
+        "none",
+    ] = "none"
+
+    implicit_mode = not (
+        use_managed_identity or use_azure_cli or use_broker_login or use_device_code
+    )
+
+    if use_managed_identity or (implicit_mode and client_id is not None):
+        if not use_managed_identity and client_secret is not None:
+            assert (
+                client_id is not None
+            ), "client_id must be specified with client_secret"
+            current_auth_mode = "client_secret"
+            identity = ClientSecretCredential(
+                client_id=client_id,
+                client_secret=client_secret,
+                tenant_id=tenant_id,
+                cache_persistence_options=token_cache_option,
+                authentication_record=auth_record,
+            )
+        else:
+            current_auth_mode = "managed_identity"
+            if client_id is None:
+                # using default managed identity
+                identity = ManagedIdentityCredential(
+                    cache_persistence_options=token_cache_option,
+                )
+            else:
+                identity = ManagedIdentityCredential(
+                    client_id=client_id,
+                    cache_persistence_options=token_cache_option,
+                )
+    elif use_azure_cli or (implicit_mode and shutil.which("az") is not None):
+        current_auth_mode = "az_cli"
+        identity = AzureCliCredential(tenant_id=tenant_id)
+    else:
+        if implicit_mode:
+            # enable broker login for known supported envs if not specified using use_device_code
+            if sys.platform.startswith("darwin") or sys.platform.startswith("win32"):
+                use_broker_login = True
+            elif os.environ.get("WSL_DISTRO_NAME", "") != "":
+                use_broker_login = True
+            elif os.environ.get("TERM_PROGRAM", "") == "vscode":
+                use_broker_login = True
+            else:
+                use_broker_login = False
+        if use_broker_login:
+            current_auth_mode = "interactive"
+            identity = InteractiveBrowserBrokerCredential(
+                tenant_id="72f988bf-86f1-41af-91ab-2d7cd011db47",
+                cache_persistence_options=token_cache_option,
+                use_default_broker_account=True,
+                parent_window_handle=msal.PublicClientApplication.CONSOLE_WINDOW_HANDLE,
+                authentication_record=auth_record,
+            )
+        else:
+            current_auth_mode = "device_code"
+            identity = DeviceCodeCredential(
+                tenant_id="72f988bf-86f1-41af-91ab-2d7cd011db47",
+                cache_persistence_options=token_cache_option,
+                authentication_record=auth_record,
+            )
+
+        try:
+            auth_record = identity.authenticate(scopes=[scope])
+            if auth_record:
+                save_auth_record(auth_record)
+
+        except Exception as e:
+            print(
+                f"failed to acquire token from AAD for CloudGPT OpenAI using {current_auth_mode}",
+                e,
+            )
+            raise e
+
+    try:
+        from azure.identity import get_bearer_token_provider
+
+        token_provider = get_bearer_token_provider(identity, scope)
+        token_verified_cache: str = ""
+
+        def token_provider_wrapper():
+            nonlocal token_verified_cache
+            token = token_provider()
+            if token != token_verified_cache:
+                if not skip_access_validation:
+                    assert _validate_token(token), "failed to validate token"
+                token_verified_cache = token
+            return token
+
+        return token_provider_wrapper
+    except Exception as e:
+        print("failed to acquire token from AAD for CloudGPT OpenAI", e)
+        raise e
+
+
+@functools.lru_cache(maxsize=3)
+async def async_get_openai_token_provider(
+    **kwargs: Any,
+) -> AsyncTokenProvider:
+    # TODO: implement async version of get_openai_token_provider
+    token_provider = get_openai_token_provider(
+        **kwargs,
+    )
+
+    async def async_token_provider() -> str:
+        return token_provider()
+
+    return async_token_provider
+
+
+@_deprecated(
+    "use get_openai_token_provider instead whenever possible "
+    "and use it as the azure_ad_token_provider parameter in AzureOpenAIClient constructor. "
+    "Please do not acquire token directly or use it elsewhere."
+)
+def get_openai_token(
+    token_cache_file: str = "cloudgpt-apim-token-cache.bin",
+    client_id: Optional[str] = None,
+    client_secret: Optional[str] = None,
+    use_azure_cli: Optional[bool] = None,
+    use_broker_login: Optional[bool] = None,
+    use_managed_identity: Optional[bool] = None,
+    use_device_code: Optional[bool] = None,
+    skip_access_validation: Optional[bool] = False,
+    **kwargs: Any,
+) -> str:
+    """
+    get access token for CloudGPT OpenAI
+    """
+    return get_openai_token_provider(
+        token_cache_file=token_cache_file,
+        client_id=client_id,
+        client_secret=client_secret,
+        use_azure_cli=use_azure_cli,
+        use_broker_login=use_broker_login,
+        use_managed_identity=use_managed_identity,
+        use_device_code=use_device_code,
+        skip_access_validation=skip_access_validation,
+        **kwargs,
+    )()
+
+
+"""
+Available models for CloudGPT OpenAI
+"""
+cloudgpt_available_models = Literal[
+    "gpt-35-turbo-20220309",
+    "gpt-35-turbo-16k-20230613",
+    "gpt-35-turbo-20230613",
+    "gpt-35-turbo-1106",
+    "gpt-4-20230321",
+    "gpt-4-20230613",
+    "gpt-4-32k-20230321",
+    "gpt-4-32k-20230613",
+    "gpt-4-1106-preview",
+    "gpt-4-0125-preview",
+    "gpt-4-visual-preview",
+    "gpt-4-turbo-20240409",
+    "gpt-4o-20240513",
+    "gpt-4o-20240806",
+    "gpt-4o-mini-20240718",
+]
+
+cloudgpt_available_realtime_models = Literal["gpt-4o-realtime-preview-20241001"]
+
+
+def encode_image(image_path: str, mime_type: Optional[str] = None) -> str:
+    """
+    Utility function to encode image to base64 for using in OpenAI API
+
+    Parameters
+    ----------
+    image_path : str
+        path to the image file
+
+    mime_type : Optional[str], optional
+        mime type of the image, by default None and will infer from the file extension if possible
+
+    Returns
+    -------
+    str
+        base64 encoded image url
+    """
+    import base64
+    import mimetypes
+
+    file_name = os.path.basename(image_path)
+    mime_type = cast(
+        Optional[str],
+        mime_type if mime_type is not None else mimetypes.guess_type(file_name)[0],  # type: ignore
+    )
+    with open(image_path, "rb") as image_file:
+        encoded_image = base64.b64encode(image_file.read()).decode("ascii")
+
+    if mime_type is None or not mime_type.startswith("image/"):
+        print(
+            "Warning: mime_type is not specified or not an image mime type. Defaulting to png."
+        )
+        mime_type = "image/png"
+
+    image_url = f"data:{mime_type};base64," + encoded_image
+    return image_url
+
+
+@functools.lru_cache(maxsize=3)
+def get_openai_client(
+    client_id: Optional[str] = None,
+    client_secret: Optional[str] = None,
+    use_azure_cli: Optional[bool] = None,
+    use_broker_login: Optional[bool] = None,
+    use_managed_identity: Optional[bool] = None,
+    use_device_code: Optional[bool] = None,
+) -> OpenAI:
+    """
+    Initialize OpenAI client for CloudGPT OpenAI.
+
+    All parameters are optional and will use the default authentication method if not specified.
+
+    Parameters
+    ----------
+    client_id : Optional[str], optional
+        client id for AAD app, by default None
+    client_secret : Optional[str], optional
+        client secret for AAD app, by default None
+    use_azure_cli : Optional[bool], optional
+        use Azure CLI for authentication, by default None. If AzCli has been installed and logged in,
+        it will be used for authentication. This is recommended for headless environments and AzCLI takes
+        care of token cache and token refresh.
+    use_broker_login : Optional[bool], optional
+        use broker login for authentication, by default None.
+        If not specified, it will be enabled for known supported environments (e.g. Windows, macOS, WSL, VSCode),
+        but sometimes it may not always could cache the token for long-term usage.
+        In such cases, you can disable it by setting it to False.
+    use_managed_identity : Optional[bool], optional
+        use managed identity for authentication, by default None.
+        If not specified, it will use user assigned managed identity if client_id is specified,
+        For use system assigned managed identity, client_id could be None but need to set use_managed_identity to True.
+    use_device_code : Optional[bool], optional
+        use device code for authentication, by default None. If not specified, it will use interactive login on supported platform.
+
+    Returns
+    -------
+    OpenAI
+        OpenAI client for CloudGPT OpenAI. Check https://github.com/openai/openai-python for more details.
+    """
+    token_provider = get_openai_token_provider(
+        client_id=client_id,
+        client_secret=client_secret,
+        use_azure_cli=use_azure_cli,
+        use_broker_login=use_broker_login,
+        use_managed_identity=use_managed_identity,
+        use_device_code=use_device_code,
+    )
+    print(token_provider())
+    client = openai.AzureOpenAI(
+        api_version="2024-06-01",
+        azure_endpoint="https://cloudgpt-openai.azure-api.net/",
+        azure_ad_token_provider=token_provider,
+    )
+    return client
+
+
+def get_chat_completion(
+    model: Optional[cloudgpt_available_models] = None,
+    client_id: Optional[str] = None,
+    client_secret: Optional[str] = None,
+    use_azure_cli: Optional[bool] = None,
+    use_broker_login: Optional[bool] = None,
+    use_managed_identity: Optional[bool] = None,
+    use_device_code: Optional[bool] = None,
+    **kwargs: Any,
+):
+    """
+    Helper function to get chat completion from OpenAI API
+    """
+
+    engine: Optional[str] = kwargs.get("engine")
+
+    model_name: Any = model
+    if model_name is None:
+        if engine is None:
+            raise ValueError("model name must be specified by 'model' parameter")
+        model_name = engine
+
+    if "engine" in kwargs:
+        del kwargs["engine"]
+
+    client = get_openai_client(
+        client_id=client_id,
+        client_secret=client_secret,
+        use_azure_cli=use_azure_cli,
+        use_broker_login=use_broker_login,
+        use_managed_identity=use_managed_identity,
+        use_device_code=use_device_code,
+    )
+
+    response: Any = client.completions.create(model=model_name, **kwargs)
+
+    return response
+
+
+def _check_rtclient():
+    try:
+        import rtclient  # type: ignore
+
+        del rtclient
+    except ImportError:
+        raise ImportError(
+            f"rtclient package is required when using realtime API`. Please install it by running \n"
+            "pip install https://github.com/Azure-Samples/aoai-realtime-audio-sdk/releases/download/py%2Fv0.5.1/rtclient-0.5.1-py3-none-any.whl"
+        )
+    return True
+
+
+if TYPE_CHECKING:
+    from rtclient import RTClient, RTLowLevelClient
+
+
+async def get_realtime_low_level_client(
+    model: cloudgpt_available_realtime_models = "gpt-4o-realtime-preview-20241001",
+    **kwargs: Any,
+) -> RTLowLevelClient:
+    """
+    Get realtime client with low level API for fined grained control
+
+    Usage:
+    ```python
+    async with await get_realtime_low_level_client() as client:
+        # use client
+        pass
+    ```
+    """
+    assert _check_rtclient()
+    from rtclient import RTLowLevelClient
+
+    class CloudGPT_AOAI_RTLowLevelClient(RTLowLevelClient):
+        def __init__(
+            self,
+            token_provider: AsyncTokenProvider,
+            url: str = "https://cloudgpt-openai.azure-api.net/",
+            azure_deployment: cloudgpt_available_realtime_models | None = None,
+        ):
+            self._async_token_provider = token_provider
+
+            from azure.core.credentials import AzureKeyCredential
+
+            key_credential = AzureKeyCredential("placeholder")
+
+            super().__init__(
+                url=url,
+                key_credential=key_credential,
+                azure_deployment=azure_deployment,
+            )
+
+        async def _get_auth(self) -> Dict[str, str]:
+            token = await self._async_token_provider()
+            return {"Authorization": f"Bearer {token}"}
+
+    token_provider = await async_get_openai_token_provider(**kwargs)
+    return CloudGPT_AOAI_RTLowLevelClient(
+        token_provider=token_provider,
+        azure_deployment=model,
+    )
+
+
+async def get_realtime_client(
+    model: cloudgpt_available_realtime_models = "gpt-4o-realtime-preview-20241001",
+    **kwargs: Any,
+) -> RTClient:
+    """
+    Get realtime client with high level API for simplified usage
+
+    Usage:
+    ```python
+    async with await get_realtime_client() as client:
+        # use client
+        pass
+    ```
+    """
+    assert _check_rtclient()
+    from rtclient import RTClient, MessageQueueWithError, Session
+
+    class CloudGPT_AOAI_RTClient(RTClient):
+        def __init__(
+            self,
+            low_level_client: Optional[RTLowLevelClient] = None,
+        ):
+            self._client = low_level_client
+
+            self._message_queue = MessageQueueWithError(
+                receive_delegate=self._receive_message,
+                error_predicate=lambda m: m is not None and (m.type == "error"),
+            )
+
+            self.session: Optional[Session] = None
+
+            self._response_map: dict[str, str] = {}
+
+    low_level_client = await get_realtime_low_level_client(model=model, **kwargs)
+    return CloudGPT_AOAI_RTClient(low_level_client=low_level_client)
+
+
+def _test_call(**kwargs: Any):
+    test_message = "What is the content?"
+
+    client = get_openai_client(**kwargs)
+
+    response = client.chat.completions.create(
+        model="gpt-4o-mini-20240718",
+        messages=[{"role": "user", "content": test_message}],
+        temperature=0.7,
+        max_tokens=100,
+        top_p=0.95,
+        frequency_penalty=0,
+        presence_penalty=0,
+    )
+
+    print(response.choices[0].message)
+
+
+if __name__ == "__main__":
+    _test_call(use_broker_login=True)

combine.py

@@ -0,0 +1,11 @@
+import json
+import os
+names = os.listdir("/home/aiscuser/fhw/data")
+lines = []
+for name in names:
+    if "_with_best_answer" in name:
+        f = open(f"/home/aiscuser/fhw/data/{name}", "r+")
+        lines.extend(f.readlines())
+fw = open("/home/aiscuser/fhw/data/all_instruct_with_answers.json", "w+")
+for line in lines:
+    fw.write(line)

config_sft_fhw.yaml

@@ -0,0 +1,41 @@
+# Model arguments
+model_name_or_path: mistralai/Mistral-7B-v0.1
+model_revision: main
+torch_dtype: bfloat16
+attn_implementation: flash_attention_2
+
+# Data training arguments
+chat_template: "{% if not add_generation_prompt is defined %}\n{% set add_generation_prompt = false %}\n{% endif %}\n{%- set ns = namespace(found=false) -%}\n{%- for message in messages -%}\n    {%- if message['role'] == 'system' -%}\n        {%- set ns.found = true -%}\n    {%- endif -%}\n{%- endfor -%}\n{{bos_token}}{%- if not ns.found -%}\n{{'You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer\\n'}}\n{%- endif %}\n{%- for message in messages %}\n    {%- if message['role'] == 'system' %}\n{{ message['content'] }}\n    {%- else %}\n        {%- if message['role'] == 'user' %}\n{{'### Instruction:\\n' + message['content'] + '\\n'}}\n        {%- else %}\n{{'### Response:\\n' + message['content'] + '\\n<|EOT|>\\n'}}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{% if add_generation_prompt %}\n{{'### Response:'}}\n{% endif %}"
+dataset_mixer:
+  data/my: 1.0
+dataset_splits:
+- train
+preprocessing_num_workers: 128
+
+# SFT trainer config
+bf16: true
+do_eval: False
+eval_strategy: epoch
+gradient_accumulation_steps: 1
+gradient_checkpointing: true
+gradient_checkpointing_kwargs:
+  use_reentrant: False
+learning_rate: 1.0e-05
+log_level: info
+logging_steps: 5  
+logging_strategy: steps
+lr_scheduler_type: cosine
+max_seq_length: 4096
+num_train_epochs: 3
+output_dir: trained_models/deepseekcoder
+overwrite_output_dir: true
+per_device_eval_batch_size: 8
+per_device_train_batch_size: 8
+push_to_hub: true
+remove_unused_columns: true
+report_to:
+- tensorboard
+save_strategy: "epoch"
+save_total_limit: 5
+seed: 42
+warmup_ratio: 0.1

crux.sh

@@ -0,0 +1,87 @@
+python main.py \
+    --model /home/aiscuser/fhw/model_weights/warriordeep448/ \
+    --trust_remote_code \
+    --tasks output_prediction \
+    --batch_size 20 \
+    --n_samples 10 \
+    --max_length_generation 4096 \
+    --precision fp16 \
+    --temperature 0.8 \
+    --save_generations \
+    --save_generations_path model_generations_raw/warriordeep448_temp0.8_output/shard_0.json \
+    --shuffle \
+    --tensor_parallel_size 8
+
+python main.py \
+    --model /home/aiscuser/fhw/model_weights/warriordeep448/ \
+    --trust_remote_code \
+    --tasks output_prediction \
+    --batch_size 20 \
+    --n_samples 10 \
+    --max_length_generation 4096 \
+    --precision fp16 \
+    --temperature 0.2 \
+    --save_generations \
+    --save_generations_path model_generations_raw/warriordeep448_temp0.2_output/shard_0.json \
+    --shuffle \
+    --tensor_parallel_size 8
+
+python main.py \
+        --model /home/aiscuser/fhw/model_weights/warriordeep448/ \
+        --trust_remote_code \
+        --tasks output_prediction \
+        --batch_size 20 \
+        --n_samples 10 \
+        --max_length_generation 4096 \
+        --precision fp16 \
+        --temperature 0.8 \
+        --save_generations \
+        --save_generations_path model_generations_raw/warriordeep448+cot_temp0.8_output/shard_0.json \
+        --cot \
+        --shuffle \
+        --tensor_parallel_size 8
+
+python main.py \
+        --model /home/aiscuser/fhw/model_weights/warriordeep448/ \
+        --trust_remote_code \
+        --tasks output_prediction \
+        --batch_size 20 \
+        --n_samples 10 \
+        --max_length_generation 4096 \
+        --precision fp16 \
+        --temperature 0.2 \
+        --save_generations \
+        --save_generations_path model_generations_raw/warriordeep448+cot_temp0.2_output/shard_0.json \
+        --cot \
+        --shuffle \
+        --tensor_parallel_size 8
+
+python main.py \
+        --model /home/aiscuser/fhw/model_weights/warriordeep448/ \
+        --trust_remote_code \
+        --tasks input_prediction \
+        --batch_size 20 \
+        --n_samples 10 \
+        --max_length_generation 4096 \
+        --precision fp16 \
+        --temperature 0.8 \
+        --save_generations \
+        --save_generations_path model_generations_raw/warriordeep448+cot_temp0.8_input/shard_0.json \
+        --cot \
+        --shuffle \
+        --tensor_parallel_size 8
+
+python main.py \
+        --model /home/aiscuser/fhw/model_weights/warriordeep448/ \
+        --trust_remote_code \
+        --tasks input_prediction \
+        --batch_size 20 \
+        --n_samples 10 \
+        --max_length_generation 4096 \
+        --precision fp16 \
+        --temperature 0.2 \
+        --save_generations \
+        --save_generations_path model_generations_raw/warriordeep448+cot_temp0.2_input/shard_0.json \
+        --cot \
+        --shuffle \
+        --tensor_parallel_size 8

evalplus/.dockerignore

@@ -0,0 +1,182 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#  nuclear option because steven uses PyCharm.
+.idea/
+
+# VSCode
+.vscode/
+
+# EvalPlus specific
+EvalPlus/
+backup/
+passrate.p*
+min_cov_dir/
+HumanEvalPlus*.gz
+MbppPlus*.gz
+evalplus/_version.py
+*mbpp.json
+*.jsonl
+*.json
+*.png
+*.pdf
+trash-bin
+.bak
+evalplus.github.io
+evalplus_results/

evalplus/.github/ISSUE_TEMPLATE/buggy_contract.yml

@@ -0,0 +1,48 @@
+name: "🐛 Report Bad Contract"
+description: Report to us that certain program contract should be repaired.
+title: "🐛 [TestRemoval] - <TASK_ID> <WHY>"
+labels: ["program contract"]
+body:
+  - type: input
+    id: version
+    attributes:
+      label: "EvalPlus version"
+      description: What is the version of EvalPlus? You can find it by running `pip show evalplus`.
+      placeholder: For example, 0.1.0
+    validations:
+      required: true
+  - type: input
+    id: cache
+    attributes:
+      label: "Output of running `ls ~/.cache/evalplus`"
+    validations:
+      required: true
+  - type: input
+    id: task_id
+    attributes:
+      label: "Task ID of the programming task"
+      placeholder: HumanEval/[??]
+    validations:
+      required: true
+  - type: textarea
+    id: original
+    attributes:
+      label: "The original wrong contract"
+      description: You can run `python -c "from evalplus.data import get_human_eval_plus; print(get_human_eval_plus()['HumanEval/❓']['contract'])"`
+      render: python
+    validations:
+      required: true
+  - type: textarea
+    id: new
+    attributes:
+      label: "Your proposed new contract"
+      render: python
+    validations:
+      required: true
+  - type: textarea
+    id: other
+    attributes:
+      label: "Other context"
+      description: (Optional) Anything else the maintainer should notice?
+    validations:
+      required: false

evalplus/.github/ISSUE_TEMPLATE/buggy_test.yml

@@ -0,0 +1,49 @@
+name: "🐛 Report Bad Test Inputs"
+description: Report to us that certain test inputs should be removed.
+title: "🐛 [TestRemoval] - <TASK_ID> <WHY>"
+labels: ["bug"]
+body:
+  - type: input
+    id: version
+    attributes:
+      label: "EvalPlus version"
+      description: What is the version of EvalPlus? You can find it by running `pip show evalplus`.
+      placeholder: For example, 0.1.0
+    validations:
+      required: true
+  - type: input
+    id: cache
+    attributes:
+      label: "Output of running `ls ~/.cache/evalplus`"
+    validations:
+      required: true
+  - type: input
+    id: task_id
+    attributes:
+      label: "Task ID of the programming task"
+      placeholder: HumanEval/[??]
+    validations:
+      required: true
+  - type: textarea
+    id: test_input
+    attributes:
+      label: "Test input"
+      description: The text form of the test input that you think should be removed
+      render: python
+    validations:
+      required: true
+  - type: textarea
+    id: description
+    attributes:
+      label: "Description"
+      description: An explicit description of why you think this test should be removed
+      placeholder: Here is a correct solution but it is incorrectly falsified by the test because ...
+    validations:
+      required: true
+  - type: textarea
+    id: other
+    attributes:
+      label: "Other context"
+      description: (Optional) Anything else the maintainer should notice?
+    validations:
+      required: false

evalplus/.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1 @@
+blank_issues_enabled: true

evalplus/.github/ISSUE_TEMPLATE/model_eval_request.yml

@@ -0,0 +1,73 @@
+name: "🤗 Model Evaluation Request"
+description: Request EvalPlus maintainers to evaluate your model independently and update it on our leaderboard.
+title: "🤗 [REQUEST] - FILL_THE_MODEL_NAME_HERE"
+labels: ["model eval"]
+body:
+  - type: textarea
+    id: about
+    attributes:
+      label: "Model introduction"
+      description: Provide a brief introduction to the model.
+      placeholder: The models is created by ... and is used for ...
+    validations:
+      required: true
+  - type: input
+    id: url
+    attributes:
+      label: "Model URL"
+      description: Indicate the URL (e.g., huggingface or other release pages) of the model
+      placeholder: https://huggingface.co/[???]/[???]
+    validations:
+      required: true
+  - type: textarea
+    id: other
+    attributes:
+      label: "Additional information (Optional)"
+      description: Special steps indicating how to run the model with preferably scripts/codes.
+      placeholder: What data type precision should be used? What is the minimal hardware requirement? Can it be accelerated by tools such as vLLM?
+    validations:
+      required: false
+  - type: textarea
+    id: decomtamination
+    attributes:
+      label: "Decontamination"
+      description: How does the authors avoid contamination for their training data?
+      placeholder: Please clarify the decontamination steps and quantify it, e.g., N-gram match of ground-truth code in the training dataset.
+    validations:
+      required: true
+  - type: dropdown
+    id: author
+    attributes:
+      label: "Author"
+      description: "Are you (one of) the author(s) of the model?"
+      multiple: false
+      options:
+        - "Yes"
+        - "No"
+    validations:
+      required: true
+  - type: dropdown
+    id: data
+    attributes:
+      label: "Data"
+      description: "Is the training/fine-tuning data available in public?"
+      multiple: false
+      options:
+        - "Yes (If so please specify in 'Additional information')"
+        - "No"
+    validations:
+      required: true
+  - type: checkboxes
+    id: security
+    attributes:
+      label: "Security"
+      options:
+        - label: "I confirm that the model is safe to run which is not designed to produce malicious code or content."
+          required: true
+  - type: checkboxes
+    id: integrity
+    attributes:
+      label: "Integrity"
+      options:
+        - label: "I confirm that the model comes from unique and original work and does not contain any plagiarism."
+          required: true

evalplus/.gitignore

@@ -0,0 +1,182 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#  nuclear option because steven uses PyCharm.
+.idea/
+
+# VSCode
+.vscode/
+
+# EvalPlus specific
+EvalPlus/
+backup/
+passrate.p*
+min_cov_dir/
+HumanEvalPlus*.gz
+MbppPlus*.gz
+evalplus/_version.py
+*mbpp.json
+*.jsonl
+*.json
+*.png
+*.pdf
+trash-bin
+.bak
+evalplus.github.io
+evalplus_results/

evalplus/.pre-commit-config.yaml

@@ -0,0 +1,20 @@
+repos:
+-   repo: https://github.com/pycqa/isort
+    rev: 5.12.0
+    hooks:
+    -   id: isort
+        name: isort (python)
+        args: ["--profile", "black"]
+-   repo: https://github.com/psf/black
+    rev: 22.6.0
+    hooks:
+    -   id: black
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.3.0
+    hooks:
+    -   id: check-yaml
+    -   id: end-of-file-fixer
+    -   id: trailing-whitespace
+exclude: (?x)^(
+        groundtruth/.*
+    )$

evalplus/CITATION.cff

@@ -0,0 +1,25 @@
+cff-version: 1.2.0
+message: "If you use this work and love it, consider citing it as below \U0001F917"
+title: EvalPlus
+authors:
+  - family-names: EvalPlus Team
+url: https://github.com/evalplus/evalplus
+doi: https://doi.org/10.48550/arXiv.2305.01210
+date-released: 2023-05-01
+license: Apache-2.0
+preferred-citation:
+  type: article
+  title: "Is Your Code Generated by ChatGPT Really Correct? Rigorous Evaluation of Large Language Models for Code Generation"
+  authors:
+    - family-names: Liu
+      given-names: Jiawei
+    - family-names: Xia
+      given-names: Chunqiu Steven
+    - family-names: Wang
+      given-names: Yuyao
+    - family-names: Zhang
+      given-names: Lingming
+  year: 2023
+  journal: "arXiv preprint arXiv:2305.01210"
+  doi: https://doi.org/10.48550/arXiv.2305.01210
+  url: https://arxiv.org/abs/2305.01210

evalplus/Dockerfile

@@ -0,0 +1,19 @@
+# Better use newer Python as generated code can use new features
+FROM python:3.11-slim
+
+# install git and c++ (required by cirronlib.cpp)
+RUN apt-get update && apt-get install -y git g++
+
+# upgrade to latest pip
+RUN pip install --upgrade pip
+
+COPY . /evalplus
+
+RUN cd /evalplus && pip install ".[perf]"
+
+# Pre-install the dataset
+RUN python3 -c "from evalplus.data import *; get_human_eval_plus(); get_mbpp_plus(); get_evalperf_data()"
+
+WORKDIR /app
+
+CMD ["bash"]

evalplus/LICENSE

@@ -0,0 +1,205 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
+-------------------------------------------------------------------------------
+The files under "evalplus/eval/" additionally complies with the MIT License for
+being built on OpenAI's HumanEval work.

evalplus/MANIFEST.in

@@ -0,0 +1 @@
+exclude evalplus/_experimental/**/*.py

evalplus/README.md

@@ -0,0 +1,325 @@
+# `EvalPlus(📖) => 📚`
+
+<p align="center">
+    <a href="https://evalplus.github.io"><img src="https://img.shields.io/badge/%F0%9F%8F%86-leaderboard-8A2BE2"></a>
+    <a href="https://openreview.net/forum?id=1qvx610Cu7"><img src="https://img.shields.io/badge/EvalPlus-NeurIPS'23-a55fed.svg"></a>
+    <a href="https://openreview.net/forum?id=IBCBMeAhmC"><img src="https://img.shields.io/badge/EvalPerf-COLM'24-a55fed.svg"></a>
+    <a href="https://huggingface.co/evalplus/"><img src="https://img.shields.io/badge/🤗%20Hugging%20Face-evalplus-%23ff8811.svg"></a>
+    <a href="https://pypi.org/project/evalplus/"><img src="https://img.shields.io/pypi/v/evalplus?color=g"></a>
+    <a href="https://hub.docker.com/r/ganler/evalplus" title="Docker"><img src="https://img.shields.io/docker/image-size/ganler/evalplus"></a>
+</p>
+
+<p align="center">
+    <a href="#-about">📙About</a> •
+    <a href="#-quick-start">🔥Quick Start</a> •
+    <a href="#-llm-backends">🚀LLM Backends</a> •
+    <a href="#-documents">📚Documents</a> •
+    <a href="#-citation">📜Citation</a> •
+    <a href="#-acknowledgement">🙏Acknowledgement</a>
+</p>
+
+## 📢 News
+
+Who's using EvalPlus datasets? EvalPlus has been used by various LLM teams, including:
+
+* [Meta Llama 3.1 and 3.3](https://ai.meta.com/blog/meta-llama-3-1/)
+* [Allen AI TÜLU 1/2/3](https://github.com/allenai/open-instruct/blob/main/docs/tulu1_tulu2.md#benchmark-based-eval)
+* [Qwen2.5-Coder](https://qwenlm.github.io/blog/qwen2.5-coder-family/)
+* [CodeQwen 1.5](https://qwenlm.github.io/blog/codeqwen1.5/)
+* [DeepSeek-Coder V2](https://arxiv.org/pdf/2406.11931)
+* [Qwen2](https://arxiv.org/pdf/2407.10671)
+* [Snowflake Arctic](https://www.snowflake.com/en/data-cloud/arctic/)
+* [StarCoder2](https://arxiv.org/pdf/2402.19173)
+* [Magicoder](https://arxiv.org/pdf/2312.02120)
+* [WizardCoder](https://arxiv.org/pdf/2306.08568)
+
+Below tracks the notable updates of EvalPlus:
+
+- **[2024-10-20 `v0.3.1`]**: EvalPlus `v0.3.1` is officially released! Highlights: *(i)* Code efficiency evaluation via EvalPerf, *(ii)* one command to run all: generation + post-processing + evaluation, *(iii)* support for more inference backends such as Google Gemini & Anthropic, etc.
+- **[2024-06-09 pre `v0.3.0`]**: Improved ground-truth solutions for MBPP+ tasks (IDs: 459, 102, 559). Thanks to [EvalArena](https://github.com/crux-eval/eval-arena).
+- **[2024-04-17 pre `v0.3.0`]**: MBPP+ is upgraded to `v0.2.0` by removing some broken tasks (399 -> 378 tasks). ~4pp pass@1 improvement could be expected.
+
+<details><summary>Earlier news <i>:: click to expand ::</i></summary>
+<div>
+
+- ([`v0.2.1`](https://github.com/evalplus/evalplus/releases/tag/v0.2.1)) You can use EvalPlus datasets via [bigcode-evaluation-harness](https://github.com/bigcode-project/bigcode-evaluation-harness)! HumanEval+ oracle fixes (32).
+- ([`v0.2.0`](https://github.com/evalplus/evalplus/releases/tag/v0.2.0)) MBPP+ is released! HumanEval contract & input fixes (0/3/9/148/114/1/2/99/28/32/35/160).
+- ([`v0.1.7`](https://github.com/evalplus/evalplus/releases/tag/v0.1.7)) [Leaderboard](https://evalplus.github.io/leaderboard.html) release; HumanEval+ contract and input fixes (32/166/126/6)
+- ([`v0.1.6`](https://github.com/evalplus/evalplus/releases/tag/v0.1.6)) Configurable and by-default-conservative timeout settings; HumanEval+ contract & ground-truth fixes (129/148/75/53/0/3/9/140)
+- ([`v0.1.5`](https://github.com/evalplus/evalplus/releases/tag/v0.1.5)) HumanEval+ mini is released for ultra-fast evaluation when you have too many samples!
+- ([`v0.1.1`](https://github.com/evalplus/evalplus/releases/tag/v0.1.1)) Optimizing user experiences: evaluation speed, PyPI package, Docker, etc.
+- ([`v0.1.0`](https://github.com/evalplus/evalplus/releases/tag/v0.1.0)) HumanEval+ is released!
+
+</div>
+</details>
+
+
+## 📙 About
+
+EvalPlus is a rigorous evaluation framework for LLM4Code, with:
+
+- ✨ **HumanEval+**: 80x more tests than the original HumanEval!
+- ✨ **MBPP+**: 35x more tests than the original MBPP!
+- ✨ **EvalPerf**: evaluating the efficiency of LLM-generated code!
+- ✨ **Framework**: our packages/images/tools can easily and safely evaluate LLMs on above benchmarks.
+
+Why EvalPlus?
+
+- ✨ **Precise evaluation**: See [our leaderboard](https://evalplus.github.io/leaderboard.html) for latest LLM rankings before & after rigorous evaluation.
+- ✨ **Coding rigorousness**: Look at the score differences! esp. before & after using EvalPlus tests! Less drop means more rigorousness in code generation; while a bigger drop means the generated code tends to be fragile.
+- ✨ **Code efficiency**: Beyond correctness, our EvalPerf dataset evaluates the efficiency of LLM-generated code via performance-exercising coding tasks and test inputs.
+
+Want to know more details? Read our papers & materials!
+
+- **EvalPlus**: [NeurIPS'23 paper](https://openreview.net/forum?id=1qvx610Cu7), [Slides](https://docs.google.com/presentation/d/1eTxzUQG9uHaU13BGhrqm4wH5NmMZiM3nI0ezKlODxKs), [Poster](https://jw-liu.xyz/assets/pdf/EvalPlus_Poster.pdf), [Leaderboard](https://evalplus.github.io/leaderboard.html)
+- **EvalPerf**: [COLM'24 paper](https://openreview.net/forum?id=IBCBMeAhmC), [Poster](https://jw-liu.xyz/assets/pdf/jiawei-colm-evalperf-poster.pdf), [Documentation](./docs/evalperf.md), [Leaderboard](https://evalplus.github.io/evalperf.html)
+
+
+## 🔥 Quick Start
+
+### Code Correctness Evaluation: HumanEval(+) or MBPP(+)
+
+```bash
+pip install --upgrade "evalplus[vllm] @ git+https://github.com/evalplus/evalplus"
+# Or `pip install "evalplus[vllm]" --upgrade` for the latest stable release
+
+evalplus.evaluate --model "ise-uiuc/Magicoder-S-DS-6.7B" \
+                  --dataset [humaneval|mbpp]             \
+                  --backend vllm                         \
+                  --greedy
+```
+
+<details><summary>🛡️ Safe code execution within Docker <i>:: click to expand ::</i></summary>
+<div>
+
+```bash
+# Local generation
+evalplus.codegen --model "ise-uiuc/Magicoder-S-DS-6.7B" \
+                 --dataset humaneval                    \
+                 --backend vllm                         \
+                 --greedy
+
+# Code execution within Docker
+docker run --rm --pull=always -v $(pwd)/evalplus_results:/app ganler/evalplus:latest \
+           evalplus.evaluate --dataset humaneval                                     \
+           --samples /app/humaneval/ise-uiuc--Magicoder-S-DS-6.7B_vllm_temp_0.0.jsonl
+```
+
+</div>
+</details>
+
+### Code Efficiency Evaluation: EvalPerf (*nix only)
+
+```bash
+pip install --upgrade "evalplus[perf,vllm] @ git+https://github.com/evalplus/evalplus"
+# Or `pip install "evalplus[perf,vllm]" --upgrade` for the latest stable release
+
+sudo sh -c 'echo 0 > /proc/sys/kernel/perf_event_paranoid' # Enable perf
+evalplus.evalperf --model "ise-uiuc/Magicoder-S-DS-6.7B" --backend vllm
+```
+
+<details><summary>🛡️ Safe code execution within Docker <i>:: click to expand ::</i></summary>
+<div>
+
+```bash
+# Local generation
+evalplus.codegen --model "ise-uiuc/Magicoder-S-DS-6.7B" \
+                 --dataset evalperf                     \
+                 --backend vllm                         \
+                 --temperature 1.0                      \
+                 --n-samples 100
+
+# Code execution within Docker
+sudo sh -c 'echo 0 > /proc/sys/kernel/perf_event_paranoid' # Enable perf
+docker run --cap-add PERFMON --rm --pull=always -v $(pwd)/evalplus_results:/app ganler/evalplus:latest \
+           evalplus.evalperf --samples /app/evalperf/ise-uiuc--Magicoder-S-DS-6.7B_vllm_temp_1.0.jsonl
+```
+
+</div>
+</details>
+
+## 🚀 LLM Backends
+
+### HuggingFace models
+
+- `transformers` backend:
+
+```bash
+evalplus.evaluate --model "ise-uiuc/Magicoder-S-DS-6.7B" \
+                  --dataset [humaneval|mbpp]             \
+                  --backend hf                           \
+                  --greedy
+```
+
+> [!Note]
+>
+> EvalPlus uses different prompts for base and chat models.
+> By default it is detected by `tokenizer.chat_template` when using `hf`/`vllm` as backend.
+> For other backends, only chat mode is allowed.
+>
+> Therefore, if your base models come with a `tokenizer.chat_template`,
+> please add `--force-base-prompt` to avoid being evaluated
+> in a chat mode.
+
+<details><summary>Enable Flash Attention 2 <i>:: click to expand ::</i></summary>
+<div>
+
+```bash
+# Install Flash Attention 2
+pip install packaging ninja
+pip install flash-attn --no-build-isolation
+# Note: if you have installation problem, consider using pre-built
+# wheels from https://github.com/Dao-AILab/flash-attention/releases
+
+# Run evaluation with FA2
+evalplus.evaluate --model "ise-uiuc/Magicoder-S-DS-6.7B"         \
+                  --dataset [humaneval|mbpp]                     \
+                  --backend hf                                   \
+                  --attn-implementation [flash_attention_2|sdpa] \
+                  --greedy
+```
+
+</div>
+</details>
+
+- `vllm` backend:
+
+```bash
+evalplus.evaluate --model "ise-uiuc/Magicoder-S-DS-6.7B" \
+                  --dataset [humaneval|mbpp]             \
+                  --backend vllm                         \
+                  --tp [TENSOR_PARALLEL_SIZE]            \
+                  --greedy
+```
+
+- `openai` compatible servers (e.g., [vLLM](https://docs.vllm.ai/en/latest/serving/openai_compatible_server.html)):
+
+```bash
+# OpenAI models
+export OPENAI_API_KEY="{KEY}" # https://platform.openai.com/settings/organization/api-keys
+evalplus.evaluate --model "gpt-4o-2024-08-06"  \
+                  --dataset [humaneval|mbpp]   \
+                  --backend openai --greedy
+
+# DeepSeek
+export OPENAI_API_KEY="{KEY}" # https://platform.deepseek.com/api_keys
+evalplus.evaluate --model "deepseek-chat"              \
+                  --dataset [humaneval|mbpp]           \
+                  --base-url https://api.deepseek.com  \
+                  --backend openai --greedy
+
+# Grok
+export OPENAI_API_KEY="{KEY}" # https://console.x.ai/
+evalplus.evaluate --model "grok-beta"             \
+                  --dataset [humaneval|mbpp]      \
+                  --base-url https://api.x.ai/v1  \
+                  --backend openai --greedy
+
+# vLLM server
+# First, launch a vLLM server: https://docs.vllm.ai/en/latest/serving/deploying_with_docker.html
+evalplus.evaluate --model "ise-uiuc/Magicoder-S-DS-6.7B" \
+                  --dataset [humaneval|mbpp]             \
+                  --base-url http://localhost:8000/v1    \
+                  --backend openai --greedy
+
+# GPTQModel
+evalplus.evaluate --model "ModelCloud/Llama-3.2-1B-Instruct-gptqmodel-4bit-vortex-v1" \
+                  --dataset [humaneval|mbpp]                                          \
+                  --backend gptqmodel --greedy
+```
+
+### OpenAI models
+
+- Access OpenAI APIs from [OpenAI Console](https://platform.openai.com/)
+
+```bash
+export OPENAI_API_KEY="[YOUR_API_KEY]"
+evalplus.evaluate --model "gpt-4o"            \
+                  --dataset [humaneval|mbpp]  \
+                  --backend openai            \
+                  --greedy
+```
+
+### Anthropic models
+
+- Access Anthropic APIs from [Anthropic Console](https://console.anthropic.com/)
+
+```bash
+export ANTHROPIC_API_KEY="[YOUR_API_KEY]"
+evalplus.evaluate --model "claude-3-haiku-20240307" \
+                  --dataset [humaneval|mbpp]        \
+                  --backend anthropic               \
+                  --greedy
+```
+
+### Google Gemini models
+
+- Access Gemini APIs from [Google AI Studio](https://aistudio.google.com/)
+
+```bash
+export GOOGLE_API_KEY="[YOUR_API_KEY]"
+evalplus.evaluate --model "gemini-1.5-pro"    \
+                  --dataset [humaneval|mbpp]  \
+                  --backend google            \
+                  --greedy
+```
+
+### Amazon Bedrock models
+
+- [Amazon Bedrock](https://aws.amazon.com/bedrock/)
+
+```bash
+export BEDROCK_ROLE_ARN="[BEDROCK_ROLE_ARN]"
+evalplus.evaluate --model "anthropic.claude-3-5-sonnet-20241022-v2:0" \
+                  --dataset [humaneval|mbpp]                          \
+                  --backend bedrock                                   \
+                  --greedy
+```
+
+You can checkout the generation and results at `evalplus_results/[humaneval|mbpp]/`
+
+<details><summary>⏬ Using EvalPlus as a local repo? <i>:: click to expand ::</i></summary>
+<div>
+
+```bash
+git clone https://github.com/evalplus/evalplus.git
+cd evalplus
+export PYTHONPATH=$PYTHONPATH:$(pwd)
+pip install -r requirements.txt
+```
+
+</div>
+</details>
+
+## 📚 Documents
+
+To learn more about how to use EvalPlus, please refer to:
+
+- [EvalPlus Commands](./docs/cli.md)
+- [EvalPerf](./docs/evalperf.md)
+- [Program Execution](./docs/execution.md)
+
+## 📜 Citation
+
+```bibtex
+@inproceedings{evalplus,
+  title = {Is Your Code Generated by Chat{GPT} Really Correct? Rigorous Evaluation of Large Language Models for Code Generation},
+  author = {Liu, Jiawei and Xia, Chunqiu Steven and Wang, Yuyao and Zhang, Lingming},
+  booktitle = {Thirty-seventh Conference on Neural Information Processing Systems},
+  year = {2023},
+  url = {https://openreview.net/forum?id=1qvx610Cu7},
+}
+
+@inproceedings{evalperf,
+  title = {Evaluating Language Models for Efficient Code Generation},
+  author = {Liu, Jiawei and Xie, Songrun and Wang, Junhao and Wei, Yuxiang and Ding, Yifeng and Zhang, Lingming},
+  booktitle = {First Conference on Language Modeling},
+  year = {2024},
+  url = {https://openreview.net/forum?id=IBCBMeAhmC},
+}
+```
+
+## 🙏 Acknowledgement
+
+- [HumanEval](https://github.com/openai/human-eval)
+- [MBPP](https://github.com/google-research/google-research/tree/master/mbpp)

evalplus/build/lib/evalplus/__init__.py

@@ -0,0 +1,4 @@
+try:
+    from evalplus._version import __version__, __version_tuple__
+except ImportError:
+    __version__ = "local-dev"

evalplus/build/lib/evalplus/_version.py

@@ -0,0 +1,16 @@
+# file generated by setuptools_scm
+# don't change, don't track in version control
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    from typing import Tuple, Union
+    VERSION_TUPLE = Tuple[Union[int, str], ...]
+else:
+    VERSION_TUPLE = object
+
+version: str
+__version__: str
+__version_tuple__: VERSION_TUPLE
+version_tuple: VERSION_TUPLE
+
+__version__ = version = '0.4.0.dev33'
+__version_tuple__ = version_tuple = (0, 4, 0, 'dev33')

evalplus/build/lib/evalplus/codegen.py

@@ -0,0 +1,272 @@
+import json
+import os
+from typing import Dict, List, Optional
+
+from evalplus.data import get_evalperf_data, get_human_eval_plus, get_mbpp_plus
+from evalplus.provider import DecoderBase, make_model
+from evalplus.sanitize import sanitize
+from evalplus.utils import progress
+
+
+def codegen(
+    target_path: str,
+    model: DecoderBase,
+    dataset: Dict,
+    greedy=False,
+    n_samples=1,
+    id_range=None,
+    resume=True,
+):
+    task2nexist = {}
+    if resume and target_path.endswith(".jsonl") and os.path.isfile(target_path):
+        with open(target_path, "r") as f:
+            for line in f:
+                if not line.strip():
+                    continue
+                task_id = json.loads(line)["task_id"]
+                task2nexist[task_id] = task2nexist.get(task_id, 0) + 1
+
+    if target_path.endswith(".jsonl"):
+        raw_target_path = target_path.replace(".jsonl", ".raw.jsonl")
+    else:
+        raw_target_path = target_path + ".raw"
+        os.makedirs(target_path, exist_ok=True)
+
+    print(f"Sanitized code outputs will be saved to {target_path}")
+    print(f"Raw outputs will be saved to {raw_target_path}")
+
+    backend_type: str = type(model).__name__
+    with progress(backend_type) as p:
+        for task_id, task in p.track(dataset.items()):
+            if id_range is not None:
+                id_num = int(task_id.split("/")[1])
+                low, high = id_range
+                if id_num < low or id_num >= high:
+                    p.console.print(f"Skipping {task_id} as it is not in {id_range}")
+                    continue
+
+            if not target_path.endswith(".jsonl"):
+                p_name = task_id.replace("/", "_")
+                os.makedirs(os.path.join(target_path, p_name), exist_ok=True)
+                task2nexist[task_id] = len(
+                    [
+                        f
+                        for f in os.listdir(os.path.join(target_path, p_name))
+                        if f.endswith(".py")
+                    ]
+                )
+
+            n_more_samples = n_samples
+            log = f"Codegen: {task_id} @ {model}"
+            if resume and task2nexist.get(task_id, 0) > 0:
+                log += f" (resuming from {task2nexist[task_id]})"
+                n_more_samples -= task2nexist[task_id]
+
+            p.console.print(log)
+
+            sidx = n_samples - n_more_samples
+            while sidx < n_samples:
+                prompt = task["prompt"].strip() + "\n"
+                outputs = model.codegen(
+                    prompt,
+                    do_sample=not greedy,
+                    num_samples=n_samples - sidx,
+                )
+                assert outputs, "No outputs from model!"
+                for impl in outputs:
+                    solution = prompt + impl if model.is_direct_completion() else impl
+                    sanitized_solution = sanitize(
+                        solution, entrypoint=task["entry_point"]
+                    )
+                    if target_path.endswith(".jsonl"):
+                        # Writing the sanitized version
+                        with open(target_path, "a") as f:
+                            f.write(
+                                json.dumps(
+                                    {"task_id": task_id, "solution": sanitized_solution}
+                                )
+                                + "\n"
+                            )
+
+                        # Writing the raw version
+                        with open(raw_target_path, "a") as f:
+                            f.write(
+                                json.dumps({"task_id": task_id, "solution": solution})
+                                + "\n"
+                            )
+                    else:
+                        # Writing the sanitized version
+                        with open(
+                            os.path.join(target_path, p_name, f"{sidx}.py"),
+                            "w",
+                            encoding="utf-8",
+                        ) as f:
+                            f.write(sanitized_solution)
+
+                        # Writing the raw version
+                        with open(
+                            os.path.join(raw_target_path, p_name, f"{sidx}.py"),
+                            "w",
+                            encoding="utf-8",
+                        ) as f:
+                            f.write(solution)
+                    sidx += 1
+
+
+def run_codegen(
+    model: str,
+    dataset: str,
+    root: str = "evalplus_results",
+    bs: Optional[int] = None,
+    n_samples: int = 1,
+    temperature: float = 0.0,
+    resume: bool = True,
+    greedy: bool = False,
+    id_range: List = None,
+    version: str = "default",
+    backend: str = "vllm",
+    force_base_prompt: bool = False,
+    base_url: str = None,
+    tp: int = 1,
+    evalperf_type: str = None,  # For EvalPerf
+    jsonl_fmt: bool = True,
+    attn_implementation: str = "eager",
+    device_map: Optional[str] = None,
+    trust_remote_code: bool = False,
+    enable_prefix_caching: bool = False,
+    enable_chunked_prefill: bool = False,
+    dtype: str = "bfloat16",
+    gptqmodel_backend: str = "auto",  # For GPTQModel
+    gguf_file: Optional[str] = None
+):
+    assert dataset in ["humaneval", "mbpp", "evalperf"], f"Invalid dataset {dataset}"
+    assert evalperf_type is None or evalperf_type in [
+        "instruct",
+        "perf-instruct",
+        "perf-CoT",
+    ]
+
+    # Make dir for codes generated by each model
+    identifier = model.strip("./").replace("/", "--") + f"_{backend}_temp_{temperature}"
+    if evalperf_type:
+        identifier += f"-{evalperf_type}"
+
+    target_path = os.path.join(root, dataset, identifier)
+    if jsonl_fmt:
+        target_path += ".jsonl"
+    else:
+        os.makedirs(target_path, exist_ok=True)
+
+    if dataset == "humaneval":
+        dataset_dict = get_human_eval_plus(version=version)
+    elif dataset == "mbpp":
+        dataset_dict = get_mbpp_plus(version=version)
+    elif dataset == "evalperf":
+        original_dataset = {**get_human_eval_plus(), **get_mbpp_plus()}
+        dataset_dict = {k: original_dataset[k] for k in get_evalperf_data()}
+        assert id_range is None, "id_range not supported for evalperf"
+    else:
+        raise ValueError(f"Invalid dataset {dataset}")
+
+    all_tasks_complete = False
+    if jsonl_fmt and os.path.isfile(target_path):
+        task_counts = {}
+        with open(target_path, "r") as f:
+            for line in f:
+                if not line.strip():
+                    continue
+                data = json.loads(line)
+                task_id = data["task_id"]
+                task_counts[task_id] = task_counts.get(task_id, 0) + 1
+
+            all_tasks_complete = all(
+                task_counts.get(task_id, 0) >= n_samples
+                for task_id in dataset_dict.keys()
+            )
+
+    if all_tasks_complete:
+        print("All samples are already cached. Skipping codegen.")
+        return target_path
+
+    if greedy and (temperature != 0 or bs != 1 or n_samples != 1):
+        temperature = 0.0
+        bs = 1
+        n_samples = 1
+        print("Greedy decoding ON (--greedy): setting bs=1, n_samples=1, temperature=0")
+
+    if id_range is not None:
+        assert len(id_range) == 2, "id_range must be a list of length 2"
+        assert id_range[0] < id_range[1], "id_range must be increasing"
+        id_range = tuple(id_range)
+
+    if bs is None:
+        bs = min(n_samples, 32)
+        print(f"Setting batch size to {bs}")
+
+    # Make project dir
+    os.makedirs(root, exist_ok=True)
+    # Make dataset dir
+    os.makedirs(os.path.join(root, dataset), exist_ok=True)
+
+    # Model instructions
+    instruction_prefix = "Please provide a self-contained Python script that solves the following problem in a markdown code block:"
+    response_prefix = "Below is a Python script with a self-contained function that solves the problem and passes corresponding tests:"
+
+    if evalperf_type == "perf-instruct":
+        instruction_prefix = "Please provide an efficient and self-contained Python script that solves the following problem in a markdown code block:"
+        response_prefix = "Below is a Python script with a self-contained function that efficiently solves the problem and passes corresponding tests:"
+    elif evalperf_type == "perf-CoT":
+        instruction_prefix = "Think step by step: please provide an efficient and self-contained Python script that solves the following problem in a markdown code block:"
+        response_prefix = "Below is a Python script with a self-contained function that efficiently solves the problem and passes corresponding tests:"
+    elif evalperf_type is not None and evalperf_type != "instruct":
+        raise ValueError(f"Invalid evalperf_type: {evalperf_type}")
+
+    # Model creation
+    model_runner = make_model(
+        model=model,
+        backend=backend,
+        batch_size=bs,
+        temperature=temperature,
+        force_base_prompt=force_base_prompt,
+        dataset=dataset,
+        base_url=base_url,
+        tp=tp,
+        instruction_prefix=instruction_prefix,
+        response_prefix=response_prefix,
+        device_map=device_map,
+        attn_implementation=attn_implementation,
+        trust_remote_code=trust_remote_code,
+        enable_prefix_caching=enable_prefix_caching,
+        enable_chunked_prefill=enable_chunked_prefill,
+        dtype=dtype,
+        gptqmodel_backend=gptqmodel_backend,
+        gguf_file=gguf_file,
+    )
+
+    codegen(
+        target_path=target_path,
+        dataset=dataset_dict,
+        greedy=greedy,
+        model=model_runner,
+        n_samples=n_samples,
+        resume=resume,
+        id_range=id_range,
+    )
+
+    # force shutdown the model runner
+    del model_runner
+    import gc
+
+    gc.collect()
+
+    return target_path
+
+
+def main():
+    from fire import Fire
+
+    Fire(run_codegen)
+
+
+if __name__ == "__main__":
+    main()

evalplus/build/lib/evalplus/config.py

@@ -0,0 +1,16 @@
+## EvalPlus
+DEFAULT_GT_TIME_LIMIT_FACTOR = 4.0
+DEFAULT_MIN_TIME_LIMIT = 1.0
+
+## EvalPerf
+
+### General
+PERF_PROFILE_ROUNDS = 1
+PERF_RAM_GB_PER_PROC = 12
+
+### Evaluation Phase
+PERF_EVAL_TIMEOUT_SECOND = 45
+
+### Curation Phase
+PERF_CURATE_TIMEOUT_SECOND = 20
+PREF_CURATE_MIN_INSTRUCTION = 10000

evalplus/build/lib/evalplus/data/__init__.py

@@ -0,0 +1,14 @@
+import json
+
+from datasets import load_dataset
+
+from evalplus.data.humaneval import get_human_eval_plus, get_human_eval_plus_hash
+from evalplus.data.mbpp import get_mbpp_plus, get_mbpp_plus_hash
+from evalplus.data.utils import load_solutions, write_directory, write_jsonl
+
+
+def get_evalperf_data():
+    dataset = load_dataset("evalplus/evalperf", split="test").to_list()
+    for d in dataset:
+        d["pe_input"] = json.loads(d["pe_input"])
+    return {task["task_id"]: task for task in dataset}

evalplus/build/lib/evalplus/data/humaneval.py

@@ -0,0 +1,96 @@
+import hashlib
+import json
+import os
+from typing import Dict
+
+from evalplus.data.utils import (
+    CACHE_DIR,
+    completeness_check,
+    get_dataset_metadata,
+    make_cache,
+    stream_jsonl,
+)
+
+HUMANEVAL_PLUS_VERSION = "v0.1.10"
+HUMANEVAL_OVERRIDE_PATH = os.environ.get("HUMANEVAL_OVERRIDE_PATH", None)
+
+
+def _ready_human_eval_plus_path(mini=False, noextreme=False, version="default") -> str:
+    if HUMANEVAL_OVERRIDE_PATH:
+        return HUMANEVAL_OVERRIDE_PATH
+
+    version = HUMANEVAL_PLUS_VERSION if version == "default" else version
+    url, plus_path = get_dataset_metadata(
+        "HumanEvalPlus", HUMANEVAL_PLUS_VERSION, mini, noextreme
+    )
+    make_cache(url, plus_path)
+
+    return plus_path
+
+
+def get_human_eval_plus_hash(mini=False, noextreme=False, version="default") -> str:
+    """Get the hash of HumanEvalPlus.
+    Returns:
+        str: The hash of HumanEvalPlus
+    """
+    plus_path = _ready_human_eval_plus_path(mini, noextreme, version="default")
+    with open(plus_path, "rb") as f:
+        plus = f.read()
+    return hashlib.md5(plus).hexdigest()
+
+
+def get_human_eval_plus(
+    err_incomplete=True, mini=False, noextreme=False, version="default"
+) -> Dict[str, Dict]:
+    """Get HumanEvalPlus locally.
+    Args:
+        err_incomplete (bool, optional): Whether to raise error if HumanEvalPlus is not complete. Defaults to True.
+        mini (bool, optional): Whether to use the mini version of HumanEvalPlus. Defaults to False.
+    Returns:
+        List[Dict[str, str]]: List of dicts with keys "task_id", "prompt", "contract", "canonical_solution", "base_input"
+    Notes:
+        "task_id" is the identifier string for the task
+        "prompt" is the function signature with docstring
+        "contract" is the assertions for the function's input (validity)
+        "canonical_solution" is the ground-truth implementation for diff-testing
+        "base_input" is the test inputs from original HumanEval
+        "plus_input" is the test inputs brought by EvalPlus
+        "atol" is the absolute tolerance for diff-testing
+    """
+    plus_path = _ready_human_eval_plus_path(
+        mini=mini, noextreme=noextreme, version=version
+    )
+    plus = {task["task_id"]: task for task in stream_jsonl(plus_path)}
+    if err_incomplete:
+        completeness_check("HumanEval+", plus)
+    return plus
+
+
+def get_human_eval() -> Dict[str, Dict]:
+    """Get HumanEval from OpenAI's github repo and return as a list of parsed dicts.
+
+    Returns:
+        List[Dict[str, str]]: List of dicts with keys "prompt", "test", "entry_point"
+
+    Notes:
+        "task_id" is the identifier string for the task.
+        "prompt" is the prompt to be used for the task (function signature with docstrings).
+        "test" is test-cases wrapped in a `check` function.
+        "entry_point" is the name of the function.
+    """
+    # Check if human eval file exists in CACHE_DIR
+    human_eval_path = os.path.join(CACHE_DIR, "HumanEval.jsonl")
+    make_cache(
+        "https://github.com/openai/human-eval/raw/master/data/HumanEval.jsonl.gz",
+        human_eval_path,
+    )
+
+    human_eval = open(human_eval_path, "r").read().split("\n")
+    human_eval = [json.loads(line) for line in human_eval if line]
+
+    # Handle 115_max_fill.py to make its docstring well-formed
+    human_eval[115]["prompt"] = "import math\n" + human_eval[115]["prompt"].replace(
+        "import math\n", ""
+    )
+
+    return {task["task_id"]: task for task in human_eval}

evalplus/build/lib/evalplus/data/mbpp.py

@@ -0,0 +1,203 @@
+import hashlib
+import json
+import os
+from typing import Dict
+
+import wget
+
+from evalplus.data.utils import (
+    CACHE_DIR,
+    completeness_check,
+    get_dataset_metadata,
+    make_cache,
+    stream_jsonl,
+)
+
+MBPP_PLUS_VERSION = "v0.2.0"
+MBPP_OVERRIDE_PATH = os.environ.get("MBPP_OVERRIDE_PATH", None)
+
+
+def _ready_mbpp_plus_path(mini=False, noextreme=False, version="default") -> str:
+    assert mini is False, "Mini version of MBPP+ is not available yet."
+
+    if MBPP_OVERRIDE_PATH:
+        return MBPP_OVERRIDE_PATH
+
+    version = MBPP_PLUS_VERSION if version == "default" else version
+
+    url, plus_path = get_dataset_metadata("MbppPlus", version, mini, noextreme)
+    make_cache(url, plus_path)
+
+    return plus_path
+
+
+def mbpp_serialize_inputs(task_id: str, inputs: list) -> list:
+    task_id = int(task_id.split("/")[-1])
+
+    if task_id == 115:
+        return [[[list(item) for item in inp[0]]] for inp in inputs]
+    elif task_id == 124:
+        return [(str(inp[0]), str(inp[1])) for inp in inputs]
+    elif task_id == 252:
+        return [[str(inp[0])] for inp in inputs]
+
+    return inputs
+
+
+def mbpp_deserialize_inputs(task_id: str, inputs: list) -> list:
+    task_id = int(task_id.split("/")[-1])
+    if task_id in [
+        2,
+        116,
+        132,
+        143,
+        222,
+        261,
+        273,
+        394,
+        399,
+        421,
+        424,
+        429,
+        470,
+        560,
+        579,
+        596,
+        616,
+        630,
+        726,
+        740,
+        744,
+        809,
+    ]:
+        modified_inputs = [[tuple(lst) for lst in inp] for inp in inputs]
+
+    elif task_id in [
+        63,
+        64,
+        70,
+        94,
+        120,
+        237,
+        272,
+        299,
+        400,
+        409,
+        417,
+        438,
+        473,
+        614,
+        780,
+    ]:
+        modified_inputs = [
+            [[tuple(lst) for lst in lst_lst] for lst_lst in inp] for inp in inputs
+        ]
+
+    elif task_id in [75, 413, 444, 753]:
+        modified_inputs = [
+            [[tuple(lst) for lst in inp[0]]] + [inp[1]] for inp in inputs
+        ]
+
+    elif task_id == 106 or task_id == 750:
+        modified_inputs = [[inp[0]] + [tuple(inp[1])] for inp in inputs]
+
+    elif task_id == 115:
+        modified_inputs = [
+            [
+                [
+                    set(item) if isinstance(item, list) and len(item) else {}
+                    for item in inp[0]
+                ]
+            ]
+            for inp in inputs
+        ]
+
+    elif task_id == 124:
+        modified_inputs = [(float(inp[0]), complex(inp[1])) for inp in inputs]
+
+    elif task_id in [250, 405, 446, 617, 720, 763, 808]:
+        modified_inputs = [[tuple(inp[0])] + [inp[1]] for inp in inputs]
+
+    elif task_id in [259, 401, 445]:
+        modified_inputs = [
+            [[tuple(lst) for lst in lst_lst] for lst_lst in inp] for inp in inputs
+        ]
+        modified_inputs = [[tuple(lst) for lst in inp] for inp in modified_inputs]
+
+    elif task_id == 278:
+        modified_inputs = [
+            [[tuple(item) if isinstance(item, list) else item for item in inp[0]]]
+            for inp in inputs
+        ]
+        modified_inputs = [[tuple(lst) for lst in inp] for inp in modified_inputs]
+
+    elif task_id == 307:
+        modified_inputs = [[tuple(inp[0])] + [inp[1], inp[2]] for inp in inputs]
+
+    elif task_id == 722:
+        modified_inputs = [
+            [{key: tuple(value) for key, value in inp[0].items()}] + inp[1:]
+            for inp in inputs
+        ]
+
+    elif task_id == 252:
+        modified_inputs = [[complex(inp[0])] for inp in inputs]
+
+    elif task_id in [580, 615, 791]:
+
+        def turn_all_list_into_tuple(inp):
+            if isinstance(inp, list):
+                return tuple([turn_all_list_into_tuple(item) for item in inp])
+            return inp
+
+        modified_inputs = [turn_all_list_into_tuple(inp) for inp in inputs]
+
+    else:
+        modified_inputs = inputs
+
+    return modified_inputs
+
+
+def get_mbpp() -> Dict[str, Dict]:
+    """Get sanitized MBPP from Google's Github repo."""
+    mbpp_path = os.path.join(CACHE_DIR, "sanitized-mbpp.json")
+
+    if not os.path.exists(mbpp_path):
+        os.makedirs(CACHE_DIR, exist_ok=True)
+
+        # Install MBPP-sanitized from scratch
+        print("Downloading original MBPP dataset...")
+        wget.download(
+            "https://github.com/google-research/google-research/raw/master/mbpp/sanitized-mbpp.json",
+            mbpp_path,
+        )
+
+    with open(mbpp_path, "r") as f:
+        mbpp = json.load(f)
+
+    return {str(task["task_id"]): task for task in mbpp}
+
+
+def get_mbpp_plus(
+    err_incomplete=True, mini=False, noextreme=False, version="default"
+) -> Dict[str, Dict]:
+    plus_path = _ready_mbpp_plus_path(mini=mini, noextreme=noextreme, version=version)
+    plus = {task["task_id"]: task for task in stream_jsonl(plus_path)}
+    for task_id, task in plus.items():
+        task["base_input"] = mbpp_deserialize_inputs(task_id, task["base_input"])
+        task["plus_input"] = mbpp_deserialize_inputs(task_id, task["plus_input"])
+
+    if err_incomplete:
+        completeness_check("MBPP+", plus)
+    return plus
+
+
+def get_mbpp_plus_hash(mini=False, noextreme=False, version="default") -> str:
+    """Get the hash of MbppPlus.
+    Returns:
+        str: The hash of MbppPlus
+    """
+    plus_path = _ready_mbpp_plus_path(mini=mini, noextreme=noextreme, version=version)
+    with open(plus_path, "rb") as f:
+        plus = f.read()
+    return hashlib.md5(plus).hexdigest()

evalplus/build/lib/evalplus/data/utils.py

@@ -0,0 +1,166 @@
+import gzip
+import json
+import os
+from os import PathLike
+from typing import Dict, Iterable
+
+import tempdir
+import wget
+from appdirs import user_cache_dir
+
+CACHE_DIR = user_cache_dir("evalplus")
+
+
+def get_dataset_metadata(name: str, version: str, mini: bool, noextreme: bool = False):
+    assert name in ["HumanEvalPlus", "MbppPlus"], f"Unknown/unsupported dataset: {name}"
+    extra = ""
+    assert not (mini and noextreme), "Cannot have both mini and noextreme"
+    if mini:
+        extra = "-Mini"
+    if noextreme:
+        extra = "-NoExtreme"
+    url = f"https://github.com/evalplus/{name.lower()}_release/releases/download/{version}/{name}{extra}.jsonl.gz"
+    cache_path = os.path.join(CACHE_DIR, f"{name}{extra}-{version}.jsonl")
+    return url, cache_path
+
+
+def make_cache(gzip_url, cache_path):
+    # Check if human eval file exists in CACHE_DIR
+    if not os.path.exists(cache_path):
+        # Install HumanEval dataset and parse as jsonl
+        print(f"Downloading dataset from {gzip_url}")
+        with tempdir.TempDir() as tmpdir:
+            plus_gz_path = os.path.join(tmpdir, f"data.jsonl.gz")
+            wget.download(gzip_url, plus_gz_path)
+
+            with gzip.open(plus_gz_path, "rb") as f:
+                plus = f.read().decode("utf-8")
+
+        # create CACHE_DIR if not exists
+        if not os.path.exists(CACHE_DIR):
+            os.makedirs(CACHE_DIR)
+
+        # Write the original human eval file to CACHE_DIR
+        with open(cache_path, "w") as f:
+            f.write(plus)
+
+
+def write_jsonl(
+    filename: str, data: Iterable[Dict], append: bool = False, drop_builtin: bool = True
+):
+    """
+    Writes an iterable of dictionaries to jsonl
+    """
+    if append:
+        mode = "ab"
+    else:
+        mode = "wb"
+    filename = os.path.expanduser(filename)
+    if filename.endswith(".gz"):
+        with open(filename, mode) as fp:
+            with gzip.GzipFile(fileobj=fp, mode="wb") as gzfp:
+                for x in data:
+                    if drop_builtin:
+                        x = {k: v for k, v in x.items() if not k.startswith("_")}
+                    gzfp.write((json.dumps(x) + "\n").encode("utf-8"))
+    else:
+        with open(filename, mode) as fp:
+            for x in data:
+                if drop_builtin:
+                    x = {k: v for k, v in x.items() if not k.startswith("_")}
+                fp.write((json.dumps(x) + "\n").encode("utf-8"))
+
+
+def stream_jsonl(filename: str) -> Iterable[Dict]:
+    """
+    Parses each jsonl line and yields it as a dictionary
+    """
+    if filename.endswith(".gz"):
+        with open(filename, "rb") as gzfp:
+            with gzip.open(gzfp, "rt") as fp:
+                for line in fp:
+                    if any(not x.isspace() for x in line):
+                        yield json.loads(line)
+    else:
+        with open(filename, "r") as fp:
+            for line in fp:
+                if any(not x.isspace() for x in line):
+                    yield json.loads(line)
+
+
+def load_solutions(sample_path: PathLike) -> Iterable[Dict]:
+    """We accept two formats of inputs.
+    + `sample.jsonl` which is the format from HumanEval, i.e., {task_id, completion or solution}.
+    + A folder which contains sub-folders named after the task_id. Each sub-folder
+    contains samples named in `[?].py` where `?` is the solution id starting with 0.
+    Different from `sample.jsonl`, the solutions must be complete (with prompt prefix).
+    """
+
+    # if it is a file
+    if os.path.isfile(sample_path):
+        for i, sample in enumerate(stream_jsonl(sample_path)):
+            assert (
+                "completion" in sample or "solution" in sample
+            ), "No completion or solution found in sample!"
+            assert "solution" not in sample or isinstance(
+                sample["solution"], str
+            ), "Solution must be a string! If you have multiple solutions, please repeat the task_id."
+            assert "completion" not in sample or isinstance(
+                sample["completion"], str
+            ), "Completion must be a string! If you have multiple solutions, please repeat the task_id."
+
+            sample["_identifier"] = (
+                sample["task_id"] + f" (line {i+1} in {sample_path})"
+            )
+            yield sample
+    else:
+        # if it is a folder
+        for task_id in os.listdir(sample_path):
+            task_path = os.path.join(sample_path, task_id)
+            if not os.path.isdir(task_path):
+                continue
+
+            for solution_id in os.listdir(task_path):
+                solution_path = os.path.join(task_path, solution_id)
+                if os.path.isfile(solution_path) and solution_path.endswith(".py"):
+                    with open(solution_path, "r") as f:
+                        completion = f.read()
+                    yield {
+                        "_identifier": solution_path,
+                        "_path": solution_path,
+                        "task_id": task_id.replace("_", "/"),
+                        "solution": completion,
+                    }
+
+
+def write_directory(directory: PathLike, data: Iterable[Dict]):
+    os.makedirs(directory, exist_ok=True)
+    counters = {}
+    for sample in data:
+        assert "solution" in sample, "Samples must come with `solution` field!"
+        task_id = sample["task_id"].replace("/", "_")
+        task_dir = os.path.join(directory, task_id)
+        os.makedirs(task_dir, exist_ok=True)
+        if task_id not in counters:
+            counters[task_id] = 0
+        sample_id = counters[task_id]
+        with open(os.path.join(task_dir, f"{sample_id}.py"), "w") as f:
+            f.write(sample["solution"])
+        counters[task_id] += 1
+
+
+def completeness_check(name, plus):
+    for task_id, task in plus.items():
+        for key in [
+            "prompt",
+            "contract",
+            "canonical_solution",
+            "base_input",
+            "plus_input",
+            "atol",
+        ]:
+            assert key in task, f"{key} not found in {name} #{task_id}!"
+
+
+def to_raw(string):
+    return string.encode("unicode-escape").decode().replace("\\\\", "\\")

evalplus/build/lib/evalplus/eval/__init__.py

@@ -0,0 +1,316 @@
+# The MIT License
+#
+# Copyright (c) OpenAI (https://openai.com)
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+import itertools
+import multiprocessing
+import os
+import time
+from multiprocessing import Array, Value
+from typing import Any, Dict, List, Optional, Tuple, Union
+
+import numpy as np
+import psutil
+
+from evalplus.config import *
+from evalplus.eval._special_oracle import (
+    MBPP_OUTPUT_NOT_NONE_TASKS,
+    MBPP_OUTPUT_SET_EQ_TASKS,
+    _digit_distance_nums,
+    _poly,
+    _surface_Area,
+)
+from evalplus.eval.utils import (
+    create_tempdir,
+    reliability_guard,
+    swallow_io,
+    time_limit,
+)
+
+
+def compatible_eval_result(results: Dict) -> Dict:
+    # compatibility
+    for task_results in results["eval"].values():
+        # update the "files" field to "nfiles"
+        if "files" in task_results and "nfiles" not in task_results:
+            task_results["nfiles"] = len(task_results.pop("files"))
+    return results
+
+
+# unbiased estimator from https://github.com/openai/human-eval
+def estimate_pass_at_k(
+    num_samples: Union[int, List[int], np.ndarray],
+    num_correct: Union[List[int], np.ndarray],
+    k: int,
+) -> np.ndarray:
+    """
+    Estimates pass@k of each problem and returns them in an array.
+    """
+
+    def estimator(n: int, c: int, k: int) -> float:
+        """
+        Calculates 1 - comb(n - c, k) / comb(n, k).
+        """
+        if n - c < k:
+            return 1.0
+        return 1.0 - np.prod(1.0 - k / np.arange(n - c + 1, n + 1))
+
+    if isinstance(num_samples, int):
+        num_samples_it = itertools.repeat(num_samples, len(num_correct))
+    else:
+        assert len(num_samples) == len(num_correct)
+        num_samples_it = iter(num_samples)
+
+    return np.array(
+        [estimator(int(n), int(c), k) for n, c in zip(num_samples_it, num_correct)]
+    )
+
+
+PASS = "pass"
+FAIL = "fail"
+TIMEOUT = "timeout"
+
+_SUCCESS = 0
+_FAILED = 1
+_TIMEOUT = 2
+_UNKNOWN = 3
+
+_mapping = {_SUCCESS: PASS, _FAILED: FAIL, _TIMEOUT: TIMEOUT, _UNKNOWN: None}
+
+
+def query_maximum_memory_bytes() -> Optional[int]:
+    # Disable functionalities that can make destructive changes to the test.
+    # allow only 4GB memory usage
+    maximum_memory_bytes = os.getenv(
+        "EVALPLUS_MAX_MEMORY_BYTES", 4 * 1024 * 1024 * 1024
+    )
+    maximum_memory_bytes = min(int(maximum_memory_bytes), psutil.virtual_memory().total)
+    if maximum_memory_bytes == -1:
+        return None
+    return maximum_memory_bytes
+
+
+def is_floats(x) -> bool:
+    # check if it is float; List[float]; Tuple[float]
+    if isinstance(x, float):
+        return True
+    if isinstance(x, (list, tuple)) and x:
+        return all(isinstance(i, float) for i in x)
+    if isinstance(x, np.ndarray):
+        return x.dtype == np.float64 or x.dtype == np.float32
+    return False
+
+
+def unsafe_execute(
+    dataset: str,
+    entry_point: str,
+    code: str,
+    inputs,
+    expected: List,
+    time_limits,
+    atol,
+    fast_check,
+    stat,  # Value
+    details,  # Array
+    progress,  # Value
+):
+    with create_tempdir():
+        # These system calls are needed when cleaning up tempdir.
+        import os
+        import shutil
+
+        rmtree = shutil.rmtree
+        rmdir = os.rmdir
+        chdir = os.chdir
+        #reliability_guard(maximum_memory_bytes=query_maximum_memory_bytes())
+        exec_globals = {}
+        try:
+            with swallow_io():
+                exec(code, exec_globals)
+                fn = exec_globals[entry_point]
+
+            for i, inp in enumerate(inputs):
+                try:
+                    with time_limit(time_limits[i]):
+                        with swallow_io():
+                            out = fn(*inp)
+
+                    exp = expected[i]
+                    exact_match = out == exp
+
+                    # ================================================ #
+                    # ============== special oracles ================= #
+                    if dataset == "mbpp":
+                        if "are_equivalent" == entry_point:  # Mbpp/164 special oracle
+                            exact_match = exact_match or True
+                        elif "sum_div" == entry_point:  # Mbpp/295 special oracle
+                            exact_match = exact_match or out == 0
+                        elif "surface_Area" == entry_point:  # Mbpp/581 special oracle
+                            exact_match = (
+                                exact_match or abs(out - _surface_Area(*inp)) <= atol
+                            )
+                        elif (
+                            "digit_distance_nums" == entry_point
+                        ):  # Mbpp/558 special oracle
+                            exact_match = exact_match or out == _digit_distance_nums(
+                                *inp
+                            )
+                        elif entry_point in MBPP_OUTPUT_SET_EQ_TASKS:
+                            exact_match = set(out) == set(exp)
+                        elif entry_point in MBPP_OUTPUT_NOT_NONE_TASKS:
+                            # exp is True  if not None
+                            #        False if None
+                            if isinstance(out, bool):
+                                exact_match = out == exp
+                            else:
+                                exact_match = exp == (out is not None)
+
+                    if dataset == "humaneval":
+                        if "find_zero" == entry_point:
+                            assert abs(_poly(*inp, out)) <= atol
+                            details[i] = True
+                            progress.value += 1
+                            continue
+                    # ============== special oracles ================= #
+                    # ================================================ #
+
+                    if atol == 0 and is_floats(exp):
+                        atol = 1e-6  # enforce atol for float comparison
+                    if not exact_match and atol != 0:
+                        # explicitly set rtol=1e-07
+                        # to match `np.testing.assert_allclose`'s default values
+                        assert type(out) == type(exp)
+                        if isinstance(exp, (list, tuple)):
+                            assert len(out) == len(exp)
+                        assert np.allclose(out, exp, rtol=1e-07, atol=atol)
+                    else:
+                        assert exact_match
+                except BaseException:
+                    details[i] = False
+                    progress.value += 1
+                    if fast_check:
+                        raise
+                    continue
+
+                details[i] = True
+                progress.value += 1
+
+            stat.value = _SUCCESS
+        except BaseException:
+            stat.value = _FAILED
+        # Needed for cleaning up.
+        shutil.rmtree = rmtree
+        os.rmdir = rmdir
+        os.chdir = chdir
+
+
+def untrusted_check(
+    dataset: str,
+    code: str,
+    inputs: List[Any],
+    entry_point: str,
+    expected,
+    atol,
+    ref_time: List[float],
+    fast_check: bool = False,
+    min_time_limit: float = DEFAULT_MIN_TIME_LIMIT,
+    gt_time_limit_factor: float = DEFAULT_GT_TIME_LIMIT_FACTOR,
+) -> Tuple[str, np.ndarray]:
+    time_limits = [max(min_time_limit, gt_time_limit_factor * t) for t in ref_time]
+    timeout = min(os.getenv("EVALPLUS_TIMEOUT_PER_TASK", 60), sum(time_limits)) + 1
+    if not fast_check:
+        timeout += 1  # extra time for data collection
+
+    # shared memory objects
+    progress = Value("i", 0)
+    stat = Value("i", _UNKNOWN)
+    details = Array("b", [False for _ in range(len(inputs))])
+
+    p = multiprocessing.Process(
+        target=unsafe_execute,
+        args=(
+            dataset,
+            entry_point,
+            code,
+            inputs,
+            expected,
+            time_limits,
+            atol,
+            fast_check,
+            # return values
+            stat,
+            details,
+            progress,
+        ),
+    )
+    p.start()
+    p.join(timeout=timeout + 1)
+    if p.is_alive():
+        p.terminate()
+        time.sleep(0.1)
+    if p.is_alive():
+        p.kill()
+        time.sleep(0.1)
+
+    stat = _mapping[stat.value]
+    details = details[: progress.value]
+
+    if not stat:
+        stat = TIMEOUT
+
+    if stat == PASS:
+        if len(details) != len(inputs) or not all(details):
+            stat = FAIL
+
+    return stat, details
+
+
+def evaluate_files(
+    dataset: str,
+    files: List[str],
+    inputs: List,
+    expected: List,
+    entry_point: str,
+    atol: float,
+    ref_time: List[float],
+    fast_check: bool = False,
+    min_time_limit: float = DEFAULT_MIN_TIME_LIMIT,
+    gt_time_limit_factor: float = DEFAULT_GT_TIME_LIMIT_FACTOR,
+) -> List[Tuple[str, List[bool]]]:
+    ret = []
+    # sort files by the id in name (i.e., "../n.py")
+    files = sorted(files, key=lambda x: int(x.split("/")[-1].split(".")[0]))
+    for file in files:
+        code = open(file, "r").read()
+        stat, det = untrusted_check(
+            dataset,
+            code,
+            inputs,
+            entry_point,
+            expected=expected,
+            atol=atol,
+            ref_time=ref_time,
+            fast_check=fast_check,
+            min_time_limit=min_time_limit,
+            gt_time_limit_factor=gt_time_limit_factor,
+        )
+        ret.append((stat, det.tolist()))
+    return ret

evalplus/build/lib/evalplus/eval/_special_oracle.py

@@ -0,0 +1,55 @@
+"""Special oracle handlings for problems where direct differential testing is not applicable."""
+
+import math
+
+# For tasks whose output are not serializable, we only check the output is not None, which
+# is also consistent with the original dataset.
+MBPP_OUTPUT_NOT_NONE_TASKS = ["check_str", "text_match_three", "text_starta_endb"]
+
+# Tasks that needs to perform set comparison over two lists
+MBPP_OUTPUT_SET_EQ_TASKS = [
+    "similar_elements",  # Mbpp/2
+    "find_char_long",  # Mbpp/7
+    "common_in_nested_lists",  # Mbpp/111
+    "extract_singly",  # Mbpp/140
+    "larg_nnum",  # Mbpp/232
+    "intersection_array",  # Mbpp/249
+    "find_dissimilar",  # Mbpp/579
+    "Diff",  # Mbpp/769
+]
+
+
+# oracle for Mbpp/581
+def _surface_Area(base_edge, height):
+    """
+    Recognizes the "height" as the perpendicular distance from the base to the apex of the pyramid
+    """
+    slant_height = math.sqrt((base_edge / 2) ** 2 + height**2)
+    base_area = base_edge**2
+    lateral_area = 4 * (base_edge * slant_height) / 2
+    total_surface_area = base_area + lateral_area
+    return round(total_surface_area)
+
+
+# oracle for Mbpp/558
+def _digit_distance_nums(num1, num2):
+    """
+    Preprocesses the two numbers to have the same length by padding with zeros
+    """
+    str_num1, str_num2 = str(num1), str(num2)
+    max_length = max(len(str_num1), len(str_num2))
+    str_num1, str_num2 = str_num1.zfill(max_length), str_num2.zfill(max_length)
+    total_difference = 0
+    for digit1, digit2 in zip(str_num1, str_num2):
+        difference = abs(int(digit1) - int(digit2))
+        total_difference += difference
+    return total_difference
+
+
+# oracle for HumaneEval/032
+def _poly(xs: list, x: float):
+    """
+    Evaluates polynomial with coefficients xs at point x.
+    return xs[0] + xs[1] * x + xs[1] * x^2 + .... xs[n] * x^n
+    """
+    return sum([coeff * math.pow(x, i) for i, coeff in enumerate(xs)])

evalplus/build/lib/evalplus/eval/utils.py

@@ -0,0 +1,187 @@
+# The MIT License
+#
+# Copyright (c) OpenAI (https://openai.com)
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+import contextlib
+import faulthandler
+import io
+import os
+import platform
+import signal
+import tempfile
+from typing import Optional
+
+
+@contextlib.contextmanager
+def swallow_io():
+    stream = WriteOnlyStringIO()
+    with contextlib.redirect_stdout(stream):
+        with contextlib.redirect_stderr(stream):
+            with redirect_stdin(stream):
+                yield
+
+
+@contextlib.contextmanager
+def time_limit(seconds: float):
+    def signal_handler(signum, frame):
+        raise TimeoutException("Timed out!")
+
+    signal.setitimer(signal.ITIMER_REAL, seconds)
+    signal.signal(signal.SIGALRM, signal_handler)
+    try:
+        yield
+    finally:
+        signal.setitimer(signal.ITIMER_REAL, 0)
+
+
+@contextlib.contextmanager
+def create_tempdir():
+    with tempfile.TemporaryDirectory() as dirname:
+        with chdir(dirname):
+            yield dirname
+
+
+@contextlib.contextmanager
+def chdir(root):
+    if root == ".":
+        yield
+        return
+    cwd = os.getcwd()
+    os.chdir(root)
+    try:
+        yield
+    except BaseException as exc:
+        raise exc
+    finally:
+        os.chdir(cwd)
+
+
+class TimeoutException(Exception):
+    pass
+
+
+class WriteOnlyStringIO(io.StringIO):
+    """StringIO that throws an exception when it's read from"""
+
+    def read(self, *args, **kwargs):
+        raise IOError
+
+    def readline(self, *args, **kwargs):
+        raise IOError
+
+    def readlines(self, *args, **kwargs):
+        raise IOError
+
+    def readable(self, *args, **kwargs):
+        """Returns True if the IO object can be read."""
+        return False
+
+
+class redirect_stdin(contextlib._RedirectStream):  # type: ignore
+    _stream = "stdin"
+
+
+def reliability_guard(maximum_memory_bytes: Optional[int] = None):
+    """
+    This disables various destructive functions and prevents the generated code
+    from interfering with the test (e.g. fork bomb, killing other processes,
+    removing filesystem files, etc.)
+
+    WARNING
+    This function is NOT a security sandbox. Untrusted code, including, model-
+    generated code, should not be blindly executed outside of one. See the
+    Codex paper for more information about OpenAI's code sandbox, and proceed
+    with caution.
+    """
+
+    if maximum_memory_bytes is not None:
+        import resource
+
+        resource.setrlimit(
+            resource.RLIMIT_AS, (maximum_memory_bytes, maximum_memory_bytes)
+        )
+        resource.setrlimit(
+            resource.RLIMIT_DATA, (maximum_memory_bytes, maximum_memory_bytes)
+        )
+        if not platform.uname().system == "Darwin":
+            resource.setrlimit(
+                resource.RLIMIT_STACK, (maximum_memory_bytes, maximum_memory_bytes)
+            )
+
+    faulthandler.disable()
+
+    import builtins
+
+    builtins.exit = None
+    builtins.quit = None
+
+    import os
+
+    os.environ["OMP_NUM_THREADS"] = "1"
+
+    os.kill = None
+    os.system = None
+    os.putenv = None
+    os.remove = None
+    os.removedirs = None
+    os.rmdir = None
+    os.fchdir = None
+    os.setuid = None
+    os.fork = None
+    os.forkpty = None
+    os.killpg = None
+    os.rename = None
+    os.renames = None
+    os.truncate = None
+    os.replace = None
+    os.unlink = None
+    os.fchmod = None
+    os.fchown = None
+    os.chmod = None
+    os.chown = None
+    os.chroot = None
+    os.fchdir = None
+    os.lchflags = None
+    os.lchmod = None
+    os.lchown = None
+    os.getcwd = None
+    os.chdir = None
+    builtins.open = None
+
+    import shutil
+
+    shutil.rmtree = None
+    shutil.move = None
+    shutil.chown = None
+
+    import subprocess
+
+    subprocess.Popen = None  # type: ignore
+
+    __builtins__["help"] = None
+
+    import sys
+
+    sys.modules["ipdb"] = None
+    sys.modules["joblib"] = None
+    sys.modules["resource"] = None
+    sys.modules["psutil"] = None
+    sys.modules["tkinter"] = None

evalplus/build/lib/evalplus/evalperf.py

@@ -0,0 +1,558 @@
+"""Compute the Differential Performance Scores (DPS) and DPS_{norm} of given samples from a model.
+
+Check our COLM paper for more details: https://www.arxiv.org/abs/2408.06450
+
+^Updates from the COLM paper:
+* Condition to activate efficiency evaluation for a task:
+  * Paper: as long as you have at least one correct solution, and we select up to 10 correct solutions for efficiency sampling
+  * Here: you need to have at least `min_correct` correct solutions, and we evaluate the efficiency of all correct solutions
+  * Updating rationale: to make the evaluation more statistically robust
+
+@inproceedings{liu2024evaluating,
+  title = {Evaluating Language Models for Efficient Code Generation},
+  author = {Liu, Jiawei and Xie, Songrun and Wang, Junhao and Wei, Yuxiang and Ding, Yifeng and Zhang, Lingming},
+  booktitle = {First Conference on Language Modeling},
+  year = {2024},
+  url = {https://openreview.net/forum?id=IBCBMeAhmC},
+}
+"""
+
+import json
+import multiprocessing
+import os
+import socket
+import time
+from collections import defaultdict
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from contextlib import closing
+from datetime import datetime
+from statistics import mean
+from typing import Dict, List, Optional, Tuple
+
+import rich
+from rich.rule import Rule
+from rich.syntax import Syntax
+from rich.table import Table
+
+from evalplus.codegen import run_codegen
+from evalplus.config import *
+from evalplus.config import PERF_EVAL_TIMEOUT_SECOND
+from evalplus.data import (
+    get_evalperf_data,
+    get_human_eval_plus,
+    get_human_eval_plus_hash,
+    get_mbpp_plus,
+    get_mbpp_plus_hash,
+)
+from evalplus.data.mbpp import mbpp_deserialize_inputs
+from evalplus.data.utils import stream_jsonl
+from evalplus.eval import PASS, untrusted_check
+from evalplus.eval._special_oracle import MBPP_OUTPUT_NOT_NONE_TASKS
+from evalplus.evaluate import get_groundtruth
+from evalplus.perf.profile import (
+    are_profiles_broken,
+    default_parallelism,
+    profile,
+    simple_test_profiler,
+)
+from evalplus.utils import progress
+
+
+def rule(msg: str):
+    rich.print(Rule(msg))
+
+
+def not_none(l: list) -> list:
+    return [x for x in l if x is not None]
+
+
+def get_free_port():
+    with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
+        s.bind(("", 0))
+        s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        return s.getsockname()[1]
+
+
+def correctness_check(
+    solution: str, dataset: str, task: Dict, expected_output: List
+) -> Tuple:
+    assert isinstance(solution, str)
+    result = untrusted_check(
+        dataset,
+        solution,
+        task["base_input"] + list(task["plus_input"]),
+        task["entry_point"],
+        expected_output["base"] + expected_output["plus"],
+        task["atol"],
+        expected_output["base_time"] + expected_output["plus_time"],
+        fast_check=True,
+        min_time_limit=DEFAULT_MIN_TIME_LIMIT,
+        gt_time_limit_factor=DEFAULT_GT_TIME_LIMIT_FACTOR,
+    )
+    return result, solution
+
+
+def get_evalplus_data():
+    problems_he = get_human_eval_plus(noextreme=True)
+    dataset_hash = get_human_eval_plus_hash(noextreme=True)
+    expected_output_human = get_groundtruth(problems_he, dataset_hash, [])
+    problems_mbpp = get_mbpp_plus(noextreme=True)
+    dataset_hash = get_mbpp_plus_hash(noextreme=True)
+    expected_output_mbpp = get_groundtruth(
+        problems_mbpp,
+        dataset_hash,
+        MBPP_OUTPUT_NOT_NONE_TASKS,
+    )
+    problems = {**problems_he, **problems_mbpp}
+    expected_output = {**expected_output_human, **expected_output_mbpp}
+    return problems, expected_output
+
+
+def table_print(table_name: str, kv: Dict):
+    table = Table(
+        title=table_name,
+        show_header=True,
+        header_style="bold",
+    )
+    for col_name in kv:
+        table.add_column(col_name)
+
+    table.add_row(*[str(v) for v in kv.values()])
+    rich.print(table)
+
+
+def correctness_worker(task_id: str, samples: list, ctask: Dict, expected_output: Dict):
+    assert isinstance(
+        samples, list
+    ), f"{task_id}: samples is not a list but {type(samples)}"
+
+    results = []
+
+    for solution in samples:
+        result, solution = correctness_check(
+            solution, task_id.split("/")[0].lower(), ctask, expected_output
+        )
+        results.append(
+            {
+                "solution": solution,
+                "pass": result[0] == PASS,
+                "profiled": False,
+                "matching_cluster_idx": None,
+                "dps": None,
+                "dps_norm": None,
+            }
+        )
+
+    return task_id, results
+
+
+def perf_worker(
+    task_id: str,
+    ptask: Dict,  # EvalPerf data
+    ret_dict: Dict,
+    lazy_evaluation: bool,
+    max_profile: int,
+):
+    rich.print(f"{task_id}: Started")
+    start_time = time.time()
+
+    ######################### Profiling Setup #########################
+    n_reference = len(ptask["reference"])
+    entry_point = ptask["entry_point"]
+    pe_input = (
+        mbpp_deserialize_inputs(task_id, ptask["pe_input"])[0]
+        if task_id.startswith("Mbpp/")
+        else ptask["pe_input"][0]
+    )
+    ####################################################################
+
+    ####################################################################
+    ############### Lazily profile reference solutions #################
+    ####################################################################
+    cache_ref_num_inst = [None] * n_reference
+
+    def get_avg_ref_profile(idx, check_order=True) -> Optional[Tuple]:
+        nonlocal cache_ref_num_inst
+
+        assert (
+            idx < n_reference - 1
+            and cache_ref_num_inst[idx + 1] is not None
+            or idx == n_reference - 1
+        ), f"Calling get_avg_ref_profile({idx}) before get_avg_ref_profile({idx+1}) is called, is not allowed! {n_reference = }"
+
+        if cache_ref_num_inst[idx] is not None:
+            return cache_ref_num_inst[idx], ptask["scores"][idx]
+
+        evaluation_time = PERF_EVAL_TIMEOUT_SECOND
+        ref_solution = ptask["reference"][idx]
+        for _ in range(2):  # at most retry twice
+            profiles = profile(
+                ref_solution,
+                entry_point,
+                [pe_input],
+                timeout_second_per_test=evaluation_time,
+            )
+
+            # Bad thing#1: timeout / failure happens
+            if are_profiles_broken(profiles):
+                print(f"{task_id}: [WARNING] Error in ref: {profiles}")
+                rich.print(Syntax(ref_solution, "python"))
+                print(f"{task_id}: Retrying w/ +10s timeout...")
+                evaluation_time += 10
+            else:
+                break
+
+        avg_profile = mean(profiles)
+        # Bad thing#2: if the current #instruction is faster than that of i+1
+        if idx < n_reference - 1 and avg_profile < cache_ref_num_inst[idx + 1]:
+            print(f"{task_id}: [WARNING] #{idx} ref faster than #{idx + 1}")
+            print(f"ref {idx}: #inst {avg_profile}\tscore {ptask['scores'][idx]:.1f}")
+            print(
+                f"ref {idx+1}: #inst {cache_ref_num_inst[idx+1]}\tscore {ptask['scores'][idx+1]:.1f}"
+            )
+            rich.print(Syntax(ref_solution, "python"))
+            if check_order:
+                return None
+
+        cache_ref_num_inst[idx] = avg_profile
+        ret_dict["ref"][idx]["_num_cpu_instructions"] = avg_profile
+        return cache_ref_num_inst[idx], ptask["scores"][idx]
+
+    ####################################################################
+    ############################## END #################################
+    ####################################################################
+
+    if not lazy_evaluation:  # compute everything ahead of time
+        for i in range(n_reference - 1, -1, -1):
+            if get_avg_ref_profile(i) is None:
+                break
+
+        assert (
+            None not in cache_ref_num_inst
+        ), f"{task_id}: Failed to profile certain reference: {cache_ref_num_inst = }"
+
+    profile_cache = {}
+
+    cur_profiled = 0
+    for result in ret_dict["results"]:
+        if cur_profiled >= max_profile:
+            rich.print(f"{task_id}: Reached max_profile limit {max_profile}, stopped")
+            break
+        if not result["pass"]:
+            continue
+
+        solution = result["solution"]
+
+        if solution in profile_cache:  # reuse cache
+            sample_profiles = profile_cache[solution]
+        else:
+            sample_profiles = profile(
+                solution,
+                entry_point,
+                [pe_input],
+                timeout_second_per_test=PERF_EVAL_TIMEOUT_SECOND,
+            )
+            profile_cache[solution] = sample_profiles  # store cache
+
+        score = 0
+        norm_score = 0
+        result["matching_cluster_idx"] = -1  # -1 means even slower than the slowest ref
+        # if the solution results in a timeout, score is 0
+        if are_profiles_broken(sample_profiles):
+            print(
+                f"{task_id}: Tested solution error'ed out: {sample_profiles} ... regarded as 0 score"
+            )
+            rich.print(Syntax(solution, "python"))
+        else:
+            avg_sample_profile = result["_num_cpu_instructions"] = mean(sample_profiles)
+            # Get profiles from fast to slow (back to front):
+            for j in range(n_reference - 1, -1, -1):
+                avg_ref_profile, ref_score = get_avg_ref_profile(j, check_order=False)
+                if avg_sample_profile <= avg_ref_profile:
+                    result["matching_cluster_idx"] = j
+                    score = ref_score
+                    norm_score = 100 * (j + 1) / n_reference
+                    break
+
+        result["dps"] = score
+        result["dps_norm"] = norm_score
+        result["profiled"] = True
+        cur_profiled += 1
+
+    ret_dict["dps"] = mean(not_none([r["dps"] for r in ret_dict["results"]]))
+    ret_dict["dps_norm"] = mean(not_none([r["dps_norm"] for r in ret_dict["results"]]))
+    ret_dict["n_profiled"] = cur_profiled
+
+    table_print(
+        f"[bold green]{task_id} Completed[/]",
+        {
+            "Duration": f"{time.time() - start_time:.1f}s",
+            "DPS": f"[green]{ret_dict['dps']:.1f}[/]",
+            "DPS_norm": f"[green]{ret_dict['dps_norm']:.1f}[/]",
+            "# Profiled": f"{cur_profiled} / {len(ret_dict['results'])}",
+            "Pass@1": f"{ret_dict['pass@1']:.1f}%",
+        },
+    )
+
+    return ret_dict
+
+
+# TODO(@ganler): OPTIMIZATION: reuse the samples from the generations of other datasets
+def script(
+    samples: Optional[str] = None,
+    min_correct: int = 10,
+    max_profile: Optional[int] = None,
+    n_samples: int = 100,
+    temperature: float = 1.0,
+    parallel: Optional[int] = None,
+    lazy_evaluation: bool = True,
+    i_just_wanna_run: bool = False,
+    **model_kwargs,
+):
+    max_profile = max_profile or min(min_correct * 2, n_samples)
+    assert min_correct <= max_profile <= n_samples
+    simple_test_profiler()  # test linux perf setup
+
+    if model_kwargs:
+        # To suppress the warning of tokenizers
+        os.environ["TOKENIZERS_PARALLELISM"] = os.environ.get(
+            "TOKENIZERS_PARALLELISM", "false"
+        )
+        # overwrite parameters
+        samples = run_codegen(
+            dataset="evalperf",
+            n_samples=n_samples,
+            temperature=temperature,
+            **model_kwargs,
+        )
+
+    assert samples is not None, "Please provide the path to the samples"
+
+    # Data loading
+    problems, expected_output = get_evalplus_data()
+    ptasks = get_evalperf_data()
+
+    # Parallelism
+    max_workers = parallel or max(1, default_parallelism(divisor=4))
+    assert 0 < max_workers < multiprocessing.cpu_count(), "Invalid max CPU workers"
+
+    if os.path.isdir(samples):
+        result_path = os.path.join(samples, "evalperf_results.json")
+    else:
+        assert samples.endswith(".jsonl")
+        result_path = samples.replace(".jsonl", "_evalperf_results.json")
+    brief_result_path = result_path.replace(
+        "evalperf_results.json", "evalperf_results.brief.json"
+    )
+
+    # resume results
+    eval_results = {}
+    if not i_just_wanna_run and os.path.exists(result_path):
+        resumed_result = json.load(open(result_path, "r"))
+        if (
+            resumed_result["n_samples"] == n_samples
+            and resumed_result["temperature"] == temperature
+            and resumed_result["min_correct"] == min_correct
+            and resumed_result["max_profile"] == max_profile
+        ):
+            eval_results = resumed_result["eval"]
+            for etask in eval_results:
+                ptasks.pop(etask, None)
+
+            rich.print(f"Resumed {len(eval_results)} results from {result_path}")
+
+    # Load model's samples: task_id -> a list of samples
+    sample_iter = stream_jsonl(samples)
+    samples = defaultdict(list)
+    for task in sample_iter:
+        samples[task["task_id"].replace("_", "/")].append(task["solution"])
+    samples = {k: v[:n_samples] for k, v in samples.items()}
+
+    # assert each task has n_samples
+    for task_id, s in samples.items():
+        assert len(s) == n_samples, f"{task_id} has {len(s)} samples != {n_samples}"
+
+    # Initialize eval_results
+    for task_id, ptask in ptasks.items():
+        eval_results[task_id] = {
+            "task_id": task_id,
+            "results": [],
+            "ref": [
+                {"solution": s, "score": r, "_num_cpu_instructions": None}
+                for s, r in zip(ptask["reference"], ptask["scores"])
+            ],
+            "dps": None,
+            "dps_norm": None,
+            "pass@1": None,
+            "n_profiled": None,
+        }
+
+    rule("Correctness Checking...")
+    with progress("Correctness") as p:
+        with ThreadPoolExecutor(max_workers=max_workers) as executor:
+            futures = [
+                executor.submit(
+                    correctness_worker,
+                    task_id,
+                    samples[task_id],
+                    problems[task_id],
+                    expected_output[task_id],
+                )
+                for task_id in ptasks
+            ]
+
+            for future in p.track(as_completed(futures), total=len(futures)):
+                task_id, results = future.result()
+                eval_results[task_id]["results"] = results
+                eval_results[task_id]["pass@1"] = (
+                    100 * len([r for r in results if r["pass"]]) / n_samples
+                )
+
+    rule("EvalPerf Configurations")
+    if lazy_evaluation:
+        rich.print(
+            "[bold yellow]Lazy evaluation is enabled[/]: "
+            "Fast evaluation without enumeratively checking reference order consistency."
+        )
+
+    table_print(
+        "Configurations",
+        {
+            "Max CPU": max_workers,
+            "#Tasks": len(ptasks),
+            "#Samples per task": n_samples,
+            "Min correct": min_correct,
+            "Max profile": max_profile,
+            "Result path": result_path,
+        },
+    )
+
+    rich.print(f"IDs of tasks to evaluate: {list(ptasks.keys())}")
+    rule("Evaluation Start")
+    undone = []
+    with progress("Profiling") as p:
+        with ThreadPoolExecutor(max_workers=max_workers) as executor:
+            futures = []
+            for task_id, ptask in ptasks.items():
+                n_pass = len([r for r in eval_results[task_id]["results"] if r["pass"]])
+                if n_pass < min_correct:
+                    rich.print(
+                        f"{task_id}: [bold yellow]{n_pass} < {min_correct} correct solutions, skipped[/]"
+                    )
+                    continue
+                futures.append(
+                    executor.submit(
+                        perf_worker,
+                        task_id,
+                        ptask,
+                        eval_results[task_id],
+                        lazy_evaluation,
+                        max_profile,
+                    )
+                )
+                undone.append(task_id)
+                rich.print(f"{task_id}: Queued")
+
+            for future in p.track(as_completed(futures), total=len(futures)):
+                result = future.result()
+                eval_results[result["task_id"]] = result
+                undone.remove(result["task_id"])
+                if undone and len(undone) < max_workers:
+                    print(f"Still running: {undone}")
+
+    rule("Evaluation Summary")
+    dps = mean(not_none([res["dps"] for res in eval_results.values()]))
+    dps_norm = mean(not_none([res["dps_norm"] for res in eval_results.values()]))
+    pass_1 = mean(not_none([res["pass@1"] for res in eval_results.values()]))
+    n_evalperfed = len(not_none([res["dps"] for res in eval_results.values()]))
+
+    table_print(
+        "EvalPerf Summary",
+        {
+            "DPS": f"{dps:.1f}",
+            "DPS_norm": f"{dps_norm:.1f}",
+            "Pass@1": f"{pass_1:.1f}%",
+            "#EvalPerf-ed tasks": f"{n_evalperfed} / {len(eval_results)}",
+            "min_correct": min_correct,
+            "n_samples": n_samples,
+            "temperature": temperature,
+        },
+    )
+
+    # Save full results
+    with open(result_path, "w") as f:
+        f.write(
+            json.dumps(
+                {
+                    "date": datetime.now().strftime("%Y-%m-%d %H:%M"),
+                    "n_samples": n_samples,
+                    "temperature": temperature,
+                    "min_correct": min_correct,
+                    "max_profile": max_profile,
+                    "eval": eval_results,
+                }
+            )
+        )
+    rich.print(f"Full results have been saved to {result_path}")
+
+    # Save brief results
+    with open(brief_result_path, "w") as f:
+        f.write(
+            json.dumps(
+                {
+                    "date": datetime.now().strftime("%Y-%m-%d %H:%M"),
+                    "config": {
+                        "n_samples": n_samples,
+                        "temperature": temperature,
+                        "min_correct": min_correct,
+                        "max_profile": max_profile,
+                    },
+                    "summary": {
+                        "dps": dps,
+                        "dps_norm": dps_norm,
+                        "pass@1": pass_1,
+                    },
+                    "eval": {
+                        task_id: {
+                            "dps": res["dps"],
+                            "dps_norm": res["dps_norm"],
+                            "pass@1": res["pass@1"],
+                            "profiled": [
+                                {
+                                    "solution": r["solution"],
+                                    "matching_cluster_idx": r["matching_cluster_idx"],
+                                }
+                                for r in res["results"]
+                                if r["profiled"]
+                            ],
+                        }
+                        for task_id, res in eval_results.items()
+                    },
+                }
+            )
+        )
+
+    rich.print(f"Brief results have been saved to {brief_result_path}")
+
+    rule("To visualize win-rates and pair-wise DPS, run:")
+    rich.print(
+        Syntax(
+            f"""\
+git clone git@github.com:evalplus/evalplus.github.io.git
+git --git-dir=evalplus.github.io/.git pull
+cp {brief_result_path} evalplus.github.io/results/evalperf
+python evalplus.github.io/results/evalperf/stats.py
+python -m http.server -d evalplus.github.io {get_free_port()}""",
+            "bash",
+        )
+    )
+
+
+def main():
+    from fire import Fire
+
+    Fire(script)
+
+
+if __name__ == "__main__":
+    main()

evalplus/build/lib/evalplus/evaluate.py

@@ -0,0 +1,375 @@
+import json
+import multiprocessing
+import os
+import pickle
+import threading
+import time
+from collections import Counter, defaultdict
+from concurrent.futures import ProcessPoolExecutor, as_completed
+from datetime import datetime
+from typing import Any, Dict, List, Optional, Tuple
+from warnings import warn
+
+import numpy as np
+from termcolor import cprint
+from tqdm import tqdm
+
+from evalplus.codegen import run_codegen
+from evalplus.config import *
+from evalplus.data import (
+    get_human_eval_plus,
+    get_human_eval_plus_hash,
+    get_mbpp_plus,
+    get_mbpp_plus_hash,
+    load_solutions,
+)
+from evalplus.data.mbpp import mbpp_serialize_inputs
+from evalplus.data.utils import CACHE_DIR
+from evalplus.eval import (
+    PASS,
+    compatible_eval_result,
+    estimate_pass_at_k,
+    untrusted_check,
+)
+from evalplus.eval._special_oracle import MBPP_OUTPUT_NOT_NONE_TASKS
+from evalplus.gen.util import trusted_exec
+
+# 1st item: the status
+# 2nd item (optional): the detailed pass/fail boolean for each input
+Result = Tuple[str, List[bool]]
+
+
+def get_groundtruth(problems, hashcode, tasks_only_output_not_none):
+    cache_file = os.path.join(CACHE_DIR, f"{hashcode}.pkl")
+    if os.path.exists(cache_file):
+        print(f"Load from ground-truth from {cache_file}")
+        with open(cache_file, "rb") as f:
+            return pickle.load(f)
+
+    os.makedirs(CACHE_DIR, exist_ok=True)
+    print("Computing expected output...")
+    tbegin = time.time()
+    expected_output = {}
+    for task_id, problem in problems.items():
+        oracle = {}
+        oracle["base"], oracle["base_time"] = trusted_exec(
+            problem["prompt"] + problem["canonical_solution"],
+            problem["base_input"],
+            problem["entry_point"],
+            record_time=True,
+            output_not_none=problem["entry_point"] in tasks_only_output_not_none,
+        )
+
+        oracle["plus"], oracle["plus_time"] = trusted_exec(
+            problem["prompt"] + problem["canonical_solution"],
+            problem["plus_input"],
+            problem["entry_point"],
+            record_time=True,
+            output_not_none=problem["entry_point"] in tasks_only_output_not_none,
+        )
+        expected_output[task_id] = oracle
+    print(f"Expected outputs computed in {time.time() - tbegin:.2f}s")
+
+    with open(cache_file, "wb") as f:
+        pickle.dump(expected_output, f)
+
+    return expected_output
+
+
+def check_correctness(
+    dataset: str,
+    completion_id: int,
+    problem: Dict[str, Any],
+    solution: str,
+    expected_output: Dict[str, List],
+    base_only=False,
+    fast_check=False,
+    identifier=None,
+    min_time_limit: float = DEFAULT_MIN_TIME_LIMIT,
+    gt_time_limit_factor: float = DEFAULT_GT_TIME_LIMIT_FACTOR,
+) -> Dict[str, Result]:  # {...}, "base" | "plus" -> (status, details)
+    ret = {
+        "completion_id": completion_id,
+        "task_id": problem["task_id"],
+        "_identifier": identifier,
+        "solution": solution,
+    }
+    ret["base"] = untrusted_check(
+        dataset,
+        solution,
+        problem["base_input"],
+        problem["entry_point"],
+        expected=expected_output["base"],
+        atol=problem["atol"],
+        ref_time=expected_output["base_time"],
+        fast_check=fast_check,
+        min_time_limit=min_time_limit,
+        gt_time_limit_factor=gt_time_limit_factor,
+    )
+
+    if not base_only:
+        ret["plus"] = untrusted_check(
+            dataset,
+            solution,
+            problem["plus_input"],
+            problem["entry_point"],
+            expected=expected_output["plus"],
+            atol=problem["atol"],
+            ref_time=expected_output["plus_time"],
+            fast_check=fast_check,
+            min_time_limit=min_time_limit,
+            gt_time_limit_factor=gt_time_limit_factor,
+        )
+
+    return ret
+
+
+def evaluate(
+    dataset: str,
+    samples: Optional[str] = None,
+    base_only: bool = False,
+    parallel: Optional[int] = None,
+    i_just_wanna_run: bool = False,
+    test_details: bool = False,
+    min_time_limit: float = DEFAULT_MIN_TIME_LIMIT,
+    gt_time_limit_factor: float = DEFAULT_GT_TIME_LIMIT_FACTOR,
+    mini: bool = False,
+    noextreme: bool = False,
+    version: str = "default",
+    output_file: Optional[str] = None,
+    gguf_file: Optional[str] = None,
+    **model_kwargs,
+):
+    if model_kwargs:
+        # To suppress the warning of tokenizers
+        os.environ["TOKENIZERS_PARALLELISM"] = os.environ.get(
+            "TOKENIZERS_PARALLELISM", "false"
+        )
+        samples = run_codegen(
+            dataset=dataset,
+            gguf_file=gguf_file,
+            **model_kwargs,
+        )
+    assert samples is not None, "No samples provided"
+
+    n_workers = parallel or max(1, multiprocessing.cpu_count() // 2)
+
+    if os.path.isdir(samples):
+        result_path = os.path.join(samples, "eval_results.json")
+    else:
+        assert samples.endswith(".jsonl")
+        # legacy compatibility
+        if os.path.exists(samples.replace(".jsonl", "_eval_results.json")):
+            result_path = samples.replace(".jsonl", "_eval_results.json")
+        else:
+            result_path = samples.replace(".jsonl", ".eval_results.json")
+
+    if output_file is not None:
+        result_path = output_file
+
+    if os.path.isfile(result_path) and not i_just_wanna_run:
+        print(f"Load from previous results from {result_path}")
+        with open(result_path, "r") as f:
+            results = json.load(f)
+
+        results = compatible_eval_result(results)
+    else:
+        if dataset == "humaneval":
+            problems = get_human_eval_plus(
+                mini=mini, noextreme=noextreme, version=version
+            )
+            dataset_hash = get_human_eval_plus_hash(
+                mini=mini, noextreme=noextreme, version=version
+            )
+            expected_output = get_groundtruth(problems, dataset_hash, [])
+        elif dataset == "mbpp":
+            problems = get_mbpp_plus(mini=mini, noextreme=noextreme, version=version)
+            dataset_hash = get_mbpp_plus_hash(
+                mini=mini, noextreme=noextreme, version=version
+            )
+            expected_output = get_groundtruth(
+                problems,
+                dataset_hash,
+                MBPP_OUTPUT_NOT_NONE_TASKS,
+            )
+
+        results = {
+            "date": datetime.now().strftime("%Y-%m-%d %H:%M"),
+            "hash": dataset_hash,
+            "eval": {},
+        }
+
+        with ProcessPoolExecutor(max_workers=n_workers) as executor:
+            futures = []
+            completion_id = Counter()
+            n_samples = 0
+            eval_results = defaultdict(list)  # task_id ->
+            remainings = set()
+
+            print("Reading samples...")
+            for sample in tqdm(load_solutions(samples)):
+                task_id = sample["task_id"]
+                if task_id not in problems:
+                    warn(
+                        f"Task {task_id} is found in the samples but not found in the dataset"
+                    )
+                    continue
+                solution = (
+                    sample["solution"]
+                    if "solution" in sample
+                    else problems[task_id]["prompt"] + sample["completion"]
+                )
+                remainings.add(sample["_identifier"])
+                args = (
+                    dataset,
+                    completion_id[task_id],
+                    problems[task_id],
+                    solution,
+                    expected_output[task_id],
+                    base_only,
+                    not test_details,  # fast_check
+                    sample["_identifier"],
+                    min_time_limit,
+                    gt_time_limit_factor,
+                )
+                futures.append(executor.submit(check_correctness, *args))
+                completion_id[task_id] += 1
+                n_samples += 1
+
+            assert n_samples == len(remainings), "Missing problems in unfinished"
+            assert len(completion_id) == len(problems), "Missing problems in samples"
+
+            def stucking_checker():
+                while remainings:
+                    last_size = len(remainings)
+                    time.sleep(20)
+                    if last_size != len(remainings) or len(remainings) == 0:
+                        continue
+                    # Potential stucking
+                    warn("No samples had finished testing in the last 20s")
+                    warn(f"{len(remainings)} samples to be tested: {remainings}")
+
+            threading.Thread(target=stucking_checker).start()
+
+            for future in tqdm(as_completed(futures), total=n_samples):
+                result = future.result()
+                remainings.remove(result["_identifier"])
+                eval_results[result["task_id"]].append(result)
+
+        # sort the results for each problem by completion_id
+        for task_id, task_results in eval_results.items():
+            task_results.sort(key=lambda x: x["completion_id"])
+            results["eval"][task_id] = []
+            for res in task_results:
+
+                def get_failed_tests(stat, details, inputs) -> List[Any]:
+                    if stat == PASS or not details:
+                        return []
+
+                    if test_details:
+                        return [
+                            inputs[i] for i in range(len(details)) if not details[i]
+                        ]
+
+                    # else => simply return the only and the last fail test
+                    return [inputs[len(details) - 1]]
+
+                base_stat, base_details = res["base"]
+                base_fail_tests = get_failed_tests(
+                    base_stat, base_details, problems[task_id]["base_input"]
+                )
+
+                # initialize plus tests
+                plus_stat = None
+                plus_fail_tests = []
+
+                # with plus tests
+                if not base_only:
+                    plus_stat, plus_details = res["plus"]
+                    plus_fail_tests = get_failed_tests(
+                        plus_stat, plus_details, problems[task_id]["plus_input"]
+                    )
+
+                if dataset == "mbpp":
+                    base_fail_tests = mbpp_serialize_inputs(task_id, base_fail_tests)
+                    plus_fail_tests = mbpp_serialize_inputs(task_id, plus_fail_tests)
+
+                results["eval"][task_id].append(
+                    {
+                        "task_id": task_id,
+                        "solution": res["solution"],
+                        "base_status": base_stat,
+                        "plus_status": plus_stat,
+                        "base_fail_tests": base_fail_tests,
+                        "plus_fail_tests": plus_fail_tests,
+                    }
+                )
+
+    # Calculate pass@k.
+    total = np.array([len(r) for r in results["eval"].values()])
+    base_correct = []
+    new_correct = []
+
+    for res in results["eval"].values():
+        bc = sum([r["base_status"] == PASS for r in res])
+        base_correct.append(bc)
+        if not base_only:
+            new_correct.append(
+                sum(
+                    [
+                        res[i]["base_status"] == res[i]["plus_status"] == PASS
+                        for i in range(len(res))
+                    ]
+                )
+            )
+    base_correct = np.array(base_correct)
+
+    pass_at_k = {
+        f"pass@{k}": estimate_pass_at_k(total, base_correct, k).mean()
+        for k in [1, 10, 100]
+        if total.min() >= k
+    }
+    cprint(f"{dataset} (base tests)", "red")
+    for k, v in pass_at_k.items():
+        cprint(f"{k}:\t{v:.3f}", "red")
+    results["pass_at_k"] = {"base": pass_at_k}
+
+    if new_correct:
+        cprint(f"{dataset}+ (base + extra tests)", "green")
+        pass_at_k = {
+            f"pass@{k}": estimate_pass_at_k(total, np.array(new_correct), k).mean()
+            for k in [1, 10, 100]
+            if (total >= k).all()
+        }
+        for k, v in pass_at_k.items():
+            cprint(f"{k}:\t{v:.3f}", "green")
+        results["pass_at_k"]["plus"] = pass_at_k
+
+    # save results
+    if os.path.isfile(result_path) and i_just_wanna_run:
+        decision = ""
+        while decision.lower() not in ["y", "n"]:
+            print(f"{result_path} already exists. Press [Y/N] to overwrite or exit...")
+            decision = input()
+
+        if decision.lower() == "y":
+            # mv the file to a backup
+            new_path = result_path + ".bak"
+            while os.path.isfile(new_path):
+                new_path += ".bak"
+            os.rename(result_path, new_path)
+            print(f"Backup {result_path} to {new_path}")
+
+    if not os.path.isfile(result_path):
+        with open(result_path, "w") as f:
+            json.dump(results, f)
+
+
+def main():
+    from fire import Fire
+
+    Fire(evaluate)
+
+
+if __name__ == "__main__":
+    main()

evalplus/build/lib/evalplus/gen/__init__.py

@@ -0,0 +1,21 @@
+import copy
+from typing import Any, List
+
+
+class BaseGen(object):
+    def __init__(self, inputs: List[Any], entry_point: str, contract: str):
+        """Initializing a input mutator.
+
+        Args:
+            inputs (List[Any]): The set of initial inputs (i.e., seeds)
+            entry_point (str): The function name to invoke with the input
+            contract (str): The contract to verify input validity
+        """
+        self.contract = contract
+        self.entry_point = entry_point
+        self.seed_pool: List[Any] = copy.deepcopy(inputs)
+        self.new_inputs = []
+        self.seed_hash = set([hash(str(x)) for x in self.seed_pool])
+
+    def generate(self, num: int) -> List[Any]:
+        raise NotImplementedError

evalplus/build/lib/evalplus/gen/chatgpt_gen.py

@@ -0,0 +1,78 @@
+import ast
+import random
+from typing import List
+
+import openai
+from openai.types.chat import ChatCompletion
+
+from evalplus.data.utils import to_raw
+from evalplus.gen import BaseGen
+from evalplus.gen.util import trusted_check_exec
+from evalplus.gen.util.openai_request import make_auto_request
+
+
+class ChatGPTGen(BaseGen):
+    def __init__(self, inputs: List, signature: str, contract_code: str, gd_code: str):
+        super().__init__(inputs, signature, contract_code)
+        self.gd_code = gd_code
+        self.prompt_messages = [
+            "Please generate complex inputs to test the function.",
+            "Please generate corner case inputs to test the function.",
+            "Please generate difficult inputs to test the function.",
+        ]
+        self.iteration = 20
+        self.client = openai.Client()
+
+    def seed_selection(self) -> List:
+        # get 5 for now.
+        return random.sample(self.seed_pool, k=min(len(self.seed_pool), 5))
+
+    @staticmethod
+    def _parse_ret(ret: ChatCompletion) -> List:
+        rets = []
+        output = ret.choices[0].message.content
+        if "```" in output:
+            for x in output.split("```")[1].splitlines():
+                if x.strip() == "":
+                    continue
+                try:
+                    # remove comments
+                    input = ast.literal_eval(f"[{x.split('#')[0].strip()}]")
+                except:  # something wrong.
+                    continue
+                rets.append(input)
+        return rets
+
+    def chatgpt_generate(self, selected_inputs: List) -> List:
+        # append the groundtruth function
+        # actually it can be any function (maybe we can generate inputs for each llm generated code individually)
+        message = f"Here is a function that we want to test:\n```\n{self.gd_code}\n```"
+        str_inputs = "\n".join(
+            [
+                ", ".join([f"'{to_raw(i)}'" if type(i) == str else str(i) for i in x])
+                for x in selected_inputs
+            ]
+        )
+        message += f"\nThese are some example inputs used to test the function:\n```\n{str_inputs}\n```"
+        message += f"\n{random.choice(self.prompt_messages)}"
+        ret = make_auto_request(
+            self.client,
+            message=message,
+            model="gpt-3.5-turbo",
+            max_tokens=256,
+            response_format={"type": "text"},
+        )
+        return self._parse_ret(ret)
+
+    def generate(self, num: int):
+        while len(self.new_inputs) < num and self.iteration >= 0:
+            seeds = self.seed_selection()
+            new_inputs = self.chatgpt_generate(seeds)
+            for new_input in new_inputs:
+                if hash(str(new_input)) not in self.seed_hash:
+                    if trusted_check_exec(self.contract, [new_input], self.entry_point):
+                        self.seed_pool.append(new_input)
+                        self.seed_hash.add(hash(str(new_input)))
+                        self.new_inputs.append(new_input)
+            self.iteration -= 1
+        return self.new_inputs[:num]

evalplus/build/lib/evalplus/gen/mut_gen.py

@@ -0,0 +1,30 @@
+import random
+from abc import abstractmethod
+from typing import Any, List
+
+from evalplus.gen import BaseGen
+from evalplus.gen.util import trusted_check_exec
+
+
+class MutateGen(BaseGen):
+    def __init__(self, inputs: List, signature: str, contract_code: str):
+        super().__init__(inputs, signature, contract_code)
+
+    def seed_selection(self):
+        # random for now.
+        return random.choice(self.seed_pool)
+
+    @abstractmethod
+    def mutate(self, seed_input: Any) -> Any:
+        pass
+
+    def generate(self, num: int) -> List[Any]:
+        while len(self.new_inputs) < num:
+            seed = self.seed_selection()
+            new_input = self.mutate(seed)
+            if hash(str(new_input)) not in self.seed_hash:
+                if trusted_check_exec(self.contract, [new_input], self.entry_point):
+                    self.seed_pool.append(new_input)
+                    self.seed_hash.add(hash(str(new_input)))
+                    self.new_inputs.append(new_input)
+        return self.new_inputs[:num]

evalplus/build/lib/evalplus/gen/type_mut.py

@@ -0,0 +1,340 @@
+import copy
+import random
+import string
+import time
+from typing import Any, Dict, List, Set, Tuple
+
+from multipledispatch import dispatch
+
+from evalplus.gen.mut_gen import MutateGen
+from evalplus.gen.util import trusted_check_exec
+
+MAX_MULTI_STEP_SIZE = 5
+MUTATE_BOUND_SIZE = 8
+
+NoneType = type(None)
+
+
+# decorator to use ingredients
+class use_ingredient:
+    def __init__(self, prob: float):
+        assert 0 <= prob <= 0.95
+        self.prob = prob
+
+    def __call__(obj, func):
+        def wrapper(self, seed_input):
+            if random.random() < obj.prob and self.ingredients[type(seed_input)]:
+                return random.choice(list(self.ingredients[type(seed_input)]))
+            else:
+                return func(self, seed_input)
+
+        return wrapper
+
+
+class TypedMutGen(MutateGen):
+    def __init__(self, inputs: List, signature: str, contract_code: str):
+        super().__init__(inputs, signature, contract_code)
+        self.timeout = 60 * 60  # 1 hour
+        self.ingredients = {
+            int: set(),
+            float: set(),
+            str: set(),
+            complex: set(),
+        }
+        for x in inputs:
+            self.fetch_ingredient(x)
+
+    def seed_selection(self):
+        # random for now.
+        return random.choice(self.seed_pool)
+
+    def mutate(self, seed_input: Any) -> List:
+        new_input = copy.deepcopy(seed_input)
+
+        patience = MUTATE_BOUND_SIZE
+        while new_input == seed_input or patience == 0:
+            new_input = self.typed_mutate(new_input)
+            patience -= 1
+
+        return new_input
+
+    #########################
+    # Type-aware generation #
+    #########################
+    @dispatch(NoneType)
+    def typed_gen(self, _):
+        return None
+
+    @dispatch(int)
+    def typed_gen(self, _):
+        @use_ingredient(0.5)
+        def _impl(*_):
+            return random.randint(-100, 100)
+
+        return _impl(self, _)
+
+    @dispatch(float)
+    def typed_gen(self, _):
+        @use_ingredient(0.5)
+        def _impl(*_):
+            return random.uniform(-100, 100)
+
+        return _impl(self, _)
+
+    @dispatch(bool)
+    def typed_gen(self, _):
+        return random.choice([True, False])
+
+    @dispatch(str)
+    def typed_gen(self, _):
+        @use_ingredient(0.5)
+        def _impl(*_):
+            return "".join(
+                random.choice(string.ascii_letters)
+                for _ in range(random.randint(0, 10))
+            )
+
+        return _impl(self, _)
+
+    def any_gen(self):
+        # weighted choose
+        choice = random.choices(
+            [
+                True,
+                1,
+                1.1,
+                "str",
+                [],  # list
+                tuple(),  # tuple
+                dict(),  # dict
+                None,  # None
+            ],
+            [0.2, 0.2, 0.2, 0.2, 0.05, 0.05, 0.05, 0.05],
+        )[0]
+        return self.typed_gen(choice)
+
+    @dispatch(list)
+    def typed_gen(self, _):
+        ret = []
+        size = random.randint(0, 10)
+        if random.randint(0, 4) == 0:  # heterogeneous
+            for _ in range(size):
+                ret.append(self.any_gen())
+        else:  # homogeneous
+            t = random.choice([bool(), int(), float(), str()])
+            for _ in range(size):
+                ret.append(self.typed_gen(t))
+        return ret
+
+    @dispatch(tuple)
+    def typed_gen(self, _):
+        return tuple(self.typed_gen([]))
+
+    # NOTE: disable set for now as Steven is too weak in Python (/s)
+    # @dispatch(set)
+    # def typed_gen(self, _):
+    #     return set(self.typed_gen([]))
+
+    @dispatch(dict)
+    def typed_gen(self, _):
+        ret = dict()
+        values = self.typed_gen([])
+        # NOTE: Assumption: nobody uses dict with heterogeneous keys
+        # NOTE: Assumption: nobody uses dict with boolean keys
+        key_type = random.choice([int(), float(), str()])
+        for v in values:
+            ret[self.typed_gen(key_type)] = self.typed_gen(v)
+        return ret
+
+    ########################
+    # Type-aware mutation  #
+    ########################
+    # Simple primitives
+    @dispatch(int)
+    def typed_mutate(self, seed_input: int):
+        @use_ingredient(0.5)
+        def _impl(_, seed_input: int):
+            return seed_input + random.randint(-1, 1)
+
+        return _impl(self, seed_input)
+
+    @dispatch(float)
+    def typed_mutate(self, seed_input: float):
+        @use_ingredient(0.5)
+        def _impl(_, seed_input: float):
+            if random.randint(0, 1):
+                return seed_input + random.uniform(-1, 1)
+            return seed_input * (1 + random.uniform(-0.5, 0.5))
+
+        return _impl(self, seed_input)
+
+    @dispatch(complex)
+    def typed_mutate(self, seed_input: complex):
+        @use_ingredient(0.5)
+        def _impl(_, seed_input: complex):
+            imag = seed_input.imag + random.uniform(-1, 1)
+            return complex(0, imag)
+
+        return _impl(self, seed_input)
+
+    @dispatch(bool)
+    def typed_mutate(self, seed_input: bool):
+        return random.choice([True, False])
+
+    @dispatch(NoneType)
+    def typed_mutate(self, seed_input: NoneType):
+        return None
+
+    # List-like
+    @dispatch(list)
+    def typed_mutate(self, seed_input: List):
+        if len(seed_input) == 0:
+            return self.typed_gen([])
+
+        choice = random.randint(0, 3)
+        idx = random.randint(0, len(seed_input) - 1)
+        if choice == 0:  # remove one element
+            seed_input.pop(random.randint(0, len(seed_input) - 1))
+        elif choice == 1 and len(seed_input) > 0:  # add one mutated element
+            seed_input.insert(
+                random.randint(0, len(seed_input) - 1),
+                self.typed_mutate(seed_input[idx]),
+            )
+        elif choice == 2 and len(seed_input) > 0:  # repeat one element
+            seed_input.append(seed_input[idx])
+        else:  # inplace element change
+            seed_input[idx] = self.typed_mutate(seed_input[idx])
+        return seed_input
+
+    @dispatch(tuple)
+    def typed_mutate(self, seed_input: Tuple):
+        return tuple(self.typed_mutate(list(seed_input)))
+
+    # String
+    @dispatch(str)
+    def typed_mutate(self, seed_input: str):
+        @use_ingredient(0.4)
+        def _impl(_, seed_input: str):
+            choice = random.randint(0, 2) if seed_input else 0
+            if choice == 0 and self.ingredients[str]:  # insert an ingredient
+                idx = random.randint(0, len(seed_input))
+                return (
+                    seed_input[:idx]
+                    + random.choice(list(self.ingredients[str]))
+                    + seed_input[idx:]
+                )
+            # other choices assume len(seed_input) > 0
+            elif choice == 1:  # replace a substring with empty or mutated string
+                start = random.randint(0, len(seed_input) - 1)
+                end = random.randint(start + 1, len(seed_input))
+                mid = (
+                    ""
+                    if random.randint(0, 1)
+                    else self.typed_mutate(seed_input[start:end])
+                )
+                return seed_input[:start] + mid + seed_input[end:]
+            elif choice == 2:  # repeat one element
+                idx = random.randint(0, len(seed_input) - 1)
+                return (
+                    seed_input[:idx]
+                    + seed_input[random.randint(0, len(seed_input) - 1)]
+                    + seed_input[idx:]
+                )
+
+            # random char
+            return self.typed_gen(str())
+
+        return _impl(self, seed_input)
+
+    # Set
+    @dispatch(set)
+    def typed_mutate(self, seed_input: Set):
+        return set(self.typed_mutate(list(seed_input)))
+
+    # Dict
+    @dispatch(dict)
+    def typed_mutate(self, seed_input: Dict):
+        if len(seed_input) == 0:
+            return self.typed_gen(dict())
+
+        choice = random.randint(0, 2)
+        if choice == 0:  # remove a kv
+            del seed_input[random.choice(list(seed_input.keys()))]
+        elif choice == 1:  # add a kv
+            k = self.typed_mutate(random.choice(list(seed_input.keys())))
+            v = self.typed_mutate(random.choice(list(seed_input.values())))
+            seed_input[k] = v
+        elif choice == 2:  # inplace value change
+            k0, v0 = random.choice(list(seed_input.items()))
+            seed_input[k0] = self.typed_mutate(v0)
+        return seed_input
+
+    ############################################
+    # Fetching ingredients to self.ingredients #
+    ############################################
+    def fetch_ingredient(self, seed_input):
+        self.typed_fetch(seed_input)
+
+    @dispatch(int)
+    def typed_fetch(self, seed_input: int):
+        self.ingredients[int].add(seed_input)
+
+    @dispatch(float)
+    def typed_fetch(self, seed_input: float):
+        self.ingredients[float].add(seed_input)
+
+    @dispatch(complex)
+    def typed_fetch(self, seed_input: complex):
+        self.ingredients[complex].add(seed_input)
+
+    @dispatch(str)
+    def typed_fetch(self, seed_input: str):
+        self.ingredients[str].add(seed_input)
+        for token in seed_input.strip().split():
+            self.ingredients[str].add(token)
+
+    # List-like
+    def _fetch_list_like(self, seed_input):
+        for x in seed_input:
+            if self.typed_fetch.dispatch(type(x)):
+                self.fetch_ingredient(x)
+
+    @dispatch(list)
+    def typed_fetch(self, seed_input: List):
+        self._fetch_list_like(seed_input)
+
+    @dispatch(tuple)
+    def typed_fetch(self, seed_input: Tuple):
+        self._fetch_list_like(seed_input)
+
+    # NOTE: disable set for now as Steven is too weak in Python (/s)
+    # @dispatch(set)
+    # def typed_fetch(self, seed_input: Set):
+    #     self._fetch_list_like(seed_input)
+
+    # Dict
+    @dispatch(dict)
+    def typed_fetch(self, seed_input: Dict):
+        self._fetch_list_like(seed_input.keys())
+        self._fetch_list_like(seed_input.values())
+
+    def generate(self, num: int):
+        start = time.time()
+        num_generated = 1
+        while len(self.new_inputs) < num and time.time() - start < self.timeout:
+            if num_generated % 1000 == 0:
+                print(
+                    f"generated {num_generated} already with {len(self.new_inputs)} new inputs ... "
+                )
+            new_input = self.seed_selection()
+            # Multi-step instead of single-step
+            for _ in range(random.randint(1, MAX_MULTI_STEP_SIZE)):
+                new_input = self.mutate(new_input)
+            num_generated += 1
+            if hash(str(new_input)) not in self.seed_hash:
+                if trusted_check_exec(self.contract, [new_input], self.entry_point):
+                    self.typed_fetch(new_input)
+                    self.seed_pool.append(new_input)
+                    self.new_inputs.append(new_input)
+                self.seed_hash.add(hash(str(new_input)))
+        return self.new_inputs[:num]

evalplus/build/lib/evalplus/gen/util/__init__.py

@@ -0,0 +1,40 @@
+import time
+from copy import deepcopy
+
+from evalplus.eval.utils import time_limit
+
+
+def trusted_exec(code, inputs, entry_point, record_time=False, output_not_none=False):
+    """Execute trusted code in place."""
+    exec_globals = {}
+    exec(code, exec_globals)
+    fn = exec_globals[entry_point]
+
+    rtime = []
+    ret = []
+    for inp in inputs:
+        inp = deepcopy(inp)
+        if record_time:
+            start = time.time()
+            ret.append(fn(*inp))
+            rtime.append(time.time() - start)
+        else:
+            ret.append(fn(*inp))
+
+    if output_not_none:
+        ret = [i is not None for i in ret]
+
+    if record_time:
+        return ret, rtime
+    else:
+        return ret
+
+
+def trusted_check_exec(code, inputs, entry_point):
+    """Check trusted_exec success."""
+    try:
+        with time_limit(seconds=1.0):
+            trusted_exec(code, inputs, entry_point)
+    except Exception:
+        return False
+    return True

evalplus/build/lib/evalplus/gen/util/anthropic_request.py

@@ -0,0 +1,47 @@
+import signal
+import time
+
+import anthropic
+from anthropic.types import Message
+
+
+def handler(signum, frame):
+    # swallow signum and frame
+    raise Exception("end of time")
+
+
+def make_auto_request(client: anthropic.Client, *args, **kwargs) -> Message:
+    ret = None
+    while ret is None:
+        try:
+            signal.signal(signal.SIGALRM, handler)
+            signal.alarm(100)
+            ret = client.messages.create(*args, top_p=0.95, **kwargs)
+            signal.alarm(0)
+        except anthropic.RateLimitError:
+            print("Rate limit exceeded. Waiting...")
+            signal.alarm(0)
+            time.sleep(5)
+        except anthropic.APIConnectionError:
+            print("API connection error. Waiting...")
+            signal.alarm(0)
+            time.sleep(5)
+        except anthropic.InternalServerError:
+            print("Internal server error. Waiting...")
+            signal.alarm(0)
+            time.sleep(5)
+        except anthropic.APIError as e:
+            print("Unknown API error")
+            print(e)
+            if (
+                e.body["error"]["message"]
+                == "Output blocked by content filtering policy"
+            ):
+                raise Exception("Content filtering policy blocked output")
+            signal.alarm(0)
+        except Exception as e:
+            print("Unknown error. Waiting...")
+            print(e)
+            signal.alarm(0)
+            time.sleep(1)
+    return ret

evalplus/build/lib/evalplus/gen/util/openai_request.py

@@ -0,0 +1,51 @@
+import time
+
+import openai
+from openai.types.chat import ChatCompletion
+
+
+def make_request(
+    client: openai.Client,
+    message: str,
+    model: str,
+    max_tokens: int = 512,
+    temperature: float = 1,
+    n: int = 1,
+    **kwargs
+) -> ChatCompletion:
+    kwargs["top_p"] = 0.95
+    kwargs["max_completion_tokens"] = max_tokens
+    if model.startswith("o1-"):  # pop top-p and max_completion_tokens
+        kwargs.pop("top_p")
+        kwargs.pop("max_completion_tokens")
+        temperature = 1.0  # o1 models do not support temperature
+
+    return client.chat.completions.create(
+        model=model,
+        messages=[
+            {"role": "user", "content": message},
+        ],
+        temperature=temperature,
+        n=n,
+        **kwargs
+    )
+
+
+def make_auto_request(*args, **kwargs) -> ChatCompletion:
+    ret = None
+    while ret is None:
+        try:
+            ret = make_request(*args, **kwargs)
+        except openai.RateLimitError:
+            print("Rate limit exceeded. Waiting...")
+            time.sleep(5)
+        except openai.APIConnectionError:
+            print("API connection error. Waiting...")
+            time.sleep(5)
+        except openai.APIError as e:
+            print(e)
+        except Exception as e:
+            print("Unknown error. Waiting...")
+            print(e)
+            time.sleep(1)
+    return ret

evalplus/build/lib/evalplus/inputgen.py

@@ -0,0 +1,108 @@
+"""Generate a .jsonl file where each line is a json object
+representing a programming problem with a task ID ("task_id")
+and a list of enhanced inputs ("inputs") for that task.
+"""
+
+import argparse
+import json
+import os
+
+from evalplus.data.mbpp import mbpp_serialize_inputs
+from evalplus.gen.chatgpt_gen import ChatGPTGen
+from evalplus.gen.type_mut import TypedMutGen
+
+
+class SetEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, set):
+            return list(obj)
+        return json.JSONEncoder.default(self, obj)
+
+
+# Used for MBPP as MBPP's prompt is not a formal function signature
+def insert_contract_into_code(entry_point, code, contract):
+    lines = code.split("\n")
+    index = lines.index(
+        next(line for line in lines if line.startswith(f"def {entry_point}"))
+    )
+    lines.insert(index + 1, contract)
+    return "\n".join(lines)
+
+
+def input_generation(args, problems):
+    with open(args.output, "w") as file:
+        for problem in problems.values():
+            new_input = {}
+            task_id = problem["task_id"]
+            print(f"generating inputs for {task_id} ...")
+            # by default we do not include constraints in the prompt (code)
+            code = problem["prompt"] + problem["canonical_solution"]
+            # but we use c_code to include contract which checks input validity at execution time
+            if args.dataset == "humaneval":
+                c_code = (
+                    problem["prompt"]
+                    + problem["contract"]
+                    + problem["canonical_solution"]
+                )
+            elif args.dataset == "mbpp":
+                c_code = problem["prompt"] + insert_contract_into_code(
+                    entry_point=problem["entry_point"],
+                    code=problem["canonical_solution"],
+                    contract=problem["contract"],
+                )
+
+            # first generate chatgpt
+            input_gen = ChatGPTGen(
+                problem["base_input"], problem["entry_point"], c_code, code
+            ).generate(args.chatgpt_len)
+            # generate mutation next
+
+            if input_gen is None or len(input_gen) == 0:
+                new_input["task_id"] = task_id
+                new_input["inputs"] = {}
+                file.write(json.dumps(new_input, cls=SetEncoder) + "\n")
+                continue
+
+            input_gen.extend(
+                TypedMutGen(input_gen, problem["entry_point"], c_code).generate(
+                    args.mut_len
+                )
+            )
+            print(f"generated {len(input_gen)} inputs")
+            new_input["task_id"] = task_id
+            if args.dataset == "mbpp":
+                new_input["inputs"] = mbpp_serialize_inputs(task_id, input_gen)
+            new_input["inputs"] = input_gen
+            file.write(json.dumps(new_input, cls=SetEncoder) + "\n")
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--dataset", required=True, type=str, choices=["humaneval", "mbpp"]
+    )
+    parser.add_argument("--chatgpt_len", required=True, type=int)
+    parser.add_argument("--mut_len", required=True, type=int)
+    parser.add_argument("--output", type=str, help="Output .jsonl path")
+    args = parser.parse_args()
+
+    problems = None
+    if args.dataset == "humaneval":
+        from evalplus.data import get_human_eval_plus
+
+        # Allow it to be incomplete
+        problems = get_human_eval_plus(err_incomplete=False)
+        args.output = args.output or "HumanEvalPlusInputs.jsonl"
+
+    if args.dataset == "mbpp":
+        from evalplus.data import get_mbpp_plus
+
+        problems = get_mbpp_plus(err_incomplete=False)
+        args.output = args.output or "MbppPlusInput.jsonl"
+
+    assert not os.path.isfile(args.output), f"{args.output} already exists!"
+    input_generation(args, problems)
+
+
+if __name__ == "__main__":
+    main()

evalplus/build/lib/evalplus/lecacy_sanitize.py

@@ -0,0 +1,201 @@
+"""Legacy version of post-processing LLM-generated Python code.
+This sanitizer is implemented using regex and string manipulation.
+You might want to use the latest tree-sitter-based sanitizer (evalplus.sanitize) instead.
+"""
+
+import os
+import pathlib
+import re
+from typing import List, Optional
+
+from tqdm import tqdm
+
+from evalplus.data import (
+    get_human_eval_plus,
+    get_mbpp_plus,
+    load_solutions,
+    write_directory,
+    write_jsonl,
+)
+from evalplus.syncheck import syntax_check
+
+
+def remove_unindented_lines(
+    code: str, protect_before: str, execeptions: List[str], trim_tails: List[str]
+) -> str:
+    lines = code.splitlines()
+    cut_idx = []
+    cut_enabled = False
+    for i, line in enumerate(lines):
+        if not cut_enabled and line.startswith(protect_before):
+            cut_enabled = True
+            continue
+        if line.strip() == "":
+            continue
+        if any(line.startswith(e) for e in execeptions):
+            continue
+
+        lspace = len(line) - len(line.lstrip())
+        if lspace == 0:
+            cut_idx.append(i)
+
+        if any(line.rstrip().startswith(t) for t in trim_tails):
+            # cut off everything behind
+            cut_idx.extend(list(range(i, len(lines))))
+            break
+
+    return "\n".join([line for i, line in enumerate(lines) if i not in cut_idx])
+
+
+def to_four_space_indents(old_code):
+    new_code = ""
+    for line in old_code.splitlines():
+        lspace = len(line) - len(line.lstrip())
+        if lspace == 3:
+            new_code += " "
+        new_code += line + "\n"
+    return new_code
+
+
+def sanitize(
+    old_code: str,
+    entry_point: str,
+    rm_prefix_lines: Optional[str] = None,
+    eofs: List = None,
+):
+    new_code = old_code
+    if rm_prefix_lines is not None:
+        new_code = "\n".join(
+            [
+                line
+                for line in old_code.splitlines()
+                if not line.startswith(rm_prefix_lines)
+            ]
+        )
+
+    new_code = "\n" + new_code
+    def_left = "def " + entry_point
+
+    # basic handling of chat output
+    new_code = new_code.replace("\n```python\n", "\n```\n")
+    for chunk in new_code.split("\n```\n"):
+        if def_left in chunk:
+            new_code = chunk
+            break
+
+    chunks = [chunk for chunk in re.split(f"{def_left}\\s*\\(", new_code)]
+    # TODO: having return does not mean this is complete
+    bodies = [chunk for chunk in chunks[1:] if "    return " in chunk.split("\ndef")[0]]
+    def_left = def_left + "("
+    new_code = def_left + def_left.join(bodies) if len(bodies) > 0 else ""  # fn + impl
+    new_code = to_four_space_indents(new_code)
+
+    for eof in eofs or []:
+        new_code = new_code.split(eof)[0]
+
+    # remove lines starting from the first unindented line after def_left
+    new_code = remove_unindented_lines(
+        new_code,
+        protect_before=def_left,
+        execeptions=["def ", "import ", "from "],
+        trim_tails=['"""', "if", "print"],
+    )
+    new_code = chunks[0] + new_code
+
+    # cut all functions that are not syntactically correct && not the entry point
+    parts = new_code.split("\ndef ")
+    includes = [parts[0]]
+    for fn in new_code.split("\ndef ")[1:]:
+        if (
+            fn.strip().startswith(entry_point + " ")
+            or fn.strip().startswith(entry_point + "(")
+            or syntax_check("\ndef " + fn)
+        ):
+            includes.append(fn)
+    new_code = "\ndef ".join(includes)
+    return new_code.strip()
+
+
+def script(
+    samples: str,
+    eofs: List[str] = [],
+    inplace: bool = False,
+    rm_prefix_lines: str = None,
+    debug_task: str = None,
+    mbpp_version: str = "default",
+):
+    # task_id -> entry_point
+    entry_point = {}
+    dataset = {**get_human_eval_plus(), **get_mbpp_plus(version=mbpp_version)}
+
+    for task_id, problem in dataset.items():
+        entry_point[task_id] = problem["entry_point"]
+
+    # make a new folder with "-sanitized" suffix
+    is_folder = os.path.isdir(samples)
+    target_path = pathlib.Path(samples)
+    if not inplace:
+        if is_folder:
+            new_name = target_path.name + "-sanitized"
+        else:
+            new_name = target_path.name.replace(".jsonl", "-sanitized.jsonl")
+        target_path = target_path.parent / new_name
+    target_path = str(target_path)
+
+    nsan = 0
+    ntotal = 0
+
+    new_solutions = []
+
+    for solution in tqdm(load_solutions(samples)):
+        task_id = solution["task_id"]
+        dbg_identifier = solution["_identifier"]
+        if debug_task is not None and task_id != debug_task:
+            continue
+
+        ntotal += 1
+        if "solution" in solution:
+            old_code = solution["solution"]
+        else:
+            assert "completion" in solution
+            old_code = dataset[task_id]["prompt"] + "\n" + solution["completion"]
+
+        old_code = old_code.strip()
+
+        new_code = sanitize(
+            old_code=old_code,
+            entry_point=entry_point[task_id],
+            rm_prefix_lines=rm_prefix_lines,
+            eofs=eofs,
+        ).strip()
+
+        # if changed, print the message
+        if new_code != old_code:
+            msg = "Sanitized: " + dbg_identifier
+            if is_folder:
+                msg += " -> " + dbg_identifier.replace(samples, target_path)
+            print(msg)
+            nsan += 1
+
+        new_solutions.append({"task_id": task_id, "solution": new_code})
+
+    if is_folder:
+        write_directory(target_path, new_solutions)
+    else:
+        write_jsonl(target_path, new_solutions)
+
+    if nsan > 0:
+        print(f"Sanitized {nsan} out of {ntotal} files.")
+    else:
+        print(f"All files seems valid -- no files are sanitized.")
+    print(f"Check the sanitized files at {target_path}")
+
+
+def main():
+    from fire import Fire
+
+    Fire(script)
+
+
+if __name__ == "__main__":
+    main()