app create

app.py

@@ -1,13 +1,56 @@
 import gradio as gr
+from selfinstruct.instance import instance_main
+import openai
 from langchain.llms.openai import OpenAI
-
 import os
-os.environ["OPENAI_API_KEY"] = "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"
+import json
+# os.environ["OPENAI_API_KEY"] = "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"
+
+def OpenaiChat(key, input, temperature):
+    print(input)
+    openai.api_key = "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" if not key else key
+    # llm = OpenAI(temperature=0.9)
+    # return llm(input)
+    MODEL = "gpt-3.5-turbo"
+    response = openai.ChatCompletion.create(
+        model=MODEL,
+        messages=[
+            {"role": "user", "content": input}
+        ],
+        temperature=temperature
+    )
+    return json.dumps(response["choices"], ensure_ascii=False)
+
+
+with gr.Blocks() as demo:
+    gr.Markdown("# Self-instruct Prompt")
+    with gr.Row():
+        key = gr.Textbox(label="Input OpenAI key", type="password")
+    with gr.Tab("Generate instance"):
+        with gr.Row():
+            with gr.Column():
+                gr.Dropdown(["davinci"], value=["davinci"], multiselect=False, label="Model")
+                text_input = gr.TextArea(lines=3, label="Input instruction:", info="格式：一行一个instruction")
+            with gr.Column():
+                text_output = gr.JSON(lines=10, label="Output instance:")
+        text_button = gr.Button("Generate")
+        gr.Markdown("## 3 Steps to generate")
+        with gr.Row():
+            text_step1 = gr.TextArea(lines=3, label="Step1 identify if classifacrion task:", )
+            text_step2 = gr.TextArea(lines=3, label="Step2 generate instance:", )
+            text_step3 = gr.TextArea(lines=3, label="Step3 Prepare for finetune:", )
+
+    with gr.Tab("Simple ChatGPT"):
+        gr.Dropdown(["GPT3.5"], value=["GPT3.5"], multiselect=False, label="Model")
+        temperature = gr.Slider(0, 1.0, value=0.5, label="Temperature", info="Choose betwen 0 and 1")
+        ChatGPT_input = gr.TextArea(lines=3, label="Input:")
+        ChatGPT_output = gr.TextArea(lines=3, label="Output:")
+        ChatGPT_button = gr.Button("Run")
 
-llm = OpenAI(temperature=0.9)
+    # with gr.Accordion("Open for More!"):
+    #     gr.Markdown("Look at me...")
 
-def openai(input):
-    return llm(input)
+    text_button.click(instance_main, inputs=[text_input, key], outputs=[text_output, text_step1, text_step2, text_step3])
+    ChatGPT_button.click(OpenaiChat, inputs=[key, ChatGPT_input, temperature], outputs=ChatGPT_output)
 
-iface = gr.Interface(fn=openai, inputs="textbox", outputs="textbox")
-iface.launch()
+demo.launch()

selfinstruct/gpt3_api.py

@@ -0,0 +1,193 @@
+import json
+import tqdm
+import os
+import random
+import openai
+from datetime import datetime
+import argparse
+import time
+    
+
+def make_requests(
+        engine, prompts, max_tokens, temperature, top_p, 
+        frequency_penalty, presence_penalty, stop_sequences, logprobs, n, best_of, retries=3, api_key=None, organization=None
+    ):
+    response = None
+    target_length = max_tokens
+    if api_key is not None:
+        openai.api_key = api_key
+    if organization is not None:
+        openai.organization = organization
+    retry_cnt = 0
+    backoff_time = 30
+    while retry_cnt <= retries:
+        try:
+            response = openai.Completion.create(
+                engine=engine,
+                prompt=prompts,
+                max_tokens=target_length,
+                temperature=temperature,
+                top_p=top_p,
+                frequency_penalty=frequency_penalty,
+                presence_penalty=presence_penalty,
+                stop=stop_sequences,
+                logprobs=logprobs,
+                n=n,
+                best_of=best_of,
+            )
+            break
+        except openai.error.OpenAIError as e:
+            print(f"OpenAIError: {e}.")
+            if "Please reduce your prompt" in str(e):
+                target_length = int(target_length * 0.8)
+                print(f"Reducing target length to {target_length}, retrying...")
+            else:
+                print(f"Retrying in {backoff_time} seconds...")
+                time.sleep(backoff_time)
+                backoff_time *= 1.5
+            retry_cnt += 1
+    
+    if isinstance(prompts, list):
+        results = []
+        for j, prompt in enumerate(prompts):
+            data = {
+                "prompt": prompt,
+                "response": {"choices": response["choices"][j * n: (j + 1) * n]} if response else None,
+                "created_at": str(datetime.now()),
+            }
+            results.append(data)
+        return results
+    else:
+        data = {
+            "prompt": prompts,
+            "response": response,
+            "created_at": str(datetime.now()),
+        }
+        return [data]
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--input_file",
+        type=str,
+        help="The input file that contains the prompts to GPT3.",
+    )
+    parser.add_argument(
+        "--output_file",
+        type=str,
+        help="The output file to save the responses from GPT3.",
+    )
+    parser.add_argument(
+        "--engine",
+        type=str,
+        help="The openai GPT3 engine to use.",
+    )
+    parser.add_argument(
+        "--max_tokens",
+        default=500,
+        type=int,
+        help="The max_tokens parameter of GPT3.",
+    )
+    parser.add_argument(
+        "--temperature",
+        default=0.7,
+        type=float,
+        help="The temprature of GPT3.",
+    )
+    parser.add_argument(
+        "--top_p",
+        default=0.5,
+        type=float,
+        help="The `top_p` parameter of GPT3.",
+    )
+    parser.add_argument(
+        "--frequency_penalty",
+        default=0,
+        type=float,
+        help="The `frequency_penalty` parameter of GPT3.",
+    )
+    parser.add_argument(
+        "--presence_penalty",
+        default=0,
+        type=float,
+        help="The `presence_penalty` parameter of GPT3.",
+    )
+    parser.add_argument(
+        "--stop_sequences",
+        default=["\n\n"],
+        nargs="+",
+        help="The `stop_sequences` parameter of GPT3.",
+    )
+    parser.add_argument(
+        "--logprobs",
+        default=5,
+        type=int,
+        help="The `logprobs` parameter of GPT3"
+    )
+    parser.add_argument(
+        "--n",
+        type=int,
+        help="The `n` parameter of GPT3. The number of responses to generate."
+    )
+    parser.add_argument(
+        "--best_of",
+        type=int,
+        help="The `best_of` parameter of GPT3. The beam size on the GPT3 server."
+    )
+    parser.add_argument(
+        "--use_existing_responses",
+        action="store_true",
+        help="Whether to use existing responses from the output file if it exists."
+    )
+    parser.add_argument(
+        "--request_batch_size",
+        default=20,
+        type=int,
+        help="The number of requests to send to GPT3 at a time."
+    )
+    return parser.parse_args()
+
+    
+if __name__ == "__main__":
+    random.seed(123)
+    args = parse_args()
+    os.makedirs(os.path.dirname(args.output_file), exist_ok=True)
+
+    # read existing file if it exists
+    existing_responses = {}
+    if os.path.exists(args.output_file) and args.use_existing_responses:
+        with open(args.output_file, "r") as fin:
+            for line in fin:
+                data = json.loads(line)
+                existing_responses[data["prompt"]] = data
+
+    # do new prompts
+    with open(args.input_file, "r") as fin:
+        if args.input_file.endswith(".jsonl"):
+            all_prompts = [json.loads(line)["prompt"] for line in fin]
+        else:
+            all_prompt = [line.strip().replace("\\n", "\n") for line in fin]
+
+    with open(args.output_file, "w") as fout:
+        for i in tqdm.tqdm(range(0, len(all_prompts), args.request_batch_size)):
+            batch_prompts = all_prompts[i: i + args.request_batch_size]
+            if all(p in existing_responses for p in batch_prompts):
+                for p in batch_prompts:
+                    fout.write(json.dumps(existing_responses[p]) + "\n")
+            else:
+                results = make_requests(
+                    engine=args.engine,
+                    prompts=batch_prompts,
+                    max_tokens=args.max_tokens,
+                    temperature=args.temperature,
+                    top_p=args.top_p,
+                    frequency_penalty=args.frequency_penalty,
+                    presence_penalty=args.presence_penalty,
+                    stop_sequences=args.stop_sequences,
+                    logprobs=args.logprobs,
+                    n=args.n,
+                    best_of=args.best_of,
+                )
+                for data in results:
+                    fout.write(json.dumps(data) + "\n")

selfinstruct/identify_clf_or_not.py

@@ -0,0 +1,131 @@
+import os
+import json
+import random
+import tqdm
+import re
+import argparse
+import pandas as pd
+from collections import OrderedDict
+from gpt3_api import make_requests as make_gpt3_requests
+from templates.clf_task_template import template_1
+
+
+random.seed(42)
+
+
+templates = {
+    "template_1": template_1
+}
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    # parser.add_argument("--template", type=str, default="template_1", help="Which template to use.")
+    parser.add_argument(
+        "--batch_dir",
+        type=str,
+        required=True,
+        help="The directory where the batch is stored.",
+    )
+    parser.add_argument(
+        "--num_instructions",
+        type=int,
+        help="if specified, only generate instance input for this many instructions",
+    )
+    parser.add_argument(
+        "--template", 
+        type=str, 
+        default="template_1", 
+        help="Which template to use. Currently only `template_1` is supported.",
+    )
+    parser.add_argument(
+        "--engine",
+        type=str,
+        default="davinci",
+        help="The engine to use."
+    )
+    parser.add_argument(
+        "--request_batch_size",
+        type=int,
+        default=5,
+        help="The number of requests to send in a batch."
+    )
+    parser.add_argument(
+        "--api_key",
+        type=str,
+        help="The API key to use. If not specified, the key will be read from the environment variable `OPENAI_API_KEY`."
+    )
+    parser.add_argument(
+        "--organization",
+        type=str,
+        help="The organization to use. If not specified, the default organization id will be used."
+    )
+    return parser.parse_args()
+
+
+if __name__ == '__main__':
+    args = parse_args()
+
+    with open(os.path.join(args.batch_dir, "machine_generated_instructions.jsonl")) as fin:
+        lines = fin.readlines()
+        if args.num_instructions is not None:
+            lines = lines[:args.num_instructions]
+
+    output_path = os.path.join(args.batch_dir, f"is_clf_or_not_{args.engine}_{args.template}.jsonl")
+    existing_requests = {}
+    if os.path.exists(output_path):
+        with open(output_path) as fin:
+            for line in tqdm.tqdm(fin):
+                try:
+                    data = json.loads(line)
+                    existing_requests[data["instruction"]] = data
+                except:
+                    pass
+        print(f"Loaded {len(existing_requests)} existing requests")
+
+    progress_bar = tqdm.tqdm(total=len(lines))
+    with open(output_path, "w") as fout:
+        for batch_idx in range(0, len(lines), args.request_batch_size):
+            batch = [json.loads(line) for line in lines[batch_idx: batch_idx + args.request_batch_size]]
+            if all(d["instruction"] in existing_requests for d in batch):
+                for d in batch:
+                    data = existing_requests[d["instruction"]]
+                    data = OrderedDict(
+                        (k, data[k]) for k in \
+                            ["instruction", "is_classification"]
+                        )
+                    fout.write(json.dumps(data, ensure_ascii=False) + "\n")
+            else:
+                # prefix = compose_prompt_prefix(human_written_tasks, batch[0]["instruction"], 8, 2)
+                prefix = templates[args.template]
+                print("***batch", batch)
+                prompts = [prefix + " " + d["instruction"].strip() + "\n" + "Is it classification?" for d in batch]
+                results = make_gpt3_requests(
+                    engine=args.engine,
+                    prompts=prompts,
+                    max_tokens=3,
+                    temperature=0,
+                    top_p=0,
+                    frequency_penalty=0,
+                    presence_penalty=0,
+                    stop_sequences=["\n", "Task"],
+                    logprobs=1,
+                    n=1,
+                    best_of=1,
+                    api_key=args.api_key,
+                    organization=args.organization)
+                for i in range(len(batch)):
+                    data = batch[i]
+                    if results[i]["response"] is not None:
+                        data["is_classification"] = results[i]["response"]["choices"][0]["text"]
+                    else:
+                        data["is_classification"] = ""
+                    data = {
+                        "instruction": data["instruction"],
+                        "is_classification": data["is_classification"]
+                    }
+                    data = OrderedDict(
+                        (k, data[k]) for k in \
+                            ["instruction", "is_classification"]
+                        )
+                    fout.write(json.dumps(data, ensure_ascii=False) + "\n")
+            progress_bar.update(len(batch))

selfinstruct/instance.py

@@ -0,0 +1,313 @@
+import os
+import json
+import random
+import tqdm
+import re
+import argparse
+import pandas as pd
+from collections import OrderedDict
+from .gpt3_api import make_requests as make_gpt3_requests
+from .templates.instance_gen_template import output_first_template_for_clf, input_first_template_for_gen
+from .templates.clf_task_template import template_1
+
+random.seed(42)
+
+engine = "davinci"
+global api_key
+api_key = "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"
+
+# def parse_args():
+#     parser = argparse.ArgumentParser()
+#     parser.add_argument(
+#         "--batch_dir",
+#         type=str,
+#         required=True,
+#         help="The directory where the batch is stored.",
+#     )
+#     parser.add_argument(
+#         "--input_file",
+#         type=str,
+#         default="machine_generated_instructions.jsonl"
+#     )
+#     parser.add_argument(
+#         "--output_file",
+#         type=str,
+#         default="machine_generated_instances.jsonl",
+#     )
+#     parser.add_argument(
+#         "--num_instructions",
+#         type=int,
+#         help="if specified, only generate instance input for this many instructions",
+#     )
+#     parser.add_argument(
+#         "--max_instances_to_generate",
+#         type=int,
+#         default=5,
+#         help="The max number of instances to generate for each instruction.",
+#     )
+#     parser.add_argument(
+#         "--generation_tasks_only",
+#         action="store_true",
+#         help="If specified, only do for generation tasks.",
+#     )
+#     parser.add_argument(
+#         "--classification_tasks_only",
+#         action="store_true",
+#         help="If specified, only do for classification tasks.",
+#     )
+#     parser.add_argument(
+#         "--engine",
+#         type=str,
+#         default="davinci",
+#         help="The engine to use."
+#     )
+#     parser.add_argument(
+#         "--request_batch_size",
+#         type=int,
+#         default=5,
+#         help="The number of requests to send in a batch."
+#     )
+#     parser.add_argument(
+#         "--api_key",
+#         type=str,
+#         help="The API key to use. If not specified, the key will be read from the environment variable OPENAI_API_KEY."
+#     )
+#     parser.add_argument(
+#         "--organization",
+#         type=str,
+#         help="The organization to use. If not specified, the default organization id will be used."
+#     )
+#     return parser.parse_args()
+
+def if_classify(instructions):
+    prefix = template_1
+    prompts = [prefix + " " + instruct.strip() + "\n" + "Is it classification?" for instruct in instructions]
+    results = make_gpt3_requests(
+        engine=engine,
+        prompts=prompts,
+        max_tokens=3,
+        temperature=0,
+        top_p=0,
+        frequency_penalty=0,
+        presence_penalty=0,
+        stop_sequences=["\n", "Task"],
+        logprobs=1,
+        n=1,
+        best_of=1,
+        api_key=api_key)
+    classify_res = []
+    for i in range(len(prompts)):
+        if results[i]["response"] is not None:
+            if results[i]["response"]["choices"][0]["text"] in ["Yes", "yes", "YES"]:
+                classify_res.append(True)
+            else:
+                classify_res.append(False)
+        else:
+            print("**分类出错，", results[i])
+            classify_res.append("Unknown")
+    return classify_res
+
+def filter_duplicate_instances(instances):
+    # if the instances have same non-empty input, but different output, we will not use such instances
+    same_input_diff_output = False
+    for i in range(1, len(instances)):
+        for j in range(0, i):
+            if instances[i][1] == "":
+                continue
+            if instances[i][1] == instances[j][1] and instances[i][2] != instances[j][2]:
+                same_input_diff_output = True
+                break
+    if same_input_diff_output:
+        return []
+
+    # remove duplicate instances
+    instances = list(set(instances))
+    return instances
+
+def filter_invalid_instances(instances):
+    filtered_instances = []
+    for instance in instances:
+        # if input and output are the same, we will not use such instances
+        if instance[1] == instance[2]:
+            continue
+        # if output is empty, we will not use such instances
+        if instance[2] == "":
+            continue
+        # if input or output ends with a colon, these are usually imcomplete generation. We will not use such instances
+        if instance[1].strip().endswith(":") or instance[2].strip().endswith(":"):
+            continue
+        filtered_instances.append(instance)
+    return filtered_instances
+
+def encode_instance(instruction, input, output, random_template=True):
+    encoding_templates_w_input = [
+        ("{instruction}\nInput: {input}\nOutput:", " {output}<|endoftext|>"),
+        ("{instruction}\n\nInput: {input}\n\nOutput:", " {output}<|endoftext|>"),
+        ("Task: {instruction}\nInput: {input}\nOutput:", " {output}<|endoftext|>"),
+        ("{instruction}\n\n{input}\n\nOutput:", " {output}<|endoftext|>"),
+        ("{instruction}\n\n{input}\n\n", "{output}<|endoftext|>"),
+        ("{instruction}\n{input}\n\n", "{output}<|endoftext|>"),
+        ("Task: {instruction}\n\n{input}\n\n", "{output}<|endoftext|>"),
+    ]
+    encoding_templates_wo_input = [
+        ("{instruction} Output:", " {output}<|endoftext|>"),
+        ("{instruction}\nOutput:", " {output}<|endoftext|>"),
+        ("{instruction}\n\nOutput:", " {output}<|endoftext|>"),
+        ("{instruction}\n", "{output}<|endoftext|>"),
+        ("{instruction}\n\n", "{output}<|endoftext|>"),
+        ("Task: {instruction}\n\n", "{output}<|endoftext|>"),
+    ]
+    if random_template:
+        if input.strip() != "":
+            prompt_template, completion_template = random.choice(encoding_templates_w_input)
+            prompt = prompt_template.format(instruction=instruction.strip(), input=input.strip())
+            completion = completion_template.format(output=output.strip())
+        else:
+            prompt_template, completion_template = random.choice(encoding_templates_wo_input)
+            prompt = prompt_template.format(instruction=instruction.strip())
+            completion = completion_template.format(output=output.strip())
+    else:
+        prompt = instruction.strip() + "\n\n" + input.strip() + "\n\n"
+        completion = output.strip() + "<|endoftext|>"
+
+    data = {
+        "prompt": prompt,
+        "completion": completion,
+        "instruction": instruction.strip(),
+        "input": input.strip(),
+        "output": output.strip(),
+    }
+    return data
+
+def parse_input_output(response_text):
+    if re.findall(r"Output\s*\d*\s*:", response_text):
+        inst_input = re.split(r"Output\s*\d*\s*:", response_text)[0].strip()
+        inst_output = re.split(r"Output\s*\d*\s*:", response_text)[1].strip()
+    else:
+        inst_input = ""
+        inst_output = response_text.strip()
+    # to avoid the case multiple input/output pairs are generated
+    if re.findall(r"Input\s*\d*\s*:", inst_output):
+        inst_output = re.split(r"Input\s*\d*\s*:", inst_output)[0].strip()
+    # remove the prefix "Input:" from the string
+    inst_input = re.sub(r"^Input\s*\d*\s*:", "", inst_input).strip()
+    return inst_input, inst_output
+
+def parse_instances_for_generation_task(raw_text, instruction, response_metadata):
+    instances = []
+    raw_text = raw_text.strip()
+    if re.findall("Example\s?\d*\.?", raw_text):
+        instance_texts = re.split(r"Example\s?\d*\.?", raw_text)
+        instance_texts = [it.strip() for it in instance_texts if it.strip() != ""]
+        for instance_text in instance_texts:
+            inst_input, inst_output = parse_input_output(instance_text)
+            instances.append((instruction.strip(), inst_input.strip(), inst_output.strip()))
+    elif re.findall(r"Output\s*\d*\s*:", raw_text):
+        # we assume only one input/output pair in this case
+        inst_input, inst_output = parse_input_output(raw_text)
+        instances.append((instruction.strip(), inst_input.strip(), inst_output.strip()))
+    else:
+        return []
+    # if the generation stops because of length, we remove the last instance
+    if response_metadata["response"]["choices"][0]["finish_reason"] == "length":
+        instances = instances[:-1]
+
+    instances = filter_invalid_instances(instances)
+    instances = filter_duplicate_instances(instances)
+    return instances
+
+def parse_instances_for_classification_task(raw_text, instruction, response_metadata):
+    instances = []
+    if not "Class label:" in raw_text:
+        return []
+    instance_texts = raw_text.split("Class label:")[1:]
+    for instance_text in instance_texts:
+        instance_text = instance_text.strip()
+        fields = instance_text.split("\n", 1)
+        if len(fields) == 2:
+            # the first field split by \n is the class label
+            class_label = fields[0].strip()
+            # the rest is the input
+            input_text = fields[1].strip()
+        elif len(fields) == 1:
+            # the first field split by \n is the input
+            class_label = fields[0].strip()
+            input_text = ""
+        else:
+            raise ValueError("Invalid instance text: {}".format(instance_text))
+        instances.append((instruction.strip(), input_text.strip(), class_label.strip()))
+
+    # if the generation stops because of length, we remove the last instance
+    if response_metadata["response"]["choices"][0]["finish_reason"] == "length":
+        instances = instances[:-1]
+    instances = filter_invalid_instances(instances)
+    instances = filter_duplicate_instances(instances)
+    return instances
+
+def generate_instance(inputs):
+    classify_res = if_classify(inputs)
+    prompts = []
+    for i in range(len(inputs)):
+        if classify_res[i] in ["Yes", "yes", "YES"]:
+            prompt = output_first_template_for_clf + " " + inputs[i].strip() + "\n"
+            prompts.append(prompt)
+        else:
+            prompt = input_first_template_for_gen + " " + inputs[i].strip() + "\n"
+            prompts.append(prompt)
+    # print("prompts", prompts)
+    results = make_gpt3_requests(
+        engine=engine,
+        prompts=prompts,
+        # because the clf template is longer, we need to decrease the max_tokens
+        max_tokens=350,
+        temperature=0,
+        top_p=0,
+        frequency_penalty=0,
+        presence_penalty=1.5,
+        stop_sequences=["Task:"],
+        logprobs=1,
+        n=1,
+        best_of=1,
+        api_key=api_key)
+    return results, classify_res
+
+def prepare_finetune(inputs):
+    instance_outputs, classify_res = generate_instance(inputs)
+    training_instances = []
+    results1, results2 = [], []
+    for i in range(len(inputs)):
+        if classify_res[i]:
+            task_instances = parse_instances_for_classification_task(instance_outputs[i]["response"]["choices"][0]["text"],
+                                                                     inputs[i].strip(), instance_outputs[i])
+        else:
+            task_instances = parse_instances_for_generation_task(instance_outputs[i]["response"]["choices"][0]["text"],
+                                                                 inputs[i].strip(),  instance_outputs[i])
+        # we only allow max 5 instances per task
+        task_instances = random.sample(task_instances, min(len(task_instances), 5))
+
+        if not task_instances:
+            continue
+
+        training_instances += task_instances
+
+        for instance in training_instances:
+            results1.append({
+                "instruction": instance[0],
+                "input": instance[1],
+                "output": instance[2],
+            })
+            results2.append(json.dumps({
+                "instruction": instance[0],
+                "input": instance[1],
+                "output": instance[2],
+            }, ensure_ascii=False))
+    return results1, classify_res, instance_outputs, results2
+
+def instance_main(inputs, key):
+    if key:
+        api_key = key
+    inputs = inputs.split('\n')
+    print("***", inputs)
+    return prepare_finetune(inputs)
+
+# instance_main()

selfinstruct/prepare_for_finetuning.py

@@ -0,0 +1,304 @@
+import os
+import json
+import argparse
+import glob
+import re
+import random
+import tqdm
+import pandas as pd
+
+
+random.seed(123)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--instance_files",
+        nargs="+",
+        default=["data/batch_221203/machine_generated_instances.jsonl"],
+        type=str,
+        help="The input files that contain the machine generated instances."
+    )
+    parser.add_argument(
+        "--classification_type_files",
+        nargs="+",
+        default=["data/batch_221203/is_clf_or_not_davinci_template_1.jsonl"],
+    )
+    parser.add_argument(
+        "--output_dir",
+        default="data/gpt3_generations/batch_221203/finetuning/",
+        type=str,
+        help="The output dir to save the cleaned version of the generated instances, so that it can be used for GPT3 finetuning."
+    )
+    parser.add_argument(
+        "--num_instructions",
+        type=int,
+        help="The number of instructions to load."
+    )
+    parser.add_argument(
+        "--include_seed_tasks",
+        action="store_true",
+        help="Whether to include the seed human-written instances in the finetuning data."
+    )
+    parser.add_argument(
+        "--seed_tasks_path",
+        type=str,
+        required=True,
+        default="data/seed_tasks.jsonl",
+        help="The path to the seed data.",
+    )
+    return parser.parse_args()
+
+
+def encode_instance(instruction, input, output, random_template=True):
+    encoding_templates_w_input = [
+        ("{instruction}\nInput: {input}\nOutput:", " {output}<|endoftext|>"),
+        ("{instruction}\n\nInput: {input}\n\nOutput:", " {output}<|endoftext|>"),
+        ("Task: {instruction}\nInput: {input}\nOutput:", " {output}<|endoftext|>"),
+        ("{instruction}\n\n{input}\n\nOutput:", " {output}<|endoftext|>"),
+        ("{instruction}\n\n{input}\n\n", "{output}<|endoftext|>"),
+        ("{instruction}\n{input}\n\n", "{output}<|endoftext|>"),
+        ("Task: {instruction}\n\n{input}\n\n", "{output}<|endoftext|>"),
+    ]
+    encoding_templates_wo_input = [
+        ("{instruction} Output:", " {output}<|endoftext|>"),
+        ("{instruction}\nOutput:", " {output}<|endoftext|>"),
+        ("{instruction}\n\nOutput:", " {output}<|endoftext|>"),
+        ("{instruction}\n", "{output}<|endoftext|>"),
+        ("{instruction}\n\n", "{output}<|endoftext|>"),
+        ("Task: {instruction}\n\n", "{output}<|endoftext|>"),
+    ]
+    if random_template:
+        if input.strip() != "":
+            prompt_template, completion_template = random.choice(encoding_templates_w_input)
+            prompt = prompt_template.format(instruction=instruction.strip(), input=input.strip())
+            completion = completion_template.format(output=output.strip())
+        else:
+            prompt_template, completion_template = random.choice(encoding_templates_wo_input)
+            prompt = prompt_template.format(instruction=instruction.strip())
+            completion = completion_template.format(output=output.strip())
+    else:
+        prompt = instruction.strip() + "\n\n" + input.strip() + "\n\n"
+        completion = output.strip() + "<|endoftext|>"
+
+    data = {
+        "prompt": prompt,
+        "completion": completion,
+        "instruction": instruction.strip(),
+        "input": input.strip(),
+        "output": output.strip(),
+    }
+    return data
+
+
+def parse_input_output(response_text):
+    if re.findall(r"Output\s*\d*\s*:", response_text):
+        inst_input = re.split(r"Output\s*\d*\s*:", response_text)[0].strip()
+        inst_output = re.split(r"Output\s*\d*\s*:", response_text)[1].strip()
+    else:
+        inst_input = ""
+        inst_output = response_text.strip()
+    # to avoid the case multiple input/output pairs are generated
+    if re.findall(r"Input\s*\d*\s*:", inst_output):
+        inst_output = re.split(r"Input\s*\d*\s*:", inst_output)[0].strip()
+    # remove the prefix "Input:" from the string
+    inst_input = re.sub(r"^Input\s*\d*\s*:", "", inst_input).strip()
+    return inst_input, inst_output
+
+
+def filter_duplicate_instances(instances):
+    # if the instances have same non-empty input, but different output, we will not use such instances
+    same_input_diff_output = False
+    for i in range(1, len(instances)):
+        for j in range(0, i):
+            if instances[i][1] == "":
+                continue
+            if instances[i][1] == instances[j][1] and instances[i][2] != instances[j][2]:
+                same_input_diff_output = True
+                break
+    if same_input_diff_output:
+        return []
+
+    # remove duplicate instances
+    instances = list(set(instances))
+    return instances
+
+def filter_invalid_instances(instances):
+    filtered_instances = []
+    for instance in instances:
+        # if input and output are the same, we will not use such instances
+        if instance[1] == instance[2]:
+            continue
+        # if output is empty, we will not use such instances
+        if instance[2] == "":
+            continue
+        # if input or output ends with a colon, these are usually imcomplete generation. We will not use such instances
+        if instance[1].strip().endswith(":") or instance[2].strip().endswith(":"):
+            continue
+        filtered_instances.append(instance)
+    return filtered_instances
+
+def parse_instances_for_generation_task(raw_text, instruction, response_metadata):
+    instances = []
+    raw_text = raw_text.strip()
+    if re.findall("Example\s?\d*\.?", raw_text):
+        instance_texts = re.split(r"Example\s?\d*\.?", raw_text)
+        instance_texts = [it.strip() for it in instance_texts if it.strip() != ""]
+        for instance_text in instance_texts:
+            inst_input, inst_output = parse_input_output(instance_text)
+            instances.append((instruction.strip(), inst_input.strip(), inst_output.strip()))
+    elif re.findall(r"Output\s*\d*\s*:", raw_text):
+        # we assume only one input/output pair in this case
+        inst_input, inst_output = parse_input_output(raw_text)
+        instances.append((instruction.strip(), inst_input.strip(), inst_output.strip()))
+    else:
+        return []
+    # if the generation stops because of length, we remove the last instance
+    if response_metadata["response"]["choices"][0]["finish_reason"] == "length":
+        instances = instances[:-1]
+    
+    instances = filter_invalid_instances(instances)
+    instances = filter_duplicate_instances(instances)
+    return instances
+
+def parse_instances_for_classification_task(raw_text, instruction, response_metadata):
+    instances = []
+    if not "Class label:" in raw_text:
+        return []
+    instance_texts = raw_text.split("Class label:")[1:]
+    for instance_text in instance_texts:
+        instance_text = instance_text.strip()
+        fields = instance_text.split("\n", 1)
+        if len(fields) == 2:
+            # the first field split by \n is the class label
+            class_label = fields[0].strip()
+            # the rest is the input
+            input_text = fields[1].strip()
+        elif len(fields) == 1:
+            # the first field split by \n is the input
+            class_label = fields[0].strip()
+            input_text = ""
+        else:
+            raise ValueError("Invalid instance text: {}".format(instance_text))
+        instances.append((instruction.strip(), input_text.strip(), class_label.strip()))
+
+    # if the generation stops because of length, we remove the last instance
+    if response_metadata["response"]["choices"][0]["finish_reason"] == "length":
+        instances = instances[:-1]
+    instances = filter_invalid_instances(instances)
+    instances = filter_duplicate_instances(instances)
+    return instances
+
+
+if __name__ == "__main__":
+    args = parse_args()
+
+    training_instances = []
+    
+    generated_tasks = []
+    for instance_file in args.instance_files:
+        with open(instance_file) as fin:
+            for line in fin:
+                generated_tasks.append(json.loads(line))
+    print(f"Loaded {len(generated_tasks)} raw generated tasks")
+
+    task_clf_types = {}
+    for file in args.classification_type_files:
+        with open(file) as fin:
+            for line in fin:
+                data = json.loads(line)
+                task_clf_types[data["instruction"]] = data["is_classification"].strip() in ["Yes", "yes", "YES"]
+
+    for task in tqdm.tqdm(generated_tasks):
+        # get instruction
+        instruction = task["instruction"]
+        task["is_classification"] = task_clf_types[instruction]
+
+        # get the instances
+        if task["is_classification"]:
+            task_instances = parse_instances_for_classification_task(task["raw_instances"], instruction, task["instance_metadata"])
+        else:
+            task_instances = parse_instances_for_generation_task(task["raw_instances"], instruction, task["instance_metadata"])
+
+        # we only allow max 5 instances per task
+        task_instances = random.sample(task_instances, min(len(task_instances), 5))
+        
+        if not task_instances:
+            continue
+
+        training_instances += task_instances
+
+
+    os.makedirs(args.output_dir, exist_ok=True)
+    with open(os.path.join(args.output_dir, "all_generated_instances.jsonl"), "w") as fout:
+        for instance in training_instances:
+            fout.write(json.dumps({
+                "instruction": instance[0],
+                "input": instance[1],
+                "output": instance[2],
+            }) + "\n")
+    print(f"Saved {len(training_instances)} instances")
+    unique_instructions = set([it[0] for it in training_instances])
+    print(f"Unique instructions: {len(unique_instructions)}")
+    clf_instructions = [instruction for instruction in unique_instructions if task_clf_types[instruction]]
+    print(f"Classification instructions: {len(clf_instructions)}")
+    non_clf_instructions = [instruction for instruction in unique_instructions if not task_clf_types[instruction]]
+    print(f"Non-classification instructions: {len(non_clf_instructions)}")
+
+    if args.num_instructions is not None:
+        print(f"Sampling {args.num_instructions} instructions")
+        sampled_instructions = random.sample(unique_instructions, args.num_instructions)
+        training_instances = [it for it in training_instances if it[0] in sampled_instructions]
+        print(f"Only using {len(training_instances)} instances for these sampled instructions.")
+        with open(os.path.join(args.output_dir, f"sampled_generated_instances_{args.num_instructions}.jsonl"), "w") as fout:
+            for instance in training_instances:
+                fout.write(json.dumps({
+                    "instruction": instance[0],
+                    "input": instance[1],
+                    "output": instance[2],
+                }) + "\n")
+
+    if args.include_seed_tasks:
+        seed_tasks = [json.loads(l) for l in open(args.seed_tasks_path, "r")]
+        for task in seed_tasks:
+            for instance in task["instances"]:
+                training_instances.append((task["instruction"], instance["input"], instance["output"]))
+        print(f"Included {len(seed_tasks)} seed tasks")
+
+    # get the prompt and completion for training gpt3
+    gpt3_instances = []
+    for instance in training_instances:
+        # get input and do preprocessing
+        inst_input = instance[1]
+        # for some tasks, we check whether the input contains colon, and if so, we remove the part before the colon
+        if random.random() < 0.5:
+            colon_words = re.findall(r"(\w+):", inst_input)
+            # if only one colon is found, we assume the instance only have one input and we remove the field name before the colon
+            if len(set(colon_words)) == 1:
+                inst_input = inst_input.split(":", 1)[1].strip()
+            else:
+                inst_input = inst_input.strip()
+            # we also replace two consecutive new lines with one new line half of the time
+            inst_input = inst_input.replace("\n\n", "\n")
+        
+        gpt3_instances.append(encode_instance(instance[0], inst_input, instance[2]))
+
+    # remove duplicates
+    filtered_instances = []
+    prompt_completion_set = set()
+    for instance in gpt3_instances:
+        instance_pair = (instance["prompt"], instance["completion"])
+        if instance_pair not in prompt_completion_set:
+            prompt_completion_set.add((instance["prompt"], instance["completion"]))
+            filtered_instances.append(instance)
+    gpt3_instances = filtered_instances
+
+    # shuffle
+    random.shuffle(gpt3_instances)
+    with open(os.path.join(args.output_dir, f"gpt3_finetuning_data_{len(gpt3_instances)}.jsonl"), "w") as fout:
+        for instance in gpt3_instances:
+            fout.write(json.dumps({
+                "prompt": instance["prompt"],
+                "completion": instance["completion"],
+            }) + "\n")

selfinstruct/simple_instance.py

@@ -0,0 +1,170 @@
+import os
+import json
+import random
+import tqdm
+import re
+import argparse
+import pandas as pd
+from collections import OrderedDict
+from gpt3_api import make_requests as make_gpt3_requests
+from templates.instance_gen_template import output_first_template_for_clf, input_first_template_for_gen
+
+random.seed(42)
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--batch_dir",
+        type=str,
+        required=True,
+        help="The directory where the batch is stored.",
+    )
+    parser.add_argument(
+        "--input_file",
+        type=str,
+        default="machine_generated_instructions.jsonl"
+    )
+    parser.add_argument(
+        "--output_file",
+        type=str,
+        default="machine_generated_instances.jsonl",
+    )
+    parser.add_argument(
+        "--num_instructions",
+        type=int,
+        help="if specified, only generate instance input for this many instructions",
+    )
+    parser.add_argument(
+        "--max_instances_to_generate",
+        type=int,
+        default=5,
+        help="The max number of instances to generate for each instruction.",
+    )
+    parser.add_argument(
+        "--generation_tasks_only",
+        action="store_true",
+        help="If specified, only do for generation tasks.",
+    )
+    parser.add_argument(
+        "--classification_tasks_only",
+        action="store_true",
+        help="If specified, only do for classification tasks.",
+    )
+    parser.add_argument(
+        "--engine",
+        type=str,
+        default="davinci",
+        help="The engine to use."
+    )
+    parser.add_argument(
+        "--request_batch_size",
+        type=int,
+        default=5,
+        help="The number of requests to send in a batch."
+    )
+    parser.add_argument(
+        "--api_key",
+        type=str,
+        help="The API key to use. If not specified, the key will be read from the environment variable OPENAI_API_KEY."
+    )
+    parser.add_argument(
+        "--organization",
+        type=str,
+        help="The organization to use. If not specified, the default organization id will be used."
+    )
+    return parser.parse_args()
+
+
+if __name__ == '__main__':
+    args = parse_args()
+
+    with open(os.path.join(args.batch_dir, args.input_file)) as fin:
+        lines = fin.readlines()
+        if args.num_instructions is not None:
+            lines = lines[:args.num_instructions]
+        tasks = []
+        for line in lines:
+            data = json.loads(line)
+            if "metadata" in data:
+                data["instruction_metadata"] = data["metadata"]
+                del data["metadata"]
+            tasks.append(data)
+
+    task_clf_types = {}
+    with open(os.path.join(args.batch_dir, "is_clf_or_not_davinci_template_1.jsonl")) as fin:
+        for line in fin:
+            data = json.loads(line)
+            task_clf_types[data["instruction"]] = data["is_classification"].strip() in ["Yes", "yes", "YES"]
+
+    if args.classification_tasks_only:
+        tasks = [task for task in tasks if task_clf_types[task["instruction"]]]
+
+    if args.generation_tasks_only:
+        tasks = [task for task in tasks if not task_clf_types[task["instruction"]]]
+
+    output_path = os.path.join(args.batch_dir, args.output_file)
+    existing_requests = {}
+    if os.path.exists(output_path):
+        with open(output_path) as fin:
+            for line in tqdm.tqdm(fin):
+                try:
+                    data = json.loads(line)
+                    existing_requests[data["instruction"]] = data
+                except:
+                    pass
+        print(f"Loaded {len(existing_requests)} existing requests")
+
+    progress_bar = tqdm.tqdm(total=len(tasks))
+    with open(output_path, "w") as fout:
+        for batch_idx in range(0, len(tasks), args.request_batch_size):
+            batch = tasks[batch_idx: batch_idx + args.request_batch_size]
+            if all(d["instruction"] in existing_requests for d in batch):
+                for d in batch:
+                    data = existing_requests[d["instruction"]]
+                    data = OrderedDict(
+                        (k, data[k]) for k in \
+                        ["instruction", "raw_instances", "instance_metadata"]
+                        # ["instruction", "raw_instances", "instance_metadata", "instruction_metadata",
+                        #  "most_similar", "avg_similarity_score"]
+                    )
+                    fout.write(json.dumps(data, ensure_ascii=False) + "\n")
+            else:
+                prompts = []
+                for task in batch:
+                    if task_clf_types[task["instruction"]]:
+                        prompt = output_first_template_for_clf + " " + task["instruction"].strip() + "\n"
+                        prompts.append(prompt)
+                    else:
+                        prompt = input_first_template_for_gen + " " + task["instruction"].strip() + "\n"
+                        prompts.append(prompt)
+                print("prompts", prompts)
+                results = make_gpt3_requests(
+                    engine=args.engine,
+                    prompts=prompts,
+                    # because the clf template is longer, we need to decrease the max_tokens
+                    max_tokens=300 if any(task_clf_types[task["instruction"]] for task in batch) else 350,
+                    temperature=0,
+                    top_p=0,
+                    frequency_penalty=0,
+                    presence_penalty=1.5,
+                    stop_sequences=[f"Example {args.max_instances_to_generate + 1}", "Task:"],
+                    logprobs=1,
+                    n=1,
+                    best_of=1,
+                    api_key=args.api_key,
+                    organization=args.organization)
+                for i in range(len(batch)):
+                    data = batch[i]
+                    data["instance_metadata"] = results[i]
+                    if results[i]["response"] is not None:
+                        data["raw_instances"] = results[i]["response"]["choices"][0]["text"]
+                    else:
+                        data["raw_instances"] = ""
+                    data = OrderedDict(
+                        (k, data[k]) for k in \
+                        ["instruction", "raw_instances", "instance_metadata"]
+                        # ["instruction", "raw_instances", "instance_metadata", "instruction_metadata",
+                        #  "most_similar", "avg_similarity_score"]
+                    )
+                    fout.write(json.dumps(data, ensure_ascii=False) + "\n")
+            progress_bar.update(len(batch))

selfinstruct/templates/clf_task_template.py

@@ -0,0 +1,96 @@
+template_1 = '''Can the following task be regarded as a classification task with finite output labels?
+
+Task: Given my personality and the job, tell me if I would be suitable.
+Is it classification? Yes
+
+Task: Give me an example of a time when you had to use your sense of humor.
+Is it classification? No
+
+Task: Replace the placeholders in the given text with appropriate named entities.
+Is it classification? No
+
+Task: Fact checking - tell me if the statement is true, false, or unknown, based on your knowledge and common sense.
+Is it classification? Yes
+
+Task: Return the SSN number for the person.
+Is it classification? No
+
+Task: Detect if the Reddit thread contains hate speech.
+Is it classification? Yes
+
+Task: Analyze the sentences below to identify biases.
+Is it classification? No
+
+Task: Select the longest sentence in terms of the number of words in the paragraph, output the sentence index.
+Is it classification? Yes
+
+Task: Find out the toxic word or phrase in the sentence.
+Is it classification? No
+
+Task: Rank these countries by their population.
+Is it classification? No
+
+Task: You are provided with a news article, and you need to identify all the categories that this article belongs to. Possible categories include: Music, Sports, Politics, Tech, Finance, Basketball, Soccer, Tennis, Entertainment, Digital Game, World News. Output its categories one by one, seperated by comma.
+Is it classification? Yes
+
+Task: Given the name of an exercise, explain how to do it.
+Is it classification? No
+
+Task: Select the oldest person from the list.
+Is it classification? Yes
+
+Task: Find the four smallest perfect numbers.
+Is it classification? No
+
+Task: Does the information in the document supports the claim? You can answer "Support" or "Unsupport".
+Is it classification? Yes
+
+Task: Create a detailed budget for the given hypothetical trip.
+Is it classification? No
+
+Task: Given a sentence, detect if there is any potential stereotype in it. If so, you should explain the stereotype. Else, output no.
+Is it classification? No
+
+Task: Explain the following idiom to me, and try to give me some examples.
+Is it classification? No
+
+Task: Is there anything I can eat for a breakfast that doesn't include eggs, yet includes protein, and has roughly 700-1000 calories?
+Is it classification? No
+
+Task: Answer the following multiple choice question. Select A, B, C, or D for the final answer.
+Is it classification? Yes
+
+Task: Decide whether the syllogism is logically sound.
+Is it classification? Yes
+
+Task: How can individuals and organizations reduce unconscious bias?
+Is it classification? No
+
+Task: What are some things you can do to de-stress?
+Is it classification? No
+
+Task: Find out the largest one from a set of numbers. Output the number directly.
+Is it classification? Yes
+
+Task: Replace the <mask> token in the text with proper words that are consistent with the context. You can use multiple words for each <mask> token.
+Is it classification? No
+
+Task: Write a cover letter based on the given facts.
+Is it classification? No
+
+Task: Identify the pos tag of the word in the given sentence.
+Is it classification? Yes
+
+Task: Write a program to compute the sum of integers from k to n.
+Is it classification? No
+
+Task: In this task, you need to compare the meaning of the two sentences and tell if they are the same. Output yes or no.
+Is it classification? Yes
+
+Task: To make the pairs have the same analogy, write the fourth word.
+Is it classification? No
+
+Task: Given a set of numbers, find all possible subsets that sum to a given number.
+Is it classification? No
+
+Task:'''

selfinstruct/templates/instance_gen_template.py

@@ -0,0 +1,237 @@
+output_first_template_for_clf = '''Given the classification task definition and the class labels, generate an input that corresponds to each of the class labels. If the task doesn't require input, just generate possible class labels.
+
+Task: Classify the sentiment of the sentence into positive, negative, or mixed.
+Class label: mixed
+Sentence: I enjoy the flavor of the restaurant but their service is too slow.
+Class label: Positive
+Sentence: I had a great day today. The weather was beautiful and I spent time with friends and family.
+Class label: Negative
+Sentence: I was really disappointed by the latest superhero movie. I would not recommend it to anyone.
+
+Task: Given a dialogue, classify whether the user is satisfied with the service. You should respond with "Satisfied" or "Unsatisfied".
+Class label: Satisfied
+Dialogue:
+- Agent: Thank you for your feedback. We will work to improve our service in the future.
+- Customer: I am happy with the service you provided. Thank you for your help.
+Class label: Unsatisfied
+Dialogue:
+- Agent: I am sorry we will cancel that order for you, and you will get a refund within 7 business days.
+- Customer: oh that takes too long. I want you to take quicker action on this.
+
+Task: Given some political opinions, classify whether the person belongs to Democrats or Republicans.
+Class label: Democrats
+Opinion: I believe that everyone should have access to quality healthcare regardless of their income level.
+Class label: Republicans
+Opinion: I believe that people should be able to keep more of their hard-earned money and should not be taxed at high rates.
+
+Task: Tell me if the following email is a promotion email or not.
+Class label: Promotion
+Email: Check out our amazing new sale! We've got discounts on all of your favorite products.
+Class label: Not Promotion
+Email: We hope you are doing well. Let us know if you need any help.
+
+Task: Detect if the Reddit thread contains hate speech.
+Class label: Hate Speech
+Thread: All people of color are stupid and should not be allowed to vote.
+Class label: Not Hate Speech
+Thread: The best way to cook a steak on the grill.
+
+Task:  Does the information in the document supports the claim? You can answer "Support" or "Unsupport".
+Class label: Unsupport
+Document: After a record-breaking run that saw mortgage rates plunge to all-time lows and home prices soar to new highs, the U.S. housing market finally is slowing. While demand and price gains are cooling, any correction is likely to be a modest one, housing economists and analysts say. No one expects price drops on the scale of the declines experienced during the Great Recession.
+Claim: The US housing market is going to crash soon.
+Class label: Support
+Document: The U.S. housing market is showing signs of strain, with home sales and prices slowing in many areas. Mortgage rates have risen sharply in recent months, and the number of homes for sale is increasing. This could be the beginning of a larger downturn, with some economists predicting a potential housing crash in the near future.
+Claim: The US housing market is going to crash soon.
+
+Task: Answer the following multiple-choice question. Select A, B, C, or D for the final answer.
+Class label: C
+Question: What is the capital of Germany?
+A. London
+B. Paris
+C. Berlin
+D. Rome
+Class label: D
+Question: What is the largest planet in our solar system?
+A) Earth
+B) Saturn
+C) Mars
+D) Jupiter
+Class label: A
+Question: What is the process by which plants make their own food through photosynthesis?
+A) Respiration
+B) Fermentation
+C) Digestion
+D) Metabolism
+Class label: B
+Question: Who wrote the novel "The Great Gatsby"?
+A) Ernest Hemingway
+B) F. Scott Fitzgerald
+C) J.D. Salinger
+D) Mark Twain
+
+Task: You need to read a code and detect if there is a syntax error or not. Output true if there is an error, output false if there is not.
+Class label: true
+Code:
+def quick_sort(arr):
+    if len(arr) < 2
+        return arr
+Class label: False
+Code:
+def calculate_average(numbers):
+    total = 0
+    for number in numbers:
+        total += number
+    return total / len(numbers)
+
+Task: You are provided with a news article, and you need to identify all the categories that this article belongs to. Possible categories include Sports and Politics. Output its categories one by one, separated by a comma.
+Class label: Sports
+Article: The Golden State Warriors have won the NBA championship for the second year in a row.
+Class label: Politics
+Article: The United States has withdrawn from the Paris Climate Agreement.
+Class label: Politics, Sports
+Article: The government has proposed cutting funding for youth sports programs.
+
+Task: Given a credit card statement, the cardholder's spending habits, and the account balance, classify whether the cardholder is at risk of defaulting on their payments or not.
+Class label: At risk
+Credit card statement: Purchases at high-end clothing stores and luxury hotels.
+Cardholder's spending habits: Frequent purchases at luxury brands and high-end establishments.
+Account balance: Over the credit limit and multiple missed payments.
+Class label: Not at risk
+Credit card statement: Purchases at grocery stores and gas stations.
+Cardholder's spending habits: Regular purchases for necessary expenses and occasional dining out.
+Account balance: Slightly below the credit limit and no missed payments.
+
+Task: Given a social media post, the hashtags used, and a topic. classify whether the post is relevant to the topic or not.
+Class label: Relevant
+Post: I can't believe the government is still not taking action on climate change. It's time for us to take matters into our own hands.
+Hashtags: #climatechange #actnow
+Topic: Climate change
+Class label: Not relevant 
+Post: I just bought the new iPhone and it is amazing!
+Hashtags: #apple #technology
+Topic: Travel
+
+Task: The answer will be 'yes' if the provided sentence contains an explicit mention that answers the given question. Otherwise, answer 'no'. 
+Class label: Yes
+Sentence: Jack played basketball for an hour after school.
+Question: How long did Jack play basketball?
+Class label: No
+Sentence: The leaders of the Department of Homeland Security now appear before 88 committees and subcommittees of Congress.
+Question: How often are they required to appear?
+
+Task: Tell me what's the second largest city by population in Canada.
+Class label: Montreal
+
+Task: Classifying different types of mathematical equations, such as linear, and quadratic equations, based on the coefficients and terms in the equation.
+Class label: Linear equation
+Equation: y = 2x + 5
+Class label: Quadratic equation
+Equation: y = x^2 - 4x + 3
+
+Task: Tell me the first number of the given list.
+Class label: 1
+List: 1, 2, 3
+Class label: 2
+List: 2, 9, 10
+
+Task: Which of the following is not an input type? (a) number (b) date (c) phone number (d) email address (e) all of these are valid inputs.
+Class label: (e)
+
+Task:'''
+
+input_first_template_for_gen = '''Come up with examples for the following tasks. Try to generate multiple examples when possible. If the task doesn't require additional input, you can generate the output directly.
+
+Task: Which exercises are best for reducing belly fat at home?
+Output:
+- Lying Leg Raises
+- Leg In And Out
+- Plank
+- Side Plank
+- Sit-ups
+
+Task: Extract all the country names in the paragraph, list them separated by commas.
+Example 1
+Paragraph: Dr. No is the sixth novel by the English author Ian Fleming to feature his British Secret Service agent James Bond. Written at Fleming's Goldeneye estate in Jamaica, it was first published in the United Kingdom by Jonathan Cape in 1958. In the novel Bond looks into the disappearance in Jamaica of two fellow MI6 operatives who had been investigating Doctor No. Bond travels to No's Caribbean island and meets Honeychile Rider, who is there to collect shells. They are captured and taken to a luxurious facility carved into a mountain. The character of Doctor No, the son of a German missionary and a Chinese woman, was influenced by Sax Rohmer's Fu Manchu stories. Dr. No was the first of Fleming's novels to face widespread negative reviews in Britain, but it was received more favourably in the United States.
+Output: English, British, Jamaica, the United Kingdom, German, Chinese, Britain, the United States.
+
+Task: Converting 85 F to Celsius.
+Output: 85°F = 29.44°C
+
+Task: Sort the given list ascendingly. 
+Example 1
+List: [10, 92, 2, 5, -4, 92, 5, 101]
+Output: [-4, 2, 5, 5, 10, 92, 92, 101]
+Example 2
+Input 2 - List: [9.99, 10, -5, -1000, 5e6, 999]
+Output: [-1000, -5, 9.99, 10, 999, 5e6]
+
+Task: Suggest a better and more professional rephrasing of the following sentence.
+Example 1
+Sentence: This house is surprisingly not constructed very well, and you probably need more money to fix it after you buy it. If you ask me, I would suggest you to consider other candidates.
+Output: This house does not seem to be constructed well, so you may need to spend more money to fix it after you purchase it. I would suggest that you look at other properties.
+Example 2
+Sentence: Just so you know, we did an experiment last week and found really surprising results - language model can improve itself!
+Output: Our experiments last week demonstrated surprising results, proving that the language model can improve itself.
+
+Task: Read the following paragraph and answer a math question about the paragraph. You need to write out the calculation for getting the final answer.
+Example 1
+Paragraph: Gun violence in the United States results in tens of thousands of deaths and injuries annually, and was the leading cause of death for children 19 and younger in 2020. In 2018, the most recent year for which data are available as of 2021, the Centers for Disease Control and Prevention's (CDC) National Center for Health Statistics reports 38,390 deaths by firearm, of which 24,432 were by suicide. The rate of firearm deaths per 100,000 people rose from 10.3 per 100,000 in 1999 to 12 per 100,000 in 2017, with 109 people dying per day or about 14,542 homicides in total, being 11.9 per 100,000 in 2018. In 2010, there were 19,392 firearm-related suicides, and 11,078 firearm-related homicides in the U.S. In 2010, 358 murders were reported involving a rifle while 6,009 were reported involving a handgun; another 1,939 were reported with an unspecified type of firearm. In 2011, a total of 478,400 fatal and nonfatal violent crimes were committed with a firearm.
+Question: How many more firearm-related deaths were there in 2018 compared to 2010?
+Output:
+38390 - (19392 + 11078) = 38390 - 30470 = 7920. 
+So, in 2018, there were 7920 more deaths by firearm than in 2010.
+
+Task: Write Python code to solve this leetcode problem.
+Example 1
+Problem: You are given two non-empty linked lists representing two non-negative integers. The digits are stored in reverse order, and each of their nodes contains a single digit. Add the two numbers and return the sum as a linked list. You may assume the two numbers do not contain any leading zero, except the number 0 itself.
+Output:
+class Solution(object):
+    def addTwoNumbers(self, l1, l2):
+        carry = 0
+        root = n = ListNode(0)
+        while l1 or l2 or carry:
+            v1 = v2 = 0
+            if l1:
+                v1 = l1.val
+                l1 = l1.next
+            if l2:
+                v2 = l2.val
+                l2 = l2.next
+            carry, val = divmod(v1+v2+carry, 10)
+            n.next = ListNode(val)
+            n = n.next
+        return root.next
+
+Task: Solve the equation and find the value of X. Show your steps.
+Example 1
+Equation: 10X + 5 = 10
+Output: 10X = 5,  X = 0.5
+Example 2
+Equation: X + Y + 120 = 100
+Output: X + Y = -20, X = -20 - Y
+
+Task: Write a program to compute the sum of integers from k to n.
+Output:
+def sum(k, n):
+    sum = 0
+    for i in range(k, n+1):
+        sum += i
+    return sum
+
+Task: Select the oldest person from the given list.
+Example 1
+List: George Washington, Confucius, Michael Jordan, Michelangelo
+Output: Confucious
+Example 2
+List: Alan Turing, Geoffrey Hinton, Yann LeCun, Yoshua Bengio
+Output: Alan Turing
+
+Task: Turn down a job offer by sending an email to a recruiter explaining the reason.
+Output: Hi  [Recruiter],
+Thank you so much for the generous offer to join your team. As we discussed, I’ve admired the company for a number of years, and am a proud endorser of its products. However, after further consideration of where I currently am in my career, I’ve decided to accept an offer at another company.
+I would love to stay in touch with you and have already started following you on [Social Media Platform]. Again, thank you so much for your time and consideration.
+Thanks again,
+[Your Name]
+
+Task:'''