selfinstruct/instance.py

import os
import json
import random
import tqdm
import re
import argparse
import pandas as pd
from collections import OrderedDict

from openai import OpenAIError

from .gpt3_api import make_requests as make_gpt3_requests
from .templates.instance_gen_template import output_first_template_for_clf, input_first_template_for_gen
from .templates.clf_task_template import template_1

random.seed(42)

engine = "davinci"

# def parse_args():
#     parser = argparse.ArgumentParser()
#     parser.add_argument(
#         "--batch_dir",
#         type=str,
#         required=True,
#         help="The directory where the batch is stored.",
#     )
#     parser.add_argument(
#         "--input_file",
#         type=str,
#         default="machine_generated_instructions.jsonl"
#     )
#     parser.add_argument(
#         "--output_file",
#         type=str,
#         default="machine_generated_instances.jsonl",
#     )
#     parser.add_argument(
#         "--num_instructions",
#         type=int,
#         help="if specified, only generate instance input for this many instructions",
#     )
#     parser.add_argument(
#         "--max_instances_to_generate",
#         type=int,
#         default=5,
#         help="The max number of instances to generate for each instruction.",
#     )
#     parser.add_argument(
#         "--generation_tasks_only",
#         action="store_true",
#         help="If specified, only do for generation tasks.",
#     )
#     parser.add_argument(
#         "--classification_tasks_only",
#         action="store_true",
#         help="If specified, only do for classification tasks.",
#     )
#     parser.add_argument(
#         "--engine",
#         type=str,
#         default="davinci",
#         help="The engine to use."
#     )
#     parser.add_argument(
#         "--request_batch_size",
#         type=int,
#         default=5,
#         help="The number of requests to send in a batch."
#     )
#     parser.add_argument(
#         "--api_key",
#         type=str,
#         help="The API key to use. If not specified, the key will be read from the environment variable OPENAI_API_KEY."
#     )
#     parser.add_argument(
#         "--organization",
#         type=str,
#         help="The organization to use. If not specified, the default organization id will be used."
#     )
#     return parser.parse_args()

def if_classify(instructions, api_key):
    prefix = template_1
    prompts = [prefix + " " + instruct.strip() + "\n" + "Is it classification?" for instruct in instructions]
    results = make_gpt3_requests(
        engine=engine,
        prompts=prompts,
        max_tokens=3,
        temperature=0,
        top_p=0,
        frequency_penalty=0,
        presence_penalty=0,
        stop_sequences=["\n", "Task"],
        logprobs=1,
        n=1,
        best_of=1,
        api_key=api_key)
    classify_res = []
    for i in range(len(prompts)):
        if results[i]["response"] is not None:
            if results[i]["response"]["choices"][0]["text"] in ["Yes", "yes", "YES"]:
                classify_res.append(True)
            else:
                classify_res.append(False)
        else:
            print("**分类出错，", results[i])
            classify_res.append("Unknown")
    return classify_res

def filter_duplicate_instances(instances):
    # if the instances have same non-empty input, but different output, we will not use such instances
    same_input_diff_output = False
    for i in range(1, len(instances)):
        for j in range(0, i):
            if instances[i][1] == "":
                continue
            if instances[i][1] == instances[j][1] and instances[i][2] != instances[j][2]:
                same_input_diff_output = True
                break
    if same_input_diff_output:
        return []

    # remove duplicate instances
    instances = list(set(instances))
    return instances

def filter_invalid_instances(instances):
    filtered_instances = []
    for instance in instances:
        # if input and output are the same, we will not use such instances
        if instance[1] == instance[2]:
            continue
        # if output is empty, we will not use such instances
        if instance[2] == "":
            continue
        # if input or output ends with a colon, these are usually imcomplete generation. We will not use such instances
        if instance[1].strip().endswith(":") or instance[2].strip().endswith(":"):
            continue
        filtered_instances.append(instance)
    return filtered_instances

def encode_instance(instruction, input, output, random_template=True):
    encoding_templates_w_input = [
        ("{instruction}\nInput: {input}\nOutput:", " {output}<|endoftext|>"),
        ("{instruction}\n\nInput: {input}\n\nOutput:", " {output}<|endoftext|>"),
        ("Task: {instruction}\nInput: {input}\nOutput:", " {output}<|endoftext|>"),
        ("{instruction}\n\n{input}\n\nOutput:", " {output}<|endoftext|>"),
        ("{instruction}\n\n{input}\n\n", "{output}<|endoftext|>"),
        ("{instruction}\n{input}\n\n", "{output}<|endoftext|>"),
        ("Task: {instruction}\n\n{input}\n\n", "{output}<|endoftext|>"),
    ]
    encoding_templates_wo_input = [
        ("{instruction} Output:", " {output}<|endoftext|>"),
        ("{instruction}\nOutput:", " {output}<|endoftext|>"),
        ("{instruction}\n\nOutput:", " {output}<|endoftext|>"),
        ("{instruction}\n", "{output}<|endoftext|>"),
        ("{instruction}\n\n", "{output}<|endoftext|>"),
        ("Task: {instruction}\n\n", "{output}<|endoftext|>"),
    ]
    if random_template:
        if input.strip() != "":
            prompt_template, completion_template = random.choice(encoding_templates_w_input)
            prompt = prompt_template.format(instruction=instruction.strip(), input=input.strip())
            completion = completion_template.format(output=output.strip())
        else:
            prompt_template, completion_template = random.choice(encoding_templates_wo_input)
            prompt = prompt_template.format(instruction=instruction.strip())
            completion = completion_template.format(output=output.strip())
    else:
        prompt = instruction.strip() + "\n\n" + input.strip() + "\n\n"
        completion = output.strip() + "<|endoftext|>"

    data = {
        "prompt": prompt,
        "completion": completion,
        "instruction": instruction.strip(),
        "input": input.strip(),
        "output": output.strip(),
    }
    return data

def parse_input_output(response_text):
    if re.findall(r"Output\s*\d*\s*:", response_text):
        inst_input = re.split(r"Output\s*\d*\s*:", response_text)[0].strip()
        inst_output = re.split(r"Output\s*\d*\s*:", response_text)[1].strip()
    else:
        inst_input = ""
        inst_output = response_text.strip()
    # to avoid the case multiple input/output pairs are generated
    if re.findall(r"Input\s*\d*\s*:", inst_output):
        inst_output = re.split(r"Input\s*\d*\s*:", inst_output)[0].strip()
    # remove the prefix "Input:" from the string
    inst_input = re.sub(r"^Input\s*\d*\s*:", "", inst_input).strip()
    return inst_input, inst_output

def parse_instances_for_generation_task(raw_text, instruction, response_metadata):
    instances = []
    raw_text = raw_text.strip()
    if re.findall("Example\s?\d*\.?", raw_text):
        instance_texts = re.split(r"Example\s?\d*\.?", raw_text)
        instance_texts = [it.strip() for it in instance_texts if it.strip() != ""]
        for instance_text in instance_texts:
            inst_input, inst_output = parse_input_output(instance_text)
            instances.append((instruction.strip(), inst_input.strip(), inst_output.strip()))
    elif re.findall(r"Output\s*\d*\s*:", raw_text):
        # we assume only one input/output pair in this case
        inst_input, inst_output = parse_input_output(raw_text)
        instances.append((instruction.strip(), inst_input.strip(), inst_output.strip()))
    else:
        return []
    # if the generation stops because of length, we remove the last instance
    if response_metadata["response"]["choices"][0]["finish_reason"] == "length":
        instances = instances[:-1]

    instances = filter_invalid_instances(instances)
    instances = filter_duplicate_instances(instances)
    return instances

def parse_instances_for_classification_task(raw_text, instruction, response_metadata):
    instances = []
    if not "Class label:" in raw_text:
        return []
    instance_texts = raw_text.split("Class label:")[1:]
    for instance_text in instance_texts:
        instance_text = instance_text.strip()
        fields = instance_text.split("\n", 1)
        if len(fields) == 2:
            # the first field split by \n is the class label
            class_label = fields[0].strip()
            # the rest is the input
            input_text = fields[1].strip()
        elif len(fields) == 1:
            # the first field split by \n is the input
            class_label = fields[0].strip()
            input_text = ""
        else:
            raise ValueError("Invalid instance text: {}".format(instance_text))
        instances.append((instruction.strip(), input_text.strip(), class_label.strip()))

    # if the generation stops because of length, we remove the last instance
    if response_metadata["response"]["choices"][0]["finish_reason"] == "length":
        instances = instances[:-1]
    instances = filter_invalid_instances(instances)
    instances = filter_duplicate_instances(instances)
    return instances

def generate_instance(inputs, api_key):
    classify_res = if_classify(inputs, api_key)
    prompts = []
    for i in range(len(inputs)):
        if classify_res[i] in ["Yes", "yes", "YES"]:
            prompt = output_first_template_for_clf + " " + inputs[i].strip() + "\n"
            prompts.append(prompt)
        else:
            prompt = input_first_template_for_gen + " " + inputs[i].strip() + "\n"
            prompts.append(prompt)
    # print("prompts", prompts)
    results = make_gpt3_requests(
        engine=engine,
        prompts=prompts,
        # because the clf template is longer, we need to decrease the max_tokens
        max_tokens=350,
        temperature=0,
        top_p=0,
        frequency_penalty=0,
        presence_penalty=1.5,
        stop_sequences=["Task:"],
        logprobs=1,
        n=1,
        best_of=1,
        api_key=api_key)
    return results, classify_res

def prepare_finetune(inputs, api_key):
    instance_outputs, classify_res = generate_instance(inputs, api_key)
    training_instances = []
    results1, results2 = [], []
    for i in range(len(inputs)):
        if classify_res[i]:
            task_instances = parse_instances_for_classification_task(instance_outputs[i]["response"]["choices"][0]["text"],
                                                                     inputs[i].strip(), instance_outputs[i])
        else:
            task_instances = parse_instances_for_generation_task(instance_outputs[i]["response"]["choices"][0]["text"],
                                                                 inputs[i].strip(),  instance_outputs[i])
        # we only allow max 5 instances per task
        task_instances = random.sample(task_instances, min(len(task_instances), 5))

        if not task_instances:
            continue

        training_instances += task_instances

        for instance in training_instances:
            results1.append({
                "instruction": instance[0],
                "input": instance[1],
                "output": instance[2],
            })
            results2.append(json.dumps({
                "instruction": instance[0],
                "input": instance[1],
                "output": instance[2],
            }, ensure_ascii=False))
    return results1, classify_res, instance_outputs, results2

def instance_main(inputs, key):
    try:
        import openai
        openai.api_key = key
        MODEL = "gpt-3.5-turbo"
        openai.ChatCompletion.create(
            model=MODEL,
            messages=[
                {"role": "user", "content": "Hi"}
            ],
            temperature=1
        )
    except OpenAIError:
        return {"Wrong": "Key!"}, " ", " ", " "
    api_key = key
    inputs = inputs.split('\n')
    print("***", inputs)
    return prepare_finetune(inputs, api_key)


# instance_main()