initial commit

build_openprompt.py

@@ -0,0 +1,46 @@
+import csv
+import random
+import json
+import numpy as np
+
+from sklearn.model_selection import ShuffleSplit
+
+samples = {
+            "x": [],
+            "y": [],
+        }
+little = False
+all_loaded_sample = 500000
+# 二十万条
+with open("./data/prompts.csv") as f:
+    csv_reader = csv.DictReader(f)
+    for row_number, row in enumerate(csv_reader):
+        # if row_number == random.randint(0, 1000):
+        #     break
+        if little:
+            if row_number > 100:
+                break
+        if row_number > all_loaded_sample:
+            break
+        
+        datum = row
+        modifiers = json.loads(datum['raw_data'])['modifiers']
+        n = random.randint(1, 11)
+        if len(modifiers) < 3:
+            continue
+        label = ",".join(modifiers) if len(modifiers) > 1 else modifiers[0]
+        if 0<n and n<=6:
+            x = modifiers[0]
+        elif n>6 and n<=9:
+            x = ",".join(modifiers[:2])
+        else:
+            x = ",".join(modifiers[:3])
+        # 小文本到大文本，因此x更小，同时x按照6:3:1的比例分配
+
+        samples["x"].append(x)
+        samples["y"].append(label)
+        
+        
+with open("./data/dataset_openprompt.json", "w") as f:
+    json.dump(samples, f, indent=4, ensure_ascii=False)
+    print("*"*40, "save train done.", "with little" if little else "", "*"*40)

gpt2_generation.py

@@ -0,0 +1,453 @@
+#!/usr/bin/env python
+# coding=utf-8
+# Copyright 2018 Google AI, Google Brain and Carnegie Mellon University Authors and the HuggingFace Inc. team.
+# Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" Conditional text generation with the auto-regressive models of the library (GPT/GPT-2/CTRL/Transformer-XL/XLNet)
+"""
+
+
+import argparse
+import inspect
+import time
+import logging
+from typing import Tuple
+
+import torch
+from accelerate import PartialState
+from accelerate.utils import set_seed
+
+from transformers import (
+    AutoTokenizer,
+    BloomForCausalLM,
+    BloomTokenizerFast,
+    CTRLLMHeadModel,
+    CTRLTokenizer,
+    GenerationMixin,
+    GPT2LMHeadModel,
+    GPT2Tokenizer,
+    GPTJForCausalLM,
+    LlamaForCausalLM,
+    LlamaTokenizer,
+    OpenAIGPTLMHeadModel,
+    OpenAIGPTTokenizer,
+    OPTForCausalLM,
+    TransfoXLLMHeadModel,
+    TransfoXLTokenizer,
+    XLMTokenizer,
+    XLMWithLMHeadModel,
+    XLNetLMHeadModel,
+    XLNetTokenizer,
+)
+from transformers.modeling_outputs import CausalLMOutputWithPast
+
+
+logging.basicConfig(
+    format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
+    datefmt="%m/%d/%Y %H:%M:%S",
+    level=logging.INFO,
+)
+logger = logging.getLogger(__name__)
+
+MAX_LENGTH = int(10000)  # Hardcoded max length to avoid infinite loop
+
+MODEL_CLASSES = {
+    "gpt2": (GPT2LMHeadModel, GPT2Tokenizer),
+    "ctrl": (CTRLLMHeadModel, CTRLTokenizer),
+    "openai-gpt": (OpenAIGPTLMHeadModel, OpenAIGPTTokenizer),
+    "xlnet": (XLNetLMHeadModel, XLNetTokenizer),
+    "transfo-xl": (TransfoXLLMHeadModel, TransfoXLTokenizer),
+    "xlm": (XLMWithLMHeadModel, XLMTokenizer),
+    "gptj": (GPTJForCausalLM, AutoTokenizer),
+    "bloom": (BloomForCausalLM, BloomTokenizerFast),
+    "llama": (LlamaForCausalLM, LlamaTokenizer),
+    "opt": (OPTForCausalLM, GPT2Tokenizer),
+}
+
+# Padding text to help Transformer-XL and XLNet with short prompts as proposed by Aman Rusia
+# in https://github.com/rusiaaman/XLNet-gen#methodology
+# and https://medium.com/@amanrusia/xlnet-speaks-comparison-to-gpt-2-ea1a4e9ba39e
+PREFIX = """In 1991, the remains of Russian Tsar Nicholas II and his family
+(except for Alexei and Maria) are discovered.
+The voice of Nicholas's young son, Tsarevich Alexei Nikolaevich, narrates the
+remainder of the story. 1883 Western Siberia,
+a young Grigori Rasputin is asked by his father and a group of men to perform magic.
+Rasputin has a vision and denounces one of the men as a horse thief. Although his
+father initially slaps him for making such an accusation, Rasputin watches as the
+man is chased outside and beaten. Twenty years later, Rasputin sees a vision of
+the Virgin Mary, prompting him to become a priest. Rasputin quickly becomes famous,
+with people, even a bishop, begging for his blessing. <eod> </s> <eos>"""
+
+
+#
+# Functions to prepare models' input
+#
+
+
+def prepare_ctrl_input(args, _, tokenizer, prompt_text):
+    if args.temperature > 0.7:
+        logger.info("CTRL typically works better with lower temperatures (and lower top_k).")
+
+    encoded_prompt = tokenizer.encode(prompt_text, add_special_tokens=False)
+    if not any(encoded_prompt[0] == x for x in tokenizer.control_codes.values()):
+        logger.info("WARNING! You are not starting your generation from a control code so you won't get good results")
+    return prompt_text
+
+
+def prepare_xlm_input(args, model, tokenizer, prompt_text):
+    # kwargs = {"language": None, "mask_token_id": None}
+
+    # Set the language
+    use_lang_emb = hasattr(model.config, "use_lang_emb") and model.config.use_lang_emb
+    if hasattr(model.config, "lang2id") and use_lang_emb:
+        available_languages = model.config.lang2id.keys()
+        if args.xlm_language in available_languages:
+            language = args.xlm_language
+        else:
+            language = None
+            while language not in available_languages:
+                language = input("Using XLM. Select language in " + str(list(available_languages)) + " >>> ")
+
+        model.config.lang_id = model.config.lang2id[language]
+        # kwargs["language"] = tokenizer.lang2id[language]
+
+    # TODO fix mask_token_id setup when configurations will be synchronized between models and tokenizers
+    # XLM masked-language modeling (MLM) models need masked token
+    # is_xlm_mlm = "mlm" in args.model_name_or_path
+    # if is_xlm_mlm:
+    #     kwargs["mask_token_id"] = tokenizer.mask_token_id
+
+    return prompt_text
+
+
+def prepare_xlnet_input(args, _, tokenizer, prompt_text):
+    prefix = args.prefix if args.prefix else args.padding_text if args.padding_text else PREFIX
+    prompt_text = prefix + prompt_text
+    return prompt_text
+
+
+def prepare_transfoxl_input(args, _, tokenizer, prompt_text):
+    prefix = args.prefix if args.prefix else args.padding_text if args.padding_text else PREFIX
+    prompt_text = prefix + prompt_text
+    return prompt_text
+
+
+PREPROCESSING_FUNCTIONS = {
+    "ctrl": prepare_ctrl_input,
+    "xlm": prepare_xlm_input,
+    "xlnet": prepare_xlnet_input,
+    "transfo-xl": prepare_transfoxl_input,
+}
+
+
+def adjust_length_to_model(length, max_sequence_length):
+    if length < 0 and max_sequence_length > 0:
+        length = max_sequence_length
+    elif 0 < max_sequence_length < length:
+        length = max_sequence_length  # No generation bigger than model size
+    elif length < 0:
+        length = MAX_LENGTH  # avoid infinite loop
+    return length
+
+
+def sparse_model_config(model_config):
+    embedding_size = None
+    if hasattr(model_config, "hidden_size"):
+        embedding_size = model_config.hidden_size
+    elif hasattr(model_config, "n_embed"):
+        embedding_size = model_config.n_embed
+    elif hasattr(model_config, "n_embd"):
+        embedding_size = model_config.n_embd
+
+    num_head = None
+    if hasattr(model_config, "num_attention_heads"):
+        num_head = model_config.num_attention_heads
+    elif hasattr(model_config, "n_head"):
+        num_head = model_config.n_head
+
+    if embedding_size is None or num_head is None or num_head == 0:
+        raise ValueError("Check the model config")
+
+    num_embedding_size_per_head = int(embedding_size / num_head)
+    if hasattr(model_config, "n_layer"):
+        num_layer = model_config.n_layer
+    elif hasattr(model_config, "num_hidden_layers"):
+        num_layer = model_config.num_hidden_layers
+    else:
+        raise ValueError("Number of hidden layers couldn't be determined from the model config")
+
+    return num_layer, num_head, num_embedding_size_per_head
+
+
+def generate_past_key_values(model, batch_size, seq_len):
+    num_block_layers, num_attention_heads, num_embedding_size_per_head = sparse_model_config(model.config)
+    if model.config.model_type == "bloom":
+        past_key_values = tuple(
+            (
+                torch.empty(int(num_attention_heads * batch_size), num_embedding_size_per_head, seq_len)
+                .to(model.dtype)
+                .to(model.device),
+                torch.empty(int(num_attention_heads * batch_size), seq_len, num_embedding_size_per_head)
+                .to(model.dtype)
+                .to(model.device),
+            )
+            for _ in range(num_block_layers)
+        )
+    else:
+        past_key_values = tuple(
+            (
+                torch.empty(batch_size, num_attention_heads, seq_len, num_embedding_size_per_head)
+                .to(model.dtype)
+                .to(model.device),
+                torch.empty(batch_size, num_attention_heads, seq_len, num_embedding_size_per_head)
+                .to(model.dtype)
+                .to(model.device),
+            )
+            for _ in range(num_block_layers)
+        )
+    return past_key_values
+
+
+def prepare_jit_inputs(inputs, model, tokenizer):
+    batch_size = len(inputs)
+    dummy_input = tokenizer.batch_encode_plus(inputs, return_tensors="pt")
+    dummy_input = dummy_input.to(model.device)
+    if model.config.use_cache:
+        dummy_input["past_key_values"] = generate_past_key_values(model, batch_size, 1)
+    dummy_input["attention_mask"] = torch.cat(
+        [
+            torch.zeros(dummy_input["attention_mask"].shape[0], 1)
+            .to(dummy_input["attention_mask"].dtype)
+            .to(model.device),
+            dummy_input["attention_mask"],
+        ],
+        -1,
+    )
+    return dummy_input
+
+
+class _ModelFallbackWrapper(GenerationMixin):
+    __slots__ = ("_optimized", "_default")
+
+    def __init__(self, optimized, default):
+        self._optimized = optimized
+        self._default = default
+
+    def __call__(self, *args, **kwargs):
+        if kwargs["past_key_values"] is None and self._default.config.use_cache:
+            kwargs["past_key_values"] = generate_past_key_values(self._default, kwargs["input_ids"].shape[0], 0)
+        kwargs.pop("position_ids", None)
+        for k in list(kwargs.keys()):
+            if kwargs[k] is None or isinstance(kwargs[k], bool):
+                kwargs.pop(k)
+        outputs = self._optimized(**kwargs)
+        lm_logits = outputs[0]
+        past_key_values = outputs[1]
+        fixed_output = CausalLMOutputWithPast(
+            loss=None,
+            logits=lm_logits,
+            past_key_values=past_key_values,
+            hidden_states=None,
+            attentions=None,
+        )
+        return fixed_output
+
+    def __getattr__(self, item):
+        return getattr(self._default, item)
+
+    def prepare_inputs_for_generation(
+        self, input_ids, past_key_values=None, inputs_embeds=None, use_cache=None, **kwargs
+    ):
+        return self._default.prepare_inputs_for_generation(
+            input_ids, past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, **kwargs
+        )
+
+    def _reorder_cache(
+        self, past_key_values: Tuple[Tuple[torch.Tensor]], beam_idx: torch.Tensor
+    ) -> Tuple[Tuple[torch.Tensor]]:
+        """
+        This function is used to re-order the `past_key_values` cache if [`~PretrainedModel.beam_search`] or
+        [`~PretrainedModel.beam_sample`] is called. This is required to match `past_key_values` with the correct
+        beam_idx at every generation step.
+        """
+        return self._default._reorder_cache(past_key_values, beam_idx)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--model_type",
+        default="gpt2",
+        type=str,
+        help="Model type selected in the list: " + ", ".join(MODEL_CLASSES.keys()),
+    )
+    parser.add_argument(
+        "--model_name_or_path",
+        default="./output/gpt2_openprpmpt/checkpoint-218500",
+        type=str,
+        help="Path to pre-trained model or shortcut name selected in the list: " + ", ".join(MODEL_CLASSES.keys()),
+    )
+
+    parser.add_argument("--prompt", type=str, default="")
+    parser.add_argument("--length", type=int, default=60)
+    parser.add_argument("--stop_token", type=str, default=None, help="Token at which text generation is stopped")
+
+    parser.add_argument(
+        "--temperature",
+        type=float,
+        default=1.0,
+        help="temperature of 1.0 has no effect, lower tend toward greedy sampling",
+    )
+    parser.add_argument(
+        "--repetition_penalty", type=float, default=1.0, help="primarily useful for CTRL model; in that case, use 1.2"
+    )
+    parser.add_argument("--k", type=int, default=3)
+    parser.add_argument("--p", type=float, default=0.9)
+
+    parser.add_argument("--prefix", type=str, default="", help="Text added prior to input.")
+    parser.add_argument("--padding_text", type=str, default="", help="Deprecated, the use of `--prefix` is preferred.")
+    parser.add_argument("--xlm_language", type=str, default="", help="Optional language when used with the XLM model.")
+
+    parser.add_argument("--seed", type=int, default=42, help="random seed for initialization")
+    parser.add_argument(
+        "--use_cpu",
+        action="store_true",
+        help="Whether or not to use cpu. If set to False, " "we will use gpu/npu or mps device if available",
+    )
+    parser.add_argument("--num_return_sequences", type=int, default=4, help="The number of samples to generate.")
+    parser.add_argument(
+        "--fp16",
+        action="store_true",
+        help="Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit",
+    )
+    parser.add_argument("--jit", action="store_true", help="Whether or not to use jit trace to accelerate inference")
+    args = parser.parse_args()
+
+    # Initialize the distributed state.
+    distributed_state = PartialState(cpu=args.use_cpu)
+
+    logger.warning(f"device: {distributed_state.device}, 16-bits inference: {args.fp16}")
+
+    if args.seed is not None:
+        set_seed(args.seed)
+
+    # Initialize the model and tokenizer
+    try:
+        args.model_type = args.model_type.lower()
+        model_class, tokenizer_class = MODEL_CLASSES[args.model_type]
+    except KeyError:
+        raise KeyError("the model {} you specified is not supported. You are welcome to add it and open a PR :)")
+
+    tokenizer = tokenizer_class.from_pretrained(args.model_name_or_path, padding_side='left')
+    if tokenizer.pad_token is None:
+        tokenizer.pad_token = tokenizer.eos_token
+        tokenizer.mask_token = tokenizer.eos_token
+    model = model_class.from_pretrained(args.model_name_or_path)
+
+    # Set the model to the right device
+    model.to(distributed_state.device)
+
+    if args.fp16:
+        model.half()
+    max_seq_length = getattr(model.config, "max_position_embeddings", 0)
+    args.length = adjust_length_to_model(args.length, max_sequence_length=max_seq_length)
+    logger.info(args)
+
+    prompt_text = args.prompt if args.prompt else input("Model prompt >>> ")
+
+    # Different models need different input formatting and/or extra arguments
+    requires_preprocessing = args.model_type in PREPROCESSING_FUNCTIONS.keys()
+    if requires_preprocessing:
+        prepare_input = PREPROCESSING_FUNCTIONS.get(args.model_type)
+        preprocessed_prompt_text = prepare_input(args, model, tokenizer, prompt_text)
+
+        if model.__class__.__name__ in ["TransfoXLLMHeadModel"]:
+            tokenizer_kwargs = {"add_space_before_punct_symbol": True}
+        else:
+            tokenizer_kwargs = {}
+
+        encoded_prompt = tokenizer.encode(
+            preprocessed_prompt_text, add_special_tokens=False, return_tensors="pt", **tokenizer_kwargs
+        )
+    else:
+        prefix = args.prefix if args.prefix else args.padding_text
+        encoded_prompt = tokenizer.encode(prefix + prompt_text, add_special_tokens=False, return_tensors="pt")
+    encoded_prompt = encoded_prompt.to(distributed_state.device)
+
+    if encoded_prompt.size()[-1] == 0:
+        input_ids = None
+    else:
+        input_ids = encoded_prompt
+
+    if args.jit:
+        jit_input_texts = ["enable jit"]
+        jit_inputs = prepare_jit_inputs(jit_input_texts, model, tokenizer)
+        torch._C._jit_set_texpr_fuser_enabled(False)
+        model.config.return_dict = False
+        if hasattr(model, "forward"):
+            sig = inspect.signature(model.forward)
+        else:
+            sig = inspect.signature(model.__call__)
+        jit_inputs = tuple(jit_inputs[key] for key in sig.parameters if jit_inputs.get(key, None) is not None)
+        traced_model = torch.jit.trace(model, jit_inputs, strict=False)
+        traced_model = torch.jit.freeze(traced_model.eval())
+        traced_model(*jit_inputs)
+        traced_model(*jit_inputs)
+
+        model = _ModelFallbackWrapper(traced_model, model)
+    t1 = time.time()
+    output_sequences = model.generate(
+        input_ids=input_ids,
+        max_length=args.length + len(encoded_prompt[0]),
+        temperature=args.temperature,
+        top_k=args.k,
+        top_p=args.p,
+        repetition_penalty=args.repetition_penalty,
+        do_sample=True,
+        num_return_sequences=args.num_return_sequences,
+    )
+
+    # Remove the batch dimension when returning multiple sequences
+    if len(output_sequences.shape) > 2:
+        output_sequences.squeeze_()
+    
+    generated_sequences = []
+
+    for generated_sequence_idx, generated_sequence in enumerate(output_sequences):
+        print(f"=== GENERATED SEQUENCE {generated_sequence_idx + 1} ===")
+        generated_sequence = generated_sequence.tolist()
+
+        # Decode text
+        text = tokenizer.decode(generated_sequence, clean_up_tokenization_spaces=True)
+
+        # Remove all text after the stop token
+        text = text[: text.find(args.stop_token) if args.stop_token else None]
+
+        # Add the prompt at the beginning of the sequence. Remove the excess text that was used for pre-processing
+        total_sequence = (
+            prompt_text + text[len(tokenizer.decode(encoded_prompt[0], clean_up_tokenization_spaces=True)) :]
+        )
+
+        generated_sequences.append(total_sequence)
+        print(total_sequence)
+
+    t2 = time.time()
+    print("*"*60)
+    print(f"Time cost: {t2-t1}")
+
+    return generated_sequences
+
+
+if __name__ == "__main__":
+    main()

rouge/README.md

@@ -0,0 +1,161 @@
+---
+title: ROUGE
+emoji: 🤗 
+colorFrom: blue
+colorTo: red
+sdk: gradio
+sdk_version: 3.19.1
+app_file: app.py
+pinned: false
+tags:
+- evaluate
+- metric
+description: >-
+  ROUGE, or Recall-Oriented Understudy for Gisting Evaluation, is a set of metrics and a software package used for
+  evaluating automatic summarization and machine translation software in natural language processing.
+  The metrics compare an automatically produced summary or translation against a reference or a set of references (human-produced) summary or translation.
+  
+  Note that ROUGE is case insensitive, meaning that upper case letters are treated the same way as lower case letters.
+  
+  This metrics is a wrapper around Google Research reimplementation of ROUGE:
+  https://github.com/google-research/google-research/tree/master/rouge
+---
+
+# Metric Card for ROUGE
+
+## Metric Description
+ROUGE, or Recall-Oriented Understudy for Gisting Evaluation, is a set of metrics and a software package used for evaluating automatic summarization and machine translation software in natural language processing. The metrics compare an automatically produced summary or translation against a reference or a set of references (human-produced) summary or translation.
+
+Note that ROUGE is case insensitive, meaning that upper case letters are treated the same way as lower case letters.
+
+This metrics is a wrapper around the [Google Research reimplementation of ROUGE](https://github.com/google-research/google-research/tree/master/rouge)
+
+## How to Use
+At minimum, this metric takes as input a list of predictions and a list of references:
+```python
+>>> rouge = evaluate.load('rouge')
+>>> predictions = ["hello there", "general kenobi"]
+>>> references = ["hello there", "general kenobi"]
+>>> results = rouge.compute(predictions=predictions,
+...                         references=references)
+>>> print(results)
+{'rouge1': 1.0, 'rouge2': 1.0, 'rougeL': 1.0, 'rougeLsum': 1.0}
+```
+
+One can also pass a custom tokenizer which is especially useful for non-latin languages.
+```python
+>>> results = rouge.compute(predictions=predictions,
+...                         references=references,
+                            tokenizer=lambda x: x.split())
+>>> print(results)
+{'rouge1': 1.0, 'rouge2': 1.0, 'rougeL': 1.0, 'rougeLsum': 1.0}
+```
+
+It can also deal with lists of references for each predictions:
+```python
+>>> rouge = evaluate.load('rouge')
+>>> predictions = ["hello there", "general kenobi"]
+>>> references = [["hello", "there"], ["general kenobi", "general yoda"]]
+>>> results = rouge.compute(predictions=predictions,
+...                         references=references)
+>>> print(results)
+{'rouge1': 0.8333, 'rouge2': 0.5, 'rougeL': 0.8333, 'rougeLsum': 0.8333}```
+```
+
+### Inputs
+- **predictions** (`list`): list of predictions to score. Each prediction
+        should be a string with tokens separated by spaces.
+- **references** (`list` or `list[list]`): list of reference for each prediction or a list of several references per prediction. Each
+        reference should be a string with tokens separated by spaces.
+- **rouge_types** (`list`): A list of rouge types to calculate. Defaults to `['rouge1', 'rouge2', 'rougeL', 'rougeLsum']`.
+    - Valid rouge types:
+        - `"rouge1"`: unigram (1-gram) based scoring
+        - `"rouge2"`: bigram (2-gram) based scoring
+        - `"rougeL"`: Longest common subsequence based scoring.
+        - `"rougeLSum"`: splits text using `"\n"`
+        - See [here](https://github.com/huggingface/datasets/issues/617) for more information
+- **use_aggregator** (`boolean`): If True, returns aggregates. Defaults to `True`.
+- **use_stemmer** (`boolean`): If `True`, uses Porter stemmer to strip word suffixes. Defaults to `False`.
+
+### Output Values
+The output is a dictionary with one entry for each rouge type in the input list `rouge_types`. If `use_aggregator=False`, each dictionary entry is a list of scores, with one score for each sentence. E.g. if `rouge_types=['rouge1', 'rouge2']` and `use_aggregator=False`, the output is:
+
+```python
+{'rouge1': [0.6666666666666666, 1.0], 'rouge2': [0.0, 1.0]}
+```
+
+If `rouge_types=['rouge1', 'rouge2']` and `use_aggregator=True`, the output is of the following format:
+```python
+{'rouge1': 1.0, 'rouge2': 1.0}
+```
+
+The ROUGE values are in the range of 0 to 1.
+
+
+#### Values from Popular Papers
+
+
+### Examples
+An example without aggregation:
+```python
+>>> rouge = evaluate.load('rouge')
+>>> predictions = ["hello goodbye", "ankh morpork"]
+>>> references = ["goodbye", "general kenobi"]
+>>> results = rouge.compute(predictions=predictions,
+...                         references=references,
+...                         use_aggregator=False)
+>>> print(list(results.keys()))
+['rouge1', 'rouge2', 'rougeL', 'rougeLsum']
+>>> print(results["rouge1"])
+[0.5, 0.0]
+```
+
+The same example, but with aggregation:
+```python
+>>> rouge = evaluate.load('rouge')
+>>> predictions = ["hello goodbye", "ankh morpork"]
+>>> references = ["goodbye", "general kenobi"]
+>>> results = rouge.compute(predictions=predictions,
+...                         references=references,
+...                         use_aggregator=True)
+>>> print(list(results.keys()))
+['rouge1', 'rouge2', 'rougeL', 'rougeLsum']
+>>> print(results["rouge1"])
+0.25
+```
+
+The same example, but only calculating `rouge_1`:
+```python
+>>> rouge = evaluate.load('rouge')
+>>> predictions = ["hello goodbye", "ankh morpork"]
+>>> references = ["goodbye", "general kenobi"]
+>>> results = rouge.compute(predictions=predictions,
+...                         references=references,
+...                         rouge_types=['rouge_1'],
+...                         use_aggregator=True)
+>>> print(list(results.keys()))
+['rouge1']
+>>> print(results["rouge1"])
+0.25
+```
+
+## Limitations and Bias
+See [Schluter (2017)](https://aclanthology.org/E17-2007/) for an in-depth discussion of many of ROUGE's limits.
+
+## Citation
+```bibtex
+@inproceedings{lin-2004-rouge,
+    title = "{ROUGE}: A Package for Automatic Evaluation of Summaries",
+    author = "Lin, Chin-Yew",
+    booktitle = "Text Summarization Branches Out",
+    month = jul,
+    year = "2004",
+    address = "Barcelona, Spain",
+    publisher = "Association for Computational Linguistics",
+    url = "https://www.aclweb.org/anthology/W04-1013",
+    pages = "74--81",
+}
+```
+
+## Further References
+- This metrics is a wrapper around the [Google Research reimplementation of ROUGE](https://github.com/google-research/google-research/tree/master/rouge)

rouge/app.py

@@ -0,0 +1,6 @@
+import evaluate
+from evaluate.utils import launch_gradio_widget
+
+
+module = evaluate.load("rouge")
+launch_gradio_widget(module)

rouge/requirements.txt

@@ -0,0 +1,4 @@
+git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER}
+absl-py
+nltk
+rouge_score>=0.1.2

rouge/rouge.py

@@ -0,0 +1,158 @@
+# Copyright 2020 The HuggingFace Evaluate Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" ROUGE metric from Google Research github repo. """
+
+# The dependencies in https://github.com/google-research/google-research/blob/master/rouge/requirements.txt
+import absl  # Here to have a nice missing dependency error message early on
+import datasets
+import nltk  # Here to have a nice missing dependency error message early on
+import numpy  # Here to have a nice missing dependency error message early on
+import six  # Here to have a nice missing dependency error message early on
+from rouge_score import rouge_scorer, scoring
+
+import evaluate
+
+
+_CITATION = """\
+@inproceedings{lin-2004-rouge,
+    title = "{ROUGE}: A Package for Automatic Evaluation of Summaries",
+    author = "Lin, Chin-Yew",
+    booktitle = "Text Summarization Branches Out",
+    month = jul,
+    year = "2004",
+    address = "Barcelona, Spain",
+    publisher = "Association for Computational Linguistics",
+    url = "https://www.aclweb.org/anthology/W04-1013",
+    pages = "74--81",
+}
+"""
+
+_DESCRIPTION = """\
+ROUGE, or Recall-Oriented Understudy for Gisting Evaluation, is a set of metrics and a software package used for
+evaluating automatic summarization and machine translation software in natural language processing.
+The metrics compare an automatically produced summary or translation against a reference or a set of references (human-produced) summary or translation.
+
+Note that ROUGE is case insensitive, meaning that upper case letters are treated the same way as lower case letters.
+
+This metrics is a wrapper around Google Research reimplementation of ROUGE:
+https://github.com/google-research/google-research/tree/master/rouge
+"""
+
+_KWARGS_DESCRIPTION = """
+Calculates average rouge scores for a list of hypotheses and references
+Args:
+    predictions: list of predictions to score. Each prediction
+        should be a string with tokens separated by spaces.
+    references: list of reference for each prediction. Each
+        reference should be a string with tokens separated by spaces.
+    rouge_types: A list of rouge types to calculate.
+        Valid names:
+        `"rouge{n}"` (e.g. `"rouge1"`, `"rouge2"`) where: {n} is the n-gram based scoring,
+        `"rougeL"`: Longest common subsequence based scoring.
+        `"rougeLsum"`: rougeLsum splits text using `"\n"`.
+        See details in https://github.com/huggingface/datasets/issues/617
+    use_stemmer: Bool indicating whether Porter stemmer should be used to strip word suffixes.
+    use_aggregator: Return aggregates if this is set to True
+Returns:
+    rouge1: rouge_1 (f1),
+    rouge2: rouge_2 (f1),
+    rougeL: rouge_l (f1),
+    rougeLsum: rouge_lsum (f1)
+Examples:
+
+    >>> rouge = evaluate.load('rouge')
+    >>> predictions = ["hello there", "general kenobi"]
+    >>> references = ["hello there", "general kenobi"]
+    >>> results = rouge.compute(predictions=predictions, references=references)
+    >>> print(results)
+    {'rouge1': 1.0, 'rouge2': 1.0, 'rougeL': 1.0, 'rougeLsum': 1.0}
+"""
+
+
+class Tokenizer:
+    """Helper class to wrap a callable into a class with a `tokenize` method as used by rouge-score."""
+
+    def __init__(self, tokenizer_func):
+        self.tokenizer_func = tokenizer_func
+
+    def tokenize(self, text):
+        return self.tokenizer_func(text)
+
+
+@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
+class Rouge(evaluate.Metric):
+    def _info(self):
+        return evaluate.MetricInfo(
+            description=_DESCRIPTION,
+            citation=_CITATION,
+            inputs_description=_KWARGS_DESCRIPTION,
+            features=[
+                datasets.Features(
+                    {
+                        "predictions": datasets.Value("string", id="sequence"),
+                        "references": datasets.Sequence(datasets.Value("string", id="sequence")),
+                    }
+                ),
+                datasets.Features(
+                    {
+                        "predictions": datasets.Value("string", id="sequence"),
+                        "references": datasets.Value("string", id="sequence"),
+                    }
+                ),
+            ],
+            codebase_urls=["https://github.com/google-research/google-research/tree/master/rouge"],
+            reference_urls=[
+                "https://en.wikipedia.org/wiki/ROUGE_(metric)",
+                "https://github.com/google-research/google-research/tree/master/rouge",
+            ],
+        )
+
+    def _compute(
+        self, predictions, references, rouge_types=None, use_aggregator=True, use_stemmer=False, tokenizer=None
+    ):
+        if rouge_types is None:
+            rouge_types = ["rouge1", "rouge2", "rougeL", "rougeLsum"]
+
+        multi_ref = isinstance(references[0], list)
+
+        if tokenizer is not None:
+            tokenizer = Tokenizer(tokenizer)
+
+        scorer = rouge_scorer.RougeScorer(rouge_types=rouge_types, use_stemmer=use_stemmer, tokenizer=tokenizer)
+        if use_aggregator:
+            aggregator = scoring.BootstrapAggregator()
+        else:
+            scores = []
+
+        for ref, pred in zip(references, predictions):
+            if multi_ref:
+                score = scorer.score_multi(ref, pred)
+            else:
+                score = scorer.score(ref, pred)
+            if use_aggregator:
+                aggregator.add_scores(score)
+            else:
+                scores.append(score)
+
+        if use_aggregator:
+            result = aggregator.aggregate()
+            for key in result:
+                result[key] = result[key].mid.fmeasure
+
+        else:
+            result = {}
+            for key in scores[0]:
+                result[key] = list(score[key].fmeasure for score in scores)
+
+        return result

sft.py

@@ -0,0 +1,92 @@
+import time
+import evaluate
+import numpy as np
+
+from transformers import DataCollatorForLanguageModeling, DataCollatorForSeq2Seq
+from transformers import TrainingArguments, Trainer
+
+from utils import (
+    get_dataset,
+    get_tok_and_model,
+    get_open_prompt_data,
+    get_dict_dataset,
+    get_advance_dataset,)
+
+base_model = "distilgpt2"
+tokenizer, model = get_tok_and_model(f"./models/{base_model}")
+tokenizer.pad_token = tokenizer.eos_token
+rouge = evaluate.load("rouge")
+
+# train_data, test_data = get_open_prompt_data("./data")
+# train_dataset, test_dataset = get_dataset(train_data, test_data)
+dict_data = get_dict_dataset("./data")
+dataset = get_advance_dataset(dict_data)
+dataset = dataset.train_test_split(test_size=0.2)
+
+def preprocess_function(examples):
+    x_inputs = [x for x in examples["x"]]
+    y_inputs = examples["y"]
+    model_inputs = tokenizer(x_inputs, max_length=128, truncation=True)
+
+    labels = tokenizer(text_target=y_inputs, max_length=128, truncation=True)
+
+    model_inputs["labels"] = model_inputs["input_ids"]
+    return model_inputs
+
+def compute_metrics(eval_pred):
+    predictions, labels = eval_pred
+    decoded_preds = tokenizer.batch_decode(predictions, skip_special_tokens=True)
+    labels = np.where(labels != -100, labels, tokenizer.pad_token_id)
+    decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)
+
+    result = rouge.compute(predictions=decoded_preds, references=decoded_labels, use_stemmer=True)
+
+    prediction_lens = [np.count_nonzero(pred != tokenizer.pad_token_id) for pred in predictions]
+    result["gen_len"] = np.mean(prediction_lens)
+
+    return {k: round(v, 4) for k, v in result.items()}
+
+# data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False)
+data_collator = DataCollatorForSeq2Seq(tokenizer=tokenizer, model=model)
+
+
+print("tokenize data...")
+t1 = time.time()
+tokenized_dataset = dataset.map(preprocess_function, batched=True, remove_columns=["x", "y"])
+t2 = time.time()
+print(f"data tokenize done. process time : {t2 - t1}")
+
+
+training_args = TrainingArguments(
+    output_dir=f"./output/{base_model}_openprpmpt",
+    evaluation_strategy="steps",
+    eval_steps=20000,
+    learning_rate=2e-5,
+    lr_scheduler_type="constant",
+    report_to="tensorboard",
+    per_device_train_batch_size=64,
+    per_device_eval_batch_size=32,
+    adam_beta1=0.9,
+    adam_beta2=0.98,
+    save_total_limit=1,
+    num_train_epochs=100,
+    fp16=True,
+    push_to_hub=False,
+)
+
+trainer = Trainer(
+    model=model,
+    args=training_args,
+    train_dataset=tokenized_dataset["train"],
+    eval_dataset=tokenized_dataset["test"],
+    tokenizer=tokenizer,
+    data_collator=data_collator,
+)
+
+trainer.train()
+
+import math
+
+eval_results = trainer.evaluate()
+print(f"Perplexity: {math.exp(eval_results['eval_loss']):.2f}")
+

utils.py

@@ -0,0 +1,59 @@
+import os
+import json
+
+from typing import Dict
+from torch.utils.data import Dataset
+from datasets import Dataset as AdvancedDataset
+from transformers import AutoTokenizer, AutoModelForCausalLM
+
+
+DEFAULT_TRAIN_DATA_NAME = "test_openprompt.json"
+DEFAULT_TEST_DATA_NAME = "train_openprompt.json"
+DEFAULT_DICT_DATA_NAME = "dataset_openprompt.json"
+
+def get_open_prompt_data(path_for_data):
+    with open(os.path.join(path_for_data, DEFAULT_TRAIN_DATA_NAME)) as f:
+        train_data = json.load(f)
+    
+    with open(os.path.join(path_for_data, DEFAULT_TEST_DATA_NAME)) as f:
+        test_data = json.load(f)
+
+    return train_data, test_data
+
+def get_tok_and_model(path_for_model):
+    if not os.path.exists(path_for_model):
+        raise RuntimeError("no cached model.")
+    tok = AutoTokenizer.from_pretrained(path_for_model, padding_side='left')
+    tok.pad_token_id = 50256
+    # default for open-ended generation
+    model = AutoModelForCausalLM.from_pretrained(path_for_model)
+    return tok, model
+
+
+class OpenPromptDataset(Dataset):
+    def __init__(self, data) -> None:
+        super().__init__()
+        self.data = data
+        
+    def __len__(self):
+        return len(self.data)
+
+    def __getitem__(self, index):
+        return self.data[index]
+
+def get_dataset(train_data, test_data):
+    train_dataset = OpenPromptDataset(train_data)
+    test_dataset = OpenPromptDataset(test_data)
+    return train_dataset, test_dataset
+
+def get_dict_dataset(path_for_data):
+    with open(os.path.join(path_for_data, DEFAULT_DICT_DATA_NAME)) as f:
+        dict_data = json.load(f)
+    return dict_data
+    
+def get_advance_dataset(dict_data):
+    if not isinstance(dict_data, Dict):
+        raise RuntimeError("dict_data is not a dict.")
+    dataset = AdvancedDataset.from_dict(dict_data)
+    
+    return dataset