ecqa.py

# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""TODO: Add a description here."""

import evaluate
import datasets
import re
import string
from tqdm import tqdm
from collections import Counter

# TODO: Add BibTeX citation
_CITATION = """\
@InProceedings{huggingface:module,
title = {A great new module},
authors={huggingface, Inc.},
year={2020}
}
"""

# TODO: Add description of the module here
_DESCRIPTION = """\
This new module is designed to solve this great ML task and is crafted with a lot of care.
"""


# TODO: Add description of the arguments of the module here
_KWARGS_DESCRIPTION = """
Calculates how good are predictions given some references, using certain scores
Args:
    predictions: list of predictions to score. Each predictions
        should be a string with tokens separated by spaces.
    references: list of reference for each prediction. Each
        reference should be a string with tokens separated by spaces.
Returns:
    accuracy: description of the first score,
    another_score: description of the second score,
Examples:
    Examples should be written in doctest format, and should illustrate how
    to use the function.

    >>> my_new_module = evaluate.load("my_new_module")
    >>> results = my_new_module.compute(references=[0, 1], predictions=[0, 1])
    >>> print(results)
    {'accuracy': 1.0}
"""

# TODO: Define external resources urls if needed
BAD_WORDS_URL = "http://url/to/external/resource/bad_words.txt"

def remove_(text: str)-> str:
    ''' 불필요한 기호 제거 '''
    text = re.sub("'", " ", text)
    text = re.sub('"', " ", text)
    text = re.sub('《', " ", text)
    text = re.sub('》', " ", text)
    text = re.sub('<', " ", text)
    text = re.sub('>', " ", text)
    text = re.sub('〈', " ", text)
    text = re.sub('〉', " ", text)
    text = re.sub("\(", " ", text)
    text = re.sub("\)", " ", text)
    text = re.sub("‘", " ", text)
    text = re.sub("’", " ", text)
    return text

def white_space_fix(text: str)-> str:
    '''연속된 공백일 경우 하나의 공백으로 대체'''
    return ' '.join(text.split())

def remove_punc(text: str)-> str:
    '''구두점 제거'''
    exclude = set(string.punctuation)
    return ''.join(ch for ch in text if ch not in exclude)

def lower(text: str)-> str:
    '''소문자 전환'''
    return text.lower()

@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
class ecqa(evaluate.Metric):
    """TODO: Short description of my evaluation module."""

    def _info(self):
        # TODO: Specifies the evaluate.EvaluationModuleInfo object
        return evaluate.MetricInfo(
            # This is the description that will appear on the modules page.
            module_type="metric",
            description=_DESCRIPTION,
            citation=_CITATION,
            inputs_description=_KWARGS_DESCRIPTION,
            # This defines the format of each prediction and reference
            features=datasets.Features({
                'predictions': datasets.Value('string'),
                'references': datasets.Value('string'),
            }),
            # Homepage of the module for documentation
            homepage="http://module.homepage",
            # Additional links to the codebase or references
            codebase_urls=["http://github.com/path/to/codebase/of/new_module"],
            reference_urls=["http://path.to.reference.url/new_module"]
        )

    def _download_and_prepare(self, dl_manager):
        """Optional: download external resources useful to compute the scores"""
        # TODO: Download external resources if needed
        pass
    
    def __normalize(self, text: str):
        text = remove_(text)
        text = lower(text)
        text = remove_punc(text)
        return white_space_fix(text)

    def __compute_f1(self, prediction: str, reference: str)-> tuple[float, float, float]:
        predicted_tokens = self.__normalize(prediction).split()
        referenced_tokens = self.__normalize(reference).split()

        predictied_chars = []
        for token in predicted_tokens:
            predictied_chars += [char for char in token]

        referenced_chars = []
        for token in referenced_tokens:
            referenced_chars += [char for char in token]

        true_positive = Counter(predictied_chars) & Counter(referenced_chars)
        n_true_positive = sum(true_positive.values())
        if n_true_positive == 0:
            return 0, 0, 0

        precision = 1.0 * n_true_positive / len(predictied_chars)
        recall = 1.0 * n_true_positive / len(referenced_chars)
        f1 = (2 * precision * recall) / (precision + recall)

        return f1, recall, precision
    
    def _compute(self, predictions: list[str], references: list[str]):
        """Returns the scores"""
        # TODO: Compute the different scores of the module
        assert isinstance(predictions, list)
        assert isinstance(references, list)
        assert len(predictions) == len(references)

        f1_acc = precision_acc = recall_acc = total = 0
        for prediction, reference in tqdm(zip(predictions, references)):
            total += 1
            f1_computed, precision_computed, recall_computed = self.__compute_f1(prediction, reference)

            f1_acc += f1_computed
            precision_acc += precision_computed
            recall_acc += recall_computed

        f1, precision, recall = [
            # average
            100.0 * computed / total

            for computed in [
                f1_acc,
                precision_acc,
                recall_acc
            ]
        ]

        return {
            "f1": f1,
            "precision": precision,
            "recall": recall
        }