coord_eval.py

# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""TODO: Add a description here."""

import evaluate
import datasets
import re
import ast
import math


# TODO: Add BibTeX citation
_CITATION = """\
@InProceedings{huggingface:module,
title = {A great new module},
authors={huggingface, Inc.},
year={2020}
}
"""

# TODO: Add description of the module here
_DESCRIPTION = """\
This metric aims to compute a coordinate accuracy between coordinates generated by an LM and a golden one. A pair of coordinates is considered correct if its haversine distance to the golden coordinates is inferior to a threeshold d.
"""


# TODO: Add description of the arguments of the module here
_KWARGS_DESCRIPTION = """
Calculates how good are predictions given some references, using certain scores
Args:
    generations: list of predictions to score. Each predictions
        should be a string generated by a LM model.
    golds: list of reference for each prediction. Each
        reference should be a list of two floats corresponding to the latitude and longitude of the ground truth (eg. [12.8, 76.9]).
Returns:
    accuracy: 1 if coordinates predicted are d distant from gold ones, O otherwise.
Examples:
    >>> my_new_module = evaluate.load("rfr2003/coord_eval")
    >>> results = my_new_module.compute(generations=["(12.7, 67.8)", "(16.7, 89.6)"], golds=[[12.7, 67.8], [10.9, 80.6]], d_range=20)
    >>> print(results)
    {'coord_accuracy': 0.5}
"""


@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
class Coord_eval(evaluate.Metric):
    """TODO: Short description of my evaluation module."""

    def _info(self):
        # TODO: Specifies the evaluate.EvaluationModuleInfo object
        return evaluate.MetricInfo(
            # This is the description that will appear on the modules page.
            module_type="metric",
            description=_DESCRIPTION,
            citation=_CITATION,
            inputs_description=_KWARGS_DESCRIPTION,
            # This defines the format of each prediction and reference
            features=datasets.Features({
                'generations': datasets.Value('string'),
                'golds': datasets.Sequence(datasets.Value('float32')),
            }),
        )

    def _download_and_prepare(self, dl_manager):
        """Optional: download external resources useful to compute the scores"""
        # TODO: Download external resources if needed
        pass

    def _haversine_distance(self, coord1, coord2):
        lat1, lon1 = coord1
        lat2, lon2 = coord2
    
        # Convert degrees to radians
        lat1, lon1, lat2, lon2 = map(math.radians, [lat1, lon1, lat2, lon2])
    
        dlat = lat2 - lat1
        dlon = lon2 - lon1
    
        # Haversine formula
        a = math.sin(dlat/2)**2 + math.cos(lat1) * math.cos(lat2) * math.sin(dlon/2)**2
        c = 2 * math.atan2(math.sqrt(a), math.sqrt(1 - a))
        R = 6371.0
    
        return R * c

    def _accuracy_coord(self, gen, gold, d_range=20):
        # 1 if gen is in a range of d_range km of gold
        d = self._haversine_distance(gold, gen)
        return int(d <= d_range)

    def _compute(self, generations, golds, d_range=20):
        assert len(generations) == len(golds)
        assert isinstance(golds, list)
    
        correct, total = 0, 0
    
        for gen, gold in zip(generations, golds):
            # Each gold must be at the format : [lat, long]
            assert len(gold) == 2
            f_gold = (float(gold[0]), float(gold[1]))
    
            try:
                f_ans = ast.literal_eval(gen)
                correct += self._accuracy_coord(f_ans, f_gold, d_range)
                
            except:
                pattern = r'[\(\[]\s*(\d+((,|\s|\.)*\d+)*)\s*[, ]\s*(\d+((,|\s|\.)*\d+)*)\s*[\)\]]'
                matches = re.findall(pattern, gen)
                
                if matches:
                    match = matches[0]
                    f_ans = (float(match[0].replace(',', '.').replace(' ', '')), float(match[3].replace(',', '.').replace(' ', '')))
                    correct += self._accuracy_coord(f_ans, f_gold, d_range)
                            
            total += 1

        metrics = {}
        metrics.update({ 
            'coord_ accuracy': correct/total,
        })
    
        return metrics