add metric functions and tests

aaa

add metric function and tests

README.md

@@ -1,12 +1,50 @@
 ---
-title: Coord Eval
-emoji: 📚
-colorFrom: gray
-colorTo: red
+title: Coord_eval
+datasets:
+- GeoBenchmark
+tags:
+- evaluate
+- metric
+description: "TODO: add a description here"
 sdk: gradio
-sdk_version: 6.5.1
+sdk_version: 3.19.1
 app_file: app.py
 pinned: false
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Metric Card for Coord_eval
+
+***Module Card Instructions:*** *Fill out the following subsections. Feel free to take a look at existing metric cards if you'd like examples.*
+
+## Metric Description
+*Give a brief overview of this metric, including what task(s) it is usually used for, if any.*
+
+## How to Use
+*Give general statement of how to use the metric*
+
+*Provide simplest possible example for using the metric*
+
+### Inputs
+*List all input arguments in the format below*
+- **input_field** *(type): Definition of input, with explanation if necessary. State any default value(s).*
+
+### Output Values
+
+*Explain what this metric outputs and provide an example of what the metric output looks like. Modules should return a dictionary with one or multiple key-value pairs, e.g. {"bleu" : 6.02}*
+
+*State the range of possible values that the metric's output can take, as well as what in that range is considered good. For example: "This metric can take on any value between 0 and 100, inclusive. Higher scores are better."*
+
+#### Values from Popular Papers
+*Give examples, preferrably with links to leaderboards or publications, to papers that have reported this metric, along with the values they have reported.*
+
+### Examples
+*Give code examples of the metric being used. Try to include examples that clear up any potential ambiguity left from the metric description above. If possible, provide a range of examples that show both typical and atypical results, as well as examples where a variety of input parameters are passed.*
+
+## Limitations and Bias
+*Note any known limitations or biases that the metric has, with links and references if possible.*
+
+## Citation
+*Cite the source where this metric was introduced.*
+
+## Further References
+*Add any useful further references.*

app.py

@@ -0,0 +1,6 @@
+import evaluate
+from evaluate.utils import launch_gradio_widget
+
+
+module = evaluate.load("rfr2003/coord_eval")
+launch_gradio_widget(module)

coord_eval.py

@@ -0,0 +1,139 @@
+# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""TODO: Add a description here."""
+
+import evaluate
+import datasets
+import re
+
+
+# TODO: Add BibTeX citation
+_CITATION = """\
+@InProceedings{huggingface:module,
+title = {A great new module},
+authors={huggingface, Inc.},
+year={2020}
+}
+"""
+
+# TODO: Add description of the module here
+_DESCRIPTION = """\
+This metric aims to compute a coordinate accuracy between coordinates generated by an LM and a golden one. A pair of coordinates is considered correct if its haversine distance to the golden coordinates is inferior to a threeshold d.
+"""
+
+
+# TODO: Add description of the arguments of the module here
+_KWARGS_DESCRIPTION = """
+Calculates how good are predictions given some references, using certain scores
+Args:
+    generations: list of predictions to score. Each predictions
+        should be a string generated by a LM model.
+    golds: list of reference for each prediction. Each
+        reference should be a list of two floats corresponding to the latitude and longitude of the ground truth (eg. [12.8, 76.9]).
+Returns:
+    accuracy: 1 if coordinates predicted are d distant from gold ones, O otherwise.
+Examples:
+    >>> my_new_module = evaluate.load("rfr2003/coord_eval")
+    >>> results = my_new_module.compute(references=["(12.7, 67.8)", "(16.7, 89.6)"], predictions=[[12.7, 67.8], [10.9, 80.6]], d=20)
+    >>> print(results)
+    {'coord_accuracy': 0.5}
+"""
+
+# TODO: Define external resources urls if needed
+BAD_WORDS_URL = "http://url/to/external/resource/bad_words.txt"
+
+
+@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
+class Coord_eval(evaluate.Metric):
+    """TODO: Short description of my evaluation module."""
+
+    def _info(self):
+        # TODO: Specifies the evaluate.EvaluationModuleInfo object
+        return evaluate.MetricInfo(
+            # This is the description that will appear on the modules page.
+            module_type="metric",
+            description=_DESCRIPTION,
+            citation=_CITATION,
+            inputs_description=_KWARGS_DESCRIPTION,
+            # This defines the format of each prediction and reference
+            features=datasets.Features({
+                'generations': datasets.Value('string'),
+                'golds': datasets.Value('float32'),
+            }),
+            # Homepage of the module for documentation
+            homepage="http://module.homepage",
+            # Additional links to the codebase or references
+            codebase_urls=["http://github.com/path/to/codebase/of/new_module"],
+            reference_urls=["http://path.to.reference.url/new_module"]
+        )
+
+    def _download_and_prepare(self, dl_manager):
+        """Optional: download external resources useful to compute the scores"""
+        # TODO: Download external resources if needed
+        pass
+
+    def _haversine_distance(self, coord1, coord2):
+        lat1, lon1 = coord1
+        lat2, lon2 = coord2
+    
+        # Convert degrees to radians
+        lat1, lon1, lat2, lon2 = map(math.radians, [lat1, lon1, lat2, lon2])
+    
+        dlat = lat2 - lat1
+        dlon = lon2 - lon1
+    
+        # Haversine formula
+        a = math.sin(dlat/2)**2 + math.cos(lat1) * math.cos(lat2) * math.sin(dlon/2)**2
+        c = 2 * math.atan2(math.sqrt(a), math.sqrt(1 - a))
+        R = 6371.0
+    
+        return R * c
+
+    def _accuracy_coord(self, gen, gold, d_range=20):
+        # 1 if gen is in a range of d_range km of gold
+        d = self._haversine_distance(gold, gen)
+        return int(d <= d_range)
+
+    def _compute(self, generations, golds, d_range=20):
+        assert len(generations) == len(golds)
+        assert isinstance(golds, list)
+    
+        correct, total = 0, 0
+    
+        for gen, gold in zip(generations, golds):
+            # Each gold must be at the format : [lat, long]
+            assert len(gold) == 2
+            f_gold = (float(gold[0]), float(gold[1]))
+    
+            try:
+                f_ans = ast.literal_eval(gen)
+                correct += self._accuracy_coord(f_ans, f_gold, d_range)
+                
+            except:
+                pattern = r'[\(\[]\s*(\d+((,|\s|\.)*\d+)*)\s*[, ]\s*(\d+((,|\s|\.)*\d+)*)\s*[\)\]]'
+                matches = re.findall(pattern, gen)
+                
+                if matches:
+                    match = matches[0]
+                    f_ans = (float(match[0].replace(',', '.').replace(' ', '')), float(match[3].replace(',', '.').replace(' ', '')))
+                    correct += accuracy_coord(f_ans, f_gold, d_range)
+                            
+            total += 1
+
+        metrics = {}
+        metrics.update({ 
+            'coord_ accuracy': correct/total,
+        })
+    
+        return metrics

requirements.txt

@@ -0,0 +1 @@
+git+https://github.com/huggingface/evaluate@main

tests.py

@@ -0,0 +1,14 @@
+test_cases = [
+    {
+        "predictions": ["(12.7, 67.8)", "(16.7, 89.6)"],
+        "references": [[12.7, 67.8], [10.9, 80.6]],
+        "d": 20,
+        "result": {"coord_accuracy": 0.5}
+    },
+    {
+        "predictions": ["(12.7, 67.8)", "(16.7, 89.6)"],
+        "references": [[12.7, 67.8], [10.9, 80.6]],
+        "d": 1000000,
+        "result": {"coord_accuracy": 1}
+    },
+]