Add new module

eval_keyphrase.py

@@ -13,6 +13,8 @@
 # limitations under the License.
 """TODO: Add a description here."""
 
+import string
+
 import evaluate
 import datasets
 
@@ -86,10 +88,50 @@ class eval_keyphrase(evaluate.Metric):
         # TODO: Download external resources if needed
         pass
 
+    def _normalize_keyphrase(self, kp):
+
+        def white_space_fix(text):
+            return ' '.join(text.split())
+
+        def remove_punc(text):
+            exclude = set(string.punctuation)
+            return ''.join(ch for ch in text if ch not in exclude)
+
+        def lower(text):
+            return text.lower()
+
+        return white_space_fix(remove_punc(lower(kp)))
+
     def _compute(self, predictions, references):
         """Returns the scores"""
-        # TODO: Compute the different scores of the module
-        accuracy = sum(i == j for i, j in zip(predictions, references)) / len(predictions)
+
+        macro_metrics = {'precision': [], 'recall': [], 'f1': [], 'num_pred': [], 'num_gold': []}
+
+        for targets, preds in zip(references, predictions):
+            targets = [self._normalize_keyphrase(tmp_key).strip() for tmp_key in targets if len(self._normalize_keyphrase(tmp_key).strip()) != 0]
+            preds = [self._normalize_keyphrase(tmp_key).strip() for tmp_key in preds if len(self._normalize_keyphrase(tmp_key).strip()) != 0]
+
+            total_tgt_set = set(targets)
+            total_preds = set(preds)
+            if len(total_tgt_set) == 0: continue
+
+            # get the total_correctly_matched indicators
+            total_correctly_matched = len(total_preds & total_tgt_set)
+
+            # macro metric calculating
+            precision = total_correctly_matched / len(total_preds) if len(total_preds) else 0.0
+            recall = total_correctly_matched / len(total_tgt_set)
+            f1 = 2 * precision * recall / (precision + recall) if total_correctly_matched > 0 else 0.0
+            macro_metrics['precision'].append(precision)
+            macro_metrics['recall'].append(recall)
+            macro_metrics['f1'].append(f1)
+            macro_metrics['num_pred'].append(len(total_preds))
+            macro_metrics['num_gold'].append(len(total_tgt_set))
+
         return {
-            "accuracy": accuracy,
+            "precision": round(sum(macro_metrics["precision"])/len(macro_metrics["precision"]), 4),
+            "recall": round(sum(macro_metrics["recall"])/len(macro_metrics["recall"]), 4),
+            "f1": round(sum(macro_metrics["f1"])/len(macro_metrics["f1"]), 4),
+            "num_pred": round(sum(macro_metrics["num_pred"])/len(macro_metrics["num_pred"]), 4),
+            "num_gold": round(sum(macro_metrics["num_gold"])/len(macro_metrics["num_gold"]), 4),
         }