Add app and module.

README.md

@@ -1,12 +1,70 @@
 ---
-title: Ccc
-emoji: 🐠
-colorFrom: gray
-colorTo: indigo
+title: ccc
+tags:
+- evaluate
+- metric
+description: "Concordance correlation coefficient"
 sdk: gradio
-sdk_version: 4.16.0
+sdk_version: 3.19.1
 app_file: app.py
 pinned: false
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Metric Card for CCC
+
+## Metric Description
+The concordance correlation coefficient measures the agreement between two sets of
+values. It is often used as a measure of inter-rater agreement when ratings have
+continuous values.
+
+## How to Use
+The inputs are two sequences of floating point values. For example:
+```python
+ccc_metric = evaluate.load("agkphysics/ccc")
+results = ccc_metric.compute(references=[0.2, 0.1], predictions=[0.1, 0.2])
+```
+
+### Inputs
+- **predictions** (list of float): model predictions
+- **references** (list of float): reference labels
+
+### Output Values
+- `ccc`: the concordance correlation coefficient. This is a value between -1 (perfect
+anti-agreement) and 1 (perfect agreement), with 0 indicating no agreement.
+
+### Examples
+```python
+>>> ccc_metric = evaluate.load("agkphysics/ccc")
+>>> results = ccc_metric.compute(references=[0.2, 0.1], predictions=[0.1, 0.2])
+>>> print(results)
+{'ccc': -1.0}
+>>> results = ccc_metric.compute(references=[0.1, 0.2], predictions=[0.1, 0.2])
+>>> print(results)
+{'ccc': 1.0}
+>>> results = ccc_metric.compute(references=[0.1, 0.3], predictions=[0.1, 0.2])
+>>> print(results)
+{'ccc': 0.666666641831399}
+```
+
+## Limitations and Bias
+*Note any known limitations or biases that the metric has, with links and references if possible.*
+
+## Citation
+```bibtex
+@article{linConcordanceCorrelationCoefficient1989,
+  title = {A {{Concordance Correlation Coefficient}} to {{Evaluate Reproducibility}}},
+  author = {Lin, Lawrence I-Kuei},
+  year = {1989},
+  journal = {Biometrics},
+  volume = {45},
+  number = {1},
+  pages = {255--268},
+  publisher = {{International Biometric Society}},
+  issn = {0006-341X},
+  url = {https://www.jstor.org/stable/2532051},
+  doi = {10.2307/2532051}
+}
+```
+
+## Further References
+Wikipedia: https://en.wikipedia.org/wiki/Concordance_correlation_coefficient

app.py

@@ -0,0 +1,6 @@
+import evaluate
+from evaluate.utils import launch_gradio_widget
+
+
+module = evaluate.load("agkphysics/ccc")
+launch_gradio_widget(module)

ccc.py

@@ -0,0 +1,107 @@
+# Copyright (C) 2024 Aaron Keesing
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# “Software”), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+"""Concordance correlation coefficient"""
+
+import datasets
+import evaluate
+import numpy as np
+
+_CITATION = """\
+@article{linConcordanceCorrelationCoefficient1989,
+  title = {A {{Concordance Correlation Coefficient}} to {{Evaluate Reproducibility}}},
+  author = {Lin, Lawrence I-Kuei},
+  year = {1989},
+  journal = {Biometrics},
+  volume = {45},
+  number = {1},
+  pages = {255--268},
+  publisher = {{International Biometric Society}},
+  issn = {0006-341X},
+  url = {https://www.jstor.org/stable/2532051},
+  doi = {10.2307/2532051}
+}
+
+"""
+
+_DESCRIPTION = """\
+A metric to measure the degree of agreement between continuous-values evaluations from two raters.
+"""
+
+
+# TODO: Add description of the arguments of the module here
+_KWARGS_DESCRIPTION = """
+Calculates the CCC between predictions and references
+Args:
+    predictions: list of predictions to score. Each prediction
+        should be a floating point value.
+    references: list of references, one for each prediction. Each
+        reference should be a floating point value.
+Returns:
+    ccc: the concordance correlation coefficient, -1 <= ccc <= 1
+Examples:
+    >>> ccc_metric = evaluate.load("agkphysics/ccc")
+    >>> results = ccc_metric.compute(references=[0.2, 0.1], predictions=[0.1, 0.2])
+    >>> print(results)
+    {'ccc': -1.0}
+
+    >>> results = ccc_metric.compute(references=[0.1, 0.2], predictions=[0.1, 0.2])
+    >>> print(results)
+    {'ccc': 1.0}
+
+    >>> results = ccc_metric.compute(references=[0.1, 0.3], predictions=[0.1, 0.2])
+    >>> print(results)
+    {'ccc': 0.666666641831399}
+"""
+
+
+@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
+class CCC(evaluate.Metric):
+    """Computes the CCC, concordance correlation coefficient."""
+
+    def _info(self):
+        return evaluate.MetricInfo(
+            module_type="metric",
+            description=_DESCRIPTION,
+            citation=_CITATION,
+            inputs_description=_KWARGS_DESCRIPTION,
+            features=datasets.Features(
+                {
+                    "predictions": datasets.Value("float32"),
+                    "references": datasets.Value("float32"),
+                }
+            ),
+            homepage="https://en.wikipedia.org/wiki/Concordance_correlation_coefficient",
+            reference_urls=[
+                "https://www.jstor.org/stable/2532051",
+                "https://en.wikipedia.org/wiki/Concordance_correlation_coefficient",
+            ],
+        )
+
+    def _compute(self, predictions, references):
+        """Returns the CCC score"""
+        sxy = np.cov(predictions, references, ddof=0)[0, 1]
+        sxx = np.var(predictions, ddof=0)
+        syy = np.var(references, ddof=0)
+        mu_x = np.mean(predictions)
+        mu_y = np.mean(references)
+        ccc = 2 * sxy / (sxx + syy + (mu_x - mu_y) ** 2)
+        return {"ccc": ccc}

requirements.txt

@@ -0,0 +1,2 @@
+git+https://github.com/huggingface/evaluate@main
+numpy

tests.py

@@ -0,0 +1,5 @@
+test_cases = [
+    {"predictions": [0.1, 0.2], "references": [0.2, 0.1], "result": {"ccc": -1}},
+    {"predictions": [0.1, 0.2], "references": [0.1, 0.2], "result": {"ccc": 1}},
+    {"predictions": [0.1, 0.2], "references": [0.1, 0.3], "result": {"ccc": 0.6666666}},
+]