Delete deepscreen/models/metrics

deepscreen/models/metrics/bedroc.py

@@ -1,45 +0,0 @@
-import torch
-from torch import Tensor
-from torchmetrics.retrieval.base import RetrievalMetric
-from torchmetrics.utilities.checks import _check_retrieval_functional_inputs
-
-from deepscreen.models.metrics.rie import calc_rie
-
-
-class BEDROC(RetrievalMetric):
-    is_differentiable: bool = False
-    higher_is_better: bool = True
-    full_state_update: bool = False
-
-    def __init__(
-            self,
-            alpha: float = 80.5,
-    ):
-        super().__init__()
-        self.alpha = alpha
-
-    def _metric(self, preds: Tensor, target: Tensor) -> Tensor:
-        preds, target = _check_retrieval_functional_inputs(preds, target)
-
-        n_total = target.size(0)
-        n_actives = target.sum()
-
-        if n_actives == 0:
-            return torch.tensor(0.0, device=preds.device)
-        elif n_actives == n_total:
-            return torch.tensor(1.0, device=preds.device)
-
-        r_a = n_actives / n_total
-        exp_a = torch.exp(torch.tensor(self.alpha))
-
-        idx = torch.argsort(preds, descending=True, stable=True)
-        active_ranks = torch.take(target, idx).nonzero() + 1
-
-        rie = calc_rie(n_total, active_ranks, r_a, exp_a)
-        rie_min = (1 - exp_a ** r_a) / (r_a * (1 - exp_a))
-        rie_max = (1 - exp_a ** (-r_a)) / (r_a * (1 - exp_a ** (-1)))
-
-        return (rie - rie_min) / (rie_max - rie_min)
-
-    def plot(self, val=None, ax=None):
-        return self._plot(val, ax)

deepscreen/models/metrics/ci.py

@@ -1,39 +0,0 @@
-import torch
-from torchmetrics import Metric
-from torchmetrics.utilities.checks import _check_same_shape
-from torchmetrics.utilities.imports import _MATPLOTLIB_AVAILABLE
-
-if not _MATPLOTLIB_AVAILABLE:
-    __doctest_skip__ = ["ConcordanceIndex.plot"]
-
-
-class ConcordanceIndex(Metric):
-    is_differentiable: bool = False
-    higher_is_better: bool = True
-    full_state_update: bool = False
-    plot_lower_bound: float = 0.5
-    plot_upper_bound: float = 1.0
-
-    def __init__(self, dist_sync_on_step=False):
-        super().__init__(dist_sync_on_step=dist_sync_on_step)
-
-        self.add_state("num_concordant", default=torch.tensor(0), dist_reduce_fx="sum")
-        self.add_state("num_valid", default=torch.tensor(0), dist_reduce_fx="sum")
-
-    def update(self, preds: torch.Tensor, target: torch.Tensor):
-        _check_same_shape(preds, target)
-
-        g = preds.unsqueeze(-1) - preds
-        g = (g == 0) * 0.5 + (g > 0)
-
-        f = (target.unsqueeze(-1) - target) > 0
-        f = torch.tril(f, diagonal=0)
-
-        self.num_concordant += torch.sum(torch.mul(g, f)).long()
-        self.num_valid += torch.sum(f).long()
-
-    def compute(self):
-        return torch.where(self.num_valid == 0, 0.0, self.num_concordant / self.num_valid)
-
-    def plot(self, val=None, ax=None):
-        return self._plot(val, ax)

deepscreen/models/metrics/ef.py

@@ -1,34 +0,0 @@
-import math
-
-from torch import Tensor, topk
-from torchmetrics.retrieval.base import RetrievalMetric
-from torchmetrics.utilities.checks import _check_retrieval_functional_inputs
-
-
-class EnrichmentFactor(RetrievalMetric):
-    is_differentiable: bool = False
-    higher_is_better: bool = True
-    full_state_update: bool = False
-
-    def __init__(
-            self,
-            alpha: float,
-    ):
-        super().__init__()
-        if alpha <= 0 or alpha > 1:
-            raise ValueError(f"Argument ``alpha`` has to be in interval (0, 1] but got {alpha}")
-        self.alpha = alpha
-
-    def _metric(self, preds: Tensor, target: Tensor) -> Tensor:
-        preds, target = _check_retrieval_functional_inputs(preds, target)
-
-        n_total = target.size(0)
-        n_sampled = math.ceil(n_total * self.alpha)
-        _, idx = topk(preds, n_sampled)
-        hits_sampled = target[idx].sum()
-        hits_total = target.sum()
-
-        return hits_sampled / (hits_total * self.alpha)
-
-    def plot(self, val=None, ax=None):
-        return self._plot(val, ax)

deepscreen/models/metrics/hit_rate.py

@@ -1,36 +0,0 @@
-import math
-
-from torch import Tensor, topk
-from torchmetrics.retrieval.base import RetrievalMetric
-from torchmetrics.utilities.checks import _check_retrieval_functional_inputs
-
-
-class HitRate(RetrievalMetric):
-    """
-    Computes hit rate for virtual screening.
-    """
-    is_differentiable: bool = False
-    higher_is_better: bool = True
-    full_state_update: bool = False
-
-    def __init__(
-            self,
-            alpha: float = 0.01,
-    ):
-        super().__init__()
-        if alpha <= 0 or alpha > 1:
-            raise ValueError(f"Argument ``alpha`` has to be in interval (0, 1] but got {alpha}")
-        self.alpha = alpha
-
-    def _metric(self, preds: Tensor, target: Tensor) -> Tensor:
-        preds, target = _check_retrieval_functional_inputs(preds, target)
-
-        n_total = target.size(0)
-        n_sampled = math.ceil(n_total * self.alpha)
-        _, idx = topk(preds, n_sampled)
-        hits_sampled = target[idx].sum()
-
-        return hits_sampled / n_sampled
-
-    def plot(self, val=None, ax=None):
-        return self._plot(val, ax)

deepscreen/models/metrics/rie.py

@@ -1,44 +0,0 @@
-import torch
-from torch import Tensor
-from torchmetrics.retrieval.base import RetrievalMetric
-from torchmetrics.utilities.checks import _check_retrieval_functional_inputs
-
-
-def calc_rie(n_total, active_ranks, r_a, exp_a):
-    numerator = (exp_a ** (- active_ranks / n_total)).sum()
-    denominator = (1 - exp_a ** (-1)) / (exp_a ** (1 / n_total) - 1)
-
-    return numerator / (r_a * denominator)
-
-
-class RIE(RetrievalMetric):
-    is_differentiable: bool = False
-    higher_is_better: bool = True
-    full_state_update: bool = False
-
-    def __init__(
-            self,
-            alpha: float = 80.5,
-    ):
-        super().__init__()
-        self.alpha = alpha
-
-    def _metric(self, preds: Tensor, target: Tensor) -> Tensor:
-        preds, target = _check_retrieval_functional_inputs(preds, target)
-
-        n_total = target.size(0)
-        n_actives = target.sum()
-
-        if n_actives == 0:
-            return torch.tensor(0.0, device=preds.device)
-
-        r_a = n_actives / n_total
-        exp_a = torch.exp(torch.tensor(-self.alpha))
-
-        idx = torch.argsort(preds, descending=True, stable=True)
-        active_ranks = torch.take(target, idx).nonzero() + 1
-
-        return calc_rie(n_total, active_ranks, r_a, exp_a)
-
-    def plot(self, val=None, ax=None):
-        return self._plot(val, ax)

deepscreen/models/metrics/sensitivity.py

@@ -1,337 +0,0 @@
-# Copyright The Lightning team.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-from typing import Any, Optional, Sequence, Union
-
-from torch import Tensor
-from torchmetrics.utilities.compute import _safe_divide, _adjust_weights_safe_divide
-from typing_extensions import Literal
-
-from torchmetrics.classification.base import _ClassificationTaskWrapper
-from torchmetrics.classification.stat_scores import BinaryStatScores, MulticlassStatScores, MultilabelStatScores
-from torchmetrics.metric import Metric
-from torchmetrics.utilities.enums import ClassificationTask
-from torchmetrics.utilities.imports import _MATPLOTLIB_AVAILABLE
-from torchmetrics.utilities.plot import _AX_TYPE, _PLOT_OUT_TYPE
-
-if not _MATPLOTLIB_AVAILABLE:
-    __doctest_skip__ = ["BinarySensitivity.plot", "MulticlassSensitivity.plot", "MultilabelSensitivity.plot"]
-
-
-class BinarySensitivity(BinaryStatScores):
-    r"""Compute `Sensitivity`_ for binary tasks.
-
-    .. math:: \text{Sensitivity} = \frac{\text{TN}}{\text{TN} + \text{FP}}
-
-    Where :math:`\text{TN}` and :math:`\text{FP}` represent the number of true negatives and false positives
-    respectively. The metric is only proper defined when :math:`\text{TN} + \text{FP} \neq 0`. If this case is
-    encountered a score of 0 is returned.
-
-    As input to ``forward`` and ``update`` the metric accepts the following input:
-
-    - ``preds`` (:class:`~torch.Tensor`): An int or float tensor of shape ``(N, ...)``. If preds is a floating point
-      tensor with values outside [0,1] range we consider the input to be logits and will auto apply sigmoid per
-      element. Addtionally, we convert to int tensor with thresholding using the value in ``threshold``.
-    - ``target`` (:class:`~torch.Tensor`): An int tensor of shape ``(N, ...)``
-
-    As output to ``forward`` and ``compute`` the metric returns the following output:
-
-    - ``bs`` (:class:`~torch.Tensor`): If ``multidim_average`` is set to ``global``, the metric returns a scalar value.
-      If ``multidim_average`` is set to ``samplewise``, the metric returns ``(N,)`` vector consisting of a scalar value
-      per sample.
-
-    Args:
-        threshold: Threshold for transforming probability to binary {0,1} predictions
-        multidim_average:
-            Defines how additionally dimensions ``...`` should be handled. Should be one of the following:
-
-            - ``global``: Additional dimensions are flatted along the batch dimension
-            - ``samplewise``: Statistic will be calculated independently for each sample on the ``N`` axis.
-              The statistics in this case are calculated over the additional dimensions.
-
-        ignore_index:
-            Specifies a target value that is ignored and does not contribute to the metric calculation
-        validate_args: bool indicating if input arguments and tensors should be validated for correctness.
-            Set to ``False`` for faster computations.
-    """
-    plot_lower_bound: float = 0.0
-    plot_upper_bound: float = 1.0
-
-    def compute(self) -> Tensor:
-        """Compute metric."""
-        tp, fp, tn, fn = self._final_state()
-        return _sensitivity_reduce(tp, fp, tn, fn, average="binary", multidim_average=self.multidim_average)
-
-    def plot(
-            self, val: Optional[Union[Tensor, Sequence[Tensor]]] = None, ax: Optional[_AX_TYPE] = None
-    ) -> _PLOT_OUT_TYPE:
-        """Plot a single or multiple values from the metric.
-
-        Args:
-            val: Either a single result from calling `metric.forward` or `metric.compute` or a list of these results.
-                If no value is provided, will automatically call `metric.compute` and plot that result.
-            ax: An matplotlib axis object. If provided will add plot to that axis
-
-        Returns:
-            Figure object and Axes object
-
-        Raises:
-            ModuleNotFoundError:
-                If `matplotlib` is not installed
-        """
-        return self._plot(val, ax)
-
-
-class MulticlassSensitivity(MulticlassStatScores):
-    r"""Compute `Sensitivity`_ for multiclass tasks.
-
-    .. math:: \text{Sensitivity} = \frac{\text{TN}}{\text{TN} + \text{FP}}
-
-    Where :math:`\text{TN}` and :math:`\text{FP}` represent the number of true negatives and false positives
-    respectively.  The metric is only proper defined when :math:`\text{TN} + \text{FP} \neq 0`. If this case is
-    encountered for any class, the metric for that class will be set to 0 and the overall metric may therefore be
-    affected in turn.
-
-    As input to ``forward`` and ``update`` the metric accepts the following input:
-
-    - ``preds`` (:class:`~torch.Tensor`): An int tensor of shape ``(N, ...)`` or float tensor of shape ``(N, C, ..)``.
-      If preds is a floating point we apply ``torch.argmax`` along the ``C`` dimension to automatically convert
-      probabilities/logits into an int tensor.
-    - ``target`` (:class:`~torch.Tensor`): An int tensor of shape ``(N, ...)``
-
-    As output to ``forward`` and ``compute`` the metric returns the following output:
-
-    - ``mcs`` (:class:`~torch.Tensor`): The returned shape depends on the ``average`` and ``multidim_average``
-      arguments:
-
-        - If ``multidim_average`` is set to ``global``:
-
-          - If ``average='micro'/'macro'/'weighted'``, the output will be a scalar tensor
-          - If ``average=None/'none'``, the shape will be ``(C,)``
-
-        - If ``multidim_average`` is set to ``samplewise``:
-
-          - If ``average='micro'/'macro'/'weighted'``, the shape will be ``(N,)``
-          - If ``average=None/'none'``, the shape will be ``(N, C)``
-
-    Args:
-        num_classes: Integer specifing the number of classes
-        average:
-            Defines the reduction that is applied over labels. Should be one of the following:
-
-            - ``micro``: Sum statistics over all labels
-            - ``macro``: Calculate statistics for each label and average them
-            - ``weighted``: calculates statistics for each label and computes weighted average using their support
-            - ``"none"`` or ``None``: calculates statistic for each label and applies no reduction
-
-        top_k:
-            Number of highest probability or logit score predictions considered to find the correct label.
-            Only works when ``preds`` contain probabilities/logits.
-        multidim_average:
-            Defines how additionally dimensions ``...`` should be handled. Should be one of the following:
-
-            - ``global``: Additional dimensions are flatted along the batch dimension
-            - ``samplewise``: Statistic will be calculated independently for each sample on the ``N`` axis.
-              The statistics in this case are calculated over the additional dimensions.
-
-        ignore_index:
-            Specifies a target value that is ignored and does not contribute to the metric calculation
-        validate_args: bool indicating if input arguments and tensors should be validated for correctness.
-            Set to ``False`` for faster computations.
-    """
-    plot_lower_bound: float = 0.0
-    plot_upper_bound: float = 1.0
-    plot_legend_name: str = "Class"
-
-    def compute(self) -> Tensor:
-        """Compute metric."""
-        tp, fp, tn, fn = self._final_state()
-        return _sensitivity_reduce(tp, fp, tn, fn, average=self.average, multidim_average=self.multidim_average)
-
-    def plot(
-            self, val: Optional[Union[Tensor, Sequence[Tensor]]] = None, ax: Optional[_AX_TYPE] = None
-    ) -> _PLOT_OUT_TYPE:
-        """Plot a single or multiple values from the metric.
-
-        Args:
-            val: Either a single result from calling `metric.forward` or `metric.compute` or a list of these results.
-                If no value is provided, will automatically call `metric.compute` and plot that result.
-            ax: An matplotlib axis object. If provided will add plot to that axis
-
-        Returns:
-            Figure object and Axes object
-
-        Raises:
-            ModuleNotFoundError:
-                If `matplotlib` is not installed
-        """
-        return self._plot(val, ax)
-
-
-class MultilabelSensitivity(MultilabelStatScores):
-    r"""Compute `Sensitivity`_ for multilabel tasks.
-
-    .. math:: \text{Sensitivity} = \frac{\text{TN}}{\text{TN} + \text{FP}}
-
-    Where :math:`\text{TN}` and :math:`\text{FP}` represent the number of true negatives and false positives
-    respectively. The metric is only proper defined when :math:`\text{TN} + \text{FP} \neq 0`. If this case is
-    encountered for any label, the metric for that label will be set to 0 and the overall metric may therefore be
-    affected in turn.
-
-    As input to ``forward`` and ``update`` the metric accepts the following input:
-
-    - ``preds`` (:class:`~torch.Tensor`): An int or float tensor of shape ``(N, C, ...)``. If preds is a floating
-      point tensor with values outside [0,1] range we consider the input to be logits and will auto apply sigmoid
-      per element. Addtionally, we convert to int tensor with thresholding using the value in ``threshold``.
-    - ``target`` (:class:`~torch.Tensor`): An int tensor of shape ``(N, C, ...)``
-
-
-    As output to ``forward`` and ``compute`` the metric returns the following output:
-
-    - ``mls`` (:class:`~torch.Tensor`): The returned shape depends on the ``average`` and ``multidim_average``
-      arguments:
-
-        - If ``multidim_average`` is set to ``global``
-
-          - If ``average='micro'/'macro'/'weighted'``, the output will be a scalar tensor
-          - If ``average=None/'none'``, the shape will be ``(C,)``
-
-        - If ``multidim_average`` is set to ``samplewise``
-
-          - If ``average='micro'/'macro'/'weighted'``, the shape will be ``(N,)``
-          - If ``average=None/'none'``, the shape will be ``(N, C)``
-
-    Args:
-        num_labels: Integer specifing the number of labels
-        threshold: Threshold for transforming probability to binary (0,1) predictions
-        average:
-            Defines the reduction that is applied over labels. Should be one of the following:
-
-            - ``micro``: Sum statistics over all labels
-            - ``macro``: Calculate statistics for each label and average them
-            - ``weighted``: calculates statistics for each label and computes weighted average using their support
-            - ``"none"`` or ``None``: calculates statistic for each label and applies no reduction
-
-        multidim_average: Defines how additionally dimensions ``...`` should be handled. Should be one of the following:
-
-            - ``global``: Additional dimensions are flatted along the batch dimension
-            - ``samplewise``: Statistic will be calculated independently for each sample on the ``N`` axis.
-              The statistics in this case are calculated over the additional dimensions.
-
-        ignore_index:
-            Specifies a target value that is ignored and does not contribute to the metric calculation
-        validate_args: bool indicating if input arguments and tensors should be validated for correctness.
-            Set to ``False`` for faster computations.
-    """
-    plot_lower_bound: float = 0.0
-    plot_upper_bound: float = 1.0
-    plot_legend_name: str = "Label"
-
-    def compute(self) -> Tensor:
-        """Compute metric."""
-        tp, fp, tn, fn = self._final_state()
-        return _sensitivity_reduce(
-            tp, fp, tn, fn, average=self.average, multidim_average=self.multidim_average, multilabel=True
-        )
-
-    def plot(
-            self, val: Optional[Union[Tensor, Sequence[Tensor]]] = None, ax: Optional[_AX_TYPE] = None
-    ) -> _PLOT_OUT_TYPE:
-        """Plot a single or multiple values from the metric.
-
-        Args:
-            val: Either a single result from calling `metric.forward` or `metric.compute` or a list of these results.
-                If no value is provided, will automatically call `metric.compute` and plot that result.
-            ax: An matplotlib axis object. If provided will add plot to that axis
-
-        Returns:
-            Figure object and Axes object
-
-        Raises:
-            ModuleNotFoundError:
-                If `matplotlib` is not installed
-        """
-        return self._plot(val, ax)
-
-
-class Sensitivity(_ClassificationTaskWrapper):
-    r"""Compute `Sensitivity`_.
-
-    .. math:: \text{Sensitivity} = \frac{\text{TN}}{\text{TN} + \text{FP}}
-
-    Where :math:`\text{TN}` and :math:`\text{FP}` represent the number of true negatives and false positives
-    respectively. The metric is only proper defined when :math:`\text{TP} + \text{FP} \neq 0`. If this case is
-    encountered for any class/label, the metric for that class/label will be set to 0 and the overall metric may
-    therefore be affected in turn.
-
-    This function is a simple wrapper to get the task specific versions of this metric, which is done by setting the
-    ``task`` argument to either ``'binary'``, ``'multiclass'`` or ``multilabel``. See the documentation of
-    :class:`~torchmetrics.classification.BinarySensitivity`, :class:`~torchmetrics.classification.MulticlassSensitivity`
-    and :class:`~torchmetrics.classification.MultilabelSensitivity` for the specific details of each argument influence
-    and examples.
-
-    Legacy Example:
-    """
-
-    def __new__(  # type: ignore[misc]
-            cls,
-            task: Literal["binary", "multiclass", "multilabel"],
-            threshold: float = 0.5,
-            num_classes: Optional[int] = None,
-            num_labels: Optional[int] = None,
-            average: Optional[Literal["micro", "macro", "weighted", "none"]] = "micro",
-            multidim_average: Optional[Literal["global", "samplewise"]] = "global",
-            top_k: Optional[int] = 1,
-            ignore_index: Optional[int] = None,
-            validate_args: bool = True,
-            **kwargs: Any,
-    ) -> Metric:
-        """Initialize task metric."""
-        task = ClassificationTask.from_str(task)
-        assert multidim_average is not None  # noqa: S101  # needed for mypy
-        kwargs.update(
-            {"multidim_average": multidim_average, "ignore_index": ignore_index, "validate_args": validate_args}
-        )
-        if task == ClassificationTask.BINARY:
-            return BinarySensitivity(threshold, **kwargs)
-        if task == ClassificationTask.MULTICLASS:
-            if not isinstance(num_classes, int):
-                raise ValueError(f"`num_classes` is expected to be `int` but `{type(num_classes)} was passed.`")
-            if not isinstance(top_k, int):
-                raise ValueError(f"`top_k` is expected to be `int` but `{type(top_k)} was passed.`")
-            return MulticlassSensitivity(num_classes, top_k, average, **kwargs)
-        if task == ClassificationTask.MULTILABEL:
-            if not isinstance(num_labels, int):
-                raise ValueError(f"`num_labels` is expected to be `int` but `{type(num_labels)} was passed.`")
-            return MultilabelSensitivity(num_labels, threshold, average, **kwargs)
-        raise ValueError(f"Task {task} not supported!")
-
-
-def _sensitivity_reduce(
-        tp: Tensor,
-        fp: Tensor,
-        tn: Tensor,
-        fn: Tensor,
-        average: Optional[Literal["binary", "micro", "macro", "weighted", "none"]],
-        multidim_average: Literal["global", "samplewise"] = "global",
-        multilabel: bool = False,
-) -> Tensor:
-    if average == "binary":
-        return _safe_divide(tp, tp + fn)
-    if average == "micro":
-        tp = tp.sum(dim=0 if multidim_average == "global" else 1)
-        fn = fn.sum(dim=0 if multidim_average == "global" else 1)
-        return _safe_divide(tp, tp + fn)
-
-    sensitivity_score = _safe_divide(tp, tp + fn)
-    return _adjust_weights_safe_divide(sensitivity_score, average, multilabel, tp, fp, fn)