File size: 6,460 Bytes
8c838e7 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 |
#!/usr/bin/env python
# encoding: utf-8
# The MIT License (MIT)
# Copyright (c) 2012-2019 CNRS
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
# AUTHORS
# Hervé BREDIN - http://herve.niderb.fr
"""Metrics for diarization"""
from typing import Optional, Dict, TYPE_CHECKING
from pyannote_audio_utils.core import Annotation, Timeline
from pyannote_audio_utils.core.utils.types import Label
from .identification import IdentificationErrorRate
from .matcher import HungarianMapper
from .types import Details, MetricComponents
if TYPE_CHECKING:
pass
# TODO: can't we put these as class attributes?
DER_NAME = 'diarization error rate'
class DiarizationErrorRate(IdentificationErrorRate):
"""Diarization error rate
First, the optimal mapping between reference and hypothesis labels
is obtained using the Hungarian algorithm. Then, the actual diarization
error rate is computed as the identification error rate with each hypothesis
label translated into the corresponding reference label.
Parameters
----------
collar : float, optional
Duration (in seconds) of collars removed from evaluation around
boundaries of reference segments.
skip_overlap : bool, optional
Set to True to not evaluate overlap regions.
Defaults to False (i.e. keep overlap regions).
Usage
-----
* Diarization error rate between `reference` and `hypothesis` annotations
>>> metric = DiarizationErrorRate()
>>> reference = Annotation(...) # doctest: +SKIP
>>> hypothesis = Annotation(...) # doctest: +SKIP
>>> value = metric(reference, hypothesis) # doctest: +SKIP
* Compute global diarization error rate and confidence interval
over multiple documents
>>> for reference, hypothesis in ... # doctest: +SKIP
... metric(reference, hypothesis) # doctest: +SKIP
>>> global_value = abs(metric) # doctest: +SKIP
>>> mean, (lower, upper) = metric.confidence_interval() # doctest: +SKIP
* Get diarization error rate detailed components
>>> components = metric(reference, hypothesis, detailed=True) #doctest +SKIP
* Get accumulated components
>>> components = metric[:] # doctest: +SKIP
>>> metric['confusion'] # doctest: +SKIP
See Also
--------
:class:`pyannote_audio_utils.metric.base.BaseMetric`: details on accumulation
:class:`pyannote_audio_utils.metric.identification.IdentificationErrorRate`: identification error rate
"""
@classmethod
def metric_name(cls) -> str:
return DER_NAME
def __init__(self, collar: float = 0.0, skip_overlap: bool = False,
**kwargs):
super().__init__(collar=collar, skip_overlap=skip_overlap, **kwargs)
self.mapper_ = HungarianMapper()
def optimal_mapping(self,
reference: Annotation,
hypothesis: Annotation,
uem: Optional[Timeline] = None) -> Dict[Label, Label]:
"""Optimal label mapping
Parameters
----------
reference : Annotation
hypothesis : Annotation
Reference and hypothesis diarization
uem : Timeline
Evaluation map
Returns
-------
mapping : dict
Mapping between hypothesis (key) and reference (value) labels
"""
# NOTE that this 'uemification' will not be called when
# 'optimal_mapping' is called from 'compute_components' as it
# has already been done in 'compute_components'
if uem:
reference, hypothesis = self.uemify(reference, hypothesis, uem=uem)
# call hungarian mapper
return self.mapper_(hypothesis, reference)
def compute_components(self,
reference: Annotation,
hypothesis: Annotation,
uem: Optional[Timeline] = None,
**kwargs) -> Details:
# crop reference and hypothesis to evaluated regions (uem)
# remove collars around reference segment boundaries
# remove overlap regions (if requested)
reference, hypothesis, uem = self.uemify(
reference, hypothesis, uem=uem,
collar=self.collar, skip_overlap=self.skip_overlap,
returns_uem=True)
# NOTE that this 'uemification' must be done here because it
# might have an impact on the search for the optimal mapping.
# make sure reference only contains string labels ('A', 'B', ...)
reference = reference.rename_labels(generator='string')
# make sure hypothesis only contains integer labels (1, 2, ...)
hypothesis = hypothesis.rename_labels(generator='int')
# optimal (int --> str) mapping
mapping = self.optimal_mapping(reference, hypothesis)
# compute identification error rate based on mapped hypothesis
# NOTE that collar is set to 0.0 because 'uemify' has already
# been applied (same reason for setting skip_overlap to False)
mapped = hypothesis.rename_labels(mapping=mapping)
return super(DiarizationErrorRate, self) \
.compute_components(reference, mapped, uem=uem,
collar=0.0, skip_overlap=False,
**kwargs)
|