File size: 8,706 Bytes
1a0e6e8 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 | """
Scoring Module
End-to-end pipeline for computing audio similarity scores.
"""
import logging
import torch
from typing import Optional, Dict
from audio_loader import AudioLoader
from wavlm_encoder import WavLMEncoder
from dtw_similarity import DTWSimilarity
logger = logging.getLogger(__name__)
class SimilarityScorer:
"""End-to-end similarity scoring pipeline."""
def __init__(
self,
model_name: str = "./wavlm-base-plus",
device: Optional[str] = None,
distance_metric: str = "cosine",
sakoe_chiba_ratio: float = 0.1,
normalize_dtw: bool = True,
score_midpoint: float = 0.3,
score_steepness: float = 10.0
):
"""
Initialize similarity scorer.
Args:
model_name: WavLM model name from HuggingFace
device: Device to run model on
distance_metric: Distance metric for DTW ("cosine" or "euclidean")
sakoe_chiba_ratio: Band radius ratio for DTW constraint
normalize_dtw: Whether to normalize DTW distance by path length
score_midpoint: Distance value that maps to 50 for the logisitic score
score_steepness: Logistic steepness for score mapping
"""
self.audio_loader = AudioLoader(target_sr=16000)
self.encoder = WavLMEncoder(model_name=model_name, device=device)
self.dtw_similarity = DTWSimilarity(distance_metric=distance_metric, sakoe_chiba_ratio=sakoe_chiba_ratio)
self.normalize_dtw = normalize_dtw
self.score_midpoint = score_midpoint
self.score_steepness = score_steepness
def compute_similarity(
self,
audio_path1: str,
audio_path2: str
) -> float:
"""
Compute similarity score between two audio files.
Args:
audio_path1: Path to first audio file (participant's recitation)
audio_path2: Path to second audio file (reference recitation)
Returns:
Similarity score (DTW distance - lower means more similar)
"""
# Load and preprocess audio
waveform1 = self.audio_loader.load_and_preprocess(audio_path1)
waveform2 = self.audio_loader.load_and_preprocess(audio_path2)
# Extract features
features1 = self.encoder.encode(waveform1)
features2 = self.encoder.encode(waveform2)
# Compute DTW similarity
dtw_distance = self.dtw_similarity.compute_similarity(
features1,
features2,
normalize=self.normalize_dtw
)
logger.info(
"compute_similarity | audio1=%s | audio2=%s | dtw_distance=%.6f",
audio_path1, audio_path2, dtw_distance,
)
return dtw_distance
def compute_similarity_score_normalized(
self,
audio_path1: str,
audio_path2: str
) -> float:
"""
Compute normalized similarity score (0-100 scale, higher is better).
Uses a logistic (sigmoid) mapping calibrated so that:
- identical pairs (d β 0.05) β score ~ 92
- very similar (d β 0.15) β score ~ 82
* borderline (d β 0.30) β score = 50
* different pairs (d β 0.35) β score ~ 38
* noise/non-speech (d > 0.50) β score < 12
Args:
audio_path1: Path to first audio file
audio_path2: Path to second audio file
Returns:
Normalized similarity score (0-100, where 100 is perfect match)
"""
# Get DTW distance (already path-normalised when self.normalize_dtw)
dtw_distance = self.compute_similarity(audio_path1, audio_path2)
# Logistic mapping (calibrated for cosine-distance DTW)
similarity_score = DTWSimilarity.distance_to_score(
dtw_distance,
midpoint=self.score_midpoint,
steepness=self.score_steepness
)
logger.info(
"score_normalized | dtw_distance=%.6f | score=%.2f",
dtw_distance, similarity_score,
)
return similarity_score
def compute_detailed_similarity(
self,
audio_path1: str,
audio_path2: str,
use_vad: bool = True,
layer_indices: Optional[list] = None
) -> Dict:
"""
Compute detailed similarity analysis (supports multiple layers).
Args:
audio_path1: Path to first audio file (reference)
audio_path2: Path to second audio file (test)
use_vad: Whether to use VAD endpoint trimming
layer_indices: List of WavLM layers to extract (defaults to [12])
Returns:
Dictionary containing detailed metrics, warping paths, DTW matrices, and original waveforms.
"""
if layer_indices is None:
layer_indices = [12]
self.audio_loader.use_vad = use_vad
# Process and retrieve both raw and VAD waveforms
waveform_ref_raw, sr_ref = self.audio_loader.load_audio(audio_path1)
waveform_test_raw, sr_test = self.audio_loader.load_audio(audio_path2)
if use_vad:
waveform_ref_vad = self.audio_loader.vad_trim_endpoints(waveform_ref_raw, sr_ref)
waveform_test_vad = self.audio_loader.vad_trim_endpoints(waveform_test_raw, sr_test)
else:
waveform_ref_vad = waveform_ref_raw
waveform_test_vad = waveform_test_raw
waveform_ref = self.audio_loader.normalize_audio(waveform_ref_vad)
waveform_test = self.audio_loader.normalize_audio(waveform_test_vad)
# Extract features for all requested layers
feat_ref_dict = self.encoder.encode(waveform_ref, extract_layers=layer_indices)
feat_test_dict = self.encoder.encode(waveform_test, extract_layers=layer_indices)
results = {}
for layer in layer_indices:
seq1 = feat_ref_dict[layer].cpu().numpy()
seq2 = feat_test_dict[layer].cpu().numpy()
# Compute DTW
raw_distance, dtw_matrix = self.dtw_similarity.dtw(seq1, seq2)
warping_path = self.dtw_similarity.compute_path(dtw_matrix)
path_length = len(warping_path)
# Normalize
if self.normalize_dtw and path_length > 0:
normalized_distance = raw_distance / path_length
else:
normalized_distance = raw_distance
# Logistic score
normalized_score = DTWSimilarity.distance_to_score(
normalized_distance,
midpoint=self.score_midpoint,
steepness=self.score_steepness
)
num_frames_ref = seq1.shape[0]
num_frames_test = seq2.shape[0]
sr = 16000
ref_duration_sec = round(num_frames_ref * 320 / sr, 3)
test_duration_sec = round(num_frames_test * 320 / sr, 3)
duration_ratio = round(test_duration_sec / ref_duration_sec, 4) if ref_duration_sec > 0 else 0.0
diagnostics = {
"raw_dtw_distance": float(raw_distance),
"normalized_distance": float(normalized_distance),
"path_length": int(path_length),
"num_frames_ref": num_frames_ref,
"num_frames_test": num_frames_test,
"feature_dimension": seq1.shape[1],
"ref_duration_sec": ref_duration_sec,
"test_duration_sec": test_duration_sec,
"duration_ratio": duration_ratio,
"distance_metric": self.dtw_similarity.distance_metric,
"normalized": self.normalize_dtw,
"sakoe_chiba_ratio": self.dtw_similarity.sakoe_chiba_ratio,
}
results[layer] = {
"score": normalized_score,
"dtw_distance": float(normalized_distance),
"raw_dtw_distance": float(raw_distance),
"warping_path": warping_path,
"dtw_matrix": dtw_matrix,
"diagnostics": diagnostics
}
return {
"results": results,
"waveforms": {
"ref_raw": waveform_ref_raw,
"ref_vad": waveform_ref_vad,
"ref_normalized": waveform_ref,
"test_raw": waveform_test_raw,
"test_vad": waveform_test_vad,
"test_normalized": waveform_test,
}
}
|