| | --- |
| | language: |
| | - en |
| | license: mit |
| | base_model: |
| | - facebook/wav2vec2-base |
| | --- |
| | |
| | SCD（Speaker Change Detection，讲者变化检测）：是指在音频或视频内容中识别出讲话者发生变化的技术。它通常被应用于多讲者的对话或演讲场景中，以此来检测何时从一个讲者切换到另一个讲者。 |
| |
|
| | 如何使用 |
| | # Note: at the time this code was originally written, transformers.Wav2Vec2ForAudioFrameClassification was incomplete |
| | # -> this adds the then-missing parts |
| | class Wav2Vec2ForAudioFrameClassification_custom(transformers.Wav2Vec2ForAudioFrameClassification, |
| | PyTorchModelHubMixin, |
| | repo_url="your-repo-url", |
| | pipeline_tag="text-to-image", |
| | license="mit",): |
| | def __init__(self, config): |
| | super().__init__(config) |
| | self.num_labels = config.num_labels |
| | |
| | if hasattr(config, "add_adapter") and config.add_adapter: |
| | raise ValueError( |
| | "Audio frame classification does not support the use of Wav2Vec2 adapters (config.add_adapter=True)" |
| | ) |
| | self.wav2vec2 = Wav2Vec2Model(config) |
| | num_layers = config.num_hidden_layers + 1 # transformer layers + input embeddings |
| | if config.use_weighted_layer_sum: |
| | self.layer_weights = nn.Parameter(torch.ones(num_layers) / num_layers) |
| | self.classifier = nn.Linear(config.hidden_size, config.num_labels) |
| | |
| | self.init_weights() |
| | |
| | def forward( |
| | self, |
| | input_values, |
| | attention_mask=None, |
| | output_attentions=None, |
| | output_hidden_states=None, |
| | return_dict=None, |
| | labels=None, # ADDED |
| | ): |
| | return_dict = return_dict if return_dict is not None else self.config.use_return_dict |
| | output_hidden_states = True if self.config.use_weighted_layer_sum else output_hidden_states |
| | |
| | outputs = self.wav2vec2( |
| | input_values, |
| | attention_mask=attention_mask, |
| | output_attentions=output_attentions, |
| | output_hidden_states=output_hidden_states, |
| | return_dict=return_dict, |
| | ) |
| | |
| | if self.config.use_weighted_layer_sum: |
| | hidden_states = outputs[_HIDDEN_STATES_START_POSITION] |
| | hidden_states = torch.stack(hidden_states, dim=1) |
| | norm_weights = nn.functional.softmax(self.layer_weights, dim=-1) |
| | hidden_states = (hidden_states * norm_weights.view(-1, 1, 1)).sum(dim=1) |
| | else: |
| | hidden_states = outputs[0] |
| | |
| | logits = self.classifier(hidden_states) |
| | labels = labels.reshape(-1,1) # 1xN -> Nx1 |
| | |
| | # ADDED |
| | loss = None |
| | if labels is not None: |
| | if self.num_labels == 1: |
| | loss_fct = MSELoss() |
| | #loss = loss_fct(logits.squeeze(), labels.squeeze()) |
| | loss = loss_fct(logits.view(-1, self.num_labels), labels) |
| | else: |
| | loss_fct = CrossEntropyLoss() |
| | loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1)) |
| | |
| | |
| | if not return_dict: |
| | output = (logits,) + outputs[_HIDDEN_STATES_START_POSITION:] |
| | return ((loss,) + output) if loss is not None else output |
| | |
| | return TokenClassifierOutput( |
| | loss=loss, |
| | logits=logits, |
| | hidden_states=outputs.hidden_states, |
| | attentions=outputs.attentions, |
| | ) |
| | |
| |
|
