File size: 2,107 Bytes
366b225 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 |
# -*- coding: utf-8 -*-
import torch
import torch.nn as nn
from transformers import BertModel
from .scalar_mix import ScalarMix
class BertEmbedding(nn.Module):
def __init__(self, model, n_layers, n_out, requires_grad=False):
proxies = {
"http": "http://10.10.1.10:3128",
"https": "https://10.10.1.10:1080",
}
super(BertEmbedding, self).__init__()
#self.bert = AutoModelForMaskedLM.from_pretrained("Sanath369/distilroberta-base-finetuned-telugu_bert1")
self.bert = BertModel.from_pretrained(model, output_hidden_states=True)
self.bert = self.bert.requires_grad_(requires_grad)
self.n_layers = n_layers
self.n_out = n_out
self.requires_grad = requires_grad
self.hidden_size = self.bert.config.hidden_size
self.scalar_mix = ScalarMix(n_layers)
self.projection = nn.Linear(self.hidden_size, n_out, False)
def __repr__(self):
s = self.__class__.__name__ + '('
s += f"n_layers={self.n_layers}, n_out={self.n_out}"
if self.requires_grad:
s += f", requires_grad={self.requires_grad}"
s += ')'
return s
def forward(self, subwords, bert_lens, bert_mask):
batch_size, seq_len = bert_lens.shape
mask = bert_lens.gt(0)
if not self.requires_grad:
self.bert.eval()
# print(subwords)
out = self.bert(subwords, attention_mask=bert_mask)
# print(out[0].shape)
# print(out[1].shape)
# print("bert_mask:", bert_mask)
_,_,bert = self.bert(subwords, attention_mask=bert_mask)
bert = bert[-self.n_layers:]
# print("first" , bert)
bert = self.scalar_mix(bert)
# print("Second" , bert)
bert = bert[bert_mask].split(bert_lens[mask].tolist())
bert = torch.stack([i.mean(0) for i in bert])
bert_embed = bert.new_zeros(batch_size, seq_len, self.hidden_size)
bert_embed = bert_embed.masked_scatter_(mask.unsqueeze(-1), bert)
bert_embed = self.projection(bert_embed)
return bert_embed
|