anlp-a2-models / src /part3 /decoding.py
Chechaaa09's picture
Upload Task 1 checkpoints, tokenizer, configurations and source
ad63418 verified
Raw History Blame Contribute Delete
6.07 kB
import torch
import torch.nn.functional as F
## greedy decoding
@torch.no_grad()
def greedy_decode(model, input_ids, max_new_tokens, eos_token_id = None, pad_token_id = None):
model.eval()
batch_size = input_ids.shape[0]
generated = input_ids.clone()
finished = torch.zeros(batch_size, dtype = torch.bool, device = input_ids.device)
for step in range(max_new_tokens):
outputs = model(generated)
logits = outputs.logits
next_token_logits = logits[:, -1, :]
next_token = torch.argmax(next_token_logits, dim = -1, keepdim = True)
if pad_token_id is not None:
next_token = torch.where(finished.unsqueeze(-1), pad_token_id, next_token)
if eos_token_id is not None:
is_eos = (next_token.squeeze(-1) == eos_token_id)
finished = finished | is_eos
generated = torch.cat([generated, next_token], dim = -1)
if finished.all():
break
return generated
# top-k decoding
@torch.no_grad()
def top_k_decoding(model, input_ids, max_new_tokens, k =50, temperature = 1.0, eos_token_id = None, pad_token_id = None):
model.eval()
batch_size = input_ids.shape[0]
generated = input_ids.clone()
finished = torch.zeros(batch_size, dtype = torch.bool, device = input_ids.device)
for step in range(max_new_tokens):
outputs = model(generated)
logits = outputs.logits
next_token_logits = logits[:, -1, :] / temperature
topk_logits = torch.topk(next_token_logits, k, dim = -1)
threshold = topk_logits.values[:, -1:]
mask = (next_token_logits < threshold)
next_token_logits = next_token_logits.masked_fill(mask, float("-inf"))
probs = F.softmax(next_token_logits, dim = -1)
next_token = torch.multinomial(probs, num_samples = 1)
if pad_token_id is not None:
next_token = torch.where(finished.unsqueeze(-1), pad_token_id, next_token)
if eos_token_id is not None:
is_eos = (next_token.squeeze(-1) == eos_token_id)
finished = finished | is_eos
generated = torch.cat([generated, next_token], dim = -1)
if finished.all():
break
return generated
# top-p decoding
@torch.no_grad()
def top_p_decoding(model, input_ids, max_new_tokens, p = 0.9, temperature = 1.0, eos_token_id = None, pad_token_id = None):
model.eval()
batch_size = input_ids.shape[0]
generated = input_ids.clone()
finished = torch.zeros(batch_size, dtype = torch.bool, device = input_ids.device)
for step in range(max_new_tokens):
outputs = model(generated)
logits = outputs.logits
next_token_logits = logits[:, -1, :] / temperature
sorted_logits, sorted_indices = torch.sort(next_token_logits, descending = True, dim = -1)
sorted_probs = F.softmax(sorted_logits, dim = -1)
cumulative_probs = torch.cumsum(sorted_probs, dim = -1)
sorted_indices_to_remove = cumulative_probs > p
sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone()
sorted_indices_to_remove[..., 0] = False
sorted_logits = sorted_logits.masked_fill(sorted_indices_to_remove, float('-inf'))
probs = F.softmax(sorted_logits, dim = -1)
sampled_sorted_index = torch.multinomial(probs, num_samples = 1)
next_token = torch.gather(sorted_indices, dim = -1, index = sampled_sorted_index)
if pad_token_id is not None:
next_token = torch.where(finished.unsqueeze(-1), pad_token_id, next_token)
if eos_token_id is not None:
is_eos = (next_token.squeeze(-1) == eos_token_id)
finished = finished | is_eos
generated = torch.cat([generated, next_token], dim = -1)
if finished.all():
break
return generated
## Beam Search Decode
@torch.no_grad()
def beam_search_decode(model, input_ids, beam_width = 1, max_new_tokens = 50, length_penalty_alpha = 1.0, eos_token_id = None):
model.eval()
batch_size = input_ids.shape[0]
beam_sequences = input_ids.clone()
beam_scores = torch.zeros(1, dtype=torch.float, device=input_ids.device)
completed = []
for step in range(max_new_tokens):
outputs = model(beam_sequences)
next_token_logits = outputs.logits[:, -1, :]
log_probs = F.log_softmax(next_token_logits, dim = -1)
candidate_scores = beam_scores.unsqueeze(-1) + log_probs
vocab_size = next_token_logits.shape[-1]
flat_scores = candidate_scores.view(-1)
top_scores, top_indices = torch.topk(flat_scores, k = beam_width, dim = -1)
parent_beam_idx = torch.div(top_indices, vocab_size, rounding_mode = "floor")
token_id = top_indices % vocab_size
next_beam_sequences = []
next_beam_scores = []
for i in range(beam_width):
b_idx = parent_beam_idx[i]
tok = token_id[i]
score = top_scores[i]
parent_seq = beam_sequences[b_idx : b_idx + 1]
new_seq = torch.cat([parent_seq, tok.view(1, 1)], dim = -1)
if eos_token_id is not None and tok.item() == eos_token_id:
gen_len = new_seq.shape[1] - input_ids.shape[1]
norm_score = score.item() / (gen_len ** length_penalty_alpha)
completed.append((new_seq, norm_score))
else:
next_beam_sequences.append(new_seq)
next_beam_scores.append(score)
if len(next_beam_sequences) == 0 or len(completed) >= beam_width:
break
beam_sequences = torch.cat(next_beam_sequences, dim = 0)
beam_scores = torch.stack(next_beam_scores)
for b in range(beam_sequences.shape[0]):
seq = beam_sequences[b : b + 1]
gen_len = max(1, seq.shape[1] - input_ids.shape[1])
norm_score = beam_scores[b].item() / (gen_len ** length_penalty_alpha)
completed.append((seq, norm_score))
completed.sort(key = lambda x: x[1], reverse = True)
return completed[0][0]