File size: 6,070 Bytes
ad63418 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 | import torch
import torch.nn.functional as F
## greedy decoding
@torch.no_grad()
def greedy_decode(model, input_ids, max_new_tokens, eos_token_id = None, pad_token_id = None):
model.eval()
batch_size = input_ids.shape[0]
generated = input_ids.clone()
finished = torch.zeros(batch_size, dtype = torch.bool, device = input_ids.device)
for step in range(max_new_tokens):
outputs = model(generated)
logits = outputs.logits
next_token_logits = logits[:, -1, :]
next_token = torch.argmax(next_token_logits, dim = -1, keepdim = True)
if pad_token_id is not None:
next_token = torch.where(finished.unsqueeze(-1), pad_token_id, next_token)
if eos_token_id is not None:
is_eos = (next_token.squeeze(-1) == eos_token_id)
finished = finished | is_eos
generated = torch.cat([generated, next_token], dim = -1)
if finished.all():
break
return generated
# top-k decoding
@torch.no_grad()
def top_k_decoding(model, input_ids, max_new_tokens, k =50, temperature = 1.0, eos_token_id = None, pad_token_id = None):
model.eval()
batch_size = input_ids.shape[0]
generated = input_ids.clone()
finished = torch.zeros(batch_size, dtype = torch.bool, device = input_ids.device)
for step in range(max_new_tokens):
outputs = model(generated)
logits = outputs.logits
next_token_logits = logits[:, -1, :] / temperature
topk_logits = torch.topk(next_token_logits, k, dim = -1)
threshold = topk_logits.values[:, -1:]
mask = (next_token_logits < threshold)
next_token_logits = next_token_logits.masked_fill(mask, float("-inf"))
probs = F.softmax(next_token_logits, dim = -1)
next_token = torch.multinomial(probs, num_samples = 1)
if pad_token_id is not None:
next_token = torch.where(finished.unsqueeze(-1), pad_token_id, next_token)
if eos_token_id is not None:
is_eos = (next_token.squeeze(-1) == eos_token_id)
finished = finished | is_eos
generated = torch.cat([generated, next_token], dim = -1)
if finished.all():
break
return generated
# top-p decoding
@torch.no_grad()
def top_p_decoding(model, input_ids, max_new_tokens, p = 0.9, temperature = 1.0, eos_token_id = None, pad_token_id = None):
model.eval()
batch_size = input_ids.shape[0]
generated = input_ids.clone()
finished = torch.zeros(batch_size, dtype = torch.bool, device = input_ids.device)
for step in range(max_new_tokens):
outputs = model(generated)
logits = outputs.logits
next_token_logits = logits[:, -1, :] / temperature
sorted_logits, sorted_indices = torch.sort(next_token_logits, descending = True, dim = -1)
sorted_probs = F.softmax(sorted_logits, dim = -1)
cumulative_probs = torch.cumsum(sorted_probs, dim = -1)
sorted_indices_to_remove = cumulative_probs > p
sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone()
sorted_indices_to_remove[..., 0] = False
sorted_logits = sorted_logits.masked_fill(sorted_indices_to_remove, float('-inf'))
probs = F.softmax(sorted_logits, dim = -1)
sampled_sorted_index = torch.multinomial(probs, num_samples = 1)
next_token = torch.gather(sorted_indices, dim = -1, index = sampled_sorted_index)
if pad_token_id is not None:
next_token = torch.where(finished.unsqueeze(-1), pad_token_id, next_token)
if eos_token_id is not None:
is_eos = (next_token.squeeze(-1) == eos_token_id)
finished = finished | is_eos
generated = torch.cat([generated, next_token], dim = -1)
if finished.all():
break
return generated
## Beam Search Decode
@torch.no_grad()
def beam_search_decode(model, input_ids, beam_width = 1, max_new_tokens = 50, length_penalty_alpha = 1.0, eos_token_id = None):
model.eval()
batch_size = input_ids.shape[0]
beam_sequences = input_ids.clone()
beam_scores = torch.zeros(1, dtype=torch.float, device=input_ids.device)
completed = []
for step in range(max_new_tokens):
outputs = model(beam_sequences)
next_token_logits = outputs.logits[:, -1, :]
log_probs = F.log_softmax(next_token_logits, dim = -1)
candidate_scores = beam_scores.unsqueeze(-1) + log_probs
vocab_size = next_token_logits.shape[-1]
flat_scores = candidate_scores.view(-1)
top_scores, top_indices = torch.topk(flat_scores, k = beam_width, dim = -1)
parent_beam_idx = torch.div(top_indices, vocab_size, rounding_mode = "floor")
token_id = top_indices % vocab_size
next_beam_sequences = []
next_beam_scores = []
for i in range(beam_width):
b_idx = parent_beam_idx[i]
tok = token_id[i]
score = top_scores[i]
parent_seq = beam_sequences[b_idx : b_idx + 1]
new_seq = torch.cat([parent_seq, tok.view(1, 1)], dim = -1)
if eos_token_id is not None and tok.item() == eos_token_id:
gen_len = new_seq.shape[1] - input_ids.shape[1]
norm_score = score.item() / (gen_len ** length_penalty_alpha)
completed.append((new_seq, norm_score))
else:
next_beam_sequences.append(new_seq)
next_beam_scores.append(score)
if len(next_beam_sequences) == 0 or len(completed) >= beam_width:
break
beam_sequences = torch.cat(next_beam_sequences, dim = 0)
beam_scores = torch.stack(next_beam_scores)
for b in range(beam_sequences.shape[0]):
seq = beam_sequences[b : b + 1]
gen_len = max(1, seq.shape[1] - input_ids.shape[1])
norm_score = beam_scores[b].item() / (gen_len ** length_penalty_alpha)
completed.append((seq, norm_score))
completed.sort(key = lambda x: x[1], reverse = True)
return completed[0][0] |