import torch import torch.nn.functional as F ## greedy decoding @torch.no_grad() def greedy_decode(model, input_ids, max_new_tokens, eos_token_id = None, pad_token_id = None): model.eval() batch_size = input_ids.shape[0] generated = input_ids.clone() finished = torch.zeros(batch_size, dtype = torch.bool, device = input_ids.device) for step in range(max_new_tokens): outputs = model(generated) logits = outputs.logits next_token_logits = logits[:, -1, :] next_token = torch.argmax(next_token_logits, dim = -1, keepdim = True) if pad_token_id is not None: next_token = torch.where(finished.unsqueeze(-1), pad_token_id, next_token) if eos_token_id is not None: is_eos = (next_token.squeeze(-1) == eos_token_id) finished = finished | is_eos generated = torch.cat([generated, next_token], dim = -1) if finished.all(): break return generated # top-k decoding @torch.no_grad() def top_k_decoding(model, input_ids, max_new_tokens, k =50, temperature = 1.0, eos_token_id = None, pad_token_id = None): model.eval() batch_size = input_ids.shape[0] generated = input_ids.clone() finished = torch.zeros(batch_size, dtype = torch.bool, device = input_ids.device) for step in range(max_new_tokens): outputs = model(generated) logits = outputs.logits next_token_logits = logits[:, -1, :] / temperature topk_logits = torch.topk(next_token_logits, k, dim = -1) threshold = topk_logits.values[:, -1:] mask = (next_token_logits < threshold) next_token_logits = next_token_logits.masked_fill(mask, float("-inf")) probs = F.softmax(next_token_logits, dim = -1) next_token = torch.multinomial(probs, num_samples = 1) if pad_token_id is not None: next_token = torch.where(finished.unsqueeze(-1), pad_token_id, next_token) if eos_token_id is not None: is_eos = (next_token.squeeze(-1) == eos_token_id) finished = finished | is_eos generated = torch.cat([generated, next_token], dim = -1) if finished.all(): break return generated # top-p decoding @torch.no_grad() def top_p_decoding(model, input_ids, max_new_tokens, p = 0.9, temperature = 1.0, eos_token_id = None, pad_token_id = None): model.eval() batch_size = input_ids.shape[0] generated = input_ids.clone() finished = torch.zeros(batch_size, dtype = torch.bool, device = input_ids.device) for step in range(max_new_tokens): outputs = model(generated) logits = outputs.logits next_token_logits = logits[:, -1, :] / temperature sorted_logits, sorted_indices = torch.sort(next_token_logits, descending = True, dim = -1) sorted_probs = F.softmax(sorted_logits, dim = -1) cumulative_probs = torch.cumsum(sorted_probs, dim = -1) sorted_indices_to_remove = cumulative_probs > p sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone() sorted_indices_to_remove[..., 0] = False sorted_logits = sorted_logits.masked_fill(sorted_indices_to_remove, float('-inf')) probs = F.softmax(sorted_logits, dim = -1) sampled_sorted_index = torch.multinomial(probs, num_samples = 1) next_token = torch.gather(sorted_indices, dim = -1, index = sampled_sorted_index) if pad_token_id is not None: next_token = torch.where(finished.unsqueeze(-1), pad_token_id, next_token) if eos_token_id is not None: is_eos = (next_token.squeeze(-1) == eos_token_id) finished = finished | is_eos generated = torch.cat([generated, next_token], dim = -1) if finished.all(): break return generated ## Beam Search Decode @torch.no_grad() def beam_search_decode(model, input_ids, beam_width = 1, max_new_tokens = 50, length_penalty_alpha = 1.0, eos_token_id = None): model.eval() batch_size = input_ids.shape[0] beam_sequences = input_ids.clone() beam_scores = torch.zeros(1, dtype=torch.float, device=input_ids.device) completed = [] for step in range(max_new_tokens): outputs = model(beam_sequences) next_token_logits = outputs.logits[:, -1, :] log_probs = F.log_softmax(next_token_logits, dim = -1) candidate_scores = beam_scores.unsqueeze(-1) + log_probs vocab_size = next_token_logits.shape[-1] flat_scores = candidate_scores.view(-1) top_scores, top_indices = torch.topk(flat_scores, k = beam_width, dim = -1) parent_beam_idx = torch.div(top_indices, vocab_size, rounding_mode = "floor") token_id = top_indices % vocab_size next_beam_sequences = [] next_beam_scores = [] for i in range(beam_width): b_idx = parent_beam_idx[i] tok = token_id[i] score = top_scores[i] parent_seq = beam_sequences[b_idx : b_idx + 1] new_seq = torch.cat([parent_seq, tok.view(1, 1)], dim = -1) if eos_token_id is not None and tok.item() == eos_token_id: gen_len = new_seq.shape[1] - input_ids.shape[1] norm_score = score.item() / (gen_len ** length_penalty_alpha) completed.append((new_seq, norm_score)) else: next_beam_sequences.append(new_seq) next_beam_scores.append(score) if len(next_beam_sequences) == 0 or len(completed) >= beam_width: break beam_sequences = torch.cat(next_beam_sequences, dim = 0) beam_scores = torch.stack(next_beam_scores) for b in range(beam_sequences.shape[0]): seq = beam_sequences[b : b + 1] gen_len = max(1, seq.shape[1] - input_ids.shape[1]) norm_score = beam_scores[b].item() / (gen_len ** length_penalty_alpha) completed.append((seq, norm_score)) completed.sort(key = lambda x: x[1], reverse = True) return completed[0][0]