Download src/part3/decoding.py from Chechaaa09/anlp-a2-models: direct link, hf CLI and curl.
- Browser
- Download file 6.07 kB
-
https://huggingface.co/Chechaaa09/anlp-a2-models/resolve/main/src/part3/decoding.py
- Command line
-
hf download hf://Chechaaa09/anlp-a2-models/src/part3/decoding.py
-
curl -L -o decoding.py https://huggingface.co/Chechaaa09/anlp-a2-models/resolve/main/src/part3/decoding.py
6.07 kB
| import torch | |
| import torch.nn.functional as F | |
| ## greedy decoding | |
| def greedy_decode(model, input_ids, max_new_tokens, eos_token_id = None, pad_token_id = None): | |
| model.eval() | |
| batch_size = input_ids.shape[0] | |
| generated = input_ids.clone() | |
| finished = torch.zeros(batch_size, dtype = torch.bool, device = input_ids.device) | |
| for step in range(max_new_tokens): | |
| outputs = model(generated) | |
| logits = outputs.logits | |
| next_token_logits = logits[:, -1, :] | |
| next_token = torch.argmax(next_token_logits, dim = -1, keepdim = True) | |
| if pad_token_id is not None: | |
| next_token = torch.where(finished.unsqueeze(-1), pad_token_id, next_token) | |
| if eos_token_id is not None: | |
| is_eos = (next_token.squeeze(-1) == eos_token_id) | |
| finished = finished | is_eos | |
| generated = torch.cat([generated, next_token], dim = -1) | |
| if finished.all(): | |
| break | |
| return generated | |
| # top-k decoding | |
| def top_k_decoding(model, input_ids, max_new_tokens, k =50, temperature = 1.0, eos_token_id = None, pad_token_id = None): | |
| model.eval() | |
| batch_size = input_ids.shape[0] | |
| generated = input_ids.clone() | |
| finished = torch.zeros(batch_size, dtype = torch.bool, device = input_ids.device) | |
| for step in range(max_new_tokens): | |
| outputs = model(generated) | |
| logits = outputs.logits | |
| next_token_logits = logits[:, -1, :] / temperature | |
| topk_logits = torch.topk(next_token_logits, k, dim = -1) | |
| threshold = topk_logits.values[:, -1:] | |
| mask = (next_token_logits < threshold) | |
| next_token_logits = next_token_logits.masked_fill(mask, float("-inf")) | |
| probs = F.softmax(next_token_logits, dim = -1) | |
| next_token = torch.multinomial(probs, num_samples = 1) | |
| if pad_token_id is not None: | |
| next_token = torch.where(finished.unsqueeze(-1), pad_token_id, next_token) | |
| if eos_token_id is not None: | |
| is_eos = (next_token.squeeze(-1) == eos_token_id) | |
| finished = finished | is_eos | |
| generated = torch.cat([generated, next_token], dim = -1) | |
| if finished.all(): | |
| break | |
| return generated | |
| # top-p decoding | |
| def top_p_decoding(model, input_ids, max_new_tokens, p = 0.9, temperature = 1.0, eos_token_id = None, pad_token_id = None): | |
| model.eval() | |
| batch_size = input_ids.shape[0] | |
| generated = input_ids.clone() | |
| finished = torch.zeros(batch_size, dtype = torch.bool, device = input_ids.device) | |
| for step in range(max_new_tokens): | |
| outputs = model(generated) | |
| logits = outputs.logits | |
| next_token_logits = logits[:, -1, :] / temperature | |
| sorted_logits, sorted_indices = torch.sort(next_token_logits, descending = True, dim = -1) | |
| sorted_probs = F.softmax(sorted_logits, dim = -1) | |
| cumulative_probs = torch.cumsum(sorted_probs, dim = -1) | |
| sorted_indices_to_remove = cumulative_probs > p | |
| sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone() | |
| sorted_indices_to_remove[..., 0] = False | |
| sorted_logits = sorted_logits.masked_fill(sorted_indices_to_remove, float('-inf')) | |
| probs = F.softmax(sorted_logits, dim = -1) | |
| sampled_sorted_index = torch.multinomial(probs, num_samples = 1) | |
| next_token = torch.gather(sorted_indices, dim = -1, index = sampled_sorted_index) | |
| if pad_token_id is not None: | |
| next_token = torch.where(finished.unsqueeze(-1), pad_token_id, next_token) | |
| if eos_token_id is not None: | |
| is_eos = (next_token.squeeze(-1) == eos_token_id) | |
| finished = finished | is_eos | |
| generated = torch.cat([generated, next_token], dim = -1) | |
| if finished.all(): | |
| break | |
| return generated | |
| ## Beam Search Decode | |
| def beam_search_decode(model, input_ids, beam_width = 1, max_new_tokens = 50, length_penalty_alpha = 1.0, eos_token_id = None): | |
| model.eval() | |
| batch_size = input_ids.shape[0] | |
| beam_sequences = input_ids.clone() | |
| beam_scores = torch.zeros(1, dtype=torch.float, device=input_ids.device) | |
| completed = [] | |
| for step in range(max_new_tokens): | |
| outputs = model(beam_sequences) | |
| next_token_logits = outputs.logits[:, -1, :] | |
| log_probs = F.log_softmax(next_token_logits, dim = -1) | |
| candidate_scores = beam_scores.unsqueeze(-1) + log_probs | |
| vocab_size = next_token_logits.shape[-1] | |
| flat_scores = candidate_scores.view(-1) | |
| top_scores, top_indices = torch.topk(flat_scores, k = beam_width, dim = -1) | |
| parent_beam_idx = torch.div(top_indices, vocab_size, rounding_mode = "floor") | |
| token_id = top_indices % vocab_size | |
| next_beam_sequences = [] | |
| next_beam_scores = [] | |
| for i in range(beam_width): | |
| b_idx = parent_beam_idx[i] | |
| tok = token_id[i] | |
| score = top_scores[i] | |
| parent_seq = beam_sequences[b_idx : b_idx + 1] | |
| new_seq = torch.cat([parent_seq, tok.view(1, 1)], dim = -1) | |
| if eos_token_id is not None and tok.item() == eos_token_id: | |
| gen_len = new_seq.shape[1] - input_ids.shape[1] | |
| norm_score = score.item() / (gen_len ** length_penalty_alpha) | |
| completed.append((new_seq, norm_score)) | |
| else: | |
| next_beam_sequences.append(new_seq) | |
| next_beam_scores.append(score) | |
| if len(next_beam_sequences) == 0 or len(completed) >= beam_width: | |
| break | |
| beam_sequences = torch.cat(next_beam_sequences, dim = 0) | |
| beam_scores = torch.stack(next_beam_scores) | |
| for b in range(beam_sequences.shape[0]): | |
| seq = beam_sequences[b : b + 1] | |
| gen_len = max(1, seq.shape[1] - input_ids.shape[1]) | |
| norm_score = beam_scores[b].item() / (gen_len ** length_penalty_alpha) | |
| completed.append((seq, norm_score)) | |
| completed.sort(key = lambda x: x[1], reverse = True) | |
| return completed[0][0] |