Ignore padding in loss method

modeling_gpt_optimized.py

@@ -195,5 +195,5 @@ class GPTOptim(GPT2PreTrainedModel):
         logits = self.model.lm_head(x) # (B, T, vocab_size)
         loss = None
         if labels is not None:
-            loss = F.cross_entropy(logits.view(-1, logits.size(-1)), labels.view(-1))
+            loss = F.cross_entropy(logits.view(-1, logits.size(-1)), labels.view(-1), ignore_index=config.eos_token_id)
         return logits, loss