Add support for transformers version>5 as well as 4 (backwards compatible).

The RobertaTokenizer v5 removed `encode_plus`, rather use `__call__` when `encode_plus` is missing.

modeling.py

@@ -172,8 +172,6 @@ class IceBertPosForTokenClassification(PreTrainedModel):
         head_mask: Optional[torch.Tensor] = None,
         inputs_embeds: Optional[torch.Tensor] = None,
         output_attentions: Optional[bool] = None,
-        output_hidden_states: Optional[bool] = None,
-        return_dict: Optional[bool] = None,
     ) -> Tuple[torch.Tensor, torch.Tensor]:
         """
         B = batch_size, L = seq_len, H = hidden_size, C = num_categories, A = num_attributes, W = max_words
@@ -187,10 +185,6 @@ class IceBertPosForTokenClassification(PreTrainedModel):
             cat_logits: Category logits (B x W x C)
             attr_logits: Attribute logits (B x W x A)
         """
-        return_dict = (
-            return_dict if return_dict is not None else self.config.use_return_dict
-        )
-
         # Get RoBERTa outputs
         outputs = self.roberta(
             input_ids,
@@ -201,7 +195,6 @@ class IceBertPosForTokenClassification(PreTrainedModel):
             inputs_embeds=inputs_embeds,
             output_attentions=output_attentions,
             output_hidden_states=True,
-            return_dict=return_dict,
         )
 
         hidden_states = outputs[0]  # (B x L x H)
@@ -370,15 +363,28 @@ class IceBertPosForTokenClassification(PreTrainedModel):
             Tuple of (input_ids, attention_mask, word_mask) without batch dimension.
         """
         # Encode with word boundary preservation
-        encoding = tokenizer.encode_plus(
-            words,
-            return_tensors="pt",
-            is_split_into_words=True,
-            add_special_tokens=True,
-            truncation=truncate,
-            # The model was probably trained with a lot shorter sequences
-            max_length=self.config.max_position_embeddings - 2,
-        )
+        # For transformers version <5.0 .encode_plus exists.
+        # For versions >=5.0, use __call__ instead.
+        if hasattr(tokenizer, "encode_plus"):
+            encoding = tokenizer.encode_plus(
+                words,
+                return_tensors="pt",
+                is_split_into_words=True,
+                add_special_tokens=True,
+                truncation=truncate,
+                # The model was probably trained with a lot shorter sequences
+                max_length=self.config.max_position_embeddings - 2,
+            )
+        else:
+            encoding = tokenizer(
+                words,
+                return_tensors="pt",
+                is_split_into_words=True,
+                add_special_tokens=True,
+                truncation=truncate,
+                # The model was probably trained with a lot shorter sequences
+                max_length=self.config.max_position_embeddings - 2,
+            )
 
         input_ids = encoding["input_ids"].squeeze(0)  # (L,)
         attention_mask = torch.ones_like(input_ids)