Update weights and modeling code to latest version

modeling_lucaone.py

@@ -1141,8 +1141,10 @@ class LucaGPLMForMaskedLM(LucaGPLMPreTrainedModel):
 
 class LucaGPLMForSequenceClassification(LucaGPLMPreTrainedModel):
     def __init__(self, config):
+        if hasattr(config, "classifier_num_labels") and config.classifier_num_labels > 0:
+            config.num_labels = config.classifier_num_labels
         super().__init__(config)
-        self.num_labels = config.classifier_num_labels
+        self.num_labels = config.num_labels
         self.task_level = config.task_level
         self.task_type = config.task_type
         assert self.task_level == "seq_level"
@@ -1247,8 +1249,10 @@ class LucaGPLMForSequenceClassification(LucaGPLMPreTrainedModel):
 
 class LucaGPLMForTokenClassification(LucaGPLMPreTrainedModel):
     def __init__(self, config):
+        if hasattr(config, "classifier_num_labels") and config.classifier_num_labels > 0:
+            config.num_labels = config.classifier_num_labels
         super().__init__(config)
-        self.num_labels = config.classifier_num_labels
+        self.num_labels = config.num_labels
         self.task_level = config.task_level
         self.task_type = config.task_type
         assert self.task_level == "token_level"

tokenization_lucaone.py

@@ -133,7 +133,7 @@ class LucaGPLMTokenizer(PreTrainedTokenizer):
         self.cls_idx = self.tok_to_idx.get("[CLS]", 2)
         self.mask_idx = self.tok_to_idx.get("[MASK]", 4)
         self.eos_idx = self.tok_to_idx.get("[SEP]", 3)
-        
+
         super().__init__(
             unk_token=unk_token,
             pad_token=pad_token,
@@ -295,7 +295,23 @@ class LucaGPLMTokenizer(PreTrainedTokenizer):
 
     def batch_encode_plus(self, *args, **kwargs):
         # 显式调用父类，或者保留你原有的实现，只要确保内部调用的是修复后的 encode_plus 即可
-        return super().batch_encode_plus(*args, **kwargs)
+        # return super().batch_encode_plus(*args, **kwargs)
+        # 修改
+        # 循环处理每一条数据
+        batch_outputs = []
+        batch_text = kwargs["text"]
+        seq_type = kwargs["seq_type"]
+        for text in batch_text:
+            batch_outputs.append(self.encode_plus(text, seq_type=seq_type, **kwargs))
+
+        # 将结果合并为 Dict[str, List[List[int]]]
+        # 这样 Dataset.map(batched=True) 才能正确解析
+        combined = {key: [] for key in batch_outputs[0].keys()}
+        for output in batch_outputs:
+            for key, value in output.items():
+                combined[key].append(value)
+
+        return combined
 
     def encode_plus(
         self,
@@ -311,7 +327,9 @@ class LucaGPLMTokenizer(PreTrainedTokenizer):
         truncation: bool = False,
         **kwargs
     ) -> Dict[str, Any]:
-        
+        # 修改
+        # 忽略掉不认识的参数，比如 text_pair
+        kwargs.pop("text_pair", None)
         # 调用修复后的 encode，它现在会正确处理截断
         token_ids = self.encode(
             text,