Upload folder using huggingface_hub

rnnlm_model/modeling_rnnlm.py

@@ -6,14 +6,18 @@ import torch.nn as nn
 try:
     from transformers import PreTrainedModel
     from transformers.modeling_outputs import CausalLMOutputWithPast
-    from transformers.generation import LogitsProcessor, LogitsProcessorList
+    from transformers.generation import GenerationMixin, LogitsProcessor, LogitsProcessorList
 except ImportError:
     from transformers.modeling_utils import PreTrainedModel
     from transformers.modeling_outputs import CausalLMOutputWithPast
     try:
-        from transformers.generation import LogitsProcessor, LogitsProcessorList
+        from transformers.generation import GenerationMixin, LogitsProcessor, LogitsProcessorList
     except ImportError:
-        from transformers.generation_utils import LogitsProcessor, LogitsProcessorList
+        try:
+            from transformers.generation_utils import GenerationMixin, LogitsProcessor, LogitsProcessorList
+        except ImportError:
+            from transformers.generation_utils import LogitsProcessor, LogitsProcessorList
+            GenerationMixin = None
 
 from .configuration_rnnlm import RNNLMConfig
 
@@ -113,6 +117,8 @@ class RNNLMForCausalLM(PreTrainedModel):
     def __init__(self, config: RNNLMConfig, **kwargs):
         super().__init__(config)
         self.config = config
+        # RNNLM has no tied weights; transformers expects this attribute (dict) for .update()
+        self.all_tied_weights_keys = {}
         self.vocab_size = config.vocab_size
         self.embedding_dim = config.embedding_dim
         self.hidden_size = config.hidden_size
@@ -299,4 +305,9 @@ class RNNLMForCausalLM(PreTrainedModel):
             logits_processor = LogitsProcessorList(logits_processor)
         logits_processor.insert(0, processor)
         kwargs["logits_processor"] = logits_processor
+        # RNNLM uses tuple cache (hidden states), not DynamicCache; avoid cache to prevent "not subscriptable" error
+        kwargs.setdefault("use_cache", False)
+        # Call GenerationMixin.generate explicitly (super() can fail in some loading contexts)
+        if GenerationMixin is not None:
+            return GenerationMixin.generate(self, inputs, **kwargs)
         return super().generate(inputs, **kwargs)

rnnlm_model/pipeline_rnnlm.py

@@ -3,6 +3,7 @@
 
 from transformers.pipelines.text_generation import TextGenerationPipeline
 from transformers.pipelines.text_generation import ReturnType
+from transformers import GenerationConfig
 
 
 class RNNLMTextGenerationPipeline(TextGenerationPipeline):
@@ -14,6 +15,23 @@ class RNNLMTextGenerationPipeline(TextGenerationPipeline):
     When the tokenizer has generalize_ents=True, entities are extracted from the
     prompt and used to replace ENT_PERSON_0, ENT_GPE_0, etc. in the generated output.
     """
+    assistant_model = None  # Class default for transformers compatibility (assisted decoding)
+    assistant_tokenizer = None
+    prefix = None  # For XLNet/TransfoXL; RNNLM doesn't use it
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        # Newer transformers expect these; RNNLM doesn't use them
+        self.assistant_model = None
+        self.assistant_tokenizer = None
+        self.prefix = getattr(self, "prefix", None)
+        if not hasattr(self, "generation_config") or self.generation_config is None:
+            self.generation_config = GenerationConfig(
+                pad_token_id=getattr(self.tokenizer, "pad_token_id", 0),
+                max_new_tokens=256,
+                do_sample=True,
+                temperature=0.7,
+            )
 
     def postprocess(
         self,

rnnlm_model/tokenization_utils.py

@@ -265,14 +265,14 @@ def detokenize_tok_seq(encoder, seq, ents=[], begin_sentence=True):
         # capitalize first-person "I" pronoun
         detok_sent = re.sub(r"(^| )i ", r"\1I ", detok_sent)
 
-        # rules for contractions
-        detok_sent = re.sub(" n\'\s*t ", "n\'t ", detok_sent)
-        detok_sent = re.sub(" \'\s*d ", "\'d ", detok_sent)
-        detok_sent = re.sub(" \'\s*s ", "\'s ", detok_sent)
-        detok_sent = re.sub(" \'\s*ve ", "\'ve ", detok_sent)
-        detok_sent = re.sub(" \'\s*ll ", "\'ll ", detok_sent)
-        detok_sent = re.sub(" \'\s*m ", "\'m ", detok_sent)
-        detok_sent = re.sub(" \'\s*re ", "\'re ", detok_sent)
+        # rules for contractions (pattern: raw string for \s; replacement: no backslash)
+        detok_sent = re.sub(r" n'\s*t ", "n't ", detok_sent)
+        detok_sent = re.sub(r" '\s*d ", "'d ", detok_sent)
+        detok_sent = re.sub(r" '\s*s ", "'s ", detok_sent)
+        detok_sent = re.sub(r" '\s*ve ", "'ve ", detok_sent)
+        detok_sent = re.sub(r" '\s*ll ", "'ll ", detok_sent)
+        detok_sent = re.sub(r" '\s*m ", "'m ", detok_sent)
+        detok_sent = re.sub(r" '\s*re ", "'re ", detok_sent)
 
         # rules for formatting punctuation
         detok_sent = re.sub(" \.", ".", detok_sent)
@@ -291,7 +291,7 @@ def detokenize_tok_seq(encoder, seq, ents=[], begin_sentence=True):
         detok_sent = re.sub("\`\`", "\"", detok_sent)
 
         # filter repetitive characters
-        detok_sent = re.sub("([\"\']\s*){2,}", "\" ", detok_sent)
+        detok_sent = re.sub(r'(["\']\s*){2,}', '" ', detok_sent)
 
         # map each opening puncutation mark to closing mark
         punc_pairs = {"\'": "\'", "\'": "\'",