Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer_config.json filter=lfs diff=lfs merge=lfs -text

config.json

@@ -0,0 +1,22 @@
+{
+  "architectures": [
+    "RNNLMForCausalLM"
+  ],
+  "embedding_dim": 300,
+  "hidden_size": 500,
+  "model_type": "rnnlm",
+  "n_feature_nodes": 100,
+  "n_pos_embedding_nodes": 25,
+  "n_pos_nodes": 100,
+  "n_pos_tags": 59,
+  "num_hidden_layers": 2,
+  "pad_token_id": 0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.46.3",
+  "unk_token_id": 1,
+  "use_cache": true,
+  "use_features": false,
+  "use_pos": false,
+  "vocab_size": 64986
+}

generation_config.json

@@ -0,0 +1,5 @@
+{
+  "_from_model_config": true,
+  "pad_token_id": 0,
+  "transformers_version": "4.46.3"
+}

handler.py

@@ -0,0 +1,95 @@
+# coding: utf-8
+"""
+Custom Inference Handler for RNNLM (creative-help) on Hugging Face Inference Endpoints.
+
+Implements EndpointHandler as described in:
+https://huggingface.co/docs/inference-endpoints/en/guides/custom_handler
+
+The handler loads the RNNLM model with entity adaptation support and serves
+text generation requests via the Inference API.
+"""
+
+import os
+import sys
+from typing import Any, Dict, List, Union
+
+
+class EndpointHandler:
+    """
+    Custom handler for RNNLM text generation on Hugging Face Inference Endpoints.
+    Loads the model, tokenizer, and pipeline at init; serves generation requests in __call__.
+    """
+
+    def __init__(self, path: str = ""):
+        """
+        Initialize the handler. Called when the Endpoint starts.
+        :param path: Path to the model repository (model weights, config, tokenizer).
+        """
+        self.path = path or "."
+        self.path = os.path.abspath(self.path)
+
+        # Add model repo to path so we can import rnnlm_model
+        if self.path not in sys.path:
+            sys.path.insert(0, self.path)
+
+        # Register custom model architecture with Transformers
+        from transformers import AutoConfig, AutoModelForCausalLM
+        from rnnlm_model import (
+            RNNLMConfig,
+            RNNLMForCausalLM,
+            RNNLMTokenizer,
+            RNNLMTextGenerationPipeline,
+        )
+
+        AutoConfig.register("rnnlm", RNNLMConfig)
+        AutoModelForCausalLM.register(RNNLMConfig, RNNLMForCausalLM)
+
+        # Load model and tokenizer
+        self.model = AutoModelForCausalLM.from_pretrained(
+            self.path,
+            trust_remote_code=True,
+        )
+        self.tokenizer = RNNLMTokenizer.from_pretrained(self.path)
+
+        # Create text generation pipeline with entity adaptation
+        self.pipeline = RNNLMTextGenerationPipeline(
+            model=self.model,
+            tokenizer=self.tokenizer,
+        )
+
+    def __call__(self, data: Dict[str, Any]) -> Union[List[Dict[str, str]], Dict[str, Any]]:
+        """
+        Handle inference requests. Called on every API request.
+        :param data: Request payload with "inputs" (prompt string or list) and optional "parameters".
+        :return: List of dicts with "generated_text" key(s), or single dict for compatibility.
+        """
+        inputs = data.pop("inputs", None)
+        if inputs is None:
+            return {"error": "Missing 'inputs' in request body"}
+
+        parameters = data.pop("parameters", data) or {}
+        if not isinstance(parameters, dict):
+            parameters = {}
+
+        # Default generation parameters
+        gen_kwargs = {
+            "max_new_tokens": parameters.get("max_new_tokens", 50),
+            "do_sample": parameters.get("do_sample", True),
+            "temperature": parameters.get("temperature", 1.0),
+            "pad_token_id": self.tokenizer.pad_token_id,
+        }
+        # Allow override of other params (top_p, top_k, repetition_penalty, etc.)
+        for k, v in parameters.items():
+            if k not in gen_kwargs:
+                gen_kwargs[k] = v
+
+        # Run generation
+        try:
+            result = self.pipeline(inputs, **gen_kwargs)
+        except Exception as e:
+            return {"error": str(e)}
+
+        # Ensure we return a list of dicts (API expects list for batch)
+        if isinstance(result, list):
+            return result
+        return [result] if isinstance(result, dict) else [{"generated_text": str(result)}]

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8ba1140559355d5160d133f9b243db038758bf3922520e2e5aab6b08fe55f07
+size 219043380

requirements.txt

@@ -0,0 +1,11 @@
+# Custom dependencies for RNNLM (creative-help) Inference Endpoint
+# Base stack (torch, transformers) is provided by the Inference Endpoints container
+
+# RNNLM tokenizer uses spaCy for tokenization and entity extraction
+spacy>=3.0
+# English spaCy model - required for RNNLMTokenizer (entity recognition, tokenization)
+# Install from GitHub release (pip cannot install spacy models via python -m spacy download in container)
+https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.7.0/en_core_web_sm-3.7.0-py3-none-any.whl
+
+# NumPy (used by tokenization_utils)
+numpy

rnnlm_model/__init__.py

@@ -0,0 +1,14 @@
+# coding: utf-8
+"""RNNLM model for HuggingFace Transformers."""
+
+from .configuration_rnnlm import RNNLMConfig
+from .modeling_rnnlm import RNNLMForCausalLM
+from .tokenization_rnnlm import RNNLMTokenizer
+from .pipeline_rnnlm import RNNLMTextGenerationPipeline
+
+__all__ = [
+    "RNNLMConfig",
+    "RNNLMForCausalLM",
+    "RNNLMTokenizer",
+    "RNNLMTextGenerationPipeline",
+]

rnnlm_model/configuration_rnnlm.py

@@ -0,0 +1,51 @@
+# coding: utf-8
+"""RNN Language Model configuration for HuggingFace Transformers."""
+
+try:
+    from transformers import PreTrainedConfig
+except ImportError:
+    try:
+        from transformers.configuration_utils import PreTrainedConfig
+    except ImportError:
+        from transformers.configuration_utils import PretrainedConfig as PreTrainedConfig
+
+
+class RNNLMConfig(PreTrainedConfig):
+    """Configuration class for RNNLM (RNN Language Model)."""
+
+    model_type = "rnnlm"
+
+    def __init__(
+        self,
+        vocab_size=50000,
+        embedding_dim=300,
+        hidden_size=250,
+        num_hidden_layers=1,
+        pad_token_id=0,
+        unk_token_id=1,
+        bos_token_id=None,
+        eos_token_id=None,
+        use_pos=False,
+        use_features=False,
+        n_pos_tags=59,
+        n_pos_embedding_nodes=25,
+        n_pos_nodes=100,
+        n_feature_nodes=100,
+        **kwargs,
+    ):
+        super().__init__(pad_token_id=pad_token_id, **kwargs)
+        self.vocab_size = vocab_size
+        self.embedding_dim = embedding_dim
+        self.hidden_size = hidden_size
+        self.num_hidden_layers = num_hidden_layers
+        self.unk_token_id = unk_token_id
+        self.bos_token_id = bos_token_id
+        self.eos_token_id = eos_token_id
+        self.use_pos = use_pos
+        self.use_features = use_features
+        self.n_pos_tags = n_pos_tags
+        self.n_pos_embedding_nodes = n_pos_embedding_nodes
+        self.n_pos_nodes = n_pos_nodes
+        self.n_feature_nodes = n_feature_nodes
+        self.use_cache = True  # Required for generation
+        self.tie_word_embeddings = False  # RNNLM uses separate embed and output layers

rnnlm_model/modeling_rnnlm.py

@@ -0,0 +1,302 @@
+# coding: utf-8
+"""RNN Language Model for HuggingFace Transformers - PyTorch implementation."""
+
+import torch
+import torch.nn as nn
+try:
+    from transformers import PreTrainedModel
+    from transformers.modeling_outputs import CausalLMOutputWithPast
+    from transformers.generation import LogitsProcessor, LogitsProcessorList
+except ImportError:
+    from transformers.modeling_utils import PreTrainedModel
+    from transformers.modeling_outputs import CausalLMOutputWithPast
+    try:
+        from transformers.generation import LogitsProcessor, LogitsProcessorList
+    except ImportError:
+        from transformers.generation_utils import LogitsProcessor, LogitsProcessorList
+
+from .configuration_rnnlm import RNNLMConfig
+
+
+class PreventUnkLogitsProcessor(LogitsProcessor):
+    """
+    Redistribute probability from pad (0) and unk (1) to other tokens before sampling.
+    Matches the original Keras model's prevent_unk behavior.
+    """
+
+    def __init__(self, pad_token_id: int = 0, unk_token_id: int = 1):
+        self.pad_token_id = pad_token_id
+        self.unk_token_id = unk_token_id
+
+    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
+        # Set pad and unk logits to very small value so they're never sampled
+        scores = scores.clone()
+        scores[:, self.pad_token_id] = -1e8
+        scores[:, self.unk_token_id] = -1e8
+        return scores
+
+
+class GRUKerasCompat(nn.Module):
+    """
+    GRU matching Keras reset_after=False (GRU v1).
+    Keras: h_new = tanh(W_h·x + W_hn·(r⊙h))
+    PyTorch default: h_new = tanh(W_h·x + r⊙(W_hn·h))
+    We implement the Keras formulation for correct conversion.
+    Uses same weight layout as nn.GRU: [r, z, n] gate order.
+    """
+
+    def __init__(self, input_size: int, hidden_size: int, batch_first: bool = True):
+        super().__init__()
+        self.input_size = input_size
+        self.hidden_size = hidden_size
+        self.batch_first = batch_first
+        self.weight_ih = nn.Parameter(torch.empty(3 * hidden_size, input_size))
+        self.weight_hh = nn.Parameter(torch.empty(3 * hidden_size, hidden_size))
+        self.bias_ih = nn.Parameter(torch.empty(3 * hidden_size))
+        self.bias_hh = nn.Parameter(torch.empty(3 * hidden_size))
+        self.reset_parameters()
+
+    def reset_parameters(self):
+        nn.init.xavier_uniform_(self.weight_ih)
+        nn.init.xavier_uniform_(self.weight_hh)
+        nn.init.zeros_(self.bias_ih)
+        nn.init.zeros_(self.bias_hh)
+
+    def forward(self, x: torch.Tensor, h_0: torch.Tensor = None):
+        if self.batch_first:
+            x = x  # (batch, seq, input)
+        else:
+            x = x.transpose(0, 1)
+        batch, seq_len, _ = x.shape
+        if h_0 is None:
+            h = x.new_zeros(batch, self.hidden_size)
+        else:
+            h = h_0.squeeze(0)  # (batch, hidden)
+
+        outputs = []
+        for t in range(seq_len):
+            x_t = x[:, t, :]  # (batch, input)
+            # Gates: weight layout [r, z, n], each (hidden, input) or (hidden, hidden)
+            r_ih = x_t @ self.weight_ih[:self.hidden_size].t() + self.bias_ih[:self.hidden_size]
+            z_ih = x_t @ self.weight_ih[self.hidden_size:2*self.hidden_size].t() + self.bias_ih[self.hidden_size:2*self.hidden_size]
+            n_ih = x_t @ self.weight_ih[2*self.hidden_size:].t() + self.bias_ih[2*self.hidden_size:]
+
+            r_hh = h @ self.weight_hh[:self.hidden_size].t() + self.bias_hh[:self.hidden_size]
+            z_hh = h @ self.weight_hh[self.hidden_size:2*self.hidden_size].t() + self.bias_hh[self.hidden_size:2*self.hidden_size]
+            n_hh = (h * torch.sigmoid(r_ih + r_hh)) @ self.weight_hh[2*self.hidden_size:].t() + self.bias_hh[2*self.hidden_size:]
+
+            r = torch.sigmoid(r_ih + r_hh)
+            z = torch.sigmoid(z_ih + z_hh)
+            n = torch.tanh(n_ih + n_hh)
+            h = (1 - z) * n + z * h
+            outputs.append(h)
+
+        output = torch.stack(outputs, dim=1)  # (batch, seq, hidden)
+        if not self.batch_first:
+            output = output.transpose(0, 1)
+        return output, h.unsqueeze(0)
+
+
+class RNNLMForCausalLM(PreTrainedModel):
+    """
+    RNN-based Causal Language Model for text generation.
+    Compatible with HuggingFace TextGenerationPipeline.
+    Supports base model (no POS, no features). POS and features require
+    additional preprocessing at generation time.
+    """
+
+    config_class = RNNLMConfig
+    base_model_prefix = "rnnlm"
+    supports_gradient_checkpointing = False
+    _no_split_modules = []
+
+    def __init__(self, config: RNNLMConfig, **kwargs):
+        super().__init__(config)
+        self.config = config
+        self.vocab_size = config.vocab_size
+        self.embedding_dim = config.embedding_dim
+        self.hidden_size = config.hidden_size
+        self.num_hidden_layers = config.num_hidden_layers
+        self.use_pos = getattr(config, "use_pos", False)
+        self.use_features = getattr(config, "use_features", False)
+
+        # Embedding layer (vocab_size + 1 for padding at index 0)
+        self.embedding = nn.Embedding(
+            config.vocab_size + 1,
+            config.embedding_dim,
+            padding_idx=0,
+        )
+
+        # GRU layers (Keras reset_after=False compatible)
+        self.gru_layers = nn.ModuleList()
+        for i in range(config.num_hidden_layers):
+            input_size = config.embedding_dim if i == 0 else config.hidden_size
+            self.gru_layers.append(
+                GRUKerasCompat(
+                    input_size=input_size,
+                    hidden_size=config.hidden_size,
+                    batch_first=True,
+                )
+            )
+
+        # Output size after GRU
+        lm_input_size = config.hidden_size
+
+        # Optional POS branch (for loading converted models - generation needs external POS)
+        if self.use_pos:
+            self.pos_embedding = nn.Embedding(
+                config.n_pos_tags + 1,
+                config.n_pos_embedding_nodes,
+                padding_idx=0,
+            )
+            self.pos_gru = nn.GRU(
+                input_size=config.n_pos_embedding_nodes,
+                hidden_size=config.n_pos_nodes,
+                num_layers=1,
+                batch_first=True,
+            )
+            lm_input_size = lm_input_size + config.n_pos_nodes
+        else:
+            self.pos_embedding = None
+            self.pos_gru = None
+
+        # Optional feature branch
+        if self.use_features:
+            self.feature_dense = nn.Sequential(
+                nn.Linear(config.vocab_size + 1, config.n_feature_nodes),
+                nn.Sigmoid(),
+            )
+            lm_input_size = lm_input_size + config.n_feature_nodes
+        else:
+            self.feature_dense = None
+
+        # Output projection
+        self.lm_head = nn.Linear(lm_input_size, config.vocab_size + 1)
+
+        # Initialize weights
+        self.apply(self._init_weights)
+
+    def _init_weights(self, module):
+        if isinstance(module, nn.Linear):
+            torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)
+            if module.bias is not None:
+                torch.nn.init.zeros_(module.bias)
+        elif isinstance(module, nn.Embedding):
+            torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)
+            if module.padding_idx is not None:
+                module.weight.data[module.padding_idx].zero_()
+
+    def get_input_embeddings(self):
+        return self.embedding
+
+    def set_input_embeddings(self, value):
+        self.embedding = value
+
+    def get_output_embeddings(self):
+        return self.lm_head
+
+    def set_output_embeddings(self, new_embeddings):
+        self.lm_head = new_embeddings
+
+    def prepare_inputs_for_generation(self, input_ids, past_key_values=None, **kwargs):
+        """
+        For RNN: past_key_values stores the hidden state tuple (h_n for each GRU layer).
+        During generation we only need the last token and the cached hidden state.
+        """
+        if past_key_values is not None:
+            input_ids = input_ids[:, -1:]
+        return {"input_ids": input_ids, "past_key_values": past_key_values}
+
+    def forward(
+        self,
+        input_ids=None,
+        attention_mask=None,
+        past_key_values=None,
+        position_ids=None,
+        pos_ids=None,
+        feature_vecs=None,
+        labels=None,
+        use_cache=None,
+        output_attentions=None,
+        output_hidden_states=None,
+        return_dict=None,
+    ):
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+        use_cache = use_cache if use_cache is not None else self.config.use_cache
+
+        # Get embeddings
+        inputs_embeds = self.embedding(input_ids)
+
+        # Run through GRU layers
+        hidden_states = inputs_embeds
+        new_past_key_values = () if use_cache else None
+
+        for i, gru_layer in enumerate(self.gru_layers):
+            if past_key_values is not None and len(past_key_values) > i:
+                h_0 = past_key_values[i]
+                hidden_states, h_n = gru_layer(hidden_states, h_0)
+            else:
+                hidden_states, h_n = gru_layer(hidden_states)
+
+            if use_cache:
+                new_past_key_values = new_past_key_values + (h_n,)
+
+        # Optional: concatenate POS hidden states (requires pos_ids at each step)
+        if self.use_pos and pos_ids is not None:
+            pos_embeds = self.pos_embedding(pos_ids)
+            _, pos_h_n = self.pos_gru(pos_embeds)
+            pos_hidden = pos_h_n.squeeze(0).unsqueeze(
+                1).expand(-1, hidden_states.size(1), -1)
+            hidden_states = torch.cat([hidden_states, pos_hidden], dim=-1)
+
+        # Optional: concatenate feature vectors
+        if self.use_features and feature_vecs is not None:
+            features = self.feature_dense(feature_vecs)
+            features = features.unsqueeze(
+                1).expand(-1, hidden_states.size(1), -1)
+            hidden_states = torch.cat([hidden_states, features], dim=-1)
+
+        # Project to vocabulary
+        logits = self.lm_head(hidden_states)
+
+        loss = None
+        if labels is not None:
+            shift_logits = logits[..., :-1, :].contiguous()
+            shift_labels = labels[..., 1:].contiguous()
+            loss_fct = nn.CrossEntropyLoss()
+            loss = loss_fct(
+                shift_logits.view(-1, shift_logits.size(-1)),
+                shift_labels.view(-1),
+            )
+
+        if not return_dict:
+            output = (logits,) + (new_past_key_values,
+                                  ) if use_cache else (logits,)
+            return ((loss,) + output) if loss is not None else output
+
+        return CausalLMOutputWithPast(
+            loss=loss,
+            logits=logits,
+            past_key_values=new_past_key_values,
+            hidden_states=None,
+            attentions=None,
+        )
+
+    @staticmethod
+    def _reorder_cache(past_key_values, beam_idx):
+        """Reorder past_key_values for beam search."""
+        return tuple(layer_past.index_select(0, beam_idx) for layer_past in past_key_values)
+
+    def generate(self, inputs=None, **kwargs):
+        """Override to add prevent_unk (pad/unk suppression) during generation."""
+        pad_id = getattr(self.config, "pad_token_id", 0)
+        unk_id = getattr(self.config, "unk_token_id", 1)
+        processor = PreventUnkLogitsProcessor(pad_token_id=pad_id, unk_token_id=unk_id)
+        logits_processor = kwargs.pop("logits_processor", None)
+        if logits_processor is None:
+            logits_processor = LogitsProcessorList()
+        elif not isinstance(logits_processor, LogitsProcessorList):
+            logits_processor = LogitsProcessorList(logits_processor)
+        logits_processor.insert(0, processor)
+        kwargs["logits_processor"] = logits_processor
+        return super().generate(inputs, **kwargs)

rnnlm_model/pipeline_rnnlm.py

@@ -0,0 +1,113 @@
+# coding: utf-8
+"""Custom TextGenerationPipeline for RNNLM with entity adaptation support."""
+
+from transformers.pipelines.text_generation import TextGenerationPipeline
+from transformers.pipelines.text_generation import ReturnType
+
+
+class RNNLMTextGenerationPipeline(TextGenerationPipeline):
+    """
+    TextGenerationPipeline that applies RNNLM-specific post-processing:
+    - Detokenization (capitalization, punctuation formatting)
+    - Entity adaptation: replaces generic ENT_* tokens with real entities from the prompt
+
+    When the tokenizer has generalize_ents=True, entities are extracted from the
+    prompt and used to replace ENT_PERSON_0, ENT_GPE_0, etc. in the generated output.
+    """
+
+    def postprocess(
+        self,
+        model_outputs,
+        return_type=ReturnType.NEW_TEXT,
+        clean_up_tokenization_spaces=False,
+        continue_final_message=None,
+    ):
+        generated_sequence = model_outputs["generated_sequence"][0]
+        input_ids = model_outputs["input_ids"]
+        prompt_text = model_outputs["prompt_text"]
+
+        # Convert to list (handle both PyTorch and TensorFlow)
+        if hasattr(generated_sequence, "cpu"):
+            generated_sequence = generated_sequence.cpu().tolist()
+        elif hasattr(generated_sequence, "numpy"):
+            generated_sequence = generated_sequence.numpy().tolist()
+        else:
+            generated_sequence = list(generated_sequence)
+
+        # Flatten if (num_return_sequences, seq_len) -> iterate over sequences
+        if generated_sequence and isinstance(generated_sequence[0], (list, tuple)):
+            sequences = generated_sequence
+        else:
+            sequences = [generated_sequence]
+
+        # Get prompt text(s) - can be str or list for batch
+        if isinstance(prompt_text, (list, tuple)):
+            prompts = list(prompt_text)
+        else:
+            prompts = [prompt_text] * len(sequences)
+
+        records = []
+        for seq_idx, sequence in enumerate(sequences):
+            if return_type == ReturnType.TENSORS:
+                record = {"generated_token_ids": sequence}
+            elif return_type in {ReturnType.NEW_TEXT, ReturnType.FULL_TEXT}:
+                # Use RNNLM-specific decode when tokenizer supports it (detokenize + entity adaptation)
+                # Entities are re-extracted from the original prompt here (prompt_text from model_outputs)
+                # and used to replace ENT_* tokens in the decoded output - no need to save from preprocess
+                tokenizer = self.tokenizer
+                prompt = prompts[seq_idx] if seq_idx < len(
+                    prompts) else (prompts[0] if prompts else "")
+                use_ents = getattr(tokenizer, "_generalize_ents", False) and isinstance(
+                    prompt, str) and prompt.strip()
+                ents = tokenizer.get_ents_for_context(
+                    prompt) if use_ents else None
+
+                # Generated text starts a new sentence if prompt ends with end-of-sentence punctuation
+                prompt_rstrip = prompt.rstrip() if isinstance(prompt, str) else ""
+                begin_sentence = prompt_rstrip.endswith((".", "!", "?"))
+
+                decode_kw = dict(
+                    skip_special_tokens=True,
+                    clean_up_tokenization_spaces=clean_up_tokenization_spaces,
+                    detokenize=True,
+                    begin_sentence=begin_sentence,
+                )
+                if use_ents and ents:
+                    decode_kw.update(
+                        adapt_ents=True, capitalize_ents=True, ents=[ents])
+
+                # Decode only the generated token IDs, then append to saved prompt
+                prompt_len = 0
+                if input_ids is not None:
+                    try:
+                        if hasattr(input_ids, "shape") and len(input_ids.shape) >= 2:
+                            pid = input_ids[seq_idx] if seq_idx < input_ids.shape[0] else input_ids[0]
+                        elif hasattr(input_ids, "__len__") and seq_idx < len(input_ids):
+                            pid = input_ids[seq_idx]
+                        else:
+                            pid = input_ids
+                        if hasattr(pid, "cpu"):
+                            pid = pid.cpu().tolist()
+                        elif hasattr(pid, "tolist"):
+                            pid = pid.tolist()
+                        else:
+                            pid = list(pid) if pid is not None else []
+                        prompt_len = len(pid) if pid else 0
+                    except (IndexError, TypeError):
+                        pass
+
+                if prompt_len > 0:
+                    generated_ids = sequence[prompt_len:]
+                    decoded_generated = tokenizer.decode(
+                        generated_ids, **decode_kw) if generated_ids else ""
+                    if return_type == ReturnType.FULL_TEXT:
+                        text = prompt.rstrip() + (decoded_generated if decoded_generated else "")
+                    else:
+                        text = decoded_generated
+                else:
+                    text = tokenizer.decode(sequence, **decode_kw)
+
+                record = {"generated_text": text}
+            records.append(record)
+
+        return records

rnnlm_model/tokenization_rnnlm.py

@@ -0,0 +1,293 @@
+# coding: utf-8
+"""RNNLM tokenizer - wraps SequenceTransformer for HuggingFace compatibility."""
+
+import json
+import os
+from typing import List, Optional, Union
+
+try:
+    from transformers import PreTrainedTokenizer
+except ImportError:
+    from transformers.tokenization_utils import PreTrainedTokenizer
+
+from .tokenization_utils import (
+    replace_ents_in_seq,
+    decode_num_seqs,
+    get_ents,
+    number_ents,
+    ent_counts_to_probs,
+)
+
+
+class RNNLMTokenizer(PreTrainedTokenizer):
+    """
+    Tokenizer for RNNLM that uses spaCy-based tokenization and a custom lexicon.
+    Compatible with the original SequenceTransformer from the narrative-prediction models.
+    """
+
+    model_input_names = ["input_ids", "attention_mask"]
+
+    def __init__(
+        self,
+        lexicon: Optional[dict] = None,
+        lexicon_lookup: Optional[list] = None,
+        unk_token="<UNK>",
+        pad_token="<pad>",
+        lemmatize=False,
+        include_tags=None,
+        prepend_start=False,
+        generalize_ents=True,
+        ent_counts=None,
+        filtered_ent_counts=None,
+        **kwargs,
+    ):
+        self._lexicon = lexicon or {}
+        self._lexicon_lookup = lexicon_lookup or [None, unk_token]
+        self._lemmatize = lemmatize
+        self._include_tags = include_tags or []
+        self._prepend_start = prepend_start
+        self._generalize_ents = generalize_ents
+        self._ent_counts = ent_counts or {}
+        self._filtered_ent_counts = filtered_ent_counts or {}
+        self._encoder = None  # Lazy load spaCy
+
+        super().__init__(
+            unk_token=unk_token,
+            pad_token=pad_token,
+            **kwargs,
+        )
+
+    @property
+    def vocab_size(self) -> int:
+        """Vocabulary size (excluding padding)."""
+        return len(self._lexicon) if self._lexicon else len(self._lexicon_lookup) - 1
+
+    def get_vocab(self) -> dict:
+        """Return token-to-id mapping. Required by PreTrainedTokenizer for save_pretrained."""
+        vocab = dict(self._lexicon) if self._lexicon else {}
+        # Ensure special tokens are in vocab (pad=0, unk=1)
+        if self.pad_token and self.pad_token not in vocab:
+            vocab[self.pad_token] = 0
+        if self.unk_token and self.unk_token not in vocab:
+            vocab[self.unk_token] = 1
+        return vocab
+
+    def _get_encoder(self):
+        """Lazy load spaCy encoder."""
+        if self._encoder is None:
+            try:
+                import spacy
+                self._encoder = spacy.load("en_core_web_sm")
+            except OSError:
+                try:
+                    import spacy
+                    self._encoder = spacy.load("en_core_web_md")
+                except OSError:
+                    raise RuntimeError(
+                        "spaCy English model required. Run: python -m spacy download en_core_web_sm"
+                    )
+        return self._encoder
+
+    def _tokenize(self, text: str) -> List[str]:
+        """Tokenize text using spaCy (matching SequenceTransformer.tokenize).
+        When generalize_ents is True, extracts entities and replaces them with generic
+        ENT_TYPE_N tokens before tokenization."""
+        encoder = self._get_encoder()
+        if self._generalize_ents:
+            # Replace named entities with generic tokens (e.g. ENT_PERSON_0)
+            text = replace_ents_in_seq(encoder, text)
+        doc = encoder(text)
+
+        # Match tokenize() from models/transformer.py
+        seq = []
+        for word in doc:
+            wtext = getattr(word, 'text', getattr(
+                word, 'string', str(word))).strip()
+            if self._include_tags and "_" not in wtext and word.tag_ not in self._include_tags:
+                continue
+            if self._lemmatize:
+                tok = word.lemma_ if not wtext.startswith("ENT_") else wtext
+            else:
+                tok = wtext.lower() if not wtext.startswith("ENT_") else wtext
+            if tok:
+                seq.append(tok)
+
+        if self._prepend_start:
+            seq.insert(0, "<START>")
+        return seq
+
+    def _convert_token_to_id(self, token: str) -> int:
+        """Convert a single token to ID. Required by PreTrainedTokenizer base class."""
+        return self._lexicon.get(token, 1) if self._lexicon else 1  # 1 = UNK
+
+    def _convert_tokens_to_ids(self, tokens: Union[str, List[str]]) -> Union[int, List[int]]:
+        """Convert tokens to IDs using lexicon."""
+        if isinstance(tokens, str):
+            return self._convert_token_to_id(tokens)
+        return [self._convert_token_to_id(t) for t in tokens]
+
+    def _convert_id_to_token(self, index: int) -> str:
+        """Convert a single ID to token. Required by PreTrainedTokenizer base class."""
+        unk = self.unk_token if hasattr(self, "unk_token") else "<UNK>"
+        if 0 <= index < len(self._lexicon_lookup) and self._lexicon_lookup[index]:
+            return self._lexicon_lookup[index]
+        return unk
+
+    def _convert_ids_to_tokens(self, ids: Union[int, List[int]]) -> Union[str, List[str]]:
+        """Convert IDs to tokens using lexicon_lookup."""
+        if isinstance(ids, int):
+            return self._convert_id_to_token(ids)
+        return [self._convert_id_to_token(i) for i in ids]
+
+    def convert_tokens_to_string(self, tokens: List[str]) -> str:
+        """Convert tokens to string (join with space)."""
+        return " ".join(tokens)
+
+    def decode(
+        self,
+        token_ids,
+        begin_sentence=True,
+        skip_special_tokens=True,
+        clean_up_tokenization_spaces=False,
+        ents=None,
+        adapt_ents=True,
+        detokenize=True,
+        capitalize_ents=True,
+        n_sents_per_seq=1,
+        eos_tokens=None,
+        **kwargs,
+    ):
+        """Decode token IDs to string. When adapt_ents=True and ents is provided,
+        replaces generic ENT_* tokens in the output with entities from the input context.
+        ents should be a list of dicts (one per sequence) mapping entity name to type
+        (e.g. {"John": "PERSON_0"} from number_ents(get_ents(...)))."""
+        if isinstance(token_ids[0], (list, tuple)):
+            seqs = token_ids
+        else:
+            seqs = [token_ids]
+        # ents must be list of dicts (one per sequence)
+        if ents is not None:
+            ents = [ents] if isinstance(ents, dict) else (
+                ents if isinstance(ents, list) else [])
+        encoder = self._get_encoder()
+        sub_ent_probs = ent_counts_to_probs(
+            self._filtered_ent_counts) if self._filtered_ent_counts else {}
+        decoded = decode_num_seqs(
+            encoder,
+            self._lexicon_lookup,
+            self.unk_token,
+            seqs,
+            n_sents_per_seq=n_sents_per_seq,
+            eos_tokens=eos_tokens or [],
+            detokenize=detokenize,
+            ents=ents or [],
+            capitalize_ents=capitalize_ents,
+            adapt_ents=adapt_ents,
+            sub_ent_probs=sub_ent_probs,
+            begin_sentence=begin_sentence,
+        )
+        result = decoded[0] if len(decoded) == 1 and not isinstance(
+            token_ids[0], (list, tuple)) else decoded
+        if clean_up_tokenization_spaces and isinstance(result, str):
+            result = result.rstrip()  # preserve leading space from detokenize_tok_seq
+        return result
+
+    def get_ents_for_context(self, text: str):
+        """Extract and number entities from context text for use with decode(..., adapt_ents=True).
+        Returns a dict mapping entity name to type (e.g. {"John": "PERSON_0"}) for a single sequence."""
+        encoder = self._get_encoder()
+        ents, ent_counts = get_ents(encoder, text)
+        return number_ents(encoder, ents, ent_counts)
+
+    def build_inputs_with_special_tokens(
+        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
+    ) -> List[int]:
+        """No special tokens for RNNLM - return as is."""
+        if token_ids_1 is None:
+            return token_ids_0
+        return token_ids_0 + token_ids_1
+
+    def get_special_tokens_mask(
+        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
+    ) -> List[int]:
+        """Return mask of 0s (no special tokens in RNNLM)."""
+        return [0] * len(token_ids_0 + (token_ids_1 or []))
+
+    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> tuple:
+        """Save lexicon and lexicon_lookup to files."""
+        if not os.path.isdir(save_directory):
+            os.makedirs(save_directory)
+
+        prefix = filename_prefix or ""
+        vocab_file = os.path.join(save_directory, f"{prefix}vocab.json")
+        lookup_file = os.path.join(
+            save_directory, f"{prefix}lexicon_lookup.json")
+
+        with open(vocab_file, "w", encoding="utf-8") as f:
+            json.dump(self._lexicon, f, ensure_ascii=False, indent=2)
+
+        with open(lookup_file, "w", encoding="utf-8") as f:
+            json.dump(self._lexicon_lookup, f, ensure_ascii=False, indent=2)
+
+        return (vocab_file, lookup_file)
+
+    @classmethod
+    def from_pretrained(cls, pretrained_model_name_or_path, *init_inputs, **kwargs):
+        """Load tokenizer - supports both HF format and paths with vocab.json + lexicon_lookup.json."""
+        save_directory = pretrained_model_name_or_path
+        if os.path.isdir(save_directory):
+            vocab_file = os.path.join(save_directory, "vocab.json")
+            lookup_file = os.path.join(save_directory, "lexicon_lookup.json")
+            if os.path.exists(vocab_file) and os.path.exists(lookup_file):
+                with open(vocab_file, "r", encoding="utf-8") as f:
+                    lexicon = json.load(f)
+                with open(lookup_file, "r", encoding="utf-8") as f:
+                    lexicon_lookup = json.load(f)
+                tokenizer_config_file = os.path.join(
+                    save_directory, "tokenizer_config.json")
+                lemmatize = False
+                include_tags = []
+                prepend_start = False
+                generalize_ents = False
+                ent_counts = {}
+                filtered_ent_counts = {}
+                if os.path.exists(tokenizer_config_file):
+                    with open(tokenizer_config_file, "r", encoding="utf-8") as f:
+                        tc = json.load(f)
+                    lemmatize = tc.get("lemmatize", False)
+                    include_tags = tc.get("include_tags", [])
+                    prepend_start = tc.get("prepend_start", False)
+                    generalize_ents = tc.get("generalize_ents", False)
+                    ent_counts = tc.get("ent_counts", {})
+                    filtered_ent_counts = tc.get("filtered_ent_counts", {})
+                return cls(
+                    lexicon=lexicon,
+                    lexicon_lookup=lexicon_lookup,
+                    lemmatize=lemmatize,
+                    include_tags=include_tags,
+                    prepend_start=prepend_start,
+                    generalize_ents=generalize_ents,
+                    ent_counts=ent_counts,
+                    filtered_ent_counts=filtered_ent_counts,
+                    **kwargs,
+                )
+        return super().from_pretrained(pretrained_model_name_or_path, *init_inputs, **kwargs)
+
+    def save_pretrained(self, save_directory: str, **kwargs):
+        """Save tokenizer - also save tokenizer config with custom attributes."""
+        super().save_pretrained(save_directory, **kwargs)
+        # Save extra config for our tokenizer
+        config_path = os.path.join(save_directory, "tokenizer_config.json")
+        if os.path.exists(config_path):
+            with open(config_path, "r", encoding="utf-8") as f:
+                config = json.load(f)
+        else:
+            config = {}
+        config["lemmatize"] = self._lemmatize
+        config["include_tags"] = self._include_tags
+        config["prepend_start"] = self._prepend_start
+        config["generalize_ents"] = self._generalize_ents
+        config["ent_counts"] = self._ent_counts
+        config["filtered_ent_counts"] = self._filtered_ent_counts
+        with open(config_path, "w", encoding="utf-8") as f:
+            json.dump(config, f, indent=2)

rnnlm_model/tokenization_utils.py

@@ -0,0 +1,357 @@
+"""Tokenization utilities for RNNLM - entity extraction, replacement, and decoding."""
+
+import re
+import numpy as np
+
+# RNG for adapt_tok_seq_ents when sampling from sub_ent_probs
+_rng = np.random.RandomState(0)
+
+
+def segment(encoder, seq):
+    doc = encoder(seq)
+    return [getattr(sent, 'text', getattr(sent, 'string', str(sent))).strip() for sent in doc.sents]
+
+
+def tokenize(encoder, seq, lowercase=True, recognize_ents=False,
+             lemmatize=False, include_tags=[], include_pos=[], prepend_start=False):
+    seq = encoder(seq)
+    if recognize_ents:  # merge named entities into single tokens
+        ent_start_idxs = {ent.start: ent for ent in seq.ents
+                          if getattr(ent, 'text', getattr(ent, 'string', '')).strip()}
+        # combine each ent into a single token; this is pretty hard to read, but it works
+        seq = [ent_start_idxs[word_idx] if word_idx in ent_start_idxs else word
+               for word_idx, word in enumerate(seq)
+               if (not word.ent_type_ or word_idx in ent_start_idxs)]
+
+    def _wtext(w):
+        return getattr(w, 'text', getattr(w, 'string', str(w))).strip()
+
+    # Don't apply POS filtering to phrases (words with underscores)
+    if include_tags:  # fine-grained POS tags
+        seq = [word for word in seq
+               if ("_" in _wtext(word) or word.tag_ in include_tags)]
+    if include_pos:  # coarse-grained POS tags
+        seq = [word for word in seq
+               if ("_" in _wtext(word) or word.pos_ in include_pos)]
+    if lemmatize:
+        seq = [word.lemma_ if not _wtext(word).startswith('ENT_')
+               else _wtext(word) for word in seq]
+    # don't lowercase if token is an entity (entities will be of type span instead of token; or will be prefixed with 'ENT_' if already transformed to types)
+    elif lowercase:
+        seq = [_wtext(word).lower() if not _wtext(word).startswith('ENT_')
+               else _wtext(word) for word in seq]
+    else:
+        seq = [_wtext(word) for word in seq]
+    # some words may be empty strings, so filter
+    seq = [word for word in seq if word]
+    if prepend_start:
+        seq.insert(0, u"<START>")
+    return seq
+
+
+def ent_counts_to_probs(ent_counts):
+    """Convert entity counts to probabilities for sampling when adapting entities."""
+    return {ent_type: {ent: count * 1.0 / sum(counts.values())
+                       for ent, count in counts.items()}
+            for ent_type, counts in ent_counts.items()}
+
+
+def get_ents(encoder, seq, include_ent_types=('PERSON', 'NORP', 'ORG', 'GPE')):
+    '''return dict of all entities in seq mapped to their entity types, optionally labeled with gender for PERSON entities'''
+
+    ents = {}
+    ent_counts = {}
+    for ent in encoder(seq).ents:
+        ent_type = ent.label_
+        if ent_type in include_ent_types:
+            ent = getattr(ent, 'text', getattr(
+                ent, 'string', str(ent))).strip()
+            if ent:  # not sure why, but whitespace can be detected as an ent, so need to check for this
+                ents[ent] = [ent_type]
+                if ent in ent_counts:
+                    ent_counts[ent] += 1
+                else:
+                    ent_counts[ent] = 1
+                ents[ent] = "_".join(ents[ent])
+    return ents, ent_counts
+
+
+def number_ents(encoder, ents, ent_counts):
+    '''return dict of all entities in seq mapped to their entity types,
+    with numerical suffixes to distinguish entities of the same type'''
+    ent_counts = sorted([(count, ent, ents[ent])
+                        for ent, count in ent_counts.items()])[::-1]
+    ent_type_counts = {}
+    num_ents = {}
+    for count, ent, ent_type in ent_counts:
+        tok_ent = tokenize(encoder, ent, lowercase=False)
+        coref_ent = [num_ent for num_ent in num_ents
+                     if (tokenize(encoder, num_ent, lowercase=False)[0] == tok_ent[0]
+                         or tokenize(encoder, num_ent, lowercase=False)[-1] == tok_ent[-1])
+                     # treat ents with same first or last word as co-referring
+                     and ents[num_ent] == ent_type]
+        if coref_ent:
+            num_ents[ent] = num_ents[coref_ent[0]]
+        else:
+            ent_type = ent_type.split("_")
+            if ent_type[0] in ent_type_counts:
+                ent_type_counts[ent_type[0]] += 1
+            else:
+                ent_type_counts[ent_type[0]] = 1
+            num_ents[ent] = ent_type
+            # insert number id after entity type (and before tag, if it exists)
+            num_ents[ent].insert(1, str(ent_type_counts[ent_type[0]] - 1))
+            num_ents[ent] = "_".join(num_ents[ent])
+    return num_ents
+
+
+def replace_ents_in_seq(encoder, seq):
+    '''extract entities from seq and replace them with their entity types'''
+    ents, ent_counts = get_ents(encoder, seq)
+    ents = number_ents(encoder, ents, ent_counts)
+    seq = tokenize(encoder, seq, lowercase=False, recognize_ents=True)
+    # word can be Token or Span; get text for lookup
+
+    def _text(w):
+        return (getattr(w, 'text', None) or getattr(w, 'string', None) or str(w)).strip()
+    seq = ['ENT_' + ents[_text(word)] if _text(word)
+           in ents else _text(word) for word in seq]
+    seq = " ".join(seq)
+    return seq
+
+
+def decode_num_seqs(encoder, lexicon_lookup, unk_word, seqs, n_sents_per_seq=None, eos_tokens=[],
+                    detokenize=False, ents=[], capitalize_ents=False, adapt_ents=False,
+                    sub_ent_probs=None, begin_sentence=True):
+    if not seqs:
+        return []
+    if type(seqs[0]) not in (list, np.ndarray, tuple):
+        seqs = [seqs]
+    decoded_seqs = []
+    # transform numerical seq back into string (seq elements are token IDs)
+    for seq_idx, seq in enumerate(seqs):
+        # Flatten to list of Python ints (handles 2D tensors from model.generate, e.g. (1, seq_len))
+        if hasattr(seq, 'cpu'):
+            seq = seq.cpu()
+        if hasattr(seq, 'tolist'):
+            seq = seq.tolist()
+        elif seq and hasattr(seq[0], 'tolist'):
+            # list(tensor) gives list of row tensors - convert each to list
+            seq = [row.tolist() for row in seq]
+        else:
+            seq = list(seq)
+        # If 2D (batch, seq_len), take each row; else single sequence
+        if seq and isinstance(seq[0], list):
+            rows = seq
+        else:
+            rows = [seq]
+
+        def _to_int(x):
+            if isinstance(x, (list, tuple)):
+                return [_to_int(v) for v in x]
+            return int(x.item()) if hasattr(x, 'item') else int(x)
+
+        for row_idx, row in enumerate(rows):
+            tok_seq = []
+            flat_row = _to_int(row) if isinstance(
+                row, (list, tuple)) else [_to_int(row)]
+            if isinstance(flat_row[0], list):
+                flat_row = [v for sub in flat_row for v in (
+                    sub if isinstance(sub, list) else [sub])]
+            for w in flat_row:
+                i = w if isinstance(w, int) else int(w)
+                tok_seq.append(
+                    lexicon_lookup[i] if (0 <= i < len(lexicon_lookup) and lexicon_lookup[i])
+                    else unk_word
+                )
+            seq = tok_seq
+            if adapt_ents:  # replace ENT_* with entities from ents, or sub_ent_probs/UNK as fallback
+                ent_idx = min(seq_idx + row_idx, len(ents) - 1) if ents else 0
+                seq_ents = ents[ent_idx] if ents else {}
+                seq = adapt_tok_seq_ents(
+                    seq, ents=seq_ents, sub_ent_probs=sub_ent_probs or {})
+            if detokenize:  # apply rules for transforming token list into formatted sequence
+                if ents and capitalize_ents:
+                    ent_idx = min(seq_idx + row_idx,
+                                  len(ents) - 1) if ents else 0
+                    seq = detokenize_tok_seq(
+                        encoder, seq, ents=ents[ent_idx], begin_sentence=begin_sentence)
+                else:
+                    seq = detokenize_tok_seq(
+                        encoder, seq, ents=[], begin_sentence=begin_sentence)
+            else:
+                # otherwise just join tokens with whitespace between each
+                seq = " ".join(seq)
+            if eos_tokens:  # if filter_n_sents is a number, filter generated sequence to only the first N=filter_n_sents sentences
+                seq = filter_gen_seq(encoder, seq, eos_tokens=eos_tokens)
+            elif n_sents_per_seq:
+                seq = filter_gen_seq(encoder, seq, n_sents=n_sents_per_seq)
+            decoded_seqs.append(seq)
+    return decoded_seqs
+
+
+def adapt_tok_seq_ents(seq, ents={}, sub_ent_probs={}):
+
+    # reverse ents so that types map to names
+    ents = {ent_type: ent for ent, ent_type in ents.items()}
+    adapted_seq_ents = {"_".join(token.split("_")[1:]): None
+                        for token in seq if token.startswith('ENT_')}
+
+    if not adapted_seq_ents:
+        return seq
+
+    for seq_ent_type in {ent_type: adapted_ent for ent_type, adapted_ent in adapted_seq_ents.items() if not adapted_ent}:
+        if seq_ent_type in ents:
+            adapted_seq_ents[seq_ent_type] = ents[seq_ent_type]
+            del ents[seq_ent_type]
+
+    if ents:
+        for seq_ent_type in {ent_type: adapted_ent for ent_type, adapted_ent in adapted_seq_ents.items() if not adapted_ent}:
+            for ent_type, ent in ents.items():
+                if seq_ent_type.split("_")[0] in ent_type.split("_")[0]:
+                    adapted_seq_ents[seq_ent_type] = ents[ent_type]
+                    del ents[ent_type]
+                    break
+
+    for seq_ent_type in {ent_type: adapted_ent for ent_type, adapted_ent in adapted_seq_ents.items() if not adapted_ent}:
+        if seq_ent_type.split("_")[0] in sub_ent_probs:
+            sub_ents, sub_probs = zip(
+                *sub_ent_probs[seq_ent_type.split("_")[0]].items())
+            rand_ent_idx = _rng.choice(len(sub_ents), p=np.array(sub_probs))
+            adapted_seq_ents[seq_ent_type] = sub_ents[rand_ent_idx]
+
+    # Use ANY available entity (any type) when no type-specific match found
+    all_entities = list(ents.values())
+    for base_type, type_ents in sub_ent_probs.items():
+        all_entities.extend(type_ents.keys())
+    for seq_ent_type in {ent_type: adapted_ent for ent_type, adapted_ent in adapted_seq_ents.items() if not adapted_ent}:
+        if all_entities:
+            adapted_seq_ents[seq_ent_type] = _rng.choice(all_entities)
+        else:
+            adapted_seq_ents[seq_ent_type] = "ENT_" + seq_ent_type
+
+    seq = [adapted_seq_ents["_".join(token.split("_")[1:])] if "_".join(
+        token.split("_")[1:]) in adapted_seq_ents else token for token in seq]
+    return seq
+
+
+def detokenize_tok_seq(encoder, seq, ents=[], begin_sentence=True):
+    '''use simple rules for transforming list of tokens back into string
+    ents is optional list of words (named entities) that should be capitalized'''
+    seq = [sent.split() for sent
+           in segment(encoder, " ".join(seq))]  # split sequence into sentences
+    detok_seq = []
+    for sent_idx, sent in enumerate(seq):
+
+        assert (type(sent) in (list, tuple))
+
+        if ents:
+            token_idx = 0
+            # capitalize all tokens that appear in cap_ents
+            while token_idx < len(sent):
+                for ent in ents:
+                    ent = ent.split()
+                    if sent[token_idx:token_idx + len(ent)] == [token.lower() for token in ent]:
+                        # import pdb;pdb.set_trace()
+                        sent[token_idx:token_idx + len(ent)] = list(ent)
+                        token_idx += len(ent) - 1
+                        break
+                token_idx += 1
+
+        detok_sent = " ".join(sent)
+
+        detok_sent = re.sub("\'", "'", detok_sent)
+
+        # capitalize first-person "I" pronoun
+        detok_sent = re.sub(r"(^| )i ", r"\1I ", detok_sent)
+
+        # rules for contractions
+        detok_sent = re.sub(" n\'\s*t ", "n\'t ", detok_sent)
+        detok_sent = re.sub(" \'\s*d ", "\'d ", detok_sent)
+        detok_sent = re.sub(" \'\s*s ", "\'s ", detok_sent)
+        detok_sent = re.sub(" \'\s*ve ", "\'ve ", detok_sent)
+        detok_sent = re.sub(" \'\s*ll ", "\'ll ", detok_sent)
+        detok_sent = re.sub(" \'\s*m ", "\'m ", detok_sent)
+        detok_sent = re.sub(" \'\s*re ", "\'re ", detok_sent)
+
+        # rules for formatting punctuation
+        detok_sent = re.sub(" \.", ".", detok_sent)
+        detok_sent = re.sub(" \!", "!", detok_sent)
+        detok_sent = re.sub(" \?", "?", detok_sent)
+        detok_sent = re.sub(" ,", ",", detok_sent)
+        detok_sent = re.sub(" \- ", "-", detok_sent)
+        detok_sent = re.sub(" :", ":", detok_sent)
+        detok_sent = re.sub(" ;", ";", detok_sent)
+        detok_sent = re.sub("\$ ", "$", detok_sent)
+        detok_sent = re.sub("\' \'", "\'\'", detok_sent)
+        detok_sent = re.sub("\` \`", "\`\`", detok_sent)
+
+        # replace repeated single quotes with double quotation mark.
+        detok_sent = re.sub("\'\'", "\"", detok_sent)
+        detok_sent = re.sub("\`\`", "\"", detok_sent)
+
+        # filter repetitive characters
+        detok_sent = re.sub("([\"\']\s*){2,}", "\" ", detok_sent)
+
+        # map each opening puncutation mark to closing mark
+        punc_pairs = {"\'": "\'", "\'": "\'",
+                      "`": "\'", "\"": "\"", "(": ")", "[": "]"}
+        open_punc = []
+        char_idx = 0
+        while char_idx < len(detok_sent):  # check for quotes and parenthesis
+            char = detok_sent[char_idx]
+            # end quote/parenthesis
+            if open_punc and char == punc_pairs[open_punc[-1]]:
+                if char_idx > 0 and detok_sent[char_idx - 1] == " ":
+                    detok_sent = detok_sent[:char_idx -
+                                            1] + detok_sent[char_idx:]
+                    open_punc.pop()
+            elif char in punc_pairs:
+                if char_idx < len(detok_sent) - 1 and detok_sent[char_idx + 1] == " ":
+                    open_punc.append(char)
+                    detok_sent = detok_sent[:char_idx +
+                                            1] + detok_sent[char_idx + 2:]
+            if char_idx < len(detok_sent) and detok_sent[char_idx] == char:
+                char_idx += 1
+
+        detok_sent = detok_sent.strip()
+        # capitalize first alphabetic character if begin_sentence is True
+        if begin_sentence:
+            for char_idx, char in enumerate(detok_sent):
+                if char.isalpha():
+                    detok_sent = detok_sent[:char_idx +
+                                            1].upper() + detok_sent[char_idx + 1:]
+                    break
+        detok_seq.append(detok_sent)
+
+    detok_seq = " ".join(detok_seq)
+    contraction_patterns = ("'s", "'re", "'ve", "'d", "'ll", "'m", "n't")
+    punctuation_patterns = (".", "!", "?", ",", "-", ":", ";", ")", "]")
+    # Only prepend space if detok_seq doesn't start with these
+    starts_with_pattern = detok_seq.startswith(
+        contraction_patterns) or detok_seq.startswith(punctuation_patterns)
+    if not starts_with_pattern and detok_seq:
+        detok_seq = " " + detok_seq
+    return detok_seq
+
+
+def filter_gen_seq(encoder, seq, n_sents=1, eos_tokens=[]):
+    '''given a generated sequence, filter so that only the first n_sents are included in final generated sequence'''
+    leading_space = seq.startswith(" ") if seq else False
+    if eos_tokens:  # if end-of-sentence tokens given, cut off sequence at first occurrence of one of these tokens; otherwise use segmenter to infer sentence boundaries
+        doc = encoder(seq)
+        for idx, word in enumerate(doc):
+            wtext = getattr(word, 'text', getattr(
+                word, 'string', str(word))).strip()
+            if wtext in eos_tokens:
+                span = doc[:idx + 1]
+                seq = getattr(span, 'text', getattr(
+                    span, 'string', str(span))).strip()
+                break
+        else:
+            seq = getattr(doc, 'text', getattr(doc, 'string', str(doc)))
+    else:
+        seq = " ".join(segment(encoder, seq)[:n_sents])
+    if leading_space and seq:
+        seq = " " + seq.lstrip()
+    return seq

special_tokens_map.json

@@ -0,0 +1,4 @@
+{
+  "pad_token": "<pad>",
+  "unk_token": "<UNK>"
+}

tokenizer_config.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d20caa40c3cb68b113ead456587ccc9308b0e4743b61aa218c5fbf8b3d88e52b
+size 14303042