add time series safetensors

0092638_seism.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2b94653c6964b630038897a27cb6d276ff866d9ecd1f6419358b9407f0df62e
+size 72128

README.md

@@ -290,7 +290,7 @@ print(json.dumps(response.model_dump(), indent=2, ensure_ascii=False))
 Time series inference is currently only supported in LMDeploy. To get started, download and deploy Intern-S2-Preview-397B with LMDeploy by following the [Model Deployment Guide](./deployment_guide.md).
 Below is an example of detecting earthquake events from a time series signal file. Additional data types and functionalities are also supported.
 
-**Please note**: this demo is slightly different from the one in [Intern-S1-Pro](https://huggingface.co/internlm/Intern-S1-Pro#time-series-demo). The main difference is that in the messages content, you need to provide time_series_url first, followed by the text prompt. Please adapt your implementation based on this demo.
+**Please note**: in the message content, the order of time_series_url and the text prompt can be arbitrary.
 
 ```
 from openai import OpenAI
@@ -413,6 +413,57 @@ print(response.choices[0].message)
 
 ```
 
+For time series forecasting, `forecast_horizon` is optional. Set it to an integer to produce a forecast of exactly that length, or set it to `None` to let the model infer the horizon from the text prompt.
+
+```
+def forecast_base64(file_path: str, forecast_horizon: int | None = None):
+    base64_ts = encode_time_series_base64(file_path)
+    messages = [
+        {
+            "role": "user",
+            "content": [
+                {
+                    "type": "text",
+                    "text": (
+                        "Please complete a electric load forecasting task. "
+                        "This dataset is based on historical electricity load data every half hour within 24 hours of the region, "
+                        "as well as data on minimum temperature, maximum temperature, humidity, air pressure, etc., "
+                        "to predict future load consumption every half hour within 24 hours. Here is the weather information for city TAS: "
+                        "Historical date weather: minimum temperature of 279.71K, maximum temperature of 285.83K, humidity of 85.0%, "
+                        "air pressure of 1003.0hPa. Forecast date weather: minimum temperature 280.54K, maximum temperature 286.47K, "
+                        "humidity 74.0%, air pressure 1007.0hPa. This data has no relevant effect information. "
+                        "Please predict the next 48 time points given information above."
+                    ),
+                },
+                {
+                    "type": "time_series_url",
+                    "time_series_url": {
+                        "url": f"data:time_series/npy;base64,{base64_ts}",
+                    },
+                },
+            ],
+        }
+    ]
+
+    return client.chat.completions.create(
+        model=model_name,
+        messages=messages,
+        temperature=0,
+        max_tokens=16,
+        extra_body={
+            "chat_template_kwargs": {"enable_thinking": False},
+            "enable_forecasting": True,
+            "forecast_horizon": forecast_horizon,
+        },
+    )
+
+
+response = forecast_base64("./load_20210803_0.npy", forecast_horizon=None)
+forecast = response.choices[0].message.ts_forecast
+print("Point forecast:", forecast.point_forecast)
+print("Quantile forecast:", forecast.quantile_forecast)
+```
+
 ## Agent Integration
 
 Intern-S2-Preview-397B can be plugged into agent frameworks in two ways: connecting to a **self-hosted deployment**, or calling the **official InternLM API**. Below we cover both, with examples for agent frameworks (OpenClaw, Hermes, etc.) and for Claude Code.

chat_template.jinja

@@ -27,6 +27,11 @@
                     {{- 'Video ' ~ video_count.value ~ ': ' }}
                 {%- endif %}
                 {{- '<|vision_start|><|video_pad|><|vision_end|>' }}
+                {%- elif 'time_series' in item or item.type == 'time_series' %}
+                {%- if is_system_content %}
+                    {{- raise_exception('System message cannot contain time series.') }}
+                {%- endif %}
+                {{- '<|ts|><TS_CONTEXT><|/ts|>' }}
             {%- elif 'text' in item %}
                 {{- item.text }}
             {%- else %}
@@ -42,6 +47,16 @@
 {%- if not messages %}
     {{- raise_exception('No messages provided.') }}
 {%- endif %}
+{%- set ts_ns = namespace(has_time_series=false) %}
+{%- for message in messages %}
+    {%- if message.content is iterable and message.content is not mapping and message.content is not string %}
+        {%- for item in message.content %}
+            {%- if 'time_series' in item or item.type == 'time_series' %}
+                {%- set ts_ns.has_time_series = true %}
+            {%- endif %}
+        {%- endfor %}
+    {%- endif %}
+{%- endfor %}
 {%- if tools and tools is iterable and tools is not mapping %}
     {{- '<|im_start|>system\n' }}
     {{- "# Tools\n\nYou have access to the following functions:\n\n<tools>" }}
@@ -139,7 +154,7 @@
 {%- endfor %}
 {%- if add_generation_prompt %}
     {{- '<|im_start|>assistant\n' }}
-    {%- if enable_thinking is defined and enable_thinking is false %}
+    {%- if ts_ns.has_time_series or (enable_thinking is defined and enable_thinking is false) %}
         {{- '<think>\n\n</think>\n\n' }}
     {%- else %}
         {{- '<think>\n' }}

config.json

@@ -161,13 +161,43 @@
     "encoder_layers": 17,
     "max_source_positions": 1500,
     "num_mel_bins": 80,
-    "out_hidden_size": 2048,
+    "out_hidden_size": 4096,
     "scale_embedding": false,
     "ts_adapt_in_dim": 256,
     "ts_adapt_out_dim": 1024,
-    "ts_hidden_dim": 1024
+    "ts_hidden_dim": 1024,
+    "chunk_size": 12800,
+    "chunk_step": 12800,
+    "subsampling_hidden_dim": 128,
+    "subsampling_nhead": 8,
+    "subsampling_patch": 50,
+    "subsampling_num_query": 2,
+    "subsampling_num_conv_layers": 0,
+    "subsampling_selfatt": false
   },
   "ts_token_id": 248093,
   "ts_start_id": 248091,
-  "ts_end_id": 248092
+  "ts_end_id": 248092,
+  "ts_forecaster_config": {
+    "model_type": "interns2_preview_time_series_forecaster",
+    "d_llm": 4096,
+    "d_ts_encoder": 1024,
+    "qformer_hidden_dim": 1280,
+    "qformer_num_query_tokens": 16,
+    "qformer_num_heads": 8,
+    "qformer_num_layers": 2,
+    "qformer_dropout": 0.0,
+    "use_horizon_head": true,
+    "horizon_max_length": 0,
+    "use_cross_attn_gate": true,
+    "max_context": 2048,
+    "max_horizon": 1024,
+    "normalize_inputs": true,
+    "use_continuous_quantile_head": true,
+    "force_flip_invariance": true,
+    "infer_is_positive": true,
+    "fix_quantile_crossing": true,
+    "return_backcast": false,
+    "default_pred_len": 720
+  }
 }

configuration_interns2_preview.py

@@ -21,6 +21,57 @@ from transformers.configuration_utils import PreTrainedConfig, layer_type_valida
 from transformers.modeling_rope_utils import RopeParameters
 
 
+class InternS2PreviewBertConfig(PreTrainedConfig):
+    model_type = "intern_s2_preview_bert"
+
+    def __init__(
+        self,
+        vocab_size=30522,
+        hidden_size=768,
+        num_hidden_layers=12,
+        num_attention_heads=12,
+        intermediate_size=3072,
+        hidden_act="gelu",
+        hidden_dropout_prob=0.1,
+        attention_probs_dropout_prob=0.1,
+        max_position_embeddings=512,
+        type_vocab_size=2,
+        initializer_range=0.02,
+        layer_norm_eps=1e-12,
+        pad_token_id=0,
+        use_cache=True,
+        classifier_dropout=None,
+        is_decoder=False,
+        add_cross_attention=False,
+        bos_token_id=None,
+        eos_token_id=None,
+        tie_word_embeddings=True,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.pad_token_id = pad_token_id
+        self.is_decoder = is_decoder
+        self.add_cross_attention = add_cross_attention
+        self.bos_token_id = bos_token_id
+        self.eos_token_id = eos_token_id
+        self.tie_word_embeddings = tie_word_embeddings
+
+        self.vocab_size = vocab_size
+        self.hidden_size = hidden_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.hidden_act = hidden_act
+        self.intermediate_size = intermediate_size
+        self.hidden_dropout_prob = hidden_dropout_prob
+        self.attention_probs_dropout_prob = attention_probs_dropout_prob
+        self.max_position_embeddings = max_position_embeddings
+        self.type_vocab_size = type_vocab_size
+        self.initializer_range = initializer_range
+        self.layer_norm_eps = layer_norm_eps
+        self.use_cache = use_cache
+        self.classifier_dropout = classifier_dropout
+
+
 class InternS2PreviewVisionConfig(PreTrainedConfig):
     model_type = "intern_s2_preview"
     base_config_key = "vision_config"
@@ -306,6 +357,14 @@ class InternS2PreviewTimeSeriesConfig(PreTrainedConfig):
         ts_adapt_in_dim: int = 256,
         ts_adapt_out_dim: int = 1024,
         ts_hidden_dim: int = 1024,
+        chunk_size: int = 12800,
+        chunk_step: int = 12800,
+        subsampling_hidden_dim: int = 128,
+        subsampling_nhead: int = 8,
+        subsampling_patch: int = 50,
+        subsampling_num_query: int = 2,
+        subsampling_num_conv_layers: int = 0,
+        subsampling_selfatt: bool = False,
         **super_kwargs,
     ):
         super().__init__(**super_kwargs)
@@ -330,10 +389,83 @@ class InternS2PreviewTimeSeriesConfig(PreTrainedConfig):
         self.ts_adapt_in_dim = ts_adapt_in_dim
         self.ts_adapt_out_dim = ts_adapt_out_dim
         self.ts_hidden_dim = ts_hidden_dim
+        self.chunk_size = chunk_size
+        self.chunk_step = chunk_step
+        self.subsampling_hidden_dim = subsampling_hidden_dim
+        self.subsampling_nhead = subsampling_nhead
+        self.subsampling_patch = subsampling_patch
+        self.subsampling_num_query = subsampling_num_query
+        self.subsampling_num_conv_layers = subsampling_num_conv_layers
+        self.subsampling_selfatt = subsampling_selfatt
 
         assert self.ts_adapt_out_dim == self.ts_hidden_dim, "ts_adapt_out_dim should be equal to ts_hidden_dim"
 
 
+class InternS2PreviewTimeSeriesForecasterConfig(PreTrainedConfig):
+    model_type = "interns2_preview_time_series_forecaster"
+
+    FORECASTER_MODEL_DIMS = 1280
+
+    def __init__(
+        self,
+        d_llm: int = 2560,
+        d_ts_encoder: int = 1024,
+        qformer_hidden_dim: int = 0,
+        qformer_num_query_tokens: int = 32,
+        qformer_num_heads: int = 8,
+        qformer_num_layers: int = 2,
+        qformer_dropout: float = 0.0,
+        use_horizon_head: bool = True,
+        horizon_max_length: int = 0,
+        use_cross_attn_gate: bool = True,
+        max_context: int = 2048,
+        max_horizon: int = 1024,
+        normalize_inputs: bool = True,
+        use_continuous_quantile_head: bool = True,
+        force_flip_invariance: bool = True,
+        infer_is_positive: bool = True,
+        fix_quantile_crossing: bool = True,
+        return_backcast: bool = False,
+        default_pred_len: int = 720,
+        **super_kwargs,
+    ):
+        # Precomputed-embedding dims (must match the consumer's LLM / TS encoder).
+        self.d_llm = int(d_llm)
+        self.d_ts_encoder = int(d_ts_encoder)
+
+        # Q-former. qformer_hidden_dim == 0 -> default to the Forecaster model dim so
+        # the cross-attention KV stream needs no further projection.
+        self.qformer_hidden_dim = int(qformer_hidden_dim) or self.FORECASTER_MODEL_DIMS
+        self.qformer_num_query_tokens = int(qformer_num_query_tokens)
+        self.qformer_num_heads = int(qformer_num_heads)
+        self.qformer_num_layers = int(qformer_num_layers)
+        self.qformer_dropout = float(qformer_dropout)
+
+        # Prediction-length head.
+        self.use_horizon_head = bool(use_horizon_head)
+        self.horizon_max_length = int(horizon_max_length)
+
+        # Forecaster cross-attention backbone build.
+        self.use_cross_attn_gate = bool(use_cross_attn_gate)
+        # Cross-attn KV dim equals the compressed-token dim.
+        self.cross_attn_kv_dim = self.qformer_hidden_dim
+
+        # Forecast knobs.
+        self.max_context = int(max_context)
+        self.max_horizon = int(max_horizon)
+        self.normalize_inputs = bool(normalize_inputs)
+        self.use_continuous_quantile_head = bool(use_continuous_quantile_head)
+        self.force_flip_invariance = bool(force_flip_invariance)
+        self.infer_is_positive = bool(infer_is_positive)
+        self.fix_quantile_crossing = bool(fix_quantile_crossing)
+        self.return_backcast = bool(return_backcast)
+
+        # Fallback horizon when the horizon head is disabled and no override given.
+        self.default_pred_len = int(default_pred_len)
+
+        super().__init__(**super_kwargs)
+
+
 class InternS2PreviewConfig(PreTrainedConfig):
     r"""
     This is the configuration class to store the configuration of a [`InternS2PreviewModel`]. It is used to instantiate a
@@ -379,6 +511,7 @@ class InternS2PreviewConfig(PreTrainedConfig):
         "vision_config": InternS2PreviewVisionConfig,
         "text_config": InternS2PreviewTextConfig,
         "ts_config": InternS2PreviewTimeSeriesConfig,
+        "ts_forecaster_config": InternS2PreviewTimeSeriesForecasterConfig,
     }
     keys_to_ignore_at_inference = ["past_key_values"]
 
@@ -395,16 +528,9 @@ class InternS2PreviewConfig(PreTrainedConfig):
         ts_token_id=248093,
         ts_start_id=248091,
         ts_end_id=248092,
+        ts_forecaster_config=None,
         **kwargs,
     ):
-        if isinstance(ts_config, dict):
-            self.ts_config = self.sub_configs["ts_config"](**ts_config)
-        elif ts_config is None:
-            self.ts_config = self.sub_configs["ts_config"]()
-
-        self.ts_token_id = ts_token_id
-        self.ts_start_id = ts_start_id
-        self.ts_end_id = ts_end_id
         if isinstance(vision_config, dict):
             self.vision_config = self.sub_configs["vision_config"](**vision_config)
         elif vision_config is None:
@@ -421,6 +547,23 @@ class InternS2PreviewConfig(PreTrainedConfig):
         self.vision_end_token_id = vision_end_token_id
         self.tie_word_embeddings = tie_word_embeddings
         super().__init__(**kwargs)
+        if isinstance(ts_config, dict):
+            self.ts_config = self.sub_configs["ts_config"](**ts_config)
+        elif ts_config is None:
+            self.ts_config = self.sub_configs["ts_config"]()
+
+        if isinstance(ts_forecaster_config, dict):
+            self.ts_forecaster_config = self.sub_configs["ts_forecaster_config"](**ts_forecaster_config)
+        elif ts_forecaster_config is None:
+            self.ts_forecaster_config = self.sub_configs["ts_forecaster_config"](
+                d_llm=self.text_config.hidden_size,
+                d_ts_encoder=self.ts_config.out_hidden_size,
+            )
+
+        self.ts_token_id = ts_token_id
+        self.ts_start_id = ts_start_id
+        self.ts_end_id = ts_end_id
+
         self.auto_map = {
             "AutoConfig": "configuration_interns2_preview.InternS2PreviewConfig",
             "AutoModelForCausalLM": "modeling_interns2_preview.InternS2PreviewForCausalLM",

load_20210803_0.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c9b6e38aca67dd36a4ba2d31d984549b0ae3a5aa72c545ef2c6662088cb6aae
+size 512

model-time-series-0001.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b82eadda0ac7fd8fedd1a013dd3ee577c3a02cce3426e94af0bb5ec5940da9e9
+size 295550856

model-time-series-0002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5debb9ec91f1b657fdb66361c8d054c6733c85bd2b1213c1c4a72fe95e4c7177
+size 102400128

model-time-series-0003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4fc8d385717a6a3f1a92cae8877c74d381a79014bbd3c998ab88130ea505fd3
+size 1001137994

processing_interns2_preview.py

@@ -21,6 +21,7 @@ import importlib
 import os
 
 import numpy as np
+import torch
 
 from transformers.feature_extraction_utils import BatchFeature
 from transformers.image_utils import ImageInput
@@ -47,7 +48,42 @@ class InternS2PreviewProcessorKwargs(ProcessingKwargs, total=False):
 
 @auto_docstring
 class InternS2PreviewProcessor(ProcessorMixin):
-    def __init__(self, image_processor=None, tokenizer=None, video_processor=None, chat_template=None, **kwargs):
+    def __init__(
+        self,
+        image_processor=None,
+        tokenizer=None,
+        video_processor=None,
+        chat_template=None,
+        chunk_size=12800,
+        patch=50,
+        num_query=2,
+        ts_signals_do_normalize=True,
+        ts_signals_do_truncate=True,
+        **kwargs,
+    ):
+        """
+        Constructs an InternS2Preview processor.
+
+        Args:
+            image_processor (`ImageProcessingMixin`, *optional*):
+                The image processor.
+            tokenizer (`PreTrainedTokenizerBase`, *optional*):
+                The tokenizer.
+            video_processor (`BaseVideoProcessor`, *optional*):
+                The video processor.
+            chat_template (`str`, *optional*):
+                The chat template used to format conversations.
+            chunk_size (`int`, *optional*, defaults to 12800):
+                Chunk size used by the time-series encoder.
+            patch (`int`, *optional*, defaults to 50):
+                Patch size used by the time-series encoder.
+            num_query (`int`, *optional*, defaults to 2):
+                Number of query tokens used by the time-series encoder.
+            ts_signals_do_normalize (`bool`, *optional*, defaults to `True`):
+                Whether to normalize each input time-series signal before it is passed to the time-series encoder and/or forecaster.
+            ts_signals_do_truncate (`bool`, *optional*, defaults to `True`):
+                Whether to truncate input time-series signals that exceed the processor's maximum supported length.
+        """
         self.image_token = "<|image_pad|>" if not hasattr(tokenizer, "image_token") else tokenizer.image_token
         self.video_token = "<|video_pad|>" if not hasattr(tokenizer, "video_token") else tokenizer.video_token
         self.image_token_id = (
@@ -95,6 +131,11 @@ class InternS2PreviewProcessor(ProcessorMixin):
             if getattr(tokenizer, "ts_token_id", None)
             else tokenizer.convert_tokens_to_ids(self.ts_token)
         )
+        self.chunk_size = chunk_size
+        self.patch = patch
+        self.num_query = num_query
+        self.ts_signals_do_normalize = ts_signals_do_normalize
+        self.ts_signals_do_truncate = ts_signals_do_truncate
 
     @auto_docstring
     def __call__(
@@ -107,6 +148,18 @@ class InternS2PreviewProcessor(ProcessorMixin):
         **kwargs: Unpack[InternS2PreviewProcessorKwargs],
     ) -> BatchFeature:
         r"""
+        Args:
+            images (`ImageInput`, *optional*):
+                Images to be processed.
+            text (`TextInput`, `PreTokenizedInput`, `list[TextInput]`, or `list[PreTokenizedInput]`, *optional*):
+                Text to be encoded.
+            videos (`VideoInput`, *optional*):
+                Videos to be processed.
+            time_series_paths (`list[str]`, *optional*):
+                Paths to time series files. Supported formats include `.wav`, `.mp3`, `.flac`, `.csv`, and `.npy`.
+            time_series_sampling_rates (`list[int]`, *optional*):
+                Sampling rates corresponding to `time_series_paths`.
+
         Returns:
             [`BatchFeature`]: A [`BatchFeature`] with the following fields:
 
@@ -160,7 +213,10 @@ class InternS2PreviewProcessor(ProcessorMixin):
                 "The number of time series signals must match the number of sampling rates."
             )
             time_series_inputs = self.time_series_processor(
-                ts_paths=time_series_paths, sampling_rates=time_series_sampling_rates
+                ts_paths=time_series_paths,
+                sampling_rates=time_series_sampling_rates,
+                do_normalize=self.ts_signals_do_normalize,
+                do_truncate=self.ts_signals_do_truncate,
             )
             num_ts_tokens = time_series_inputs.pop("num_ts_tokens")
             assert len(num_ts_tokens) == len(text), (
@@ -368,6 +424,7 @@ class InternS2PreviewProcessor(ProcessorMixin):
         ts_values = []
         ts_sr = []
         ts_lens = []
+        ts_channels = []
 
         for idx, ts_path in enumerate(ts_paths):
             sr = sampling_rates[idx]
@@ -397,6 +454,7 @@ class InternS2PreviewProcessor(ProcessorMixin):
                 ts_input = ts_input[:, None]  # [T,C]
 
             ts_len = ts_input.shape[0]
+            ts_channel = ts_input.shape[1]
 
             if sr is None or sr == 0:  # if no sr provided
                 sr = ts_len / 4
@@ -404,19 +462,38 @@ class InternS2PreviewProcessor(ProcessorMixin):
             ts_values.append(ts_input)
             ts_sr.append(sr)
             ts_lens.append(ts_len)
+            ts_channels.append(ts_channel)
 
+        ts_channels = np.array(ts_channels)
         ts_lens = np.array(ts_lens)
         ts_sr = np.array(ts_sr)
         num_ts_tokens = self._get_num_ts_tokens(sampling_rates=ts_sr, ts_lens=ts_lens)
         return BatchFeature(
-            data={"ts_values": ts_values, "ts_sr": ts_sr, "ts_lens": ts_lens, "num_ts_tokens": num_ts_tokens}
+            data={
+                "ts_values": ts_values,
+                "ts_sr": ts_sr,
+                "ts_lens": ts_lens,
+                "ts_channels": ts_channels,
+                "num_ts_tokens": num_ts_tokens,
+            }
         )
 
     def _get_num_ts_tokens(self, sampling_rates, ts_lens):
-        strides = np.floor(160 / ((1 + np.exp(-sampling_rates / 100)) ** 6))
-        patch_sizes = strides * 2
-        embed_lengths = (np.ceil((ts_lens - patch_sizes) / strides) + 1).astype(np.int64)
-        num_ts_tokens = [(embed_length // 2 + 1) // 2 for embed_length in embed_lengths]
+        chunk_size, num_qformer_query = self.chunk_size, self.num_query
+        ts_len = torch.from_numpy(ts_lens)
+        chunk_num = ts_len // chunk_size
+        tail_len = ts_len - chunk_num * chunk_size
+        subrate = torch.clamp(ts_len / 500, min=1.0)
+        stride = subrate * num_qformer_query
+        patch_size = torch.ceil(stride)
+        num_ts_tokens = (
+            (
+                chunk_num * ((torch.ceil((chunk_size - patch_size) / stride + 1) * num_qformer_query + 1) // 2)
+                + (torch.ceil((tail_len - patch_size) / stride + 1) * num_qformer_query + 1) // 2
+            )
+            .long()
+            .tolist()
+        )
         return num_ts_tokens
 
 

preprocessor_config.json → processor_config.json

@@ -16,8 +16,10 @@
     0.5,
     0.5
   ],
-  "processor_class": "Qwen3VLProcessor",
-  "image_processor_type": "Qwen2VLImageProcessorFast",
+  "chunk_size": 12800,
+  "patch": 50,
+  "num_query": 2,
+  "ts_signals_do_normalize": false,
   "auto_map": {
     "AutoProcessor": "processing_interns2_preview.InternS2PreviewProcessor"
   }