开源SpaceExploreAI模型：基于Transformer模型训练的5M的大语言模型

config.json

@@ -0,0 +1,23 @@
+{
+  "model_type": "space_explore_ai_financial",
+  "architectures": [
+    "SpaceExploreAIFinancialModel"
+  ],
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "gelu",
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-12,
+  "pad_token_id": 0,
+  "unk_token_id": 3,
+  "transformers_version": "4.36.0",
+  "use_cache": true,
+  "vocab_size": 69,
+  "tokenizer_class": "FinancialFeatureProcessor",
+  "model_name": "SpaceExploreAI",
+  "hidden_size": 768,
+  "num_hidden_layers": 12,
+  "num_attention_heads": 12,
+  "max_position_embeddings": 1024,
+  "model_max_length": 1024
+}

special_tokens_map.json

@@ -0,0 +1,6 @@
+{
+  "pad_token": "<pad>",
+  "unk_token": "<unk>",
+  "bos_token": "<bos>",
+  "eos_token": "<eos>"
+}

tokenizer.json

@@ -0,0 +1,715 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 0,
+      "special": true,
+      "content": "<pad>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 1,
+      "special": true,
+      "content": "<bos>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 2,
+      "special": true,
+      "content": "<eos>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 3,
+      "special": true,
+      "content": "<unk>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 4,
+      "special": false,
+      "content": "date",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 5,
+      "special": false,
+      "content": "close",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 6,
+      "special": false,
+      "content": "high",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 7,
+      "special": false,
+      "content": "low",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 8,
+      "special": false,
+      "content": "open",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 9,
+      "special": false,
+      "content": "volume",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 10,
+      "special": false,
+      "content": "day_of_week",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 11,
+      "special": false,
+      "content": "day_of_month",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 12,
+      "special": false,
+      "content": "week_of_year",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 13,
+      "special": false,
+      "content": "month",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 14,
+      "special": false,
+      "content": "quarter",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 15,
+      "special": false,
+      "content": "year",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 16,
+      "special": false,
+      "content": "is_month_start",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 17,
+      "special": false,
+      "content": "is_month_end",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 18,
+      "special": false,
+      "content": "is_week_start",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 19,
+      "special": false,
+      "content": "is_week_end",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 20,
+      "special": false,
+      "content": "close_lag_1",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 21,
+      "special": false,
+      "content": "close_lag_2",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 22,
+      "special": false,
+      "content": "close_lag_3",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 23,
+      "special": false,
+      "content": "close_lag_5",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 24,
+      "special": false,
+      "content": "close_lag_10",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 25,
+      "special": false,
+      "content": "high_lag_1",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 26,
+      "special": false,
+      "content": "high_lag_2",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 27,
+      "special": false,
+      "content": "high_lag_3",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 28,
+      "special": false,
+      "content": "high_lag_5",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 29,
+      "special": false,
+      "content": "high_lag_10",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 30,
+      "special": false,
+      "content": "low_lag_1",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 31,
+      "special": false,
+      "content": "low_lag_2",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 32,
+      "special": false,
+      "content": "low_lag_3",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 33,
+      "special": false,
+      "content": "low_lag_5",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 34,
+      "special": false,
+      "content": "low_lag_10",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 35,
+      "special": false,
+      "content": "volume_lag_1",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 36,
+      "special": false,
+      "content": "volume_lag_2",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 37,
+      "special": false,
+      "content": "volume_lag_3",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 38,
+      "special": false,
+      "content": "volume_lag_5",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 39,
+      "special": false,
+      "content": "volume_lag_10",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 40,
+      "special": false,
+      "content": "future_return_1d",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 41,
+      "special": false,
+      "content": "future_return_2d",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 42,
+      "special": false,
+      "content": "future_return_10d",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 43,
+      "special": false,
+      "content": "past_return_1d",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 44,
+      "special": false,
+      "content": "past_return_2d",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 45,
+      "special": false,
+      "content": "past_return_5d",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 46,
+      "special": false,
+      "content": "past_return_10d",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 47,
+      "special": false,
+      "content": "daily_range",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 48,
+      "special": false,
+      "content": "daily_range_abs",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 49,
+      "special": false,
+      "content": "gap",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 50,
+      "special": false,
+      "content": "close_to_open",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 51,
+      "special": false,
+      "content": "close_to_high",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 52,
+      "special": false,
+      "content": "close_to_low",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 53,
+      "special": false,
+      "content": "volatility_5d",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 54,
+      "special": false,
+      "content": "volatility_10d",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 55,
+      "special": false,
+      "content": "volatility_21d",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 56,
+      "special": false,
+      "content": "atr_5d",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 57,
+      "special": false,
+      "content": "atr_10d",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 58,
+      "special": false,
+      "content": "atr_21d",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 59,
+      "special": false,
+      "content": "momentum_5d",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 60,
+      "special": false,
+      "content": "momentum_10d",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 61,
+      "special": false,
+      "content": "momentum_21d",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 62,
+      "special": false,
+      "content": "volume_change_5d",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 63,
+      "special": false,
+      "content": "volume_change_10d",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 64,
+      "special": false,
+      "content": "volume_change_21d",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 65,
+      "special": false,
+      "content": "close_to_ma_5d",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 66,
+      "special": false,
+      "content": "close_to_ma_10d",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 67,
+      "special": false,
+      "content": "close_to_ma_21d",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    },
+    {
+      "id": 68,
+      "special": false,
+      "content": "volume_change_rate",
+      "single_word": true,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false
+    }
+  ],
+  "normalizer": {
+    "type": "FinancialFeatureNormalizer"
+  },
+  "pre_tokenizer": {
+    "type": "FinancialFeaturePreTokenizer"
+  },
+  "post_processor": {
+    "type": "FinancialFeaturePostProcessor"
+  },
+  "decoder": {
+    "type": "FinancialFeatureDecoder"
+  },
+  "model": {
+    "type": "FinancialFeatureProcessor",
+    "vocab": {
+      "<pad>": 0,
+      "<bos>": 1,
+      "<eos>": 2,
+      "<unk>": 3,
+      "date": 4,
+      "close": 5,
+      "high": 6,
+      "low": 7,
+      "open": 8,
+      "volume": 9,
+      "day_of_week": 10,
+      "day_of_month": 11,
+      "week_of_year": 12,
+      "month": 13,
+      "quarter": 14,
+      "year": 15,
+      "is_month_start": 16,
+      "is_month_end": 17,
+      "is_week_start": 18,
+      "is_week_end": 19,
+      "close_lag_1": 20,
+      "close_lag_2": 21,
+      "close_lag_3": 22,
+      "close_lag_5": 23,
+      "close_lag_10": 24,
+      "high_lag_1": 25,
+      "high_lag_2": 26,
+      "high_lag_3": 27,
+      "high_lag_5": 28,
+      "high_lag_10": 29,
+      "low_lag_1": 30,
+      "low_lag_2": 31,
+      "low_lag_3": 32,
+      "low_lag_5": 33,
+      "low_lag_10": 34,
+      "volume_lag_1": 35,
+      "volume_lag_2": 36,
+      "volume_lag_3": 37,
+      "volume_lag_5": 38,
+      "volume_lag_10": 39,
+      "future_return_1d": 40,
+      "future_return_2d": 41,
+      "future_return_10d": 42,
+      "past_return_1d": 43,
+      "past_return_2d": 44,
+      "past_return_5d": 45,
+      "past_return_10d": 46,
+      "daily_range": 47,
+      "daily_range_abs": 48,
+      "gap": 49,
+      "close_to_open": 50,
+      "close_to_high": 51,
+      "close_to_low": 52,
+      "volatility_5d": 53,
+      "volatility_10d": 54,
+      "volatility_21d": 55,
+      "atr_5d": 56,
+      "atr_10d": 57,
+      "atr_21d": 58,
+      "momentum_5d": 59,
+      "momentum_10d": 60,
+      "momentum_21d": 61,
+      "volume_change_5d": 62,
+      "volume_change_10d": 63,
+      "volume_change_21d": 64,
+      "close_to_ma_5d": 65,
+      "close_to_ma_10d": 66,
+      "close_to_ma_21d": 67,
+      "volume_change_rate": 68
+    },
+    "feature_dim": 69
+  }
+}

tokenizer_config.json

@@ -0,0 +1,85 @@
+{
+  "feature_dim": 64,
+  "feature_groups": [
+    "price",
+    "time",
+    "lag",
+    "return",
+    "volatility",
+    "volume"
+  ],
+  "feature_names": [
+    "date",
+    "close",
+    "high",
+    "low",
+    "open",
+    "volume",
+    "day_of_week",
+    "day_of_month",
+    "week_of_year",
+    "month",
+    "quarter",
+    "year",
+    "is_month_start",
+    "is_month_end",
+    "is_week_start",
+    "is_week_end",
+    "close_lag_1",
+    "close_lag_2",
+    "close_lag_3",
+    "close_lag_5",
+    "close_lag_10",
+    "high_lag_1",
+    "high_lag_2",
+    "high_lag_3",
+    "high_lag_5",
+    "high_lag_10",
+    "low_lag_1",
+    "low_lag_2",
+    "low_lag_3",
+    "low_lag_5",
+    "low_lag_10",
+    "volume_lag_1",
+    "volume_lag_2",
+    "volume_lag_3",
+    "volume_lag_5",
+    "volume_lag_10",
+    "future_return_1d",
+    "future_return_2d",
+    "future_return_10d",
+    "past_return_1d",
+    "past_return_2d",
+    "past_return_5d",
+    "past_return_10d",
+    "daily_range",
+    "daily_range_abs",
+    "gap",
+    "close_to_open",
+    "close_to_high",
+    "close_to_low",
+    "volatility_5d",
+    "volatility_10d",
+    "volatility_21d",
+    "atr_5d",
+    "atr_10d",
+    "atr_21d",
+    "momentum_5d",
+    "momentum_10d",
+    "momentum_21d",
+    "volume_change_5d",
+    "volume_change_10d",
+    "volume_change_21d",
+    "close_to_ma_5d",
+    "close_to_ma_10d",
+    "close_to_ma_21d",
+    "volume_change_rate"
+  ],
+  "do_lower_case": false,
+  "model_max_length": 1024,
+  "tokenizer_class": "FinancialFeatureProcessor",
+  "bos_token": "<bos>",
+  "eos_token": "<eos>",
+  "pad_token": "<pad>",
+  "unk_token": "<unk>"
+}

usage.md

@@ -0,0 +1,55 @@
+
+# SpaceExploreAI-Small-Base-Regression-5M
+
+## 模型描述
+
+SpaceExploreAI-Small-Base-Regression-5M 是一个基于深度学习的金融时序预测模型，专为股票价格趋势分析和预测而设计。这是 SpaceExploreAI 系列的小型版本，具有约 5M 参数，针对regression任务进行了优化。
+
+### 主要特点
+
+- **轻量化设计**：仅有 5M 参数，适合资源受限环境
+- **回归预测：专为价格预测等回归任务优化**
+- **Transformer 架构**：基于 Transformer 架构，集成了 RoPE 旋转位置编码技术
+- **多头gqa注意力**：使用先进的多头注意力机制捕捉时间序列数据模式
+
+## 技术规格
+
+- **参数量**：约 5M
+- **模型类型**：Transformer
+- **隐藏层大小**：256
+- **隐藏层数量**：4
+- **注意力头数量**：4
+- **注意力类型**：gqa
+- **归一化类型**：rmsnorm
+- **最大序列长度**：32
+- **预测类型**：regression
+- **使用 MoE**：是，混合专家模型增强了模型的表达能力
+- **MOE配置**：8个专家，每个token使用2个专家，中间层大小1024
+
+## 使用示例
+
+```python
+from transformers import AutoTokenizer, AutoModel
+import torch
+
+# 加载模型和分词器
+model_name = "SpaceExploreAI/SpaceExploreAI-Small-Base-Regression-5M"
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+model = AutoModel.from_pretrained(model_name)
+
+# 准备输入数据 (假设您已经有了处理好的金融数据)
+inputs = torch.tensor([[0.1, 0.2, 0.3, ...]])  # 您的金融序列数据
+
+# 进行预测
+with torch.no_grad():
+    outputs = model(inputs)
+    predictions = outputs.last_hidden_state
+```
+
+## 免责声明
+
+SpaceExploreAI仅供大模型AI学习、量化交易学习，不可以用于商业用途、不可以以此为投资逻辑，后果自负。
+
+## 许可证
+
+[Apache License 2.0](LICENSE)

vocab.txt

@@ -0,0 +1,69 @@
+<pad>
+<bos>
+<eos>
+<unk>
+date
+close
+high
+low
+open
+volume
+day_of_week
+day_of_month
+week_of_year
+month
+quarter
+year
+is_month_start
+is_month_end
+is_week_start
+is_week_end
+close_lag_1
+close_lag_2
+close_lag_3
+close_lag_5
+close_lag_10
+high_lag_1
+high_lag_2
+high_lag_3
+high_lag_5
+high_lag_10
+low_lag_1
+low_lag_2
+low_lag_3
+low_lag_5
+low_lag_10
+volume_lag_1
+volume_lag_2
+volume_lag_3
+volume_lag_5
+volume_lag_10
+future_return_1d
+future_return_2d
+future_return_10d
+past_return_1d
+past_return_2d
+past_return_5d
+past_return_10d
+daily_range
+daily_range_abs
+gap
+close_to_open
+close_to_high
+close_to_low
+volatility_5d
+volatility_10d
+volatility_21d
+atr_5d
+atr_10d
+atr_21d
+momentum_5d
+momentum_10d
+momentum_21d
+volume_change_5d
+volume_change_10d
+volume_change_21d
+close_to_ma_5d
+close_to_ma_10d
+close_to_ma_21d
+volume_change_rate