Upload 2 files

TSEncoder_orion_icl_fullv1.0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:94bbbc334e931ee145d05542b9813267b8559dd6e959dd9764fb37c372e19e01
+size 241293576

TSEncoder_orion_icl_model_hparamsv1.0.json

@@ -0,0 +1,78 @@
+{
+  "adapter": {
+    "dropout": 0.1,
+    "hidden_dim": 1024,
+    "icl_dim": 512,
+    "tsencoder_dim": 512,
+    "mlp": {
+      "layer_types": [
+        "LayerNorm",
+        "Linear",
+        "GELU",
+        "Dropout",
+        "Linear"
+      ],
+      "linear_layers": [
+        {
+          "bias": true,
+          "in_features": 512,
+          "out_features": 1024
+        },
+        {
+          "bias": true,
+          "in_features": 1024,
+          "out_features": 512
+        }
+      ],
+      "num_layers": 5
+    },
+    "use_layernorm": true
+  },
+  "tsencoder": {
+    "batch_size": 512,
+    "ckpt": "checkpoints/CaukerImpro-data100k_emb512_100epochs.pt",
+    "hidden_dim": 512,
+    "tsencoder_dim": 512,
+    "seq_len": 512
+  },
+  "tsencoder_details": {
+    "modules": [
+      "ViTUnit",
+      "TokenGeneratorUnit"
+    ],
+    "tokgen_convs": 2,
+    "tokgen_scalar_encoder_groups": 2,
+    "transformer_layers": 6
+  },
+  "model": "_TSEncoderAdapterPlusOrionICL",
+  "orion": {
+    "ckpt": "/data0/fangjuntao2025/TIC-FS/code/checkpoints/ICL.ckpt",
+    "icl_predictor": {
+      "config": {
+        "dropout": 0.0,
+        "embed_dim": 128,
+        "ff_factor": 2,
+        "icl_nhead": 4,
+        "icl_num_blocks": 12,
+        "max_classes": 10,
+        "norm_first": true,
+        "perc_layers": 2,
+        "perc_num_latents": 32
+      },
+      "decoder": "Sequential(\n  (0): Linear(in_features=512, out_features=1024, bias=True)\n  (1): GELU(approximate='none')\n  (2): Linear(in_features=1024, out_features=10, bias=True)\n)",
+      "memory": {
+        "perc_num_latents": 32,
+        "read_layers": 2,
+        "write_layers": 2
+      },
+      "tf_icl_blocks": 12,
+      "y_encoder": "OneHotAndLinear(in_features=10, out_features=512, bias=True)"
+    }
+  },
+  "structure": {
+    "adapter": "TokenMLPAdapter(\n  (net): Sequential(\n    (0): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n    (1): Linear(in_features=512, out_features=1024, bias=True)\n    (2): GELU(approximate='none')\n    (3): Dropout(p=0.11824302592075059, inplace=False)\n    (4): Linear(in_features=1024, out_features=512, bias=True)\n  )\n)",
+    "icl_predictor": "ICLearning(\n  (tf_icl): Encoder(\n    (blocks): ModuleList(\n      (0-11): 12 x MultiheadAttentionBlock(\n        (linear1): Linear(in_features=512, out_features=1024, bias=True)\n        (dropout): Dropout(p=0.0, inplace=False)\n        (linear2): Linear(in_features=1024, out_features=512, bias=True)\n        (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n        (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n        (dropout1): Dropout(p=0.0, inplace=False)\n        (dropout2): Dropout(p=0.0, inplace=False)\n        (attn): MultiheadAttention(\n          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n        )\n      )\n    )\n  )\n  (ln): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n  (y_encoder): OneHotAndLinear(in_features=10, out_features=512, bias=True)\n  (decoder): Sequential(\n    (0): Linear(in_features=512, out_features=1024, bias=True)\n    (1): GELU(approximate='none')\n    (2): Linear(in_features=1024, out_features=10, bias=True)\n  )\n  (memory): PerceiverMemory(\n    (write_layers): ModuleList(\n      (0-1): 2 x CrossAttnBlock(\n        (q_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n        (kv_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n        (attn): MultiheadAttentionBlock(\n          (linear1): Linear(in_features=512, out_features=1024, bias=True)\n          (dropout): Dropout(p=0.0, inplace=False)\n          (linear2): Linear(in_features=1024, out_features=512, bias=True)\n          (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n          (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n          (dropout1): Dropout(p=0.0, inplace=False)\n          (dropout2): Dropout(p=0.0, inplace=False)\n          (attn): MultiheadAttention(\n            (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n          )\n        )\n        (ffn): TransformerEncoderLayer(\n          (self_attn): MultiheadAttention(\n            (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n          )\n          (linear1): Linear(in_features=512, out_features=1024, bias=True)\n          (dropout): Dropout(p=0.0, inplace=False)\n          (linear2): Linear(in_features=1024, out_features=512, bias=True)\n          (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n          (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n          (dropout1): Dropout(p=0.0, inplace=False)\n          (dropout2): Dropout(p=0.0, inplace=False)\n        )\n      )\n    )\n    (read_layers): ModuleList(\n      (0-1): 2 x CrossAttnBlock(\n        (q_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n        (kv_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n        (attn): MultiheadAttentionBlock(\n          (linear1): Linear(in_features=512, out_features=1024, bias=True)\n          (dropout): Dropout(p=0.0, inplace=False)\n          (linear2): Linear(in_features=1024, out_features=512, bias=True)\n          (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n          (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n          (dropout1): Dropout(p=0.0, inplace=False)\n          (dropout2): Dropout(p=0.0, inplace=False)\n          (attn): MultiheadAttention(\n            (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n          )\n        )\n        (ffn): TransformerEncoderLayer(\n          (self_attn): MultiheadAttention(\n            (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n          )\n          (linear1): Linear(in_features=512, out_features=1024, bias=True)\n          (dropout): Dropout(p=0.0, inplace=False)\n          (linear2): Linear(in_features=1024, out_features=512, bias=True)\n          (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n          (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n          (dropout1): Dropout(p=0.0, inplace=False)\n          (dropout2): Dropout(p=0.0, inplace=False)\n        )\n      )\n    )\n  )\n)",
+    "tsencoder_model": "TSEncoder8M(\n  (tokgen_unit): TokenGeneratorUnit(\n    (convs): ModuleList(\n      (0-1): 2 x Convolution(\n        (conv): Conv1d(1, 512, kernel_size=(17,), stride=(1,), padding=(8,))\n      )\n    )\n    (layer_norms): ModuleList(\n      (0-1): 2 x LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n    )\n    (scalar_encoders): ModuleList(\n      (0-1): 2 x MultiScaledScalarEncoder(\n        (encoders): ModuleList(\n          (0-8): 9 x ScalarEncoder(\n            (layer_norm): LayerNorm((32,), eps=1e-15, elementwise_affine=True)\n          )\n        )\n      )\n    )\n    (linear_encoder): LinearEncoder(\n      (linear): Linear(in_features=1088, out_features=512, bias=True)\n      (layer_norm): LayerNorm((512,), eps=1e-15, elementwise_affine=True)\n    )\n  )\n  (vit_unit): ViTUnit(\n    (pos_encoder): PositionalEncoding(\n      (dropout): Dropout(p=0.1, inplace=False)\n    )\n    (transformer): Transformer(\n      (layers): ModuleList(\n        (0-5): 6 x ModuleList(\n          (0): PreNorm(\n            (norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n            (fn): Attention(\n              (attend): Softmax(dim=-1)\n              (dropout): Dropout(p=0.1, inplace=False)\n              (to_qkv): Linear(in_features=512, out_features=3072, bias=False)\n              (to_out): Sequential(\n                (0): Linear(in_features=1024, out_features=512, bias=True)\n                (1): Dropout(p=0.1, inplace=False)\n              )\n            )\n          )\n          (1): PreNorm(\n            (norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n            (fn): FeedForward(\n              (net): Sequential(\n                (0): Linear(in_features=512, out_features=512, bias=True)\n                (1): GELU(approximate='none')\n                (2): Dropout(p=0.1, inplace=False)\n                (3): Linear(in_features=512, out_features=512, bias=True)\n                (4): Dropout(p=0.1, inplace=False)\n              )\n            )\n          )\n        )\n      )\n    )\n  )\n  (prj): Sequential(\n    (0): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n    (1): Linear(in_features=512, out_features=512, bias=True)\n  )\n)",
+    "repr": "_TSEncoderAdapterPlusOrionICL(\n  (tsencoder_model): TSEncoder8M(\n    (tokgen_unit): TokenGeneratorUnit(\n      (convs): ModuleList(\n        (0-1): 2 x Convolution(\n          (conv): Conv1d(1, 512, kernel_size=(17,), stride=(1,), padding=(8,))\n        )\n      )\n      (layer_norms): ModuleList(\n        (0-1): 2 x LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n      )\n      (scalar_encoders): ModuleList(\n        (0-1): 2 x MultiScaledScalarEncoder(\n          (encoders): ModuleList(\n            (0-8): 9 x ScalarEncoder(\n              (layer_norm): LayerNorm((32,), eps=1e-15, elementwise_affine=True)\n            )\n          )\n        )\n      )\n      (linear_encoder): LinearEncoder(\n        (linear): Linear(in_features=1088, out_features=512, bias=True)\n        (layer_norm): LayerNorm((512,), eps=1e-15, elementwise_affine=True)\n      )\n    )\n    (vit_unit): ViTUnit(\n      (pos_encoder): PositionalEncoding(\n        (dropout): Dropout(p=0.1, inplace=False)\n      )\n      (transformer): Transformer(\n        (layers): ModuleList(\n          (0-5): 6 x ModuleList(\n            (0): PreNorm(\n              (norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n              (fn): Attention(\n                (attend): Softmax(dim=-1)\n                (dropout): Dropout(p=0.1, inplace=False)\n                (to_qkv): Linear(in_features=512, out_features=3072, bias=False)\n                (to_out): Sequential(\n                  (0): Linear(in_features=1024, out_features=512, bias=True)\n                  (1): Dropout(p=0.1, inplace=False)\n                )\n              )\n            )\n            (1): PreNorm(\n              (norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n              (fn): FeedForward(\n                (net): Sequential(\n                  (0): Linear(in_features=512, out_features=512, bias=True)\n                  (1): GELU(approximate='none')\n                  (2): Dropout(p=0.1, inplace=False)\n                  (3): Linear(in_features=512, out_features=512, bias=True)\n                  (4): Dropout(p=0.1, inplace=False)\n                )\n              )\n            )\n          )\n        )\n      )\n    )\n    (prj): Sequential(\n      (0): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n      (1): Linear(in_features=512, out_features=512, bias=True)\n    )\n  )\n  (adapter): TokenMLPAdapter(\n    (net): Sequential(\n      (0): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n      (1): Linear(in_features=512, out_features=1024, bias=True)\n      (2): GELU(approximate='none')\n      (3): Dropout(p=0.11824302592075059, inplace=False)\n      (4): Linear(in_features=1024, out_features=512, bias=True)\n    )\n  )\n  (icl_predictor): ICLearning(\n    (tf_icl): Encoder(\n      (blocks): ModuleList(\n        (0-11): 12 x MultiheadAttentionBlock(\n          (linear1): Linear(in_features=512, out_features=1024, bias=True)\n          (dropout): Dropout(p=0.0, inplace=False)\n          (linear2): Linear(in_features=1024, out_features=512, bias=True)\n          (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n          (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n          (dropout1): Dropout(p=0.0, inplace=False)\n          (dropout2): Dropout(p=0.0, inplace=False)\n          (attn): MultiheadAttention(\n            (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n          )\n        )\n      )\n    )\n    (ln): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n    (y_encoder): OneHotAndLinear(in_features=10, out_features=512, bias=True)\n    (decoder): Sequential(\n      (0): Linear(in_features=512, out_features=1024, bias=True)\n      (1): GELU(approximate='none')\n      (2): Linear(in_features=1024, out_features=10, bias=True)\n    )\n    (memory): PerceiverMemory(\n      (write_layers): ModuleList(\n        (0-1): 2 x CrossAttnBlock(\n          (q_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n          (kv_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n          (attn): MultiheadAttentionBlock(\n            (linear1): Linear(in_features=512, out_features=1024, bias=True)\n            (dropout): Dropout(p=0.0, inplace=False)\n            (linear2): Linear(in_features=1024, out_features=512, bias=True)\n            (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n            (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n            (dropout1): Dropout(p=0.0, inplace=False)\n            (dropout2): Dropout(p=0.0, inplace=False)\n            (attn): MultiheadAttention(\n              (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n            )\n          )\n          (ffn): TransformerEncoderLayer(\n            (self_attn): MultiheadAttention(\n              (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n            )\n            (linear1): Linear(in_features=512, out_features=1024, bias=True)\n            (dropout): Dropout(p=0.0, inplace=False)\n            (linear2): Linear(in_features=1024, out_features=512, bias=True)\n            (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n            (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n            (dropout1): Dropout(p=0.0, inplace=False)\n            (dropout2): Dropout(p=0.0, inplace=False)\n          )\n        )\n      )\n      (read_layers): ModuleList(\n        (0-1): 2 x CrossAttnBlock(\n          (q_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n          (kv_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n          (attn): MultiheadAttentionBlock(\n            (linear1): Linear(in_features=512, out_features=1024, bias=True)\n            (dropout): Dropout(p=0.0, inplace=False)\n            (linear2): Linear(in_features=1024, out_features=512, bias=True)\n            (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n            (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n            (dropout1): Dropout(p=0.0, inplace=False)\n            (dropout2): Dropout(p=0.0, inplace=False)\n            (attn): MultiheadAttention(\n              (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n            )\n          )\n          (ffn): TransformerEncoderLayer(\n            (self_attn): MultiheadAttention(\n              (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n            )\n            (linear1): Linear(in_features=512, out_features=1024, bias=True)\n            (dropout): Dropout(p=0.0, inplace=False)\n            (linear2): Linear(in_features=1024, out_features=512, bias=True)\n            (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n            (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n            (dropout1): Dropout(p=0.0, inplace=False)\n            (dropout2): Dropout(p=0.0, inplace=False)\n          )\n        )\n      )\n    )\n  )\n)"
+  }
+}