TSEncoder_orion_icl_model_hparamsv1.0.json

{
  "adapter": {
    "dropout": 0.1,
    "hidden_dim": 1024,
    "icl_dim": 512,
    "tsencoder_dim": 512,
    "mlp": {
      "layer_types": [
        "LayerNorm",
        "Linear",
        "GELU",
        "Dropout",
        "Linear"
      ],
      "linear_layers": [
        {
          "bias": true,
          "in_features": 512,
          "out_features": 1024
        },
        {
          "bias": true,
          "in_features": 1024,
          "out_features": 512
        }
      ],
      "num_layers": 5
    },
    "use_layernorm": true
  },
  "tsencoder": {
    "batch_size": 512,
    "ckpt": "checkpoints/CaukerImpro-data100k_emb512_100epochs.pt",
    "hidden_dim": 512,
    "tsencoder_dim": 512,
    "seq_len": 512
  },
  "tsencoder_details": {
    "modules": [
      "ViTUnit",
      "TokenGeneratorUnit"
    ],
    "tokgen_convs": 2,
    "tokgen_scalar_encoder_groups": 2,
    "transformer_layers": 6
  },
  "model": "_TSEncoderAdapterPlusOrionICL",
  "orion": {
    "ckpt": "/data0/fangjuntao2025/TIC-FS/code/checkpoints/ICL.ckpt",
    "icl_predictor": {
      "config": {
        "dropout": 0.0,
        "embed_dim": 128,
        "ff_factor": 2,
        "icl_nhead": 4,
        "icl_num_blocks": 12,
        "max_classes": 10,
        "norm_first": true,
        "perc_layers": 2,
        "perc_num_latents": 32
      },
      "decoder": "Sequential(\n  (0): Linear(in_features=512, out_features=1024, bias=True)\n  (1): GELU(approximate='none')\n  (2): Linear(in_features=1024, out_features=10, bias=True)\n)",
      "memory": {
        "perc_num_latents": 32,
        "read_layers": 2,
        "write_layers": 2
      },
      "tf_icl_blocks": 12,
      "y_encoder": "OneHotAndLinear(in_features=10, out_features=512, bias=True)"
    }
  },
  "structure": {
    "adapter": "TokenMLPAdapter(\n  (net): Sequential(\n    (0): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n    (1): Linear(in_features=512, out_features=1024, bias=True)\n    (2): GELU(approximate='none')\n    (3): Dropout(p=0.11824302592075059, inplace=False)\n    (4): Linear(in_features=1024, out_features=512, bias=True)\n  )\n)",
    "icl_predictor": "ICLearning(\n  (tf_icl): Encoder(\n    (blocks): ModuleList(\n      (0-11): 12 x MultiheadAttentionBlock(\n        (linear1): Linear(in_features=512, out_features=1024, bias=True)\n        (dropout): Dropout(p=0.0, inplace=False)\n        (linear2): Linear(in_features=1024, out_features=512, bias=True)\n        (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n        (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n        (dropout1): Dropout(p=0.0, inplace=False)\n        (dropout2): Dropout(p=0.0, inplace=False)\n        (attn): MultiheadAttention(\n          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n        )\n      )\n    )\n  )\n  (ln): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n  (y_encoder): OneHotAndLinear(in_features=10, out_features=512, bias=True)\n  (decoder): Sequential(\n    (0): Linear(in_features=512, out_features=1024, bias=True)\n    (1): GELU(approximate='none')\n    (2): Linear(in_features=1024, out_features=10, bias=True)\n  )\n  (memory): PerceiverMemory(\n    (write_layers): ModuleList(\n      (0-1): 2 x CrossAttnBlock(\n        (q_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n        (kv_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n        (attn): MultiheadAttentionBlock(\n          (linear1): Linear(in_features=512, out_features=1024, bias=True)\n          (dropout): Dropout(p=0.0, inplace=False)\n          (linear2): Linear(in_features=1024, out_features=512, bias=True)\n          (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n          (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n          (dropout1): Dropout(p=0.0, inplace=False)\n          (dropout2): Dropout(p=0.0, inplace=False)\n          (attn): MultiheadAttention(\n            (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n          )\n        )\n        (ffn): TransformerEncoderLayer(\n          (self_attn): MultiheadAttention(\n            (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n          )\n          (linear1): Linear(in_features=512, out_features=1024, bias=True)\n          (dropout): Dropout(p=0.0, inplace=False)\n          (linear2): Linear(in_features=1024, out_features=512, bias=True)\n          (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n          (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n          (dropout1): Dropout(p=0.0, inplace=False)\n          (dropout2): Dropout(p=0.0, inplace=False)\n        )\n      )\n    )\n    (read_layers): ModuleList(\n      (0-1): 2 x CrossAttnBlock(\n        (q_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n        (kv_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n        (attn): MultiheadAttentionBlock(\n          (linear1): Linear(in_features=512, out_features=1024, bias=True)\n          (dropout): Dropout(p=0.0, inplace=False)\n          (linear2): Linear(in_features=1024, out_features=512, bias=True)\n          (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n          (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n          (dropout1): Dropout(p=0.0, inplace=False)\n          (dropout2): Dropout(p=0.0, inplace=False)\n          (attn): MultiheadAttention(\n            (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n          )\n        )\n        (ffn): TransformerEncoderLayer(\n          (self_attn): MultiheadAttention(\n            (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n          )\n          (linear1): Linear(in_features=512, out_features=1024, bias=True)\n          (dropout): Dropout(p=0.0, inplace=False)\n          (linear2): Linear(in_features=1024, out_features=512, bias=True)\n          (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n          (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n          (dropout1): Dropout(p=0.0, inplace=False)\n          (dropout2): Dropout(p=0.0, inplace=False)\n        )\n      )\n    )\n  )\n)",
    "tsencoder_model": "TSEncoder8M(\n  (tokgen_unit): TokenGeneratorUnit(\n    (convs): ModuleList(\n      (0-1): 2 x Convolution(\n        (conv): Conv1d(1, 512, kernel_size=(17,), stride=(1,), padding=(8,))\n      )\n    )\n    (layer_norms): ModuleList(\n      (0-1): 2 x LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n    )\n    (scalar_encoders): ModuleList(\n      (0-1): 2 x MultiScaledScalarEncoder(\n        (encoders): ModuleList(\n          (0-8): 9 x ScalarEncoder(\n            (layer_norm): LayerNorm((32,), eps=1e-15, elementwise_affine=True)\n          )\n        )\n      )\n    )\n    (linear_encoder): LinearEncoder(\n      (linear): Linear(in_features=1088, out_features=512, bias=True)\n      (layer_norm): LayerNorm((512,), eps=1e-15, elementwise_affine=True)\n    )\n  )\n  (vit_unit): ViTUnit(\n    (pos_encoder): PositionalEncoding(\n      (dropout): Dropout(p=0.1, inplace=False)\n    )\n    (transformer): Transformer(\n      (layers): ModuleList(\n        (0-5): 6 x ModuleList(\n          (0): PreNorm(\n            (norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n            (fn): Attention(\n              (attend): Softmax(dim=-1)\n              (dropout): Dropout(p=0.1, inplace=False)\n              (to_qkv): Linear(in_features=512, out_features=3072, bias=False)\n              (to_out): Sequential(\n                (0): Linear(in_features=1024, out_features=512, bias=True)\n                (1): Dropout(p=0.1, inplace=False)\n              )\n            )\n          )\n          (1): PreNorm(\n            (norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n            (fn): FeedForward(\n              (net): Sequential(\n                (0): Linear(in_features=512, out_features=512, bias=True)\n                (1): GELU(approximate='none')\n                (2): Dropout(p=0.1, inplace=False)\n                (3): Linear(in_features=512, out_features=512, bias=True)\n                (4): Dropout(p=0.1, inplace=False)\n              )\n            )\n          )\n        )\n      )\n    )\n  )\n  (prj): Sequential(\n    (0): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n    (1): Linear(in_features=512, out_features=512, bias=True)\n  )\n)",
    "repr": "_TSEncoderAdapterPlusOrionICL(\n  (tsencoder_model): TSEncoder8M(\n    (tokgen_unit): TokenGeneratorUnit(\n      (convs): ModuleList(\n        (0-1): 2 x Convolution(\n          (conv): Conv1d(1, 512, kernel_size=(17,), stride=(1,), padding=(8,))\n        )\n      )\n      (layer_norms): ModuleList(\n        (0-1): 2 x LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n      )\n      (scalar_encoders): ModuleList(\n        (0-1): 2 x MultiScaledScalarEncoder(\n          (encoders): ModuleList(\n            (0-8): 9 x ScalarEncoder(\n              (layer_norm): LayerNorm((32,), eps=1e-15, elementwise_affine=True)\n            )\n          )\n        )\n      )\n      (linear_encoder): LinearEncoder(\n        (linear): Linear(in_features=1088, out_features=512, bias=True)\n        (layer_norm): LayerNorm((512,), eps=1e-15, elementwise_affine=True)\n      )\n    )\n    (vit_unit): ViTUnit(\n      (pos_encoder): PositionalEncoding(\n        (dropout): Dropout(p=0.1, inplace=False)\n      )\n      (transformer): Transformer(\n        (layers): ModuleList(\n          (0-5): 6 x ModuleList(\n            (0): PreNorm(\n              (norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n              (fn): Attention(\n                (attend): Softmax(dim=-1)\n                (dropout): Dropout(p=0.1, inplace=False)\n                (to_qkv): Linear(in_features=512, out_features=3072, bias=False)\n                (to_out): Sequential(\n                  (0): Linear(in_features=1024, out_features=512, bias=True)\n                  (1): Dropout(p=0.1, inplace=False)\n                )\n              )\n            )\n            (1): PreNorm(\n              (norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n              (fn): FeedForward(\n                (net): Sequential(\n                  (0): Linear(in_features=512, out_features=512, bias=True)\n                  (1): GELU(approximate='none')\n                  (2): Dropout(p=0.1, inplace=False)\n                  (3): Linear(in_features=512, out_features=512, bias=True)\n                  (4): Dropout(p=0.1, inplace=False)\n                )\n              )\n            )\n          )\n        )\n      )\n    )\n    (prj): Sequential(\n      (0): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n      (1): Linear(in_features=512, out_features=512, bias=True)\n    )\n  )\n  (adapter): TokenMLPAdapter(\n    (net): Sequential(\n      (0): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n      (1): Linear(in_features=512, out_features=1024, bias=True)\n      (2): GELU(approximate='none')\n      (3): Dropout(p=0.11824302592075059, inplace=False)\n      (4): Linear(in_features=1024, out_features=512, bias=True)\n    )\n  )\n  (icl_predictor): ICLearning(\n    (tf_icl): Encoder(\n      (blocks): ModuleList(\n        (0-11): 12 x MultiheadAttentionBlock(\n          (linear1): Linear(in_features=512, out_features=1024, bias=True)\n          (dropout): Dropout(p=0.0, inplace=False)\n          (linear2): Linear(in_features=1024, out_features=512, bias=True)\n          (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n          (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n          (dropout1): Dropout(p=0.0, inplace=False)\n          (dropout2): Dropout(p=0.0, inplace=False)\n          (attn): MultiheadAttention(\n            (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n          )\n        )\n      )\n    )\n    (ln): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n    (y_encoder): OneHotAndLinear(in_features=10, out_features=512, bias=True)\n    (decoder): Sequential(\n      (0): Linear(in_features=512, out_features=1024, bias=True)\n      (1): GELU(approximate='none')\n      (2): Linear(in_features=1024, out_features=10, bias=True)\n    )\n    (memory): PerceiverMemory(\n      (write_layers): ModuleList(\n        (0-1): 2 x CrossAttnBlock(\n          (q_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n          (kv_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n          (attn): MultiheadAttentionBlock(\n            (linear1): Linear(in_features=512, out_features=1024, bias=True)\n            (dropout): Dropout(p=0.0, inplace=False)\n            (linear2): Linear(in_features=1024, out_features=512, bias=True)\n            (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n            (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n            (dropout1): Dropout(p=0.0, inplace=False)\n            (dropout2): Dropout(p=0.0, inplace=False)\n            (attn): MultiheadAttention(\n              (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n            )\n          )\n          (ffn): TransformerEncoderLayer(\n            (self_attn): MultiheadAttention(\n              (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n            )\n            (linear1): Linear(in_features=512, out_features=1024, bias=True)\n            (dropout): Dropout(p=0.0, inplace=False)\n            (linear2): Linear(in_features=1024, out_features=512, bias=True)\n            (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n            (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n            (dropout1): Dropout(p=0.0, inplace=False)\n            (dropout2): Dropout(p=0.0, inplace=False)\n          )\n        )\n      )\n      (read_layers): ModuleList(\n        (0-1): 2 x CrossAttnBlock(\n          (q_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n          (kv_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n          (attn): MultiheadAttentionBlock(\n            (linear1): Linear(in_features=512, out_features=1024, bias=True)\n            (dropout): Dropout(p=0.0, inplace=False)\n            (linear2): Linear(in_features=1024, out_features=512, bias=True)\n            (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n            (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n            (dropout1): Dropout(p=0.0, inplace=False)\n            (dropout2): Dropout(p=0.0, inplace=False)\n            (attn): MultiheadAttention(\n              (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n            )\n          )\n          (ffn): TransformerEncoderLayer(\n            (self_attn): MultiheadAttention(\n              (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n            )\n            (linear1): Linear(in_features=512, out_features=1024, bias=True)\n            (dropout): Dropout(p=0.0, inplace=False)\n            (linear2): Linear(in_features=1024, out_features=512, bias=True)\n            (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n            (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n            (dropout1): Dropout(p=0.0, inplace=False)\n            (dropout2): Dropout(p=0.0, inplace=False)\n          )\n        )\n      )\n    )\n  )\n)"
  }
}