File size: 15,788 Bytes
5f317e2 8501abe 5f317e2 8501abe 5f317e2 8501abe 5f317e2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 | {
"adapter": {
"dropout": 0.1,
"hidden_dim": 1024,
"icl_dim": 512,
"tsencoder_dim": 512,
"mlp": {
"layer_types": [
"LayerNorm",
"Linear",
"GELU",
"Dropout",
"Linear"
],
"linear_layers": [
{
"bias": true,
"in_features": 512,
"out_features": 1024
},
{
"bias": true,
"in_features": 1024,
"out_features": 512
}
],
"num_layers": 5
},
"use_layernorm": true
},
"tsencoder": {
"batch_size": 512,
"ckpt": "checkpoints/CaukerImpro-data100k_emb512_100epochs.pt",
"hidden_dim": 512,
"tsencoder_dim": 512,
"seq_len": 512
},
"tsencoder_details": {
"modules": [
"ViTUnit",
"TokenGeneratorUnit"
],
"tokgen_convs": 2,
"tokgen_scalar_encoder_groups": 2,
"transformer_layers": 6
},
"model": "_TSEncoderAdapterPlusOrionICL",
"orion": {
"ckpt": "/data0/fangjuntao2025/TIC-FS/code/checkpoints/ICL.ckpt",
"icl_predictor": {
"config": {
"dropout": 0.0,
"embed_dim": 128,
"ff_factor": 2,
"icl_nhead": 4,
"icl_num_blocks": 12,
"max_classes": 10,
"norm_first": true,
"perc_layers": 2,
"perc_num_latents": 32
},
"decoder": "Sequential(\n (0): Linear(in_features=512, out_features=1024, bias=True)\n (1): GELU(approximate='none')\n (2): Linear(in_features=1024, out_features=10, bias=True)\n)",
"memory": {
"perc_num_latents": 32,
"read_layers": 2,
"write_layers": 2
},
"tf_icl_blocks": 12,
"y_encoder": "OneHotAndLinear(in_features=10, out_features=512, bias=True)"
}
},
"structure": {
"adapter": "TokenMLPAdapter(\n (net): Sequential(\n (0): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n (1): Linear(in_features=512, out_features=1024, bias=True)\n (2): GELU(approximate='none')\n (3): Dropout(p=0.11824302592075059, inplace=False)\n (4): Linear(in_features=1024, out_features=512, bias=True)\n )\n)",
"icl_predictor": "ICLearning(\n (tf_icl): Encoder(\n (blocks): ModuleList(\n (0-11): 12 x MultiheadAttentionBlock(\n (linear1): Linear(in_features=512, out_features=1024, bias=True)\n (dropout): Dropout(p=0.0, inplace=False)\n (linear2): Linear(in_features=1024, out_features=512, bias=True)\n (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n (dropout1): Dropout(p=0.0, inplace=False)\n (dropout2): Dropout(p=0.0, inplace=False)\n (attn): MultiheadAttention(\n (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n )\n )\n )\n )\n (ln): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n (y_encoder): OneHotAndLinear(in_features=10, out_features=512, bias=True)\n (decoder): Sequential(\n (0): Linear(in_features=512, out_features=1024, bias=True)\n (1): GELU(approximate='none')\n (2): Linear(in_features=1024, out_features=10, bias=True)\n )\n (memory): PerceiverMemory(\n (write_layers): ModuleList(\n (0-1): 2 x CrossAttnBlock(\n (q_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n (kv_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n (attn): MultiheadAttentionBlock(\n (linear1): Linear(in_features=512, out_features=1024, bias=True)\n (dropout): Dropout(p=0.0, inplace=False)\n (linear2): Linear(in_features=1024, out_features=512, bias=True)\n (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n (dropout1): Dropout(p=0.0, inplace=False)\n (dropout2): Dropout(p=0.0, inplace=False)\n (attn): MultiheadAttention(\n (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n )\n )\n (ffn): TransformerEncoderLayer(\n (self_attn): MultiheadAttention(\n (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n )\n (linear1): Linear(in_features=512, out_features=1024, bias=True)\n (dropout): Dropout(p=0.0, inplace=False)\n (linear2): Linear(in_features=1024, out_features=512, bias=True)\n (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n (dropout1): Dropout(p=0.0, inplace=False)\n (dropout2): Dropout(p=0.0, inplace=False)\n )\n )\n )\n (read_layers): ModuleList(\n (0-1): 2 x CrossAttnBlock(\n (q_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n (kv_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n (attn): MultiheadAttentionBlock(\n (linear1): Linear(in_features=512, out_features=1024, bias=True)\n (dropout): Dropout(p=0.0, inplace=False)\n (linear2): Linear(in_features=1024, out_features=512, bias=True)\n (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n (dropout1): Dropout(p=0.0, inplace=False)\n (dropout2): Dropout(p=0.0, inplace=False)\n (attn): MultiheadAttention(\n (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n )\n )\n (ffn): TransformerEncoderLayer(\n (self_attn): MultiheadAttention(\n (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n )\n (linear1): Linear(in_features=512, out_features=1024, bias=True)\n (dropout): Dropout(p=0.0, inplace=False)\n (linear2): Linear(in_features=1024, out_features=512, bias=True)\n (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n (dropout1): Dropout(p=0.0, inplace=False)\n (dropout2): Dropout(p=0.0, inplace=False)\n )\n )\n )\n )\n)",
"tsencoder_model": "TSEncoder8M(\n (tokgen_unit): TokenGeneratorUnit(\n (convs): ModuleList(\n (0-1): 2 x Convolution(\n (conv): Conv1d(1, 512, kernel_size=(17,), stride=(1,), padding=(8,))\n )\n )\n (layer_norms): ModuleList(\n (0-1): 2 x LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n )\n (scalar_encoders): ModuleList(\n (0-1): 2 x MultiScaledScalarEncoder(\n (encoders): ModuleList(\n (0-8): 9 x ScalarEncoder(\n (layer_norm): LayerNorm((32,), eps=1e-15, elementwise_affine=True)\n )\n )\n )\n )\n (linear_encoder): LinearEncoder(\n (linear): Linear(in_features=1088, out_features=512, bias=True)\n (layer_norm): LayerNorm((512,), eps=1e-15, elementwise_affine=True)\n )\n )\n (vit_unit): ViTUnit(\n (pos_encoder): PositionalEncoding(\n (dropout): Dropout(p=0.1, inplace=False)\n )\n (transformer): Transformer(\n (layers): ModuleList(\n (0-5): 6 x ModuleList(\n (0): PreNorm(\n (norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n (fn): Attention(\n (attend): Softmax(dim=-1)\n (dropout): Dropout(p=0.1, inplace=False)\n (to_qkv): Linear(in_features=512, out_features=3072, bias=False)\n (to_out): Sequential(\n (0): Linear(in_features=1024, out_features=512, bias=True)\n (1): Dropout(p=0.1, inplace=False)\n )\n )\n )\n (1): PreNorm(\n (norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n (fn): FeedForward(\n (net): Sequential(\n (0): Linear(in_features=512, out_features=512, bias=True)\n (1): GELU(approximate='none')\n (2): Dropout(p=0.1, inplace=False)\n (3): Linear(in_features=512, out_features=512, bias=True)\n (4): Dropout(p=0.1, inplace=False)\n )\n )\n )\n )\n )\n )\n )\n (prj): Sequential(\n (0): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n (1): Linear(in_features=512, out_features=512, bias=True)\n )\n)",
"repr": "_TSEncoderAdapterPlusOrionICL(\n (tsencoder_model): TSEncoder8M(\n (tokgen_unit): TokenGeneratorUnit(\n (convs): ModuleList(\n (0-1): 2 x Convolution(\n (conv): Conv1d(1, 512, kernel_size=(17,), stride=(1,), padding=(8,))\n )\n )\n (layer_norms): ModuleList(\n (0-1): 2 x LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n )\n (scalar_encoders): ModuleList(\n (0-1): 2 x MultiScaledScalarEncoder(\n (encoders): ModuleList(\n (0-8): 9 x ScalarEncoder(\n (layer_norm): LayerNorm((32,), eps=1e-15, elementwise_affine=True)\n )\n )\n )\n )\n (linear_encoder): LinearEncoder(\n (linear): Linear(in_features=1088, out_features=512, bias=True)\n (layer_norm): LayerNorm((512,), eps=1e-15, elementwise_affine=True)\n )\n )\n (vit_unit): ViTUnit(\n (pos_encoder): PositionalEncoding(\n (dropout): Dropout(p=0.1, inplace=False)\n )\n (transformer): Transformer(\n (layers): ModuleList(\n (0-5): 6 x ModuleList(\n (0): PreNorm(\n (norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n (fn): Attention(\n (attend): Softmax(dim=-1)\n (dropout): Dropout(p=0.1, inplace=False)\n (to_qkv): Linear(in_features=512, out_features=3072, bias=False)\n (to_out): Sequential(\n (0): Linear(in_features=1024, out_features=512, bias=True)\n (1): Dropout(p=0.1, inplace=False)\n )\n )\n )\n (1): PreNorm(\n (norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n (fn): FeedForward(\n (net): Sequential(\n (0): Linear(in_features=512, out_features=512, bias=True)\n (1): GELU(approximate='none')\n (2): Dropout(p=0.1, inplace=False)\n (3): Linear(in_features=512, out_features=512, bias=True)\n (4): Dropout(p=0.1, inplace=False)\n )\n )\n )\n )\n )\n )\n )\n (prj): Sequential(\n (0): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n (1): Linear(in_features=512, out_features=512, bias=True)\n )\n )\n (adapter): TokenMLPAdapter(\n (net): Sequential(\n (0): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n (1): Linear(in_features=512, out_features=1024, bias=True)\n (2): GELU(approximate='none')\n (3): Dropout(p=0.11824302592075059, inplace=False)\n (4): Linear(in_features=1024, out_features=512, bias=True)\n )\n )\n (icl_predictor): ICLearning(\n (tf_icl): Encoder(\n (blocks): ModuleList(\n (0-11): 12 x MultiheadAttentionBlock(\n (linear1): Linear(in_features=512, out_features=1024, bias=True)\n (dropout): Dropout(p=0.0, inplace=False)\n (linear2): Linear(in_features=1024, out_features=512, bias=True)\n (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n (dropout1): Dropout(p=0.0, inplace=False)\n (dropout2): Dropout(p=0.0, inplace=False)\n (attn): MultiheadAttention(\n (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n )\n )\n )\n )\n (ln): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n (y_encoder): OneHotAndLinear(in_features=10, out_features=512, bias=True)\n (decoder): Sequential(\n (0): Linear(in_features=512, out_features=1024, bias=True)\n (1): GELU(approximate='none')\n (2): Linear(in_features=1024, out_features=10, bias=True)\n )\n (memory): PerceiverMemory(\n (write_layers): ModuleList(\n (0-1): 2 x CrossAttnBlock(\n (q_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n (kv_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n (attn): MultiheadAttentionBlock(\n (linear1): Linear(in_features=512, out_features=1024, bias=True)\n (dropout): Dropout(p=0.0, inplace=False)\n (linear2): Linear(in_features=1024, out_features=512, bias=True)\n (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n (dropout1): Dropout(p=0.0, inplace=False)\n (dropout2): Dropout(p=0.0, inplace=False)\n (attn): MultiheadAttention(\n (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n )\n )\n (ffn): TransformerEncoderLayer(\n (self_attn): MultiheadAttention(\n (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n )\n (linear1): Linear(in_features=512, out_features=1024, bias=True)\n (dropout): Dropout(p=0.0, inplace=False)\n (linear2): Linear(in_features=1024, out_features=512, bias=True)\n (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n (dropout1): Dropout(p=0.0, inplace=False)\n (dropout2): Dropout(p=0.0, inplace=False)\n )\n )\n )\n (read_layers): ModuleList(\n (0-1): 2 x CrossAttnBlock(\n (q_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n (kv_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n (attn): MultiheadAttentionBlock(\n (linear1): Linear(in_features=512, out_features=1024, bias=True)\n (dropout): Dropout(p=0.0, inplace=False)\n (linear2): Linear(in_features=1024, out_features=512, bias=True)\n (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n (dropout1): Dropout(p=0.0, inplace=False)\n (dropout2): Dropout(p=0.0, inplace=False)\n (attn): MultiheadAttention(\n (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n )\n )\n (ffn): TransformerEncoderLayer(\n (self_attn): MultiheadAttention(\n (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n )\n (linear1): Linear(in_features=512, out_features=1024, bias=True)\n (dropout): Dropout(p=0.0, inplace=False)\n (linear2): Linear(in_features=1024, out_features=512, bias=True)\n (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n (dropout1): Dropout(p=0.0, inplace=False)\n (dropout2): Dropout(p=0.0, inplace=False)\n )\n )\n )\n )\n )\n)"
}
} |