File size: 1,122 Bytes
1de51cd | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 | arch:
ACT_threshold: 0
H_cycles: 3
H_layers: 0
L_cycles: 4
L_layers: 2
expansion: 4
forward_dtype: bfloat16
group_emb_len: 1
halt_exploration_prob: 0.1
halt_max_steps: 16
hidden_size: 512
loss:
alpha: 0.5
loss_type: stablemax_cross_entropy
name: losses@ACTLossHead
mlp_t: false
name: recursive_reasoning.trm@TinyRecursiveReasoningModel_ACTV1
no_ACT_continue: true
num_heads: 8
pos_encodings: rope
puzzle_emb_len: 16
puzzle_emb_ndim: 512
universal_emb_len: 1
beta1: 0.9
beta2: 0.95
checkpoint_every_eval: true
checkpoint_path: ./checkpoint/miniarc/mini_GC_losses@ACTLossHead
data_paths:
- data/miniARC-aug-1000
data_paths_test: []
ema: true
ema_rate: 0.999
epochs: 100000
eval_interval: 10000
eval_save_outputs: []
evaluators: []
freeze_weights: false
global_batch_size: 4096
group_emb_lr: 0.001
group_emb_weight_decay: 0.1
load_checkpoint: null
lr: 0.0001
lr_min_ratio: 1.0
lr_warmup_steps: 2000
min_eval_interval: 0
project_name: Miniarc-aug-1000-ACT-torch
puzzle_emb_lr: 0.01
puzzle_emb_weight_decay: 0.1
run_name: mini_GC_losses@ACTLossHead
seed: 0
weight_decay: 0.1
|