suchirsalhan's picture
final: log, final.json, probes
2de2d16 verified
Raw History Blame Contribute Delete
2.69 kB
model:
architecture: hnet_moe_router
d_enc: 32
d_main: 48
n_enc: 1
n_dec: 1
n_main: 1
n_heads_enc: 2
n_heads_main: 2
seq_len: 128
dropout: 0.0
target_ratio: 6.0
max_chunk_mult: 3.0
dechunk: ema
router_noise: 0.0
rope_base: 10000.0
moe_router: {n_experts: 5, gate: script, expert_families: null, common: inherit,
default_family: latin, expert_init: clone, target_ratios: null, unilid_path: null,
unilid_window: 32, unilid_temperature: 1.0, unilid_lang_experts: null, gate_temperature: 1.0,
balance_coef: 0.01, detach_gate_in_ratio: true}
memory:
ali:
budget_B: 512
staging_capacity: 2048
policy: utility
value_mode: learned
candidate_types: [surface, split, merge]
write_rule: mean
eta_min: 0.0
memory_scope: persistent
consolidate_every: 5
max_span: 64
gate: vector
retrieval: true
read_dropout: 0.1
gate_hidden: 0
type_dim: 16
gate_bias_init: -2.0
probe_seqs: 1
probe_reps: 3
horizon: 32
predictor: mlp
lambda_util: 0.1
enabled: true
inventory_kwargs: {min_obs: 1, min_age: 0}
probe_mix: null
util_batch_slots: 1024
pkm: {n_keys: 16, topk: 4, heads: 1, d_key: null, value_lr_mult: 1.0}
objective: {name: ratio, schedule: null, alpha: 0.03, alpha_parity_cap: 0.06, global_anchor: 0.0,
premium_clip: 4.0, premium_kind: flores_per_sent, tau1: 0.4, tau2: 0.7, freeze_scope: router,
cpc_decay: 0.9, genreg_target_entropy: null, genreg_eta: 0.05, flex_targets: premium,
flex_schedule: inverse_linear, flex_start_ratio: null, flex_ramp_start: 0.0, flex_ramp_end: 0.8,
flex_freeze_at: null, flex_bits_source: flores_ngram, flex_bits_order: 5, flex_bits_anchor: en}
data:
corpus: synthetic
langs: [en, de, hi]
mixture: null
doc_filters: null
byte_classes: null
training:
seed: 0
max_steps: 20
batch_size: 4
grad_accum: 1
lr: 0.0003
min_lr_frac: 0.1
warmup: 2
weight_decay: 0.1
grad_clip: 1.0
betas: [0.9, 0.95]
router_lr_mult: 1.0
log_every: 5
amp: none
compile: false
device: cpu
checkpointing:
run_name: smoke_moe-router-script
runs_dir: /private/tmp/claude-501/-Users-suchirsalhan-Documents-PHD-PHD-beetle-hnet/3f003e5e-086f-4ff2-a93f-1f86ed43e9bd/scratchpad/runs
ckpt_every: 10
keep: false
milestones: []
resume: auto
keep_schedule: log2
keep_extra_offsets: [2, 4, 6, 8]
hub_repo: AdaptiveChunking/{run_name}
hub_private: false
keep_local: false
evaluation:
eval_every: 10
eval_batches: 1
probe_sents: 4
probe_bpb_batches: 1
log_spaced_probe: true
dense_early_every: 0
dense_early_until: 0
dump_reps: false
dump_reps_thresholds: [0.3, 0.4, 0.5, 0.6, 0.7]