suchirsalhan's picture
main <- step 20
71e11b3 verified
Raw History Blame Contribute Delete
4.57 kB
{
"arch": "hnet_moe_router",
"step": 20,
"model_config": {
"vocab_size": 256,
"d_enc": 32,
"d_main": 48,
"n_enc": 1,
"n_dec": 1,
"n_main": 1,
"n_heads_enc": 2,
"n_heads_main": 2,
"seq_len": 128,
"dropout": 0.0,
"target_ratio": 6.0,
"max_chunk_mult": 3.0,
"dechunk": "ema",
"router_noise": 0.0,
"rope_base": 10000.0
},
"ali_config": null,
"tied_weights": {
"head.weight": "emb.weight"
},
"config": {
"model": {
"architecture": "hnet_moe_router",
"d_enc": 32,
"d_main": 48,
"n_enc": 1,
"n_dec": 1,
"n_main": 1,
"n_heads_enc": 2,
"n_heads_main": 2,
"seq_len": 128,
"dropout": 0.0,
"target_ratio": 6.0,
"max_chunk_mult": 3.0,
"dechunk": "ema",
"router_noise": 0.0,
"rope_base": 10000.0,
"moe_router": {
"n_experts": 5,
"gate": "script",
"expert_families": null,
"common": "inherit",
"default_family": "latin",
"expert_init": "clone",
"target_ratios": null,
"unilid_path": null,
"unilid_window": 32,
"unilid_temperature": 1.0,
"unilid_lang_experts": null,
"gate_temperature": 1.0,
"balance_coef": 0.01,
"detach_gate_in_ratio": true
}
},
"memory": {
"ali": {
"budget_B": 512,
"staging_capacity": 2048,
"policy": "utility",
"value_mode": "learned",
"candidate_types": [
"surface",
"split",
"merge"
],
"write_rule": "mean",
"eta_min": 0.0,
"memory_scope": "persistent",
"consolidate_every": 5,
"max_span": 64,
"gate": "vector",
"retrieval": true,
"read_dropout": 0.1,
"gate_hidden": 0,
"type_dim": 16,
"gate_bias_init": -2.0,
"probe_seqs": 1,
"probe_reps": 3,
"horizon": 32,
"predictor": "mlp",
"lambda_util": 0.1,
"enabled": true,
"inventory_kwargs": {
"min_obs": 1,
"min_age": 0
},
"probe_mix": null,
"util_batch_slots": 1024
},
"pkm": {
"n_keys": 16,
"topk": 4,
"heads": 1,
"d_key": null,
"value_lr_mult": 1.0
}
},
"objective": {
"name": "ratio",
"schedule": null,
"alpha": 0.03,
"alpha_parity_cap": 0.06,
"global_anchor": 0.0,
"premium_clip": 4.0,
"premium_kind": "flores_per_sent",
"tau1": 0.4,
"tau2": 0.7,
"freeze_scope": "router",
"cpc_decay": 0.9,
"genreg_target_entropy": null,
"genreg_eta": 0.05,
"flex_targets": "premium",
"flex_schedule": "inverse_linear",
"flex_start_ratio": null,
"flex_ramp_start": 0.0,
"flex_ramp_end": 0.8,
"flex_freeze_at": null,
"flex_bits_source": "flores_ngram",
"flex_bits_order": 5,
"flex_bits_anchor": "en"
},
"data": {
"corpus": "synthetic",
"langs": [
"en",
"de",
"hi"
],
"mixture": null,
"doc_filters": null,
"byte_classes": null
},
"training": {
"seed": 0,
"max_steps": 20,
"batch_size": 4,
"grad_accum": 1,
"lr": 0.0003,
"min_lr_frac": 0.1,
"warmup": 2,
"weight_decay": 0.1,
"grad_clip": 1.0,
"betas": [
0.9,
0.95
],
"router_lr_mult": 1.0,
"log_every": 5,
"amp": "none",
"compile": false,
"device": "cpu"
},
"checkpointing": {
"run_name": "smoke_moe-router-script",
"runs_dir": "/private/tmp/claude-501/-Users-suchirsalhan-Documents-PHD-PHD-beetle-hnet/3f003e5e-086f-4ff2-a93f-1f86ed43e9bd/scratchpad/runs",
"ckpt_every": 10,
"keep": false,
"milestones": [],
"resume": "auto",
"keep_schedule": "log2",
"keep_extra_offsets": [
2,
4,
6,
8
],
"hub_repo": "AdaptiveChunking/{run_name}",
"hub_private": false,
"keep_local": false
},
"evaluation": {
"eval_every": 10,
"eval_batches": 1,
"probe_sents": 4,
"probe_bpb_batches": 1,
"log_spaced_probe": true,
"dense_early_every": 0,
"dense_early_until": 0,
"dump_reps": false,
"dump_reps_thresholds": [
0.3,
0.4,
0.5,
0.6,
0.7
]
}
},
"format": "beetle-hnet step branch v1 (src/checkpointing/hub.py)"
}