File size: 2,774 Bytes
89888d1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 | model:
architecture: ali
d_enc: 32
d_main: 48
n_enc: 1
n_dec: 1
n_main: 1
n_heads_enc: 2
n_heads_main: 2
seq_len: 128
dropout: 0.0
target_ratio: 6.0
max_chunk_mult: 3.0
dechunk: ema
router_noise: 0.0
rope_base: 10000.0
moe_router:
n_experts: 5
gate: script
expert_families: null
common: inherit
default_family: latin
expert_init: auto
target_ratios: null
unilid_path: null
unilid_window: 32
unilid_temperature: 1.0
unilid_lang_experts: null
gate_temperature: 1.0
balance_coef: 0.01
detach_gate_in_ratio: true
memory:
ali:
budget_B: 512
staging_capacity: 2048
policy: utility
value_mode: learned
candidate_types:
- surface
- split
- merge
write_rule: mean
eta_min: 0.0
memory_scope: persistent
consolidate_every: 5
max_span: 64
gate: vector
retrieval: true
read_dropout: 0.1
gate_hidden: 0
type_dim: 16
gate_bias_init: -2.0
probe_seqs: 1
probe_reps: 3
horizon: 32
predictor: mlp
lambda_util: 0.1
enabled: true
inventory_kwargs:
min_obs: 1
min_age: 0
probe_mix: null
util_batch_slots: 1024
pkm:
n_keys: 16
topk: 4
heads: 1
d_key: null
value_lr_mult: 1.0
objective:
name: ratio
schedule: null
alpha: 0.03
alpha_parity_cap: 0.06
global_anchor: 0.0
premium_clip: 4.0
premium_kind: flores_per_sent
tau1: 0.4
tau2: 0.7
freeze_scope: router
cpc_decay: 0.9
genreg_target_entropy: null
genreg_eta: 0.05
flex_targets: premium
flex_schedule: inverse_linear
flex_start_ratio: null
flex_ramp_start: 0.0
flex_ramp_end: 0.8
flex_freeze_at: null
flex_bits_source: flores_ngram
flex_bits_order: 5
flex_bits_anchor: en
data:
corpus: synthetic
langs:
- en
- de
- hi
mixture: null
doc_filters: null
byte_classes: null
training:
seed: 0
max_steps: 6
batch_size: 4
grad_accum: 1
lr: 0.0003
min_lr_frac: 0.1
warmup: 2
weight_decay: 0.1
grad_clip: 1.0
betas:
- 0.9
- 0.95
router_lr_mult: 1.0
log_every: 5
amp: none
compile: false
device: cpu
checkpointing:
run_name: alitiny
runs_dir: /private/tmp/claude-501/-Users-suchirsalhan-Documents-PHD-PHD-beetle-hnet/3f003e5e-086f-4ff2-a93f-1f86ed43e9bd/scratchpad/runs
ckpt_every: 10
keep: false
milestones: []
resume: auto
keep_schedule: log2
keep_extra_offsets:
- 2
- 4
- 6
- 8
hub_repo: AdaptiveChunking/{run_name}
hub_private: false
keep_local: false
evaluation:
eval_every: 10
eval_batches: 1
probe_sents: 4
probe_bpb_batches: 1
log_spaced_probe: true
dense_early_every: 0
dense_early_until: 0
dump_reps: false
dump_reps_thresholds:
- 0.3
- 0.4
- 0.5
- 0.6
- 0.7
|