suchirsalhan commited on
Commit
89888d1
·
verified ·
1 Parent(s): 13e12ca

main <- step 0

Browse files
Files changed (4) hide show
  1. README.md +31 -0
  2. config.json +222 -0
  3. model.safetensors +3 -0
  4. training_config.yaml +150 -0
README.md ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ license: apache-2.0
3
+ library_name: pytorch
4
+ tags: [dynamic-chunking, tokenization, hnet, byte-level, beetle-hnet]
5
+ ---
6
+
7
+ # alitiny
8
+
9
+ A byte-level H-Net run from [beetle-hnet](https://github.com/suchirsalhan/beetle-hnet)
10
+ (`architecture=ali`, `objective=ratio`
11
+ `schedule=None`, `dechunk=ema`, seed 0,
12
+ 6 steps x 4 x 128 bytes, langs ['en', 'de', 'hi']).
13
+
14
+ ## Layout (Beetle-style)
15
+
16
+ - **`step-N` branches**, one per kept checkpoint on a phase-aware log2 schedule (0, 1, 2, 4, 6,
17
+ 8, 16, ..., 2^k from every objective-phase start, plus the final step). Each holds
18
+ `model.safetensors`, `config.json`, `training_config.yaml` and the full resumable trainer
19
+ state `training_state/checkpoint.pt` (for inventory arms this includes the inventory).
20
+ - **`main`**: the latest step's weights and config, plus `log.jsonl`, `final.json` and the
21
+ FLORES+ boundary probes `probe/` once the run has finished.
22
+
23
+ ```python
24
+ from huggingface_hub import hf_hub_download
25
+ import torch
26
+ p = hf_hub_download("AdaptiveChunking/alitiny", "training_state/checkpoint.pt", revision="step-1024")
27
+ state = torch.load(p, map_location="cpu", weights_only=False)
28
+ ```
29
+
30
+ `model.safetensors` stores each tied tensor once; `config.json["tied_weights"]` maps the
31
+ dropped names to the stored ones.
config.json ADDED
@@ -0,0 +1,222 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "arch": "ali",
3
+ "step": 0,
4
+ "model_config": {
5
+ "vocab_size": 256,
6
+ "d_enc": 32,
7
+ "d_main": 48,
8
+ "n_enc": 1,
9
+ "n_dec": 1,
10
+ "n_main": 1,
11
+ "n_heads_enc": 2,
12
+ "n_heads_main": 2,
13
+ "seq_len": 128,
14
+ "dropout": 0.0,
15
+ "target_ratio": 6.0,
16
+ "max_chunk_mult": 3.0,
17
+ "dechunk": "ema",
18
+ "router_noise": 0.0,
19
+ "rope_base": 10000.0
20
+ },
21
+ "ali_config": {
22
+ "enabled": true,
23
+ "budget_B": 512,
24
+ "staging_capacity": 2048,
25
+ "policy": "utility",
26
+ "value_mode": "learned",
27
+ "candidate_types": [
28
+ "surface",
29
+ "split",
30
+ "merge"
31
+ ],
32
+ "gate": "vector",
33
+ "retrieval": true,
34
+ "memory_scope": "persistent",
35
+ "read_dropout": 0.1,
36
+ "write_rule": "mean",
37
+ "eta_min": 0.0,
38
+ "consolidate_every": 5,
39
+ "probe_seqs": 1,
40
+ "probe_reps": 3,
41
+ "horizon": 32,
42
+ "predictor": "mlp",
43
+ "lambda_util": 0.1,
44
+ "max_span": 64,
45
+ "byte_classes_path": null,
46
+ "gate_hidden": 0,
47
+ "type_dim": 16,
48
+ "gate_bias_init": -2.0
49
+ },
50
+ "tied_weights": {
51
+ "head.weight": "emb.weight"
52
+ },
53
+ "config": {
54
+ "model": {
55
+ "architecture": "ali",
56
+ "d_enc": 32,
57
+ "d_main": 48,
58
+ "n_enc": 1,
59
+ "n_dec": 1,
60
+ "n_main": 1,
61
+ "n_heads_enc": 2,
62
+ "n_heads_main": 2,
63
+ "seq_len": 128,
64
+ "dropout": 0.0,
65
+ "target_ratio": 6.0,
66
+ "max_chunk_mult": 3.0,
67
+ "dechunk": "ema",
68
+ "router_noise": 0.0,
69
+ "rope_base": 10000.0,
70
+ "moe_router": {
71
+ "n_experts": 5,
72
+ "gate": "script",
73
+ "expert_families": null,
74
+ "common": "inherit",
75
+ "default_family": "latin",
76
+ "expert_init": "auto",
77
+ "target_ratios": null,
78
+ "unilid_path": null,
79
+ "unilid_window": 32,
80
+ "unilid_temperature": 1.0,
81
+ "unilid_lang_experts": null,
82
+ "gate_temperature": 1.0,
83
+ "balance_coef": 0.01,
84
+ "detach_gate_in_ratio": true
85
+ }
86
+ },
87
+ "memory": {
88
+ "ali": {
89
+ "budget_B": 512,
90
+ "staging_capacity": 2048,
91
+ "policy": "utility",
92
+ "value_mode": "learned",
93
+ "candidate_types": [
94
+ "surface",
95
+ "split",
96
+ "merge"
97
+ ],
98
+ "write_rule": "mean",
99
+ "eta_min": 0.0,
100
+ "memory_scope": "persistent",
101
+ "consolidate_every": 5,
102
+ "max_span": 64,
103
+ "gate": "vector",
104
+ "retrieval": true,
105
+ "read_dropout": 0.1,
106
+ "gate_hidden": 0,
107
+ "type_dim": 16,
108
+ "gate_bias_init": -2.0,
109
+ "probe_seqs": 1,
110
+ "probe_reps": 3,
111
+ "horizon": 32,
112
+ "predictor": "mlp",
113
+ "lambda_util": 0.1,
114
+ "enabled": true,
115
+ "inventory_kwargs": {
116
+ "min_obs": 1,
117
+ "min_age": 0
118
+ },
119
+ "probe_mix": null,
120
+ "util_batch_slots": 1024
121
+ },
122
+ "pkm": {
123
+ "n_keys": 16,
124
+ "topk": 4,
125
+ "heads": 1,
126
+ "d_key": null,
127
+ "value_lr_mult": 1.0
128
+ }
129
+ },
130
+ "objective": {
131
+ "name": "ratio",
132
+ "schedule": null,
133
+ "alpha": 0.03,
134
+ "alpha_parity_cap": 0.06,
135
+ "global_anchor": 0.0,
136
+ "premium_clip": 4.0,
137
+ "premium_kind": "flores_per_sent",
138
+ "tau1": 0.4,
139
+ "tau2": 0.7,
140
+ "freeze_scope": "router",
141
+ "cpc_decay": 0.9,
142
+ "genreg_target_entropy": null,
143
+ "genreg_eta": 0.05,
144
+ "flex_targets": "premium",
145
+ "flex_schedule": "inverse_linear",
146
+ "flex_start_ratio": null,
147
+ "flex_ramp_start": 0.0,
148
+ "flex_ramp_end": 0.8,
149
+ "flex_freeze_at": null,
150
+ "flex_bits_source": "flores_ngram",
151
+ "flex_bits_order": 5,
152
+ "flex_bits_anchor": "en"
153
+ },
154
+ "data": {
155
+ "corpus": "synthetic",
156
+ "langs": [
157
+ "en",
158
+ "de",
159
+ "hi"
160
+ ],
161
+ "mixture": null,
162
+ "doc_filters": null,
163
+ "byte_classes": null
164
+ },
165
+ "training": {
166
+ "seed": 0,
167
+ "max_steps": 6,
168
+ "batch_size": 4,
169
+ "grad_accum": 1,
170
+ "lr": 0.0003,
171
+ "min_lr_frac": 0.1,
172
+ "warmup": 2,
173
+ "weight_decay": 0.1,
174
+ "grad_clip": 1.0,
175
+ "betas": [
176
+ 0.9,
177
+ 0.95
178
+ ],
179
+ "router_lr_mult": 1.0,
180
+ "log_every": 5,
181
+ "amp": "none",
182
+ "compile": false,
183
+ "device": "cpu"
184
+ },
185
+ "checkpointing": {
186
+ "run_name": "alitiny",
187
+ "runs_dir": "/private/tmp/claude-501/-Users-suchirsalhan-Documents-PHD-PHD-beetle-hnet/3f003e5e-086f-4ff2-a93f-1f86ed43e9bd/scratchpad/runs",
188
+ "ckpt_every": 10,
189
+ "keep": false,
190
+ "milestones": [],
191
+ "resume": "auto",
192
+ "keep_schedule": "log2",
193
+ "keep_extra_offsets": [
194
+ 2,
195
+ 4,
196
+ 6,
197
+ 8
198
+ ],
199
+ "hub_repo": "AdaptiveChunking/{run_name}",
200
+ "hub_private": false,
201
+ "keep_local": false
202
+ },
203
+ "evaluation": {
204
+ "eval_every": 10,
205
+ "eval_batches": 1,
206
+ "probe_sents": 4,
207
+ "probe_bpb_batches": 1,
208
+ "log_spaced_probe": true,
209
+ "dense_early_every": 0,
210
+ "dense_early_until": 0,
211
+ "dump_reps": false,
212
+ "dump_reps_thresholds": [
213
+ 0.3,
214
+ 0.4,
215
+ 0.5,
216
+ 0.6,
217
+ 0.7
218
+ ]
219
+ }
220
+ },
221
+ "format": "beetle-hnet step branch v1 (src/checkpointing/hub.py)"
222
+ }
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e0953584b641f6ed8167ff96024dee0d6cb63cbd5e7ecc7df1929ec579f78e62
3
+ size 290688
training_config.yaml ADDED
@@ -0,0 +1,150 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ model:
2
+ architecture: ali
3
+ d_enc: 32
4
+ d_main: 48
5
+ n_enc: 1
6
+ n_dec: 1
7
+ n_main: 1
8
+ n_heads_enc: 2
9
+ n_heads_main: 2
10
+ seq_len: 128
11
+ dropout: 0.0
12
+ target_ratio: 6.0
13
+ max_chunk_mult: 3.0
14
+ dechunk: ema
15
+ router_noise: 0.0
16
+ rope_base: 10000.0
17
+ moe_router:
18
+ n_experts: 5
19
+ gate: script
20
+ expert_families: null
21
+ common: inherit
22
+ default_family: latin
23
+ expert_init: auto
24
+ target_ratios: null
25
+ unilid_path: null
26
+ unilid_window: 32
27
+ unilid_temperature: 1.0
28
+ unilid_lang_experts: null
29
+ gate_temperature: 1.0
30
+ balance_coef: 0.01
31
+ detach_gate_in_ratio: true
32
+ memory:
33
+ ali:
34
+ budget_B: 512
35
+ staging_capacity: 2048
36
+ policy: utility
37
+ value_mode: learned
38
+ candidate_types:
39
+ - surface
40
+ - split
41
+ - merge
42
+ write_rule: mean
43
+ eta_min: 0.0
44
+ memory_scope: persistent
45
+ consolidate_every: 5
46
+ max_span: 64
47
+ gate: vector
48
+ retrieval: true
49
+ read_dropout: 0.1
50
+ gate_hidden: 0
51
+ type_dim: 16
52
+ gate_bias_init: -2.0
53
+ probe_seqs: 1
54
+ probe_reps: 3
55
+ horizon: 32
56
+ predictor: mlp
57
+ lambda_util: 0.1
58
+ enabled: true
59
+ inventory_kwargs:
60
+ min_obs: 1
61
+ min_age: 0
62
+ probe_mix: null
63
+ util_batch_slots: 1024
64
+ pkm:
65
+ n_keys: 16
66
+ topk: 4
67
+ heads: 1
68
+ d_key: null
69
+ value_lr_mult: 1.0
70
+ objective:
71
+ name: ratio
72
+ schedule: null
73
+ alpha: 0.03
74
+ alpha_parity_cap: 0.06
75
+ global_anchor: 0.0
76
+ premium_clip: 4.0
77
+ premium_kind: flores_per_sent
78
+ tau1: 0.4
79
+ tau2: 0.7
80
+ freeze_scope: router
81
+ cpc_decay: 0.9
82
+ genreg_target_entropy: null
83
+ genreg_eta: 0.05
84
+ flex_targets: premium
85
+ flex_schedule: inverse_linear
86
+ flex_start_ratio: null
87
+ flex_ramp_start: 0.0
88
+ flex_ramp_end: 0.8
89
+ flex_freeze_at: null
90
+ flex_bits_source: flores_ngram
91
+ flex_bits_order: 5
92
+ flex_bits_anchor: en
93
+ data:
94
+ corpus: synthetic
95
+ langs:
96
+ - en
97
+ - de
98
+ - hi
99
+ mixture: null
100
+ doc_filters: null
101
+ byte_classes: null
102
+ training:
103
+ seed: 0
104
+ max_steps: 6
105
+ batch_size: 4
106
+ grad_accum: 1
107
+ lr: 0.0003
108
+ min_lr_frac: 0.1
109
+ warmup: 2
110
+ weight_decay: 0.1
111
+ grad_clip: 1.0
112
+ betas:
113
+ - 0.9
114
+ - 0.95
115
+ router_lr_mult: 1.0
116
+ log_every: 5
117
+ amp: none
118
+ compile: false
119
+ device: cpu
120
+ checkpointing:
121
+ run_name: alitiny
122
+ runs_dir: /private/tmp/claude-501/-Users-suchirsalhan-Documents-PHD-PHD-beetle-hnet/3f003e5e-086f-4ff2-a93f-1f86ed43e9bd/scratchpad/runs
123
+ ckpt_every: 10
124
+ keep: false
125
+ milestones: []
126
+ resume: auto
127
+ keep_schedule: log2
128
+ keep_extra_offsets:
129
+ - 2
130
+ - 4
131
+ - 6
132
+ - 8
133
+ hub_repo: AdaptiveChunking/{run_name}
134
+ hub_private: false
135
+ keep_local: false
136
+ evaluation:
137
+ eval_every: 10
138
+ eval_batches: 1
139
+ probe_sents: 4
140
+ probe_bpb_batches: 1
141
+ log_spaced_probe: true
142
+ dense_early_every: 0
143
+ dense_early_until: 0
144
+ dump_reps: false
145
+ dump_reps_thresholds:
146
+ - 0.3
147
+ - 0.4
148
+ - 0.5
149
+ - 0.6
150
+ - 0.7