suchirsalhan commited on
Commit
2de2d16
·
verified ·
1 Parent(s): 71e11b3

final: log, final.json, probes

Browse files
config.yaml ADDED
@@ -0,0 +1,97 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ model:
2
+ architecture: hnet_moe_router
3
+ d_enc: 32
4
+ d_main: 48
5
+ n_enc: 1
6
+ n_dec: 1
7
+ n_main: 1
8
+ n_heads_enc: 2
9
+ n_heads_main: 2
10
+ seq_len: 128
11
+ dropout: 0.0
12
+ target_ratio: 6.0
13
+ max_chunk_mult: 3.0
14
+ dechunk: ema
15
+ router_noise: 0.0
16
+ rope_base: 10000.0
17
+ moe_router: {n_experts: 5, gate: script, expert_families: null, common: inherit,
18
+ default_family: latin, expert_init: clone, target_ratios: null, unilid_path: null,
19
+ unilid_window: 32, unilid_temperature: 1.0, unilid_lang_experts: null, gate_temperature: 1.0,
20
+ balance_coef: 0.01, detach_gate_in_ratio: true}
21
+ memory:
22
+ ali:
23
+ budget_B: 512
24
+ staging_capacity: 2048
25
+ policy: utility
26
+ value_mode: learned
27
+ candidate_types: [surface, split, merge]
28
+ write_rule: mean
29
+ eta_min: 0.0
30
+ memory_scope: persistent
31
+ consolidate_every: 5
32
+ max_span: 64
33
+ gate: vector
34
+ retrieval: true
35
+ read_dropout: 0.1
36
+ gate_hidden: 0
37
+ type_dim: 16
38
+ gate_bias_init: -2.0
39
+ probe_seqs: 1
40
+ probe_reps: 3
41
+ horizon: 32
42
+ predictor: mlp
43
+ lambda_util: 0.1
44
+ enabled: true
45
+ inventory_kwargs: {min_obs: 1, min_age: 0}
46
+ probe_mix: null
47
+ util_batch_slots: 1024
48
+ pkm: {n_keys: 16, topk: 4, heads: 1, d_key: null, value_lr_mult: 1.0}
49
+ objective: {name: ratio, schedule: null, alpha: 0.03, alpha_parity_cap: 0.06, global_anchor: 0.0,
50
+ premium_clip: 4.0, premium_kind: flores_per_sent, tau1: 0.4, tau2: 0.7, freeze_scope: router,
51
+ cpc_decay: 0.9, genreg_target_entropy: null, genreg_eta: 0.05, flex_targets: premium,
52
+ flex_schedule: inverse_linear, flex_start_ratio: null, flex_ramp_start: 0.0, flex_ramp_end: 0.8,
53
+ flex_freeze_at: null, flex_bits_source: flores_ngram, flex_bits_order: 5, flex_bits_anchor: en}
54
+ data:
55
+ corpus: synthetic
56
+ langs: [en, de, hi]
57
+ mixture: null
58
+ doc_filters: null
59
+ byte_classes: null
60
+ training:
61
+ seed: 0
62
+ max_steps: 20
63
+ batch_size: 4
64
+ grad_accum: 1
65
+ lr: 0.0003
66
+ min_lr_frac: 0.1
67
+ warmup: 2
68
+ weight_decay: 0.1
69
+ grad_clip: 1.0
70
+ betas: [0.9, 0.95]
71
+ router_lr_mult: 1.0
72
+ log_every: 5
73
+ amp: none
74
+ compile: false
75
+ device: cpu
76
+ checkpointing:
77
+ run_name: smoke_moe-router-script
78
+ runs_dir: /private/tmp/claude-501/-Users-suchirsalhan-Documents-PHD-PHD-beetle-hnet/3f003e5e-086f-4ff2-a93f-1f86ed43e9bd/scratchpad/runs
79
+ ckpt_every: 10
80
+ keep: false
81
+ milestones: []
82
+ resume: auto
83
+ keep_schedule: log2
84
+ keep_extra_offsets: [2, 4, 6, 8]
85
+ hub_repo: AdaptiveChunking/{run_name}
86
+ hub_private: false
87
+ keep_local: false
88
+ evaluation:
89
+ eval_every: 10
90
+ eval_batches: 1
91
+ probe_sents: 4
92
+ probe_bpb_batches: 1
93
+ log_spaced_probe: true
94
+ dense_early_every: 0
95
+ dense_early_until: 0
96
+ dump_reps: false
97
+ dump_reps_thresholds: [0.3, 0.4, 0.5, 0.6, 0.7]
final.json ADDED
@@ -0,0 +1,107 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "event": "final",
3
+ "run": "smoke_moe-router-script",
4
+ "arch": "hnet_moe_router",
5
+ "objective": "ratio",
6
+ "condition": "A",
7
+ "step": 20,
8
+ "params": 75152,
9
+ "seed": 0,
10
+ "per_lang": {
11
+ "en": {
12
+ "cpc": 1.1517857142857142,
13
+ "cps": 64.5,
14
+ "bpb": 7.901662406011139,
15
+ "bytes_per_chunk": 2.094069529652352
16
+ },
17
+ "de": {
18
+ "cpc": 0.9630996309963099,
19
+ "cps": 65.25,
20
+ "bpb": 7.85908982061656,
21
+ "bytes_per_chunk": 1.9212007504690432
22
+ },
23
+ "hi": {
24
+ "cpc": 0.9090909090909091,
25
+ "cps": 60.0,
26
+ "bpb": 7.883620738005027,
27
+ "bytes_per_chunk": 2.2164502164502164
28
+ }
29
+ },
30
+ "bpb_mean": 7.881457654877575,
31
+ "gini_cpc": 0.05350456590476349,
32
+ "gini_cps": 0.01844532279314892,
33
+ "bpic": 2.0237154150197627,
34
+ "mean_boundary_entropy": 0.672665536403656,
35
+ "dechunk": "ema",
36
+ "router_noise": 0.0,
37
+ "transitions": [
38
+ {
39
+ "step": 0,
40
+ "phase": 1,
41
+ "objective": "global",
42
+ "router_trainable": true,
43
+ "encoder_trainable": true,
44
+ "frozen_router_params": 0
45
+ },
46
+ {
47
+ "step": 8,
48
+ "phase": 2,
49
+ "objective": "global",
50
+ "router_trainable": true,
51
+ "encoder_trainable": true,
52
+ "frozen_router_params": 0
53
+ },
54
+ {
55
+ "step": 14,
56
+ "phase": 3,
57
+ "objective": "global",
58
+ "router_trainable": true,
59
+ "encoder_trainable": true,
60
+ "frozen_router_params": 0
61
+ }
62
+ ],
63
+ "probe_overhead_sec_total": 0.0,
64
+ "probe_overhead_seq_frac": 0.0,
65
+ "wall_sec": 1.1627311706542969,
66
+ "flops": {
67
+ "flops_total": 3815989632.0,
68
+ "flops_main": 1173578112.0,
69
+ "per_lang_main_flops": {
70
+ "en": 277736256.0,
71
+ "de": 367411968.0,
72
+ "hi": 412927200.0
73
+ }
74
+ },
75
+ "compute_received": {
76
+ "per_lang": {
77
+ "en": {
78
+ "cps": 64.5,
79
+ "bytes_per_sent": 128.0,
80
+ "main_flops_per_sent": 4362264.0,
81
+ "byte_flops_per_sent": 11010048.0,
82
+ "total_flops_per_sent": 15372312.0
83
+ },
84
+ "de": {
85
+ "cps": 65.25,
86
+ "bytes_per_sent": 128.0,
87
+ "main_flops_per_sent": 4417686.0,
88
+ "byte_flops_per_sent": 11010048.0,
89
+ "total_flops_per_sent": 15427734.0
90
+ },
91
+ "hi": {
92
+ "cps": 60.0,
93
+ "bytes_per_sent": 128.0,
94
+ "main_flops_per_sent": 4032000.0,
95
+ "byte_flops_per_sent": 11010048.0,
96
+ "total_flops_per_sent": 15042048.0
97
+ }
98
+ },
99
+ "gini_main_flops": 0.020069076135951214,
100
+ "gini_total_flops": 0.005608906085311061,
101
+ "main_flops_parity_vs_ref": {
102
+ "en": 1.0,
103
+ "de": 1.0127048706818293,
104
+ "hi": 0.9242906894218231
105
+ }
106
+ }
107
+ }
hub.jsonl ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {"event": "hub_push", "step": 0, "repo": "AdaptiveChunking/smoke_moe-router-script", "branch": "step-0", "sec": 68.7, "local_deleted": true, "t": "2026-11-14 20:26:37"}
2
+ {"event": "hub_push", "step": 1, "repo": "AdaptiveChunking/smoke_moe-router-script", "branch": "step-1", "sec": 94.0, "local_deleted": true, "t": "2026-11-14 20:28:11"}
3
+ {"event": "hub_push", "step": 2, "repo": "AdaptiveChunking/smoke_moe-router-script", "branch": "step-2", "sec": 81.8, "local_deleted": true, "t": "2026-11-14 20:29:33"}
4
+ {"event": "hub_push", "step": 4, "repo": "AdaptiveChunking/smoke_moe-router-script", "branch": "step-4", "sec": 68.6, "local_deleted": true, "t": "2026-11-14 20:30:42"}
5
+ {"event": "hub_push", "step": 6, "repo": "AdaptiveChunking/smoke_moe-router-script", "branch": "step-6", "sec": 54.1, "local_deleted": true, "t": "2026-11-14 20:31:36"}
6
+ {"event": "hub_push", "step": 8, "repo": "AdaptiveChunking/smoke_moe-router-script", "branch": "step-8", "sec": 68.1, "local_deleted": true, "t": "2026-11-14 20:32:44"}
7
+ {"event": "hub_push", "step": 16, "repo": "AdaptiveChunking/smoke_moe-router-script", "branch": "step-16", "sec": 51.0, "local_deleted": true, "t": "2026-11-14 20:33:35"}
8
+ {"event": "hub_push", "step": 20, "repo": "AdaptiveChunking/smoke_moe-router-script", "branch": "step-20", "sec": 78.6, "local_deleted": true, "t": "2026-11-14 20:34:54"}
log.jsonl ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"event": "hub", "repo": "AdaptiveChunking/smoke_moe-router-script", "keep_steps": [0, 1, 2, 4, 6, 8, 16, 20], "t": 0.7}
2
+ {"event": "start", "run": "smoke_moe-router-script", "arch": "hnet_moe_router", "objective": "ratio", "condition": "A", "params": 75152, "langs": ["en", "de", "hi"], "premium": {"en": 1.0, "de": 1.05, "hi": 2.4}, "schedule": {"tau1": 0.4, "tau2": 0.7, "p1_objective": "global", "p2_objective": "global", "p3_objective": "global", "freeze_in_p3": false, "freeze_scope": "router"}, "corpus": "<synthetic>", "premium_kind": "flores_per_sent", "doc_filtered": {}, "weights": {"en": 0.3333, "de": 0.3333, "hi": 0.3333}, "device": "cpu", "t": 0.7}
3
+ {"event": "probe_schedule", "n": 9, "steps": [0, 1, 2, 4, 6, 8, 10, 16, 20], "t": 0.7}
4
+ {"event": "probe", "step": 0, "phase": 1, "objective": "global", "router_trainable": true, "per_lang": {"en": {"chunks": 271.0, "chars": 224.0, "bytes": 512.0, "cpc": 1.209821, "cps": 67.75, "mean_p": 0.51202, "boundary_entropy": 0.673043, "bpb": 8.003899}, "de": {"chunks": 267.0, "chars": 271.0, "bytes": 512.0, "cpc": 0.98524, "cps": 66.75, "mean_p": 0.506525, "boundary_entropy": 0.669965, "bpb": 7.998187}, "hi": {"chunks": 289.0, "chars": 264.0, "bytes": 512.0, "cpc": 1.094697, "cps": 72.25, "mean_p": 0.509041, "boundary_entropy": 0.673906, "bpb": 8.011275}}, "bpb": {"en": 8.0039, "de": 7.99819, "hi": 8.01127}, "gini_cpc": 0.045511, "gini_cps": 0.017735, "bpic": 1.857316, "worst_lang": "en", "cpc_parity_vs_en": {"en": 1.0, "de": 0.8143679960784848, "hi": 0.9048417756904842}, "t": 0.7}
5
+ {"event": "train", "step": 0, "arch": "hnet_moe_router", "phase": 1, "objective": "global", "lm_loss": 5.55016, "bpb": 8.00718, "chunk_loss": 0.055461, "F": 0.51172, "G": 0.50726, "chunks": 65.5, "bytes_per_chunk": 1.954, "boundary_entropy": 0.67384, "router_noise": 0.0, "grad_norm": 0.8822, "lr": 0.00015, "router_trainable": true, "sec_per_step": 0.006, "moe": {"gate_mean": [0.853515625, 0.126953125, 0.0, 0.0, 0.01953125], "bytes_per_chunk": [1.942222237586975, 2.1666667461395264, 0.0, 0.0, 1.4285714626312256], "per_lang": {"en": [0.8046875, 0.1692708283662796, 0.0, 0.0, 0.02604166604578495], "de": [1.0, 0.0, 0.0, 0.0, 0.0]}}, "flops_total": 192081024.0, "flops_main": 59960448.0, "alpha": 0.03, "t": 0.7}
6
+ {"event": "probe", "step": 1, "phase": 1, "objective": "global", "router_trainable": true, "per_lang": {"en": {"chunks": 269.0, "chars": 224.0, "bytes": 512.0, "cpc": 1.200893, "cps": 67.25, "mean_p": 0.51177, "boundary_entropy": 0.673017, "bpb": 8.00014}, "de": {"chunks": 267.0, "chars": 271.0, "bytes": 512.0, "cpc": 0.98524, "cps": 66.75, "mean_p": 0.506892, "boundary_entropy": 0.669978, "bpb": 7.994909}, "hi": {"chunks": 288.0, "chars": 264.0, "bytes": 512.0, "cpc": 1.090909, "cps": 72.0, "mean_p": 0.509105, "boundary_entropy": 0.673953, "bpb": 8.011791}}, "bpb": {"en": 8.00014, "de": 7.99491, "hi": 8.01179}, "gini_cpc": 0.043871, "gini_cps": 0.01699, "bpic": 1.864078, "worst_lang": "en", "cpc_parity_vs_en": {"en": 1.0, "de": 0.8204227767184734, "hi": 0.9084150050692801}, "t": 0.7}
7
+ {"event": "probe", "step": 2, "phase": 1, "objective": "global", "router_trainable": true, "per_lang": {"en": {"chunks": 272.0, "chars": 224.0, "bytes": 512.0, "cpc": 1.214286, "cps": 68.0, "mean_p": 0.512095, "boundary_entropy": 0.672953, "bpb": 7.994883}, "de": {"chunks": 270.0, "chars": 271.0, "bytes": 512.0, "cpc": 0.99631, "cps": 67.5, "mean_p": 0.507608, "boundary_entropy": 0.670026, "bpb": 7.984517}, "hi": {"chunks": 287.0, "chars": 264.0, "bytes": 512.0, "cpc": 1.087121, "cps": 71.75, "mean_p": 0.508377, "boundary_entropy": 0.674058, "bpb": 8.001162}}, "bpb": {"en": 7.99488, "de": 7.98452, "hi": 8.00116}, "gini_cpc": 0.044066, "gini_cps": 0.013671, "bpic": 1.852835, "worst_lang": "en", "cpc_parity_vs_en": {"en": 1.0, "de": 0.8204905578467551, "hi": 0.895276292335116}, "t": 0.8}
8
+ {"event": "probe", "step": 4, "phase": 1, "objective": "global", "router_trainable": true, "per_lang": {"en": {"chunks": 277.0, "chars": 224.0, "bytes": 512.0, "cpc": 1.236607, "cps": 69.25, "mean_p": 0.511598, "boundary_entropy": 0.672927, "bpb": 7.976854}, "de": {"chunks": 271.0, "chars": 271.0, "bytes": 512.0, "cpc": 1.0, "cps": 67.75, "mean_p": 0.508214, "boundary_entropy": 0.670041, "bpb": 7.968397}, "hi": {"chunks": 278.0, "chars": 264.0, "bytes": 512.0, "cpc": 1.05303, "cps": 69.5, "mean_p": 0.506854, "boundary_entropy": 0.674398, "bpb": 7.977031}}, "bpb": {"en": 7.97685, "de": 7.9684, "hi": 7.97703}, "gini_cpc": 0.04795, "gini_cps": 0.00565, "bpic": 1.859564, "worst_lang": "en", "cpc_parity_vs_en": {"en": 1.0, "de": 0.8086642599277979, "hi": 0.8515479706815446}, "t": 0.8}
9
+ {"event": "train", "step": 5, "arch": "hnet_moe_router", "phase": 1, "objective": "global", "lm_loss": 5.52334, "bpb": 7.9685, "chunk_loss": 0.059413, "F": 0.56445, "G": 0.50894, "chunks": 72.25, "bytes_per_chunk": 1.772, "boundary_entropy": 0.67218, "router_noise": 0.0, "grad_norm": 0.7793, "lr": 0.0002819, "router_trainable": true, "sec_per_step": 0.025, "moe": {"gate_mean": [0.93359375, 0.0390625, 0.0, 0.0, 0.02734375], "bytes_per_chunk": [1.7769516706466675, 1.6666666269302368, 0.0, 0.0, 1.75], "per_lang": {"en": [0.984375, 0.015625, 0.0, 0.0, 0.0], "de": [0.890625, 0.0, 0.0, 0.0, 0.109375], "hi": [0.9296875, 0.0703125, 0.0, 0.0, 0.0]}}, "flops_total": 1156193280.0, "flops_main": 363469824.0, "alpha": 0.03, "t": 0.9}
10
+ {"event": "probe", "step": 6, "phase": 1, "objective": "global", "router_trainable": true, "per_lang": {"en": {"chunks": 280.0, "chars": 224.0, "bytes": 512.0, "cpc": 1.25, "cps": 70.0, "mean_p": 0.510988, "boundary_entropy": 0.672828, "bpb": 7.958306}, "de": {"chunks": 271.0, "chars": 271.0, "bytes": 512.0, "cpc": 1.0, "cps": 67.75, "mean_p": 0.509085, "boundary_entropy": 0.670061, "bpb": 7.944668}, "hi": {"chunks": 271.0, "chars": 264.0, "bytes": 512.0, "cpc": 1.026515, "cps": 67.75, "mean_p": 0.504988, "boundary_entropy": 0.67459, "bpb": 7.951548}}, "bpb": {"en": 7.95831, "de": 7.94467, "hi": 7.95155}, "gini_cpc": 0.050867, "gini_cps": 0.007299, "bpic": 1.868613, "worst_lang": "en", "cpc_parity_vs_en": {"en": 1.0, "de": 0.8, "hi": 0.8212121212121213}, "t": 0.9}
11
+ {"event": "probe", "step": 8, "phase": 2, "objective": "global", "router_trainable": true, "per_lang": {"en": {"chunks": 275.0, "chars": 224.0, "bytes": 512.0, "cpc": 1.227679, "cps": 68.75, "mean_p": 0.510457, "boundary_entropy": 0.672781, "bpb": 7.943742}, "de": {"chunks": 272.0, "chars": 271.0, "bytes": 512.0, "cpc": 1.00369, "cps": 68.0, "mean_p": 0.510301, "boundary_entropy": 0.670184, "bpb": 7.917672}, "hi": {"chunks": 257.0, "chars": 264.0, "bytes": 512.0, "cpc": 0.973485, "cps": 64.25, "mean_p": 0.502897, "boundary_entropy": 0.674666, "bpb": 7.927948}}, "bpb": {"en": 7.94374, "de": 7.91767, "hi": 7.92795}, "gini_cpc": 0.052877, "gini_cps": 0.014925, "bpic": 1.910448, "worst_lang": "en", "cpc_parity_vs_en": {"en": 1.0, "de": 0.8175511573297551, "hi": 0.7929476584022039}, "t": 0.9}
12
+ {"event": "phase_transition", "step": 8, "phase": 2, "objective": "global", "router_trainable": true, "encoder_trainable": true, "frozen_router_params": 0, "t": 0.9}
13
+ {"event": "probe", "step": 10, "phase": 2, "objective": "global", "router_trainable": true, "per_lang": {"en": {"chunks": 274.0, "chars": 224.0, "bytes": 512.0, "cpc": 1.223214, "cps": 68.5, "mean_p": 0.5094, "boundary_entropy": 0.672753, "bpb": 7.929703}, "de": {"chunks": 269.0, "chars": 271.0, "bytes": 512.0, "cpc": 0.99262, "cps": 67.25, "mean_p": 0.51093, "boundary_entropy": 0.670403, "bpb": 7.897897}, "hi": {"chunks": 249.0, "chars": 264.0, "bytes": 512.0, "cpc": 0.943182, "cps": 62.25, "mean_p": 0.500878, "boundary_entropy": 0.674733, "bpb": 7.909755}}, "bpb": {"en": 7.9297, "de": 7.8979, "hi": 7.90975}, "gini_cpc": 0.059097, "gini_cps": 0.021044, "bpic": 1.939394, "worst_lang": "en", "cpc_parity_vs_en": {"en": 1.0, "de": 0.8114849031701995, "hi": 0.7710683477106834}, "t": 1.0}
14
+ {"event": "eval", "step": 10, "phase": 2, "bpb": {"en": 7.9297, "de": 7.8979, "hi": 7.90975}, "bytes_per_chunk": {"en": 2.032, "de": 1.882, "hi": 1.977}, "bpb_mean": 7.91245, "frozen_inventory": false, "t": 1.0}
15
+ {"event": "train", "step": 10, "arch": "hnet_moe_router", "phase": 2, "objective": "global", "lm_loss": 5.46312, "bpb": 7.88161, "chunk_loss": 0.057702, "F": 0.53516, "G": 0.51338, "chunks": 68.5, "bytes_per_chunk": 1.869, "boundary_entropy": 0.67209, "router_noise": 0.0, "grad_norm": 0.7729, "lr": 0.0001884, "router_trainable": true, "sec_per_step": 0.027, "moe": {"gate_mean": [0.876953125, 0.08984375, 0.0, 0.0, 0.033203125], "bytes_per_chunk": [1.8477365970611572, 2.4210526943206787, 0.0, 0.0, 1.4166666269302368], "per_lang": {"en": [0.8125, 0.1796875, 0.0, 0.0, 0.0078125], "de": [0.94140625, 0.0, 0.0, 0.0, 0.05859375]}}, "flops_total": 2104989696.0, "flops_main": 651663360.0, "alpha": 0.03, "t": 1.0}
16
+ {"event": "phase_transition", "step": 14, "phase": 3, "objective": "global", "router_trainable": true, "encoder_trainable": true, "frozen_router_params": 0, "t": 1.0}
17
+ {"event": "train", "step": 15, "arch": "hnet_moe_router", "phase": 3, "objective": "global", "lm_loss": 5.47419, "bpb": 7.89759, "chunk_loss": 0.054394, "F": 0.50586, "G": 0.4986, "chunks": 64.75, "bytes_per_chunk": 1.977, "boundary_entropy": 0.6721, "router_noise": 0.0, "grad_norm": 0.8214, "lr": 7.82e-05, "router_trainable": true, "sec_per_step": 0.011, "moe": {"gate_mean": [0.9453125, 0.0546875, 0.0, 0.0, 0.0], "bytes_per_chunk": [2.0, 1.6470588445663452, 0.0, 0.0, 0.0], "per_lang": {"en": [0.9453125, 0.0546875, 0.0, 0.0, 0.0], "hi": [0.9453125, 0.0546875, 0.0, 0.0, 0.0]}}, "flops_total": 3065526144.0, "flops_main": 951596928.0, "alpha": 0.03, "t": 1.0}
18
+ {"event": "probe", "step": 16, "phase": 3, "objective": "global", "router_trainable": true, "per_lang": {"en": {"chunks": 262.0, "chars": 224.0, "bytes": 512.0, "cpc": 1.169643, "cps": 65.5, "mean_p": 0.505081, "boundary_entropy": 0.672719, "bpb": 7.905046}, "de": {"chunks": 264.0, "chars": 271.0, "bytes": 512.0, "cpc": 0.97417, "cps": 66.0, "mean_p": 0.51027, "boundary_entropy": 0.670987, "bpb": 7.86367}, "hi": {"chunks": 242.0, "chars": 264.0, "bytes": 512.0, "cpc": 0.916667, "cps": 60.5, "mean_p": 0.497167, "boundary_entropy": 0.674739, "bpb": 7.886533}}, "bpb": {"en": 7.90505, "de": 7.86367, "hi": 7.88653}, "gini_cpc": 0.055106, "gini_cps": 0.019097, "bpic": 2.0, "worst_lang": "en", "cpc_parity_vs_en": {"en": 1.0, "de": 0.8328779471000817, "hi": 0.7837150127226462}, "t": 1.1}
19
+ {"event": "train", "step": 19, "arch": "hnet_moe_router", "phase": 3, "objective": "global", "lm_loss": 5.45326, "bpb": 7.86739, "chunk_loss": 0.054398, "F": 0.50391, "G": 0.50143, "chunks": 64.5, "bytes_per_chunk": 1.984, "boundary_entropy": 0.67267, "router_noise": 0.0, "grad_norm": 0.5891, "lr": 3.21e-05, "router_trainable": true, "sec_per_step": 0.013, "moe": {"gate_mean": [0.955078125, 0.02734375, 0.0, 0.0, 0.017578125], "bytes_per_chunk": [2.012345790863037, 1.75, 0.0, 0.0, 1.2857142686843872], "per_lang": {"en": [0.9609375, 0.0390625, 0.0, 0.0, 0.0], "de": [0.953125, 0.0, 0.0, 0.0, 0.046875], "hi": [0.953125, 0.03515625, 0.0, 0.0, 0.01171875]}}, "flops_total": 3815989632.0, "flops_main": 1173578112.0, "alpha": 0.03, "t": 1.1}
20
+ {"event": "probe", "step": 20, "phase": 3, "objective": "global", "router_trainable": true, "per_lang": {"en": {"chunks": 258.0, "chars": 224.0, "bytes": 512.0, "cpc": 1.151786, "cps": 64.5, "mean_p": 0.503649, "boundary_entropy": 0.672715, "bpb": 7.899652}, "de": {"chunks": 261.0, "chars": 271.0, "bytes": 512.0, "cpc": 0.9631, "cps": 65.25, "mean_p": 0.509357, "boundary_entropy": 0.671118, "bpb": 7.858703}, "hi": {"chunks": 240.0, "chars": 264.0, "bytes": 512.0, "cpc": 0.909091, "cps": 60.0, "mean_p": 0.495954, "boundary_entropy": 0.674707, "bpb": 7.878624}}, "bpb": {"en": 7.89965, "de": 7.8587, "hi": 7.87862}, "gini_cpc": 0.053505, "gini_cps": 0.018445, "bpic": 2.023715, "worst_lang": "en", "cpc_parity_vs_en": {"en": 1.0, "de": 0.8361795245859436, "hi": 0.7892882311486963}, "t": 1.1}
21
+ {"event": "eval", "step": 20, "phase": 3, "bpb": {"en": 7.89965, "de": 7.8587, "hi": 7.87862}, "bytes_per_chunk": {"en": 2.207, "de": 1.939, "hi": 2.169}, "bpb_mean": 7.87899, "frozen_inventory": false, "t": 1.1}
22
+ {"event": "final", "run": "smoke_moe-router-script", "arch": "hnet_moe_router", "objective": "ratio", "condition": "A", "step": 20, "params": 75152, "seed": 0, "per_lang": {"en": {"cpc": 1.1517857142857142, "cps": 64.5, "bpb": 7.901662406011139, "bytes_per_chunk": 2.094069529652352}, "de": {"cpc": 0.9630996309963099, "cps": 65.25, "bpb": 7.85908982061656, "bytes_per_chunk": 1.9212007504690432}, "hi": {"cpc": 0.9090909090909091, "cps": 60.0, "bpb": 7.883620738005027, "bytes_per_chunk": 2.2164502164502164}}, "bpb_mean": 7.881457654877575, "gini_cpc": 0.05350456590476349, "gini_cps": 0.01844532279314892, "bpic": 2.0237154150197627, "mean_boundary_entropy": 0.672665536403656, "dechunk": "ema", "router_noise": 0.0, "transitions": [{"step": 0, "phase": 1, "objective": "global", "router_trainable": true, "encoder_trainable": true, "frozen_router_params": 0}, {"step": 8, "phase": 2, "objective": "global", "router_trainable": true, "encoder_trainable": true, "frozen_router_params": 0}, {"step": 14, "phase": 3, "objective": "global", "router_trainable": true, "encoder_trainable": true, "frozen_router_params": 0}], "probe_overhead_sec_total": 0.0, "probe_overhead_seq_frac": 0.0, "wall_sec": 1.1627311706542969, "flops": {"flops_total": 3815989632.0, "flops_main": 1173578112.0, "per_lang_main_flops": {"en": 277736256.0, "de": 367411968.0, "hi": 412927200.0}}, "compute_received": {"per_lang": {"en": {"cps": 64.5, "bytes_per_sent": 128.0, "main_flops_per_sent": 4362264.0, "byte_flops_per_sent": 11010048.0, "total_flops_per_sent": 15372312.0}, "de": {"cps": 65.25, "bytes_per_sent": 128.0, "main_flops_per_sent": 4417686.0, "byte_flops_per_sent": 11010048.0, "total_flops_per_sent": 15427734.0}, "hi": {"cps": 60.0, "bytes_per_sent": 128.0, "main_flops_per_sent": 4032000.0, "byte_flops_per_sent": 11010048.0, "total_flops_per_sent": 15042048.0}}, "gini_main_flops": 0.020069076135951214, "gini_total_flops": 0.005608906085311061, "main_flops_parity_vs_ref": {"en": 1.0, "de": 1.0127048706818293, "hi": 0.9242906894218231}}, "t": 1.2}
probe/000000__de.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:13c75cd54c78385f01cd5acde0f7e4b0d57d63a3eff466de99e95b61c174b1c1
3
+ size 7877
probe/000000__en.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:eb5cd6839353bc56e9da6a9ff1964f2118a1043a201444ca61c5ef0aba697e07
3
+ size 7874
probe/000000__hi.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:05ffdfa6810d86a3f2c5ee092f64ff7f0f8e4af531641bb9d103651aa1730d24
3
+ size 7867
probe/000001__de.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5404ca8ad575a0e006c0e790e40cfc105ddfadde9f8004e725a91b87b719eb81
3
+ size 7885
probe/000001__en.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c3718eb3c36a501cfc962ddd28a30ac4423d04aaf7ed053912c6c286860e2665
3
+ size 7864
probe/000001__hi.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:24df7c45a4bc0d2475bab281a66dafd425c4f4faceb5d044aa6bc0c470fa3129
3
+ size 7864
probe/000002__de.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9af851ca9960b04586d95a6d01fb1b069fba6bd8e7fd1edd9fcbe2d9dd3c13e0
3
+ size 7904
probe/000002__en.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ee21e5687cc2a7a8898205231bdd484fbc89294026c12f4caf7918c29b11486d
3
+ size 7869
probe/000002__hi.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2ddd3445d2865fbb62511fc7e67d9db9d8ab52a742354ce4c9ff6d14d576d6cb
3
+ size 7848
probe/000004__de.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1eb51e518de68f3566f332201925fb01164bb5305fc0f67d992df1c9d8096702
3
+ size 7878
probe/000004__en.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e92d9d46a0508cdd7d1f94f3bff912711883fe3c5f0c48df028ebdcf5b6ca69c
3
+ size 7870
probe/000004__hi.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3b659c5c86342b17535dc2ead31007a43e99869b5be68d967e6ec83cf4392da6
3
+ size 7876
probe/000006__de.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3bb1878c369682da79cb6197ef5df4acff5f4bcb39c4b0d1a618c011da89df3a
3
+ size 7889
probe/000006__en.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0682d9201c24b43fdf5b1b23a8cf4064e48a99ad7fd302718d0cc080536f28f1
3
+ size 7858
probe/000006__hi.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8cfccd0b827d729cab9df2385ef0a35e30d67af57407fcd64898f344867b0b59
3
+ size 7882
probe/000008__de.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3aeb7b146ad5e0793529f7da671dd3ac03748ff04c9357e82949cd64796f2d43
3
+ size 7873
probe/000008__en.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9410ad964eb65d20ab3b414b25a2a636cf4b1175b2b6003063771b17744e2884
3
+ size 7890
probe/000008__hi.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:71135f2c33df079ab75bf6a457bd82103761cc2e3be0ba44a4141560d147afdc
3
+ size 7890
probe/000010__de.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6255bdc0fd2c5facce6d20b22934c9233e019617b8ae44c1cc464768921affbf
3
+ size 7887
probe/000010__en.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6c740493f78cb02740ac838ca31653362e3eb5762653b8692aeef6c46f49332e
3
+ size 7858
probe/000010__hi.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3c01a555c4a1b39ae5700c015c00132a78cd84906ab0f0c1ca35bdeb00b47070
3
+ size 7881
probe/000016__de.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1b880749f71248df63cde824f892837d639a18e6dc20717aa071a83fe6d29ca2
3
+ size 7875
probe/000016__en.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:563b3f0e9f037d22377ee7c0972ba600e16e729534ded404ce4ee3246e1d16d0
3
+ size 7877
probe/000016__hi.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:713941852df3ac76cd369f08f8aeb1f7d23f3d6c800c85de44c5174adb6f4e79
3
+ size 7892
probe/000020__de.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:85e79166cc6043e6ffe481fbfa58a1a73c203af98228cde722c4ad038a27c0f3
3
+ size 7874
probe/000020__en.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4a7f7d5655f2eac5ffc77351cf8a827541aa02f46a4dc5d6f0f0db78fbf71b57
3
+ size 7875
probe/000020__hi.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d954bb3f09f128123f7ee2c2f7700b207d70fb7c83e262428d49a4497c757c4a
3
+ size 7889
probe/probe_text.jsonl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ {"lang": "en", "n_sent": 4, "sent_offsets": [0, 128, 256, 384, 512], "texts": ["ܿX\\1٪!3sFﶪ0uw'1t3(t׍((ƖXF;عFX!ѭܩ(#\\Zttڪ0 $ﭤ( tX;$F$Xu'", "|\\ک!XFZ tܖ;s11w|r׈0ѭF&\\wuZ'\\|湖ws\\0FXt!\\ w!ܵX!$w\\t$ Fؤ׏& Ʀ#w(Ѳ", "wܭasFZ؏'a0&1FwZu!Ʋ|0!!ܲ&\\׿Faw cFϦ\\r!ш$ #0\\1a1٦t ڭt$", "$X0#ѭ1э$tѭ1Fٖ!ϵܭ#sƭa aa'Zu$\\c\\F!$&јF $0\\a1ڤ1FttcF($ua\\331"]}
2
+ {"lang": "de", "n_sent": 4, "sent_offsets": [0, 128, 256, 384, 512], "texts": ["?Ž_239sWQo23mQ\\FW3?[.]\\cN\"Nn:N3vR?F9&W?.0?cٟ\"@Q[Um:nnv9?N[&Ÿ槡KNF\"o]3", "2nmW:\"&3sRR\"F2vsc\\3_Ks-s.\\¯9N&]N.\\]Oco2.N[3v?o0mK˧ccv\"oo9N.m왖N]Q..짧?-\\", "\\@:]\"32vmFNRv.oF3&FF:N2UW:FQnާ@Q.WO?n0.FFKnގ9&FO\\sUm@3U-&O-NOR3\\KN\"n", ":F?얧-FRRK3&䠡&F@[NNWK\\&[v\"_Wco0@QFo2U3FU2cv]Wc9_W\\nR_ocFvK\\0c@WRn"]}
3
+ {"lang": "hi", "n_sent": 4, "sent_offsets": [0, 128, 256, 384, 512], "texts": ["w@lIlI@5$s&խB=p[eGZCoNJmgӌC;sqboNoN;sZbiÅ'ZܲJmyOҷp[;svv@vlIlIp[@_3;v_ޯ", "C#WGƗ5oN&}Ɨҷ\"wI{giß\\z;sz@:z{gAޯR4O#WҷxƗJmv\"w̐y#WҷyA@;s&bgIƗƗ", "v@UB=bҷlI􇶩ߡ̳B=lI;slIlIƗ@w@Jm̷xַ̏xߌCvUm􇜩ҷҷ@'%\\\\w@zҷp[zw@lI#W\\bei{", ":%B=\\GC'B=ޯGZ'oNvx{gܲ@U\"w;si@UOlIAҷebeޯҷƗ$sZp[Oޯ%$s@U@UƗiqbUm&Z"]}