suchirsalhan commited on
Commit
90feaa8
·
verified ·
1 Parent(s): c36c068

run records: final.json, log, probes

Browse files
Files changed (1) hide show
  1. log.jsonl +1 -0
log.jsonl CHANGED
@@ -321,3 +321,4 @@
321
  {"event": "eval", "step": 6000, "phase": 3, "bpb": {"en": 1.83968, "tr": 1.99263, "fi": 2.12006}, "bytes_per_chunk": {"en": 4.644, "tr": 5.773, "fi": 4.492}, "bpb_mean": 1.98412, "frozen_inventory": true, "t": 1911.8}
322
  {"event": "reps_dumped", "dirs": ["/local/scratch/sas245/beetle-hnet/runs/ali_mem-split_s1/probe/reps/th0.30", "/local/scratch/sas245/beetle-hnet/runs/ali_mem-split_s1/probe/reps/th0.40", "/local/scratch/sas245/beetle-hnet/runs/ali_mem-split_s1/probe/reps/th0.50", "/local/scratch/sas245/beetle-hnet/runs/ali_mem-split_s1/probe/reps/th0.60", "/local/scratch/sas245/beetle-hnet/runs/ali_mem-split_s1/probe/reps/th0.70"], "t": 1914.4}
323
  {"event": "final", "run": "ali_mem-split_s1", "arch": "ali", "objective": "ratio", "condition": "A", "step": 6000, "params": 22785088, "seed": 1, "per_lang": {"en": {"cpc": 0.20599896569556972, "cps": 28.0078125, "bpb": 1.8429593685735581, "bytes_per_chunk": 4.633320371875994, "reads_per_chunk": 1.8014069072784475}, "tr": {"cpc": 0.18897420317074518, "cps": 26.7265625, "bpb": 1.9840548275014076, "bytes_per_chunk": 5.779634671987477, "reads_per_chunk": 1.590124899408023}, "fi": {"cpc": 0.23475625963531976, "cps": 34.5, "bpb": 2.138425337028634, "bytes_per_chunk": 4.477879129513854, "reads_per_chunk": 1.7251204263605617}}, "bpb_mean": 1.9884798443678668, "gini_cpc": 0.048467436322620516, "gini_cps": 0.058075059826066644, "bpic": 4.966905970933286, "mean_boundary_entropy": 0.11218297481536865, "dechunk": "ema", "router_noise": 0.0, "transitions": [{"step": 0, "phase": 1, "objective": "global", "router_trainable": true, "encoder_trainable": true, "frozen_router_params": 0}, {"step": 2400, "phase": 2, "objective": "global", "router_trainable": true, "encoder_trainable": true, "frozen_router_params": 0}, {"step": 4200, "phase": 3, "objective": "global", "router_trainable": true, "encoder_trainable": true, "frozen_router_params": 0}], "probe_overhead_sec_total": 433.98393273353577, "probe_overhead_seq_frac": 0.4999166666666667, "wall_sec": 1917.5672039985657, "inventory": {"size": 65536, "admitted": 65536, "staged": 0, "budget_B": 65536, "staging_capacity": 0, "collisions": 0, "dropped": 0, "type_rejected": 0, "config_keys": 0, "admitted_surface": 0, "staged_surface": 0, "admitted_split": 65536, "staged_split": 0, "admitted_merge": 0, "staged_merge": 0, "admitted_skel": 0, "staged_skel": 0, "mean_count": 1104.2510986328125, "n_util_obs": 870, "util_count_spearman": 0.04138498818384197}, "ali_params": 268096, "ali_config": {"enabled": true, "budget_B": 65536, "staging_capacity": 262144, "policy": "frequency", "value_mode": "learned", "candidate_types": ["split"], "gate": "vector", "retrieval": true, "memory_scope": "persistent", "read_dropout": 0.1, "write_rule": "mean", "eta_min": 0.0, "consolidate_every": 50, "probe_seqs": 2, "probe_reps": 8, "horizon": 32, "predictor": "mlp", "lambda_util": 0.1, "max_span": 64, "byte_classes_path": "/local/scratch/sas245/beetle-hnet/scratch/data/byte_classes.json", "gate_hidden": 0, "type_dim": 16, "gate_bias_init": -2.0}, "util_vs_frequency_spearman": 0.04138498818384197, "paired_vs_dropout": {"n": 1817, "pearson": 0.08266490306155061, "spearman": 0.07086633564890017, "mean_abs_discrepancy": 0.6155990958213806, "mean_paired": 0.08697881549596786, "mean_dropout": 0.08633054047822952}, "predictor_calibration": {"n": 4058, "all": {"n": 4058, "pearson": 0.13256802448049135, "spearman": 0.16531389590389728, "mae": 0.1590728610754013, "reliability": [{"pred": -0.0028653182089328766, "obs": 0.017683785408735275, "n": 406}, {"pred": 0.013966542668640614, "obs": 0.05502212047576904, "n": 406}, {"pred": 0.022499751299619675, "obs": 0.06273537129163742, "n": 406}, {"pred": 0.02940893918275833, "obs": 0.06503923237323761, "n": 405}, {"pred": 0.035803183913230896, "obs": 0.059896066784858704, "n": 406}, {"pred": 0.04160938039422035, "obs": 0.07249367237091064, "n": 406}, {"pred": 0.04843706637620926, "obs": 0.10839352756738663, "n": 405}, {"pred": 0.056011125445365906, "obs": 0.09082813560962677, "n": 406}, {"pred": 0.06582339853048325, "obs": 0.10861404240131378, "n": 406}, {"pred": 0.08516347408294678, "obs": 0.14782555401325226, "n": 406}]}, "admitted": {"n": 4058, "pearson": 0.13256802448049135, "spearman": 0.16531389590389728, "mae": 0.1590728610754013, "reliability": [{"pred": -0.0028653182089328766, "obs": 0.017683785408735275, "n": 406}, {"pred": 0.013966542668640614, "obs": 0.05502212047576904, "n": 406}, {"pred": 0.022499751299619675, "obs": 0.06273537129163742, "n": 406}, {"pred": 0.02940893918275833, "obs": 0.06503923237323761, "n": 405}, {"pred": 0.035803183913230896, "obs": 0.059896066784858704, "n": 406}, {"pred": 0.04160938039422035, "obs": 0.07249367237091064, "n": 406}, {"pred": 0.04843706637620926, "obs": 0.10839352756738663, "n": 405}, {"pred": 0.056011125445365906, "obs": 0.09082813560962677, "n": 406}, {"pred": 0.06582339853048325, "obs": 0.10861404240131378, "n": 406}, {"pred": 0.08516347408294678, "obs": 0.14782555401325226, "n": 406}]}, "staged": {"n": 0, "pearson": NaN, "spearman": NaN, "mae": NaN}}, "t": 1918.5}
 
 
321
  {"event": "eval", "step": 6000, "phase": 3, "bpb": {"en": 1.83968, "tr": 1.99263, "fi": 2.12006}, "bytes_per_chunk": {"en": 4.644, "tr": 5.773, "fi": 4.492}, "bpb_mean": 1.98412, "frozen_inventory": true, "t": 1911.8}
322
  {"event": "reps_dumped", "dirs": ["/local/scratch/sas245/beetle-hnet/runs/ali_mem-split_s1/probe/reps/th0.30", "/local/scratch/sas245/beetle-hnet/runs/ali_mem-split_s1/probe/reps/th0.40", "/local/scratch/sas245/beetle-hnet/runs/ali_mem-split_s1/probe/reps/th0.50", "/local/scratch/sas245/beetle-hnet/runs/ali_mem-split_s1/probe/reps/th0.60", "/local/scratch/sas245/beetle-hnet/runs/ali_mem-split_s1/probe/reps/th0.70"], "t": 1914.4}
323
  {"event": "final", "run": "ali_mem-split_s1", "arch": "ali", "objective": "ratio", "condition": "A", "step": 6000, "params": 22785088, "seed": 1, "per_lang": {"en": {"cpc": 0.20599896569556972, "cps": 28.0078125, "bpb": 1.8429593685735581, "bytes_per_chunk": 4.633320371875994, "reads_per_chunk": 1.8014069072784475}, "tr": {"cpc": 0.18897420317074518, "cps": 26.7265625, "bpb": 1.9840548275014076, "bytes_per_chunk": 5.779634671987477, "reads_per_chunk": 1.590124899408023}, "fi": {"cpc": 0.23475625963531976, "cps": 34.5, "bpb": 2.138425337028634, "bytes_per_chunk": 4.477879129513854, "reads_per_chunk": 1.7251204263605617}}, "bpb_mean": 1.9884798443678668, "gini_cpc": 0.048467436322620516, "gini_cps": 0.058075059826066644, "bpic": 4.966905970933286, "mean_boundary_entropy": 0.11218297481536865, "dechunk": "ema", "router_noise": 0.0, "transitions": [{"step": 0, "phase": 1, "objective": "global", "router_trainable": true, "encoder_trainable": true, "frozen_router_params": 0}, {"step": 2400, "phase": 2, "objective": "global", "router_trainable": true, "encoder_trainable": true, "frozen_router_params": 0}, {"step": 4200, "phase": 3, "objective": "global", "router_trainable": true, "encoder_trainable": true, "frozen_router_params": 0}], "probe_overhead_sec_total": 433.98393273353577, "probe_overhead_seq_frac": 0.4999166666666667, "wall_sec": 1917.5672039985657, "inventory": {"size": 65536, "admitted": 65536, "staged": 0, "budget_B": 65536, "staging_capacity": 0, "collisions": 0, "dropped": 0, "type_rejected": 0, "config_keys": 0, "admitted_surface": 0, "staged_surface": 0, "admitted_split": 65536, "staged_split": 0, "admitted_merge": 0, "staged_merge": 0, "admitted_skel": 0, "staged_skel": 0, "mean_count": 1104.2510986328125, "n_util_obs": 870, "util_count_spearman": 0.04138498818384197}, "ali_params": 268096, "ali_config": {"enabled": true, "budget_B": 65536, "staging_capacity": 262144, "policy": "frequency", "value_mode": "learned", "candidate_types": ["split"], "gate": "vector", "retrieval": true, "memory_scope": "persistent", "read_dropout": 0.1, "write_rule": "mean", "eta_min": 0.0, "consolidate_every": 50, "probe_seqs": 2, "probe_reps": 8, "horizon": 32, "predictor": "mlp", "lambda_util": 0.1, "max_span": 64, "byte_classes_path": "/local/scratch/sas245/beetle-hnet/scratch/data/byte_classes.json", "gate_hidden": 0, "type_dim": 16, "gate_bias_init": -2.0}, "util_vs_frequency_spearman": 0.04138498818384197, "paired_vs_dropout": {"n": 1817, "pearson": 0.08266490306155061, "spearman": 0.07086633564890017, "mean_abs_discrepancy": 0.6155990958213806, "mean_paired": 0.08697881549596786, "mean_dropout": 0.08633054047822952}, "predictor_calibration": {"n": 4058, "all": {"n": 4058, "pearson": 0.13256802448049135, "spearman": 0.16531389590389728, "mae": 0.1590728610754013, "reliability": [{"pred": -0.0028653182089328766, "obs": 0.017683785408735275, "n": 406}, {"pred": 0.013966542668640614, "obs": 0.05502212047576904, "n": 406}, {"pred": 0.022499751299619675, "obs": 0.06273537129163742, "n": 406}, {"pred": 0.02940893918275833, "obs": 0.06503923237323761, "n": 405}, {"pred": 0.035803183913230896, "obs": 0.059896066784858704, "n": 406}, {"pred": 0.04160938039422035, "obs": 0.07249367237091064, "n": 406}, {"pred": 0.04843706637620926, "obs": 0.10839352756738663, "n": 405}, {"pred": 0.056011125445365906, "obs": 0.09082813560962677, "n": 406}, {"pred": 0.06582339853048325, "obs": 0.10861404240131378, "n": 406}, {"pred": 0.08516347408294678, "obs": 0.14782555401325226, "n": 406}]}, "admitted": {"n": 4058, "pearson": 0.13256802448049135, "spearman": 0.16531389590389728, "mae": 0.1590728610754013, "reliability": [{"pred": -0.0028653182089328766, "obs": 0.017683785408735275, "n": 406}, {"pred": 0.013966542668640614, "obs": 0.05502212047576904, "n": 406}, {"pred": 0.022499751299619675, "obs": 0.06273537129163742, "n": 406}, {"pred": 0.02940893918275833, "obs": 0.06503923237323761, "n": 405}, {"pred": 0.035803183913230896, "obs": 0.059896066784858704, "n": 406}, {"pred": 0.04160938039422035, "obs": 0.07249367237091064, "n": 406}, {"pred": 0.04843706637620926, "obs": 0.10839352756738663, "n": 405}, {"pred": 0.056011125445365906, "obs": 0.09082813560962677, "n": 406}, {"pred": 0.06582339853048325, "obs": 0.10861404240131378, "n": 406}, {"pred": 0.08516347408294678, "obs": 0.14782555401325226, "n": 406}]}, "staged": {"n": 0, "pearson": NaN, "spearman": NaN, "mae": NaN}}, "t": 1918.5}
324
+ {"event": "hub_done", "repo": "AdaptiveChunking/ali_mem-split_s1", "pushed": [0, 1, 2, 4, 6, 8, 16, 32, 64, 128, 256, 512, 1000, 1024, 2048, 3000, 4096, 6000], "failed": [], "ok": true, "t": 1932.8}