Spaces:
Sleeping
feat: add OLMoE-1B-7B-0924 training dynamics tab
Browse filesNew files:
core/olmoe_scanner.py OLMoE checkpoint scanner (386 lines)
ui/tab_olmoe.py Training dynamics tab UI (291 lines)
Modified:
app.py +1 import, +1 tab registration
Key differences from pythia_scanner.py:
- Q/K stored separately (q_proj/k_proj), not merged QKV
- BF16 dtype
- 3-shard safetensors, index.json routing
- branch format: step{N}-tokens{M}B (not step{N})
- x-axis: tokens_B (aligns with arxiv 2506.21551 grokking intervals)
- Grokking regions annotated: Common sense/Code/Math
Model: allenai/OLMoE-1B-7B-0924
16 layers, 16 heads, d_model=2048, d_head=128
244 checkpoints, step5000-step1220000 (5000-step intervals)
All checkpoints verified available (status=200)
Reference: arxiv 2506.21551 (ICLR 2026)
Grokking confirmed in OLMoE pretraining
Used as theoretical backing for fig3 (Q3 quadrant)
- app.py +2 -0
- core/olmoe_scanner.py +386 -0
- ui/tab_olmoe.py +291 -0
|
@@ -13,6 +13,7 @@ from ui.tab_database import build_tab_database
|
|
| 13 |
from ui.tab_plot import build_tab_plot
|
| 14 |
from ui.tab_tables import build_tab_tables
|
| 15 |
from ui.tab_pythia import build_tab_pythia
|
|
|
|
| 16 |
|
| 17 |
# ไธดๆถ๏ผๅ ้ค้่ฏฏ็2.8bๆฐๆฎ๏ผ่ทไธๆฌกๅๅ ๆ่ฟไธค่ก
|
| 18 |
import os
|
|
@@ -93,6 +94,7 @@ with gr.Blocks(
|
|
| 93 |
build_tab_plot()
|
| 94 |
build_tab_tables()
|
| 95 |
build_tab_pythia()
|
|
|
|
| 96 |
|
| 97 |
# โโ Tab1 โ Tab2 ่ๅจ โโโโโโโโโโโโโโโโโโโโโโ
|
| 98 |
inspect_model_id.change(
|
|
|
|
| 13 |
from ui.tab_plot import build_tab_plot
|
| 14 |
from ui.tab_tables import build_tab_tables
|
| 15 |
from ui.tab_pythia import build_tab_pythia
|
| 16 |
+
from ui.tab_olmoe import build_tab_olmoe
|
| 17 |
|
| 18 |
# ไธดๆถ๏ผๅ ้ค้่ฏฏ็2.8bๆฐๆฎ๏ผ่ทไธๆฌกๅๅ ๆ่ฟไธค่ก
|
| 19 |
import os
|
|
|
|
| 94 |
build_tab_plot()
|
| 95 |
build_tab_tables()
|
| 96 |
build_tab_pythia()
|
| 97 |
+
build_tab_olmoe()
|
| 98 |
|
| 99 |
# โโ Tab1 โ Tab2 ่ๅจ โโโโโโโโโโโโโโโโโโโโโโ
|
| 100 |
inspect_model_id.change(
|
|
@@ -0,0 +1,386 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# core/olmoe_scanner.py
|
| 2 |
+
"""
|
| 3 |
+
OLMoE Checkpoint SSR/UniIso ๅจๆๆซๆ
|
| 4 |
+
======================================
|
| 5 |
+
ๆซๆ allenai/OLMoE-1B-7B-0924 ็ๅคไธช่ฎญ็ป checkpoint๏ผ
|
| 6 |
+
่ฎก็ฎๆฏๅฑๆฏๅคด็ SSRใUniIsoใeff_rank ็ญๆๆ ใ
|
| 7 |
+
|
| 8 |
+
ๆถๆ๏ผๆ ๅMHA๏ผQ/Kๅๅผๅญๅจ๏ผBF16๏ผ๏ผ
|
| 9 |
+
OLMoE-1B-7B-0924: 16ๅฑ 16ๅคด d_model=2048 d_head=128
|
| 10 |
+
|
| 11 |
+
ๅ pythia_scanner.py ็ๅ
ณ้ฎๅบๅซ๏ผ
|
| 12 |
+
Pythia๏ผquery_key_valueๅๅนถ [3*d_model, d_model]
|
| 13 |
+
OLMoE๏ผ q_proj/k_projๅๅผ [d_model, d_model] ร 2
|
| 14 |
+
Pythia๏ผbranch = step{N}
|
| 15 |
+
OLMoE๏ผ branch = step{N}-tokens{M}B
|
| 16 |
+
|
| 17 |
+
่พๅบ๏ผ
|
| 18 |
+
/data/olmoe_{model_name}_ssr_{run_ts}.csv
|
| 19 |
+
ๆๆ ๅฎไนไธ pythia_scanner.py ๅฎๅ
จๅฏน้ฝ
|
| 20 |
+
"""
|
| 21 |
+
|
| 22 |
+
import math
|
| 23 |
+
import time
|
| 24 |
+
import os
|
| 25 |
+
import csv
|
| 26 |
+
import re
|
| 27 |
+
import numpy as np
|
| 28 |
+
import torch
|
| 29 |
+
from datetime import datetime
|
| 30 |
+
from core.fetcher import read_safetensors_header, load_tensors_batch
|
| 31 |
+
from core.debug import dprint
|
| 32 |
+
|
| 33 |
+
# โโ OLMoE ๆจกๅ้
็ฝฎ โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
|
| 34 |
+
OLMOE_CONFIGS = {
|
| 35 |
+
"OLMoE-1B-7B-0924": {
|
| 36 |
+
"model_id": "allenai/OLMoE-1B-7B-0924",
|
| 37 |
+
"n_layers": 16,
|
| 38 |
+
"n_heads": 16,
|
| 39 |
+
"d_model": 2048,
|
| 40 |
+
"d_head": 128,
|
| 41 |
+
"n_kv_heads": 16, # MHA๏ผKVๅคดๆฐ=Qๅคดๆฐ
|
| 42 |
+
"dtype": "BF16",
|
| 43 |
+
"q_key_fmt": "model.layers.{l}.self_attn.q_proj.weight",
|
| 44 |
+
"k_key_fmt": "model.layers.{l}.self_attn.k_proj.weight",
|
| 45 |
+
"n_shards": 3, # model-0000{1,2,3}-of-00003.safetensors
|
| 46 |
+
},
|
| 47 |
+
}
|
| 48 |
+
|
| 49 |
+
DATA_DIR = "/data"
|
| 50 |
+
ENERGY_THRESHOLD = 0.90
|
| 51 |
+
|
| 52 |
+
# ้ป่ฎคๆซๆๆญฅ้ชค๏ผๅๅ้ๆ ท20ไธช๏ผ่ฆ็ๅฎๆด่ฎญ็ป
|
| 53 |
+
# ๅฎ้
branchๅๅจ่ฟ่กๆถๅจๆ่งฃๆ๏ผstepโbranchๆ ๅฐ๏ผ
|
| 54 |
+
DEFAULT_STEPS_OLMOE = [
|
| 55 |
+
5000, 25000, 50000, 75000, 100000,
|
| 56 |
+
150000, 200000, 300000, 400000, 500000,
|
| 57 |
+
600000, 700000, 800000, 900000, 1000000,
|
| 58 |
+
1050000, 1100000, 1150000, 1200000, 1220000,
|
| 59 |
+
]
|
| 60 |
+
|
| 61 |
+
# โโ ๆๆ ๅฝๆฐ๏ผไธ pythia_scanner.py ๅฎๅ
จๅฏน้ฝ๏ผโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
|
| 62 |
+
|
| 63 |
+
def effective_rank(sv: np.ndarray, threshold: float = ENERGY_THRESHOLD) -> int:
|
| 64 |
+
sv2 = sv ** 2
|
| 65 |
+
total = sv2.sum()
|
| 66 |
+
if total < 1e-12:
|
| 67 |
+
return 1
|
| 68 |
+
cumvar = np.cumsum(sv2) / total
|
| 69 |
+
return int(np.searchsorted(cumvar, threshold) + 1)
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def uni_iso(W: np.ndarray) -> float:
|
| 73 |
+
G = W @ W.T
|
| 74 |
+
G_f = np.linalg.norm(G, 'fro') + 1e-12
|
| 75 |
+
d_h = W.shape[0]
|
| 76 |
+
I_f = math.sqrt(d_h)
|
| 77 |
+
return float(np.linalg.norm(G / G_f - np.eye(d_h) / I_f, 'fro'))
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def theory_uniiso(d_h: int, k: int) -> float:
|
| 81 |
+
return math.sqrt(max(0.0, 2.0 - 2.0 * math.sqrt(k) / math.sqrt(d_h)))
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
def sv_entropy(sv: np.ndarray) -> float:
|
| 85 |
+
p = sv / (sv.sum() + 1e-12)
|
| 86 |
+
return float(-np.sum(p * np.log(p + 1e-12)))
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
def compute_ssr(sq: np.ndarray, sk: np.ndarray) -> float:
|
| 90 |
+
n = min(len(sq), len(sk))
|
| 91 |
+
sq = sq[:n] / (np.linalg.norm(sq[:n]) + 1e-12)
|
| 92 |
+
sk = sk[:n] / (np.linalg.norm(sk[:n]) + 1e-12)
|
| 93 |
+
return float(np.mean(np.abs(sq - sk)))
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
# โโ Branch ๆ ๅฐ๏ผstep โ branchๅ๏ผโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
|
| 97 |
+
|
| 98 |
+
def build_step_branch_map(model_id: str) -> dict:
|
| 99 |
+
"""
|
| 100 |
+
ไปHFๆๅๆๆbranch๏ผๅปบ็ซ {step: branch_name} ๆ ๅฐใ
|
| 101 |
+
branchๆ ผๅผ๏ผstep{N}-tokens{M}B
|
| 102 |
+
"""
|
| 103 |
+
from huggingface_hub import list_repo_refs
|
| 104 |
+
refs = list_repo_refs(model_id)
|
| 105 |
+
step_map = {}
|
| 106 |
+
for b in refs.branches:
|
| 107 |
+
m = re.match(r'step(\d+)-tokens(\d+)B', b.name)
|
| 108 |
+
if m:
|
| 109 |
+
step = int(m.group(1))
|
| 110 |
+
step_map[step] = b.name
|
| 111 |
+
dprint(f"[OLMOE] {len(step_map)} checkpoints found")
|
| 112 |
+
return step_map
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
# โโ ๅ็ๅค็ โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
|
| 116 |
+
|
| 117 |
+
def get_shard_urls(model_id: str, branch: str, cfg: dict) -> dict:
|
| 118 |
+
"""
|
| 119 |
+
่ฏปๅ model.safetensors.index.json๏ผ่ฟๅ
|
| 120 |
+
{shard_filename: (url, header, header_size)}
|
| 121 |
+
ๅชๅ ่ฝฝๅ
ๅซQKๆ้็ๅ็ใ
|
| 122 |
+
"""
|
| 123 |
+
import requests
|
| 124 |
+
n_layers = cfg["n_layers"]
|
| 125 |
+
|
| 126 |
+
# ๆถ้ๆๆ้่ฆ็key
|
| 127 |
+
needed_keys = set()
|
| 128 |
+
for l in range(n_layers):
|
| 129 |
+
needed_keys.add(cfg["q_key_fmt"].format(l=l))
|
| 130 |
+
needed_keys.add(cfg["k_key_fmt"].format(l=l))
|
| 131 |
+
|
| 132 |
+
# ่ฏปindex.json
|
| 133 |
+
idx_url = (f"https://huggingface.co/{model_id}"
|
| 134 |
+
f"/resolve/{branch}/model.safetensors.index.json")
|
| 135 |
+
r = requests.get(idx_url, timeout=30)
|
| 136 |
+
r.raise_for_status()
|
| 137 |
+
wmap = r.json()["weight_map"]
|
| 138 |
+
|
| 139 |
+
# ๆๅ็ๅ็ป
|
| 140 |
+
shard_to_keys = {}
|
| 141 |
+
for k in needed_keys:
|
| 142 |
+
shard = wmap.get(k)
|
| 143 |
+
if shard:
|
| 144 |
+
shard_to_keys.setdefault(shard, []).append(k)
|
| 145 |
+
|
| 146 |
+
# ่ฏปๅๅ็header
|
| 147 |
+
shard_info = {}
|
| 148 |
+
for shard_name, keys in shard_to_keys.items():
|
| 149 |
+
url = (f"https://huggingface.co/{model_id}"
|
| 150 |
+
f"/resolve/{branch}/{shard_name}")
|
| 151 |
+
header, header_size = read_safetensors_header(url)
|
| 152 |
+
shard_info[shard_name] = (url, header, header_size, keys)
|
| 153 |
+
|
| 154 |
+
return shard_info
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
# โโ ๅไธช checkpoint ๆซๆ โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
|
| 158 |
+
|
| 159 |
+
def scan_checkpoint_olmoe(
|
| 160 |
+
model_id: str,
|
| 161 |
+
branch: str,
|
| 162 |
+
step: int,
|
| 163 |
+
cfg: dict,
|
| 164 |
+
token: str = None,
|
| 165 |
+
) -> list:
|
| 166 |
+
"""
|
| 167 |
+
ๆซๆไธไธชOLMoE checkpoint็ๆๆๅฑๆๆๅคดใ
|
| 168 |
+
QๅKๅๅผ่ฏปๅ๏ผๅ่ช่ฎก็ฎๆๆ ๅๅๅนถใ
|
| 169 |
+
่ฟๅ list of dict๏ผๆฏไธชdictๅฏนๅบไธไธช(layer, head)ใ
|
| 170 |
+
"""
|
| 171 |
+
n_layers = cfg["n_layers"]
|
| 172 |
+
n_heads = cfg["n_heads"]
|
| 173 |
+
d_head = cfg["d_head"]
|
| 174 |
+
d_model = cfg["d_model"]
|
| 175 |
+
|
| 176 |
+
dprint(f"[SCAN] step={step} branch={branch}")
|
| 177 |
+
|
| 178 |
+
# ่ฏปๅ็ไฟกๆฏ
|
| 179 |
+
shard_info = get_shard_urls(model_id, branch, cfg)
|
| 180 |
+
|
| 181 |
+
# ไปๅๅ็ๆน้่ฏปๅQKๆ้
|
| 182 |
+
all_tensors = {}
|
| 183 |
+
for shard_name, (url, header, header_size, keys) in shard_info.items():
|
| 184 |
+
t = load_tensors_batch(url, keys, header, header_size, token=token)
|
| 185 |
+
all_tensors.update(t)
|
| 186 |
+
|
| 187 |
+
records = []
|
| 188 |
+
for layer in range(n_layers):
|
| 189 |
+
q_key = cfg["q_key_fmt"].format(l=layer)
|
| 190 |
+
k_key = cfg["k_key_fmt"].format(l=layer)
|
| 191 |
+
|
| 192 |
+
if q_key not in all_tensors or k_key not in all_tensors:
|
| 193 |
+
dprint(f"[SCAN] layer {layer} ็ผบๅคฑQK๏ผ่ทณ่ฟ")
|
| 194 |
+
continue
|
| 195 |
+
|
| 196 |
+
# shape: [d_model, d_model] โ ๆheadๅ็
|
| 197 |
+
# OLMoE็q_proj: [n_heads*d_head, d_model]
|
| 198 |
+
W_Q = all_tensors[q_key].numpy() # [2048, 2048]
|
| 199 |
+
W_K = all_tensors[k_key].numpy()
|
| 200 |
+
|
| 201 |
+
for head in range(n_heads):
|
| 202 |
+
# ๅๅบๅไธชhead
|
| 203 |
+
Wq = W_Q[head*d_head:(head+1)*d_head, :] # [128, 2048]
|
| 204 |
+
Wk = W_K[head*d_head:(head+1)*d_head, :]
|
| 205 |
+
|
| 206 |
+
sq = np.linalg.svd(Wq, compute_uv=False)
|
| 207 |
+
sk = np.linalg.svd(Wk, compute_uv=False)
|
| 208 |
+
|
| 209 |
+
q_eff_rank = effective_rank(sq)
|
| 210 |
+
q_uni_iso = uni_iso(Wq)
|
| 211 |
+
q_theory = theory_uniiso(d_head, q_eff_rank)
|
| 212 |
+
q_gap = q_uni_iso - q_theory
|
| 213 |
+
q_ssr = compute_ssr(sq, sk)
|
| 214 |
+
q_sv_ent = sv_entropy(sq)
|
| 215 |
+
q_sv_maxr = float(sq[0] / (sq[1] + 1e-12))
|
| 216 |
+
|
| 217 |
+
k_eff_rank = effective_rank(sk)
|
| 218 |
+
k_uni_iso = uni_iso(Wk)
|
| 219 |
+
k_sv_ent = sv_entropy(sk)
|
| 220 |
+
|
| 221 |
+
records.append({
|
| 222 |
+
"layer": layer,
|
| 223 |
+
"head": head,
|
| 224 |
+
"Q_eff_rank": q_eff_rank,
|
| 225 |
+
"Q_uni_iso": round(q_uni_iso, 6),
|
| 226 |
+
"Q_theory_uniiso": round(q_theory, 6),
|
| 227 |
+
"Q_gap": round(q_gap, 6),
|
| 228 |
+
"ssr": round(q_ssr, 8),
|
| 229 |
+
"Q_sv_entropy": round(q_sv_ent, 4),
|
| 230 |
+
"Q_sv_max_ratio": round(q_sv_maxr, 4),
|
| 231 |
+
"Q_sv1": round(float(sq[0]), 4),
|
| 232 |
+
"Q_sv2": round(float(sq[1]), 4),
|
| 233 |
+
"Q_sv3": round(float(sq[2]), 4),
|
| 234 |
+
"K_eff_rank": k_eff_rank,
|
| 235 |
+
"K_uni_iso": round(k_uni_iso, 6),
|
| 236 |
+
"K_sv_entropy": round(k_sv_ent, 4),
|
| 237 |
+
})
|
| 238 |
+
|
| 239 |
+
del all_tensors[q_key], all_tensors[k_key]
|
| 240 |
+
|
| 241 |
+
return records
|
| 242 |
+
|
| 243 |
+
|
| 244 |
+
# โโ CSVๅญๆฎต โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
|
| 245 |
+
|
| 246 |
+
CSV_FIELDS = [
|
| 247 |
+
"run_ts", "model", "step", "branch", "tokens_B", "layer", "head",
|
| 248 |
+
"Q_eff_rank", "Q_uni_iso", "Q_theory_uniiso", "Q_gap",
|
| 249 |
+
"ssr", "Q_sv_entropy", "Q_sv_max_ratio",
|
| 250 |
+
"Q_sv1", "Q_sv2", "Q_sv3",
|
| 251 |
+
"K_eff_rank", "K_uni_iso", "K_sv_entropy",
|
| 252 |
+
]
|
| 253 |
+
|
| 254 |
+
|
| 255 |
+
# โโ ๆญ็น็ปญ่ท โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
|
| 256 |
+
|
| 257 |
+
def load_done_steps(csv_path: str) -> set:
|
| 258 |
+
if not os.path.exists(csv_path):
|
| 259 |
+
return set()
|
| 260 |
+
done = set()
|
| 261 |
+
try:
|
| 262 |
+
with open(csv_path, "r", newline="") as f:
|
| 263 |
+
for row in csv.DictReader(f):
|
| 264 |
+
done.add(int(row["step"]))
|
| 265 |
+
except Exception:
|
| 266 |
+
pass
|
| 267 |
+
return done
|
| 268 |
+
|
| 269 |
+
|
| 270 |
+
# โโ ไธปๆซๆๅฝๆฐ โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
|
| 271 |
+
|
| 272 |
+
def scan_olmoe(
|
| 273 |
+
model_name: str = "OLMoE-1B-7B-0924",
|
| 274 |
+
steps: list = None,
|
| 275 |
+
token: str = None,
|
| 276 |
+
progress_fn=None,
|
| 277 |
+
) -> str:
|
| 278 |
+
"""
|
| 279 |
+
ๆซๆOLMoEๆๆๆๅฎcheckpointใ
|
| 280 |
+
|
| 281 |
+
ๅๆฐ๏ผ
|
| 282 |
+
model_name : OLMOE_CONFIGS็key
|
| 283 |
+
steps : ่ฆๆซๆ็stepๅ่กจ๏ผ้ป่ฎคDEFAULT_STEPS_OLMOE
|
| 284 |
+
token : HF token๏ผOLMoEๆฏๅ
ฌๅผๆจกๅ๏ผ้ๅธธไธ้่ฆ๏ผ
|
| 285 |
+
progress_fn : Gradio่ฟๅบฆๅ่ฐ
|
| 286 |
+
|
| 287 |
+
่ฟๅ๏ผcsv_path
|
| 288 |
+
"""
|
| 289 |
+
if steps is None:
|
| 290 |
+
steps = DEFAULT_STEPS_OLMOE
|
| 291 |
+
|
| 292 |
+
cfg = OLMOE_CONFIGS[model_name]
|
| 293 |
+
model_id = cfg["model_id"]
|
| 294 |
+
run_ts = datetime.now().strftime("%Y%m%d_%H%M%S")
|
| 295 |
+
|
| 296 |
+
os.makedirs(DATA_DIR, exist_ok=True)
|
| 297 |
+
|
| 298 |
+
# ๅปบ็ซstepโbranchๆ ๅฐ
|
| 299 |
+
print(f"[OLMOE] ่ทๅcheckpointๅ่กจ...", flush=True)
|
| 300 |
+
step_map = build_step_branch_map(model_id)
|
| 301 |
+
available = set(step_map.keys())
|
| 302 |
+
steps_valid = [s for s in steps if s in available]
|
| 303 |
+
steps_missing = [s for s in steps if s not in available]
|
| 304 |
+
if steps_missing:
|
| 305 |
+
print(f"[OLMOE] ไปฅไธstepไธๅญๅจ๏ผ่ทณ่ฟ๏ผ{steps_missing}", flush=True)
|
| 306 |
+
|
| 307 |
+
# ๆญ็น็ปญ่ท
|
| 308 |
+
existing = sorted([
|
| 309 |
+
f for f in os.listdir(DATA_DIR)
|
| 310 |
+
if f.startswith(f"olmoe_{model_name}_ssr_") and f.endswith(".csv")
|
| 311 |
+
])
|
| 312 |
+
if existing:
|
| 313 |
+
csv_path = os.path.join(DATA_DIR, existing[-1])
|
| 314 |
+
done_steps = load_done_steps(csv_path)
|
| 315 |
+
steps_todo = [s for s in steps_valid if s not in done_steps]
|
| 316 |
+
print(f"[OLMOE] ็ปญ่ท๏ผๅทฒๅฎๆ{len(done_steps)}ๆญฅ๏ผๅพ
่ท{len(steps_todo)}ๆญฅ",
|
| 317 |
+
flush=True)
|
| 318 |
+
file_mode = "a"
|
| 319 |
+
else:
|
| 320 |
+
csv_path = os.path.join(DATA_DIR,
|
| 321 |
+
f"olmoe_{model_name}_ssr_{run_ts}.csv")
|
| 322 |
+
done_steps = set()
|
| 323 |
+
steps_todo = steps_valid
|
| 324 |
+
file_mode = "w"
|
| 325 |
+
|
| 326 |
+
if not steps_todo:
|
| 327 |
+
print("[OLMOE] ๆๆstepๅทฒๅฎๆ", flush=True)
|
| 328 |
+
return csv_path
|
| 329 |
+
|
| 330 |
+
with open(csv_path, file_mode, newline="") as f:
|
| 331 |
+
writer = csv.DictWriter(f, fieldnames=CSV_FIELDS)
|
| 332 |
+
if file_mode == "w":
|
| 333 |
+
writer.writeheader()
|
| 334 |
+
|
| 335 |
+
total = len(steps_todo)
|
| 336 |
+
for idx, step in enumerate(steps_todo):
|
| 337 |
+
branch = step_map[step]
|
| 338 |
+
tokens_B = int(re.search(r'tokens(\d+)B', branch).group(1))
|
| 339 |
+
|
| 340 |
+
print(f"\n[OLMOE] === step {step} ({idx+1}/{total}) "
|
| 341 |
+
f"branch={branch} ===", flush=True)
|
| 342 |
+
t0 = time.time()
|
| 343 |
+
|
| 344 |
+
try:
|
| 345 |
+
records = scan_checkpoint_olmoe(
|
| 346 |
+
model_id, branch, step, cfg, token=token)
|
| 347 |
+
except Exception as e:
|
| 348 |
+
print(f"[OLMOE] step {step} ๅคฑ่ดฅ: {e}", flush=True)
|
| 349 |
+
if progress_fn:
|
| 350 |
+
progress_fn(idx+1, total, f"step {step} ๅคฑ่ดฅ: {e}")
|
| 351 |
+
continue
|
| 352 |
+
|
| 353 |
+
for rec in records:
|
| 354 |
+
row = {
|
| 355 |
+
"run_ts": run_ts,
|
| 356 |
+
"model": model_id,
|
| 357 |
+
"step": step,
|
| 358 |
+
"branch": branch,
|
| 359 |
+
"tokens_B": tokens_B,
|
| 360 |
+
}
|
| 361 |
+
row.update(rec)
|
| 362 |
+
writer.writerow(row)
|
| 363 |
+
f.flush()
|
| 364 |
+
|
| 365 |
+
elapsed = time.time() - t0
|
| 366 |
+
if records:
|
| 367 |
+
import pandas as pd
|
| 368 |
+
df_s = pd.DataFrame(records)
|
| 369 |
+
pb = df_s.groupby("layer")["Q_uni_iso"].median()
|
| 370 |
+
ui_m = float(pb.median())
|
| 371 |
+
rk_m = float(df_s.groupby("layer")["Q_eff_rank"]
|
| 372 |
+
.median().median())
|
| 373 |
+
ssr_m = float(df_s.groupby("layer")["ssr"].median().median())
|
| 374 |
+
print(f"[OLMOE] {len(records)} heads "
|
| 375 |
+
f"Q_uni_iso(pb)={ui_m:.4f} "
|
| 376 |
+
f"Q_eff_rank(pb)={rk_m:.1f} "
|
| 377 |
+
f"ssr(pb)={ssr_m:.6f} "
|
| 378 |
+
f"่ๆถ={elapsed:.1f}s", flush=True)
|
| 379 |
+
|
| 380 |
+
if progress_fn:
|
| 381 |
+
progress_fn(idx+1, total,
|
| 382 |
+
f"step {step} ({tokens_B}B tokens) ๅฎๆ "
|
| 383 |
+
f"่ๆถ={elapsed:.1f}s")
|
| 384 |
+
|
| 385 |
+
print(f"\n[OLMOE] ๅฎๆ๏ผ่พๅบ: {csv_path}")
|
| 386 |
+
return csv_path
|
|
@@ -0,0 +1,291 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# ui/tab_olmoe.py
|
| 2 |
+
"""
|
| 3 |
+
Tab: OLMoE Training Dynamics
|
| 4 |
+
================================
|
| 5 |
+
ๆซๆ allenai/OLMoE-1B-7B-0924 ็่ฎญ็ป checkpoint๏ผ
|
| 6 |
+
ๅฑ็คบ SSR / UniIso / eff_rank ้่ฎญ็ปๆญฅๆฐ๏ผtokensๆฐ๏ผ็ๅจๆๅๅใ
|
| 7 |
+
|
| 8 |
+
ๅ tab_pythia.py ็ๅบๅซ๏ผ
|
| 9 |
+
x่ฝด็จtokens_B๏ผๆดๆ็ฉ็ๆไน๏ผๅgrokkingๆ็ฎๅฏน้ฝ๏ผ
|
| 10 |
+
ๆ ๆณจๆฅ่ช arxiv 2506.21551 ็ grokking ๅบ้ด
|
| 11 |
+
"""
|
| 12 |
+
|
| 13 |
+
import os
|
| 14 |
+
import gradio as gr
|
| 15 |
+
import pandas as pd
|
| 16 |
+
import numpy as np
|
| 17 |
+
import plotly.graph_objects as go
|
| 18 |
+
import plotly.express as px
|
| 19 |
+
|
| 20 |
+
from core.olmoe_scanner import (
|
| 21 |
+
scan_olmoe, OLMOE_CONFIGS, DEFAULT_STEPS_OLMOE, DATA_DIR
|
| 22 |
+
)
|
| 23 |
+
|
| 24 |
+
# โโ grokkingๅบ้ด๏ผๆฅ่ชarxiv 2506.21551๏ผโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
|
| 25 |
+
# ๅไฝ๏ผtokens๏ผB๏ผ
|
| 26 |
+
GROKKING_REGIONS = {
|
| 27 |
+
"Common sense": (210, 420, "#2166AC"), # step50K-100K
|
| 28 |
+
"Code": (420, 840, "#D6604D"), # step100K-200K
|
| 29 |
+
"Math": (840, 1680, "#762A83"), # step200K-400K
|
| 30 |
+
}
|
| 31 |
+
|
| 32 |
+
# โโ ๆๆ ๆ ๅฐ โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
|
| 33 |
+
METRICS = {
|
| 34 |
+
"Q_uni_iso": "UniIso (Q)",
|
| 35 |
+
"Q_eff_rank": "eff_rank (Q)",
|
| 36 |
+
"ssr": "SSR (Q vs K)",
|
| 37 |
+
"Q_sv_max_ratio": "sv_max_ratio ฯโ/ฯโ (Q)",
|
| 38 |
+
"Q_sv_entropy": "sv_entropy (Q)",
|
| 39 |
+
"K_uni_iso": "UniIso (K)",
|
| 40 |
+
}
|
| 41 |
+
|
| 42 |
+
# โโ ็ปๅพๅฝๆฐ โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
|
| 43 |
+
|
| 44 |
+
def plot_training_curves(
|
| 45 |
+
df: pd.DataFrame,
|
| 46 |
+
metric: str,
|
| 47 |
+
show_grokking: bool = True,
|
| 48 |
+
) -> go.Figure:
|
| 49 |
+
"""
|
| 50 |
+
่ฎญ็ปๅจๆๆ็บฟๅพใ
|
| 51 |
+
x่ฝด๏ผtokens_B๏ผๅgrokkingๆ็ฎๅฏน้ฝ๏ผ
|
| 52 |
+
pseudo-bulk๏ผๆฏๅฑ16ๅคดๅmedian
|
| 53 |
+
ๆ ๆณจgrokkingๅบ้ด๏ผๆฅ่ชarxiv 2506.21551๏ผ
|
| 54 |
+
"""
|
| 55 |
+
if df.empty or metric not in df.columns:
|
| 56 |
+
return go.Figure().update_layout(title="ๆ ๆฐๆฎ")
|
| 57 |
+
|
| 58 |
+
cfg = list(OLMOE_CONFIGS.values())[0]
|
| 59 |
+
n_layers = cfg["n_layers"]
|
| 60 |
+
colors = px.colors.sample_colorscale("Viridis", n_layers)
|
| 61 |
+
|
| 62 |
+
# pseudo-bulk
|
| 63 |
+
pb = df.groupby(["tokens_B", "layer"])[metric].median().reset_index()
|
| 64 |
+
|
| 65 |
+
fig = go.Figure()
|
| 66 |
+
|
| 67 |
+
# grokkingๅบ้ดๆ ๆณจ
|
| 68 |
+
if show_grokking:
|
| 69 |
+
for domain, (t_start, t_end, color) in GROKKING_REGIONS.items():
|
| 70 |
+
fig.add_vrect(
|
| 71 |
+
x0=t_start, x1=t_end,
|
| 72 |
+
fillcolor=color, opacity=0.08,
|
| 73 |
+
layer="below", line_width=0,
|
| 74 |
+
annotation_text=domain,
|
| 75 |
+
annotation_position="top left",
|
| 76 |
+
annotation_font_size=9,
|
| 77 |
+
)
|
| 78 |
+
|
| 79 |
+
# ๅๅฑๆฒ็บฟ
|
| 80 |
+
for layer in range(n_layers):
|
| 81 |
+
ld = pb[pb["layer"] == layer].sort_values("tokens_B")
|
| 82 |
+
if ld.empty:
|
| 83 |
+
continue
|
| 84 |
+
fig.add_trace(go.Scatter(
|
| 85 |
+
x=ld["tokens_B"],
|
| 86 |
+
y=ld[metric],
|
| 87 |
+
mode="lines+markers",
|
| 88 |
+
name=f"L{layer}",
|
| 89 |
+
line=dict(color=colors[layer], width=1.5),
|
| 90 |
+
marker=dict(size=4),
|
| 91 |
+
))
|
| 92 |
+
|
| 93 |
+
fig.update_layout(
|
| 94 |
+
title=(f"OLMoE-1B-7B: {METRICS.get(metric, metric)} vs Training Tokens<br>"
|
| 95 |
+
f"<sup>Shaded regions: grokking intervals from arxiv 2506.21551</sup>"),
|
| 96 |
+
xaxis=dict(title="Training Tokens (B)", type="linear"),
|
| 97 |
+
yaxis=dict(title=METRICS.get(metric, metric)),
|
| 98 |
+
legend=dict(title="Layer", font=dict(size=9)),
|
| 99 |
+
height=500,
|
| 100 |
+
margin=dict(l=60, r=20, t=70, b=50),
|
| 101 |
+
)
|
| 102 |
+
return fig
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
def plot_heatmap(
|
| 106 |
+
df: pd.DataFrame,
|
| 107 |
+
metric: str,
|
| 108 |
+
tokens_B: float,
|
| 109 |
+
) -> go.Figure:
|
| 110 |
+
"""ๅฑรๅคด็ญๅๅพ๏ผ้ๅฎtokensๆฐ็ๅฟซ็
ง"""
|
| 111 |
+
# ๆพๆ่ฟ็tokens_B
|
| 112 |
+
available = sorted(df["tokens_B"].unique())
|
| 113 |
+
if not available:
|
| 114 |
+
return go.Figure().update_layout(title="ๆ ๆฐๆฎ")
|
| 115 |
+
nearest = min(available, key=lambda t: abs(t - tokens_B))
|
| 116 |
+
sub = df[df["tokens_B"] == nearest]
|
| 117 |
+
|
| 118 |
+
cfg = list(OLMOE_CONFIGS.values())[0]
|
| 119 |
+
n_layers = cfg["n_layers"]
|
| 120 |
+
n_heads = cfg["n_heads"]
|
| 121 |
+
|
| 122 |
+
mat = np.full((n_layers, n_heads), np.nan)
|
| 123 |
+
for _, row in sub.iterrows():
|
| 124 |
+
l, h = int(row["layer"]), int(row["head"])
|
| 125 |
+
if 0 <= l < n_layers and 0 <= h < n_heads:
|
| 126 |
+
mat[l, h] = row[metric]
|
| 127 |
+
|
| 128 |
+
branch = sub["branch"].iloc[0] if len(sub) else ""
|
| 129 |
+
fig = go.Figure(data=go.Heatmap(
|
| 130 |
+
z=mat,
|
| 131 |
+
x=[f"h{i}" for i in range(n_heads)],
|
| 132 |
+
y=[f"L{i}" for i in range(n_layers)],
|
| 133 |
+
colorscale="Viridis",
|
| 134 |
+
colorbar=dict(title=METRICS.get(metric, metric)),
|
| 135 |
+
))
|
| 136 |
+
fig.update_layout(
|
| 137 |
+
title=f"OLMoE {branch} ({nearest}B tokens) {METRICS.get(metric, metric)}",
|
| 138 |
+
xaxis_title="Head",
|
| 139 |
+
yaxis_title="Layer",
|
| 140 |
+
height=400,
|
| 141 |
+
margin=dict(l=60, r=20, t=50, b=50),
|
| 142 |
+
)
|
| 143 |
+
return fig
|
| 144 |
+
|
| 145 |
+
|
| 146 |
+
# โโ CSV ๅ ่ฝฝ โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
|
| 147 |
+
|
| 148 |
+
def load_latest_csv() -> pd.DataFrame:
|
| 149 |
+
if not os.path.exists(DATA_DIR):
|
| 150 |
+
return pd.DataFrame()
|
| 151 |
+
files = sorted([
|
| 152 |
+
f for f in os.listdir(DATA_DIR)
|
| 153 |
+
if f.startswith("olmoe_OLMoE") and f.endswith(".csv")
|
| 154 |
+
])
|
| 155 |
+
if not files:
|
| 156 |
+
return pd.DataFrame()
|
| 157 |
+
try:
|
| 158 |
+
return pd.read_csv(os.path.join(DATA_DIR, files[-1]))
|
| 159 |
+
except Exception:
|
| 160 |
+
return pd.DataFrame()
|
| 161 |
+
|
| 162 |
+
|
| 163 |
+
# โโ Gradio Tab โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
|
| 164 |
+
|
| 165 |
+
def build_tab_olmoe():
|
| 166 |
+
with gr.Tab("๐ง OLMoE Training Dynamics"):
|
| 167 |
+
gr.Markdown("""
|
| 168 |
+
### OLMoE-1B-7B Checkpoint Scan
|
| 169 |
+
ๆซๆ [allenai/OLMoE-1B-7B-0924](https://huggingface.co/allenai/OLMoE-1B-7B-0924)
|
| 170 |
+
่ฎญ็ป่ฟ็จไธญ็ attention ็ฉ้ต่ฐฑๆๆ ใ
|
| 171 |
+
|
| 172 |
+
้ดๅฝฑๅบๅๆ ๆณจๆฅ่ช **arxiv 2506.21551** ็ grokking ๆถ้ด็ชๅฃใ
|
| 173 |
+
> ๆฐๆฎๆไน
ๅ่ณ `/data/olmoe_OLMoE-1B-7B-0924_ssr_{ts}.csv`๏ผๆฏๆๆญ็น็ปญ่ทใ
|
| 174 |
+
""")
|
| 175 |
+
|
| 176 |
+
with gr.Row():
|
| 177 |
+
dd_metric = gr.Dropdown(
|
| 178 |
+
choices=list(METRICS.keys()),
|
| 179 |
+
value="ssr",
|
| 180 |
+
label="Metric",
|
| 181 |
+
)
|
| 182 |
+
cb_grok = gr.Checkbox(
|
| 183 |
+
value=True,
|
| 184 |
+
label="ๆพ็คบgrokkingๅบ้ด๏ผarxiv 2506.21551๏ผ",
|
| 185 |
+
)
|
| 186 |
+
btn_scan = gr.Button("๐ Run Scan", variant="primary")
|
| 187 |
+
btn_load = gr.Button("๐ Load Existing CSV")
|
| 188 |
+
btn_dl = gr.DownloadButton("โฌ๏ธ Download CSV", visible=False)
|
| 189 |
+
|
| 190 |
+
status = gr.Textbox(label="Status", lines=3, interactive=False)
|
| 191 |
+
|
| 192 |
+
with gr.Row():
|
| 193 |
+
plot_curve = gr.Plot(label="Training Dynamics (pseudo-bulk per layer)")
|
| 194 |
+
|
| 195 |
+
with gr.Row():
|
| 196 |
+
sl_tokens = gr.Slider(
|
| 197 |
+
minimum=20, maximum=5117, step=20, value=1000,
|
| 198 |
+
label="Heatmap: select tokens (B)",
|
| 199 |
+
)
|
| 200 |
+
plot_heat = gr.Plot(label="Layer ร Head Heatmap")
|
| 201 |
+
|
| 202 |
+
state_df = gr.State(pd.DataFrame())
|
| 203 |
+
state_csv = gr.State("")
|
| 204 |
+
|
| 205 |
+
# โโ Run Scan โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
|
| 206 |
+
def on_scan(metric, show_grok, progress=gr.Progress()):
|
| 207 |
+
log_lines = []
|
| 208 |
+
def prog_fn(cur, total, desc):
|
| 209 |
+
log_lines.append(f"[{cur}/{total}] {desc}")
|
| 210 |
+
progress(cur / total, desc=desc)
|
| 211 |
+
|
| 212 |
+
try:
|
| 213 |
+
csv_path = scan_olmoe(
|
| 214 |
+
model_name="OLMoE-1B-7B-0924",
|
| 215 |
+
steps=DEFAULT_STEPS_OLMOE,
|
| 216 |
+
token=None,
|
| 217 |
+
progress_fn=prog_fn,
|
| 218 |
+
)
|
| 219 |
+
except Exception as e:
|
| 220 |
+
return (f"โ ๆซๆๅคฑ่ดฅ: {e}", pd.DataFrame(), "",
|
| 221 |
+
go.Figure(), go.Figure(), gr.update(visible=False))
|
| 222 |
+
|
| 223 |
+
df = load_latest_csv()
|
| 224 |
+
if df.empty:
|
| 225 |
+
return ("โ ๏ธ CSVไธบ็ฉบ", df, csv_path,
|
| 226 |
+
go.Figure(), go.Figure(), gr.update(visible=False))
|
| 227 |
+
|
| 228 |
+
fig_c = plot_training_curves(df, metric, show_grok)
|
| 229 |
+
fig_h = plot_heatmap(df, metric, float(df["tokens_B"].max()))
|
| 230 |
+
summary = "\n".join(log_lines[-8:])
|
| 231 |
+
summary += f"\n\nโ
ๅฎๆ {df['step'].nunique()} steps {len(df)} rows"
|
| 232 |
+
return (summary, df, csv_path,
|
| 233 |
+
fig_c, fig_h, gr.update(visible=True, value=csv_path))
|
| 234 |
+
|
| 235 |
+
btn_scan.click(
|
| 236 |
+
fn=on_scan,
|
| 237 |
+
inputs=[dd_metric, cb_grok],
|
| 238 |
+
outputs=[status, state_df, state_csv,
|
| 239 |
+
plot_curve, plot_heat, btn_dl],
|
| 240 |
+
)
|
| 241 |
+
|
| 242 |
+
# โโ Load CSV โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
|
| 243 |
+
def on_load(metric, show_grok):
|
| 244 |
+
df = load_latest_csv()
|
| 245 |
+
if df.empty:
|
| 246 |
+
return ("โ ๏ธ /data ไธๆชๆพๅฐOLMoE CSV",
|
| 247 |
+
df, "", go.Figure(), go.Figure(),
|
| 248 |
+
gr.update(visible=False))
|
| 249 |
+
csv_path = os.path.join(DATA_DIR, sorted([
|
| 250 |
+
f for f in os.listdir(DATA_DIR)
|
| 251 |
+
if f.startswith("olmoe_OLMoE") and f.endswith(".csv")
|
| 252 |
+
])[-1])
|
| 253 |
+
fig_c = plot_training_curves(df, metric, show_grok)
|
| 254 |
+
fig_h = plot_heatmap(df, metric, float(df["tokens_B"].max()))
|
| 255 |
+
return (f"โ
{df['step'].nunique()} steps {len(df)} rows\nโ {csv_path}",
|
| 256 |
+
df, csv_path,
|
| 257 |
+
fig_c, fig_h, gr.update(visible=True, value=csv_path))
|
| 258 |
+
|
| 259 |
+
btn_load.click(
|
| 260 |
+
fn=on_load,
|
| 261 |
+
inputs=[dd_metric, cb_grok],
|
| 262 |
+
outputs=[status, state_df, state_csv,
|
| 263 |
+
plot_curve, plot_heat, btn_dl],
|
| 264 |
+
)
|
| 265 |
+
|
| 266 |
+
# โโ Metric / Tokens ๅๆด้็ป โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
|
| 267 |
+
def on_metric(df, metric, show_grok):
|
| 268 |
+
if df.empty:
|
| 269 |
+
return go.Figure()
|
| 270 |
+
return plot_training_curves(df, metric, show_grok)
|
| 271 |
+
|
| 272 |
+
def on_tokens(df, tokens_B, metric):
|
| 273 |
+
if df.empty:
|
| 274 |
+
return go.Figure()
|
| 275 |
+
return plot_heatmap(df, metric, tokens_B)
|
| 276 |
+
|
| 277 |
+
dd_metric.change(
|
| 278 |
+
fn=on_metric,
|
| 279 |
+
inputs=[state_df, dd_metric, cb_grok],
|
| 280 |
+
outputs=[plot_curve],
|
| 281 |
+
)
|
| 282 |
+
sl_tokens.change(
|
| 283 |
+
fn=on_tokens,
|
| 284 |
+
inputs=[state_df, sl_tokens, dd_metric],
|
| 285 |
+
outputs=[plot_heat],
|
| 286 |
+
)
|
| 287 |
+
cb_grok.change(
|
| 288 |
+
fn=on_metric,
|
| 289 |
+
inputs=[state_df, dd_metric, cb_grok],
|
| 290 |
+
outputs=[plot_curve],
|
| 291 |
+
)
|