Buckets:

illusion-brain
/

brain-domains

Files

xet

illusion-brain/brain-domains / scripts /build_framework_index.py

redmingwei

7 days ago

download

raw

11.1 kB

#!/usr/bin/env python3
"""build_framework_index — 扫描 /domains，把每个成员里的**每个对象**登记成框架索引。
关键改动（v2）：索引的行是"对象"而不是"目录"。同一个成员目录里可以同时有
  - 窗口张量 (n_win, 64, 8)
  - 载荷矩阵 (64, n_units)
  - 图嵌入   (n_nodes, 64)
  - ragged 扁平数组 + 索引表
它们阶数不同，必须分开登记，否则"超维度"就被压回一个形状里了。
设计约束（来自 exp47/48 审计）：
  - 不补零、不强行共享单元轴
  - 标签（species/region/domain）只作结构键，不当作数据轴
  - 每个对象记录它是怎么被构造的（construction），native 与 derived 分开
输出: /domains/framework/{framework_index.parquet, framework_index.csv, framework.json}
"""
from __future__ import annotations

import glob
import json
import sys
from pathlib import Path

import numpy as np
import pandas as pd

if len(sys.argv) > 1 and sys.argv[1] == "summary":
    df = pd.read_parquet("/domains/framework/framework_index.parquet")
    df["sl"] = df["shape"].apply(lambda s: json.loads(s) if isinstance(s, str) and s else [])
    print(f"members={df.member.nunique()}  objects={len(df)}  "
          f"rank={df['rank'].value_counts().to_dict()}")
    print(f"\n{'member':30s} {'obj':11s} {'rk':>2s} {'files':>5s}  {'shape family':34s} axes")
    for (mem, obj, rk, ax, dt), sub in df.groupby(["member", "obj", "rank", "axes", "dtype"],
                                                  dropna=False, sort=True):
        shapes = [s for s in sub.sl if s]
        if not shapes:
            fam = "n/a"
        else:
            nd = len(shapes[0])
            parts = []
            for i in range(nd):
                vals = [s[i] for s in shapes if len(s) > i]
                lo, hi = min(vals), max(vals)
                parts.append(str(lo) if lo == hi else f"{lo}-{hi}")
            fam = " x ".join(parts)
        print(f"{mem:30s} {str(obj)[:11]:11s} {int(rk):2d} {len(sub):5d}  {fam[:34]:34s} {ax}")
    raise SystemExit(0)
if len(sys.argv) > 1 and sys.argv[1] == "show":
    df = pd.read_parquet("/domains/framework/framework_index.parquet")
    cols = ["member", "obj", "rank", "shape", "dtype", "n_files", "axes", "construction"]
    print(df[cols].to_string(index=False, max_colwidth=46))
    raise SystemExit(0)

ROOT = Path("/domains")
OUT = ROOT / "framework"
OUT.mkdir(parents=True, exist_ok=True)

REG = [
    dict(member="steinmetz", path="steinmetz", source="mouse_steinmetz",
         species="mus_musculus", modality="extracellular_spike",
         axes=["unit", "time"], bin_ms=50.0, construction="counts_native", license="CC-BY-4.0"),
    dict(member="flywire", path="flywire", source="flywire",
         species="drosophila_melanogaster", modality="connectome",
         axes=["node", "feature"], bin_ms=None, construction="graph_native", license="CC-BY-4.0"),
    dict(member="primate_macaque_nlb", path="primate/macaque_nlb", source="macaque_nlb",
         species="macaca_mulatta", modality="extracellular_spike",
         axes=["unit", "time"], bin_ms=20.0, construction="counts_native", license="CC-BY-4.0"),
    dict(member="primate_human_eeg", path="primate/human_eeg", source="human_eeg",
         species="homo_sapiens", modality="scalp_eeg",
         axes=["trial", "channel", "time"], bin_ms=4.0, construction="eeg_native", license="CC-BY-4.0"),
    dict(member="primate_dmfc_rsg", path="primate/dmfc_rsg", source="dmfc_rsg",
         species="macaca_mulatta", modality="extracellular_spike",
         axes=["trial", "time", "unit"], bin_ms=20.0, construction="counts_native_nwb",
         license="CC-BY-4.0"),
    dict(member="omnibrain_spike_view", path="omnibrain/spike_view", source="mouse_steinmetz",
         species="mus_musculus", modality="extracellular_spike",
         axes=["window", "component", "bin"], bin_ms=100.0,
         construction="pca_gram_pad64", license="CC-BY-4.0"),
    dict(member="omnibrain_spike_view_macaque", path="omnibrain/spike_view_macaque",
         source="macaque_nlb", species="macaca_mulatta", modality="extracellular_spike",
         axes=["window", "component", "bin"], bin_ms=100.0,
         construction="pca_gram_pad64", license="CC-BY-4.0"),
    dict(member="omnibrain_spike_view_human", path="omnibrain/spike_view_human",
         source="human_intracortical", species="homo_sapiens", modality="intracortical",
         axes=["window", "component", "bin"], bin_ms=100.0,
         construction="pca_gram_pad64", license="CC-BY-4.0"),
    dict(member="omnibrain_eeg_view", path="omnibrain/eeg_view", source="human_eeg",
         species="homo_sapiens", modality="scalp_eeg",
         axes=["window", "component", "bin"], bin_ms=100.0,
         construction="identity_pad64", license="CC-BY-4.0"),
    dict(member="omnibrain_graph_view", path="omnibrain/graph_view", source="flywire",
         species="drosophila_melanogaster", modality="connectome",
         axes=["node", "embedding"], bin_ms=None,
         construction="graph_propagation_64", license="CC-BY-4.0"),
    dict(member="omnibrain_multi", path="omnibrain-multi", source="4 sources",
         species="mixed", modality="mixed", axes=["window", "component", "bin"],
         bin_ms=100.0, construction="mixed_views", license="CC-BY-4.0"),
    dict(member="omnibrain4d_eeg_test", path="omnibrain4d/eeg", source="human_eeg",
         species="homo_sapiens", modality="scalp_eeg",
         axes=["image", "repetition", "channel", "time"], bin_ms=4.0,
         construction="eeg_4d_native", license="CC-BY-4.0"),
    dict(member="omnibrain4d_eeg_train", path="omnibrain4d/eeg_train", source="human_eeg",
         species="homo_sapiens", modality="scalp_eeg",
         axes=["image", "repetition", "channel", "time"], bin_ms=4.0,
         construction="eeg_4d_native", license="CC-BY-4.0"),
    dict(member="counts_v4_ragged", path="counts-v4", source="steinmetz+macaque+human",
         species="mixed", modality="extracellular_spike", axes=["time", "unit"],
         bin_ms="per_row", construction="ragged_index_family", license="CC-BY-4.0"),
]

LOADING_KEYS = {"comps", "comp", "enc", "loadings", "basis", "encoder", "mu", "sd"}


def norm_axes(m, key, ndim):
    tpl = m["axes"]
    if key and key.lower() in LOADING_KEYS and ndim == 2:
        return "component|unit"
    if ndim == len(tpl):
        return "|".join(tpl)
    if ndim == 1:
        return "flat"
    return f"{ndim}D_unregistered"


def collect(m):
    d = ROOT / m["path"]
    if not d.exists():
        return [dict(member=m["member"], obj="<missing>", rank=0, shape=None,
                     dtype=None, n_files=0, axes=None, example=None)]
    files = []
    for pat in ("*.npz", "*.npy", "*.parquet", "*.nwb"):
        files += glob.glob(str(d / "**" / pat), recursive=True)
    files = sorted(files)
    agg = {}
    for f in files:
        ext = Path(f).suffix.lower()
        try:
            if ext == ".npz":
                z = np.load(f, mmap_mode="r")
                for k in z.keys():
                    a = z[k]
                    sig = (k, tuple(a.shape), str(a.dtype))
                    agg.setdefault(sig, []).append(Path(f).name)
            elif ext == ".npy":
                a = np.load(f, mmap_mode="r")
                sig = ("", tuple(a.shape), str(a.dtype))
                agg.setdefault(sig, []).append(Path(f).name)
            elif ext == ".parquet":
                df = pd.read_parquet(f)
                sig = ("", tuple(df.shape), "parquet")
                agg.setdefault(sig, []).append(Path(f).name)
            else:
                sig = ("", None, "nwb")
                agg.setdefault(sig, []).append(Path(f).name)
        except Exception as e:  # noqa
            sig = ("", None, f"ERR:{type(e).__name__}")
            agg.setdefault(sig, []).append(Path(f).name)
    rows = []
    for (k, shp, dt), fl in agg.items():
        nd = len(shp) if shp else 0
        rows.append(dict(member=m["member"], obj=(k or Path(fl[0]).stem), rank=nd,
                         shape=json.dumps(list(shp)) if shp else None,
                         dtype=dt, n_files=len(fl),
                         axes=norm_axes(m, k, nd), example=fl[0]))
    return rows


allrows = []
for m in REG:
    rs = collect(m)
    for r in rs:
        r.update(source=m["source"], species=m["species"], modality=m["modality"],
                 construction=m["construction"], bin_ms=str(m["bin_ms"]),
                 license=m["license"], path=m["path"])
    allrows += rs

df = pd.DataFrame(allrows)
df = df.sort_values(["order"] if "order" in df else ["member"], kind="stable")
df = df[["member", "obj", "rank", "axes", "shape", "dtype", "n_files", "example",
         "source", "species", "modality", "bin_ms", "construction", "license", "path"]]

print(f"members={df.member.nunique()}  objects={len(df)}  "
      f"rank distribution={df['rank'].value_counts().to_dict()}")
print(f"\n{'member':30s} {'obj':12s} {'rk':>2s} {'shape':28s} {'dtype':8s} {'n':>4s}  axes")
for _, r in df.iterrows():
    print(f"{r['member']:30s} {str(r['obj'])[:12]:12s} {int(r['rank']):2d} "
          f"{str(r['shape'])[:28]:28s} {str(r['dtype'])[:8]:8s} {int(r['n_files']):4d}  "
          f"{r['axes']}")

manifest = {
    "schema_version": "1.0",
    "name": "hyperdimensional_brain_domain_framework",
    "principle": {
        "index_rows_are_objects": "一行 = 一个对象（某种键/形状的组合），不是一个目录",
        "no_padding": "成员保留原生阶数与形状；不补零到共同宽度",
        "no_shared_unit_axis": "跨成员的 unit 列不是同一个神经元，禁止拼接",
        "labels_are_structure_only": "species/region/domain 只作结构键；实测 Pi_label 能量 0.05%",
        "bin_ms_is_per_row": "bin 宽属于成员自己的属性，不是数据集全局属性",
    },
    "constructions": {
        "counts_native": "整数计数，不重采样（1/8 缩放已乘回）",
        "counts_native_nwb": "NWB 原始格式，未转换",
        "ragged_index_family": "offset+n_elements 索引族，per-row bin_ms",
        "pca_gram_pad64": "逐 session Gram PCA → 64 维，零填充到 64（derived，含 padding）",
        "identity_pad64": "63 电极零填充到 64（derived，含 padding）",
        "graph_propagation_64": "图信号传播 → 64 维（不读解剖/功能标签）",
        "graph_native": "节点特征表，原生",
        "eeg_native": "(trial, channel, time) 原生",
        "eeg_4d_native": "(image, repetition, channel, time) 原生四轴",
        "mixed_views": "多来源多视图混合，含 padding 视图",
    },
    "members": df.to_dict(orient="records"),
}
(OUT / "framework.json").write_text(json.dumps(manifest, indent=2, ensure_ascii=False,
                                               default=str), encoding="utf-8")
df.to_parquet(OUT / "framework_index.parquet", index=False)
df.to_csv(OUT / "framework_index.csv", index=False)
print(f"\nwrote {OUT}/framework_index.parquet  rows={len(df)}")

Xet Storage Details

Size:: 11.1 kB
Xet hash:: c3f7d94aeae3268a20deda35195ff094ba548e6a588a639fee53d39e15a5eae1

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.

	#!/usr/bin/env python3
	"""build_framework_index — 扫描 /domains，把每个成员里的每个对象登记成框架索引。

	关键改动（v2）：索引的行是"对象"而不是"目录"。同一个成员目录里可以同时有
	- 窗口张量 (n_win, 64, 8)
	- 载荷矩阵 (64, n_units)
	- 图嵌入 (n_nodes, 64)
	- ragged 扁平数组 + 索引表
	它们阶数不同，必须分开登记，否则"超维度"就被压回一个形状里了。

	设计约束（来自 exp47/48 审计）：
	- 不补零、不强行共享单元轴
	- 标签（species/region/domain）只作结构键，不当作数据轴
	- 每个对象记录它是怎么被构造的（construction），native 与 derived 分开

	输出: /domains/framework/{framework_index.parquet, framework_index.csv, framework.json}
	"""
	from __future__ import annotations

	import glob
	import json
	import sys
	from pathlib import Path

	import numpy as np
	import pandas as pd

	if len(sys.argv) > 1 and sys.argv[1] == "summary":
	df = pd.read_parquet("/domains/framework/framework_index.parquet")
	df["sl"] = df["shape"].apply(lambda s: json.loads(s) if isinstance(s, str) and s else [])
	print(f"members={df.member.nunique()} objects={len(df)} "
	f"rank={df['rank'].value_counts().to_dict()}")
	print(f"\n{'member':30s} {'obj':11s} {'rk':>2s} {'files':>5s} {'shape family':34s} axes")
	for (mem, obj, rk, ax, dt), sub in df.groupby(["member", "obj", "rank", "axes", "dtype"],
	dropna=False, sort=True):
	shapes = [s for s in sub.sl if s]
	if not shapes:
	fam = "n/a"
	else:
	nd = len(shapes[0])
	parts = []
	for i in range(nd):
	vals = [s[i] for s in shapes if len(s) > i]
	lo, hi = min(vals), max(vals)
	parts.append(str(lo) if lo == hi else f"{lo}-{hi}")
	fam = " x ".join(parts)
	print(f"{mem:30s} {str(obj)[:11]:11s} {int(rk):2d} {len(sub):5d} {fam[:34]:34s} {ax}")
	raise SystemExit(0)
	if len(sys.argv) > 1 and sys.argv[1] == "show":
	df = pd.read_parquet("/domains/framework/framework_index.parquet")
	cols = ["member", "obj", "rank", "shape", "dtype", "n_files", "axes", "construction"]
	print(df[cols].to_string(index=False, max_colwidth=46))
	raise SystemExit(0)

	ROOT = Path("/domains")
	OUT = ROOT / "framework"
	OUT.mkdir(parents=True, exist_ok=True)

	REG = [
	dict(member="steinmetz", path="steinmetz", source="mouse_steinmetz",
	species="mus_musculus", modality="extracellular_spike",
	axes=["unit", "time"], bin_ms=50.0, construction="counts_native", license="CC-BY-4.0"),
	dict(member="flywire", path="flywire", source="flywire",
	species="drosophila_melanogaster", modality="connectome",
	axes=["node", "feature"], bin_ms=None, construction="graph_native", license="CC-BY-4.0"),
	dict(member="primate_macaque_nlb", path="primate/macaque_nlb", source="macaque_nlb",
	species="macaca_mulatta", modality="extracellular_spike",
	axes=["unit", "time"], bin_ms=20.0, construction="counts_native", license="CC-BY-4.0"),
	dict(member="primate_human_eeg", path="primate/human_eeg", source="human_eeg",
	species="homo_sapiens", modality="scalp_eeg",
	axes=["trial", "channel", "time"], bin_ms=4.0, construction="eeg_native", license="CC-BY-4.0"),
	dict(member="primate_dmfc_rsg", path="primate/dmfc_rsg", source="dmfc_rsg",
	species="macaca_mulatta", modality="extracellular_spike",
	axes=["trial", "time", "unit"], bin_ms=20.0, construction="counts_native_nwb",
	license="CC-BY-4.0"),
	dict(member="omnibrain_spike_view", path="omnibrain/spike_view", source="mouse_steinmetz",
	species="mus_musculus", modality="extracellular_spike",
	axes=["window", "component", "bin"], bin_ms=100.0,
	construction="pca_gram_pad64", license="CC-BY-4.0"),
	dict(member="omnibrain_spike_view_macaque", path="omnibrain/spike_view_macaque",
	source="macaque_nlb", species="macaca_mulatta", modality="extracellular_spike",
	axes=["window", "component", "bin"], bin_ms=100.0,
	construction="pca_gram_pad64", license="CC-BY-4.0"),
	dict(member="omnibrain_spike_view_human", path="omnibrain/spike_view_human",
	source="human_intracortical", species="homo_sapiens", modality="intracortical",
	axes=["window", "component", "bin"], bin_ms=100.0,
	construction="pca_gram_pad64", license="CC-BY-4.0"),
	dict(member="omnibrain_eeg_view", path="omnibrain/eeg_view", source="human_eeg",
	species="homo_sapiens", modality="scalp_eeg",
	axes=["window", "component", "bin"], bin_ms=100.0,
	construction="identity_pad64", license="CC-BY-4.0"),
	dict(member="omnibrain_graph_view", path="omnibrain/graph_view", source="flywire",
	species="drosophila_melanogaster", modality="connectome",
	axes=["node", "embedding"], bin_ms=None,
	construction="graph_propagation_64", license="CC-BY-4.0"),
	dict(member="omnibrain_multi", path="omnibrain-multi", source="4 sources",
	species="mixed", modality="mixed", axes=["window", "component", "bin"],
	bin_ms=100.0, construction="mixed_views", license="CC-BY-4.0"),
	dict(member="omnibrain4d_eeg_test", path="omnibrain4d/eeg", source="human_eeg",
	species="homo_sapiens", modality="scalp_eeg",
	axes=["image", "repetition", "channel", "time"], bin_ms=4.0,
	construction="eeg_4d_native", license="CC-BY-4.0"),
	dict(member="omnibrain4d_eeg_train", path="omnibrain4d/eeg_train", source="human_eeg",
	species="homo_sapiens", modality="scalp_eeg",
	axes=["image", "repetition", "channel", "time"], bin_ms=4.0,
	construction="eeg_4d_native", license="CC-BY-4.0"),
	dict(member="counts_v4_ragged", path="counts-v4", source="steinmetz+macaque+human",
	species="mixed", modality="extracellular_spike", axes=["time", "unit"],
	bin_ms="per_row", construction="ragged_index_family", license="CC-BY-4.0"),
	]

	LOADING_KEYS = {"comps", "comp", "enc", "loadings", "basis", "encoder", "mu", "sd"}


	def norm_axes(m, key, ndim):
	tpl = m["axes"]
	if key and key.lower() in LOADING_KEYS and ndim == 2:
	return "component\|unit"
	if ndim == len(tpl):
	return "\|".join(tpl)
	if ndim == 1:
	return "flat"
	return f"{ndim}D_unregistered"


	def collect(m):
	d = ROOT / m["path"]
	if not d.exists():
	return [dict(member=m["member"], obj="<missing>", rank=0, shape=None,
	dtype=None, n_files=0, axes=None, example=None)]
	files = []
	for pat in (".npz", ".npy", ".parquet", ".nwb"):
	files += glob.glob(str(d / "**" / pat), recursive=True)
	files = sorted(files)
	agg = {}
	for f in files:
	ext = Path(f).suffix.lower()
	try:
	if ext == ".npz":
	z = np.load(f, mmap_mode="r")
	for k in z.keys():
	a = z[k]
	sig = (k, tuple(a.shape), str(a.dtype))
	agg.setdefault(sig, []).append(Path(f).name)
	elif ext == ".npy":
	a = np.load(f, mmap_mode="r")
	sig = ("", tuple(a.shape), str(a.dtype))
	agg.setdefault(sig, []).append(Path(f).name)
	elif ext == ".parquet":
	df = pd.read_parquet(f)
	sig = ("", tuple(df.shape), "parquet")
	agg.setdefault(sig, []).append(Path(f).name)
	else:
	sig = ("", None, "nwb")
	agg.setdefault(sig, []).append(Path(f).name)
	except Exception as e: # noqa
	sig = ("", None, f"ERR:{type(e).__name__}")
	agg.setdefault(sig, []).append(Path(f).name)
	rows = []
	for (k, shp, dt), fl in agg.items():
	nd = len(shp) if shp else 0
	rows.append(dict(member=m["member"], obj=(k or Path(fl[0]).stem), rank=nd,
	shape=json.dumps(list(shp)) if shp else None,
	dtype=dt, n_files=len(fl),
	axes=norm_axes(m, k, nd), example=fl[0]))
	return rows


	allrows = []
	for m in REG:
	rs = collect(m)
	for r in rs:
	r.update(source=m["source"], species=m["species"], modality=m["modality"],
	construction=m["construction"], bin_ms=str(m["bin_ms"]),
	license=m["license"], path=m["path"])
	allrows += rs

	df = pd.DataFrame(allrows)
	df = df.sort_values(["order"] if "order" in df else ["member"], kind="stable")
	df = df[["member", "obj", "rank", "axes", "shape", "dtype", "n_files", "example",
	"source", "species", "modality", "bin_ms", "construction", "license", "path"]]

	print(f"members={df.member.nunique()} objects={len(df)} "
	f"rank distribution={df['rank'].value_counts().to_dict()}")
	print(f"\n{'member':30s} {'obj':12s} {'rk':>2s} {'shape':28s} {'dtype':8s} {'n':>4s} axes")
	for _, r in df.iterrows():
	print(f"{r['member']:30s} {str(r['obj'])[:12]:12s} {int(r['rank']):2d} "
	f"{str(r['shape'])[:28]:28s} {str(r['dtype'])[:8]:8s} {int(r['n_files']):4d} "
	f"{r['axes']}")

	manifest = {
	"schema_version": "1.0",
	"name": "hyperdimensional_brain_domain_framework",
	"principle": {
	"index_rows_are_objects": "一行 = 一个对象（某种键/形状的组合），不是一个目录",
	"no_padding": "成员保留原生阶数与形状；不补零到共同宽度",
	"no_shared_unit_axis": "跨成员的 unit 列不是同一个神经元，禁止拼接",
	"labels_are_structure_only": "species/region/domain 只作结构键；实测 Pi_label 能量 0.05%",
	"bin_ms_is_per_row": "bin 宽属于成员自己的属性，不是数据集全局属性",
	},
	"constructions": {
	"counts_native": "整数计数，不重采样（1/8 缩放已乘回）",
	"counts_native_nwb": "NWB 原始格式，未转换",
	"ragged_index_family": "offset+n_elements 索引族，per-row bin_ms",
	"pca_gram_pad64": "逐 session Gram PCA → 64 维，零填充到 64（derived，含 padding）",
	"identity_pad64": "63 电极零填充到 64（derived，含 padding）",
	"graph_propagation_64": "图信号传播 → 64 维（不读解剖/功能标签）",
	"graph_native": "节点特征表，原生",
	"eeg_native": "(trial, channel, time) 原生",
	"eeg_4d_native": "(image, repetition, channel, time) 原生四轴",
	"mixed_views": "多来源多视图混合，含 padding 视图",
	},
	"members": df.to_dict(orient="records"),
	}
	(OUT / "framework.json").write_text(json.dumps(manifest, indent=2, ensure_ascii=False,
	default=str), encoding="utf-8")
	df.to_parquet(OUT / "framework_index.parquet", index=False)
	df.to_csv(OUT / "framework_index.csv", index=False)
	print(f"\nwrote {OUT}/framework_index.parquet rows={len(df)}")