Buckets:
| #!/usr/bin/env python3 | |
| """build_duplicate_map — L1 系统去重。 | |
| 三条线索，逐条给出证据： | |
| A. counts_v4 的 175 段 → 源 npz（索引里有 file 字段），并用数值核对： | |
| uint8 计数与源 npz（1/8 缩放）应满足 sum(uint8) == sum(npz)*8。 | |
| B. omnibrain_multi 的窗 → omnibrain_spike_view 系列（各来源窗口数应一致）， | |
| 抽样比对窗口内容。 | |
| C. omnibrain4d 的 band → eeg（同源，band 是派生），标 derived 而非 same_record。 | |
| 输出 framework/duplicate_map.parquet / .csv | |
| """ | |
| from __future__ import annotations | |
| import glob | |
| from pathlib import Path | |
| import numpy as np | |
| import pandas as pd | |
| ROOT = Path("/domains") | |
| OUT = ROOT / "framework" | |
| rows = [] | |
| def add(src_m, src_s, dst_m, dst_s, relation, evidence, verified): | |
| rows.append(dict(src_member=src_m, src_slice=src_s, dst_member=dst_m, | |
| dst_slice=dst_s, relation=relation, evidence=evidence, | |
| verified=bool(verified))) | |
| # ---------------- A. counts_v4 → 源 npz | |
| print("[A] counts_v4 → 源 npz", flush=True) | |
| ci = pd.read_parquet(ROOT / "counts-v4/counts_index.parquet") | |
| flat = np.load(ROOT / "counts-v4/counts_ragged.npy", mmap_mode="r") | |
| ok_n = 0 | |
| for _, r in ci.iterrows(): | |
| src = Path(str(r.file)).stem | |
| src_l = str(r.source).lower() | |
| is_mouse = ("steinmetz" in src_l) or ("mouse" in src_l) | |
| member = "steinmetz" if is_mouse else "primate_macaque_nlb" | |
| off, nel = int(r.offset), int(r.n_elements) | |
| nb, nu = int(r.n_bins_20ms), int(r.n_units) | |
| seg = np.asarray(flat[off:off + nel], dtype=np.float64).reshape(nb, nu) | |
| f = ROOT / ("steinmetz" if member == "steinmetz" else "primate/macaque_nlb") / f"{src}.npz" | |
| verified = False | |
| ev = f"missing {f.name}" | |
| if f.exists(): | |
| z = np.load(f, mmap_mode="r") | |
| k = "spikes" if "spikes" in z.keys() else list(z.keys())[0] | |
| a = np.asarray(z[k], dtype=np.float64) | |
| x = a.T if a.shape[0] < a.shape[1] else a | |
| if x.shape == seg.shape: | |
| s1, s2 = float(seg.sum()), float(x.sum()) | |
| # npz 可能是整数计数，也可能是 1/8 缩放；两种假设都试，取吻合的那个 | |
| rel1 = abs(s1 - s2) / max(s1, 1.0) | |
| rel2 = abs(s1 - s2 * 8) / max(s1, 1.0) | |
| if min(rel1, rel2) < 0.02: | |
| verified = True | |
| scale = "1" if rel1 <= rel2 else "8" | |
| ev = f"scale={scale} sum_uint8={s1:.0f} sum_npz={s2:.0f} rel={min(rel1, rel2):.5f}" | |
| else: | |
| ev = f"sum_uint8={s1:.0f} sum_npz={s2:.0f} rel1={rel1:.3f} rel8={rel2:.3f}" | |
| else: | |
| ev = f"shape seg={seg.shape} npz={x.shape} bin_ms={r.bin_ms}" | |
| add("counts_v4_ragged", f"seg{int(r.segment_idx):03d}", member, src, | |
| "same_record", ev, verified) | |
| ok_n += int(verified) | |
| print(f" {len(ci)} 段，数值核对通过 {ok_n}", flush=True) | |
| # ---------------- B. multi 窗 vs spike_view | |
| print("\n[B] omnibrain_multi → spike_view", flush=True) | |
| mi = pd.read_parquet(ROOT / "omnibrain-multi/index_timeseries.parquet") | |
| ts = np.load(ROOT / "omnibrain-multi/timeseries.npy", mmap_mode="r") | |
| VIEWS = {"mouse_steinmetz": "omnibrain_spike_view", | |
| "macaque_nlb": "omnibrain_spike_view_macaque", | |
| "human_intracortical": "omnibrain_spike_view_human", | |
| "human_eeg": "omnibrain_eeg_view"} | |
| NP = 300 | |
| for src, view in VIEWS.items(): | |
| pos = np.flatnonzero((mi.source == src).to_numpy()) | |
| if not len(pos): | |
| continue | |
| a = np.asarray(ts[pos[:NP]], dtype=np.float32) | |
| # 取同源 view 的前若干窗口 | |
| files = sorted(glob.glob(str(ROOT / "omnibrain" / view.replace("omnibrain_", "") / "**" / "*.npz"), | |
| recursive=True)) | |
| chunks, got = [], 0 | |
| for f in files: | |
| z = np.load(f, mmap_mode="r") | |
| if "windows" not in z.keys(): | |
| continue | |
| w = np.asarray(z["windows"], dtype=np.float32) | |
| take = min(len(w), NP - got) | |
| chunks.append(w[:take]) | |
| got += take | |
| if got >= NP: | |
| break | |
| if got < 10: | |
| add("omnibrain_multi", src, view, "-", "unverified", "no windows found", False) | |
| continue | |
| b = np.concatenate(chunks, axis=0) | |
| n = min(len(a), len(b)) | |
| same = bool(np.allclose(a[:n], b[:n], atol=1e-3)) | |
| ev = f"n_rows_multi={len(pos)} n_win_view={got}; first {n} rows equal={same}" | |
| add("omnibrain_multi", src, view, "-", | |
| "reencoded_identical" if same else "related_same_source", ev, same) | |
| print(f" {src}: multi={len(pos)} view_sample={got} equal={same}", flush=True) | |
| # ---------------- C. 4D band → eeg | |
| print("\n[C] omnibrain4d band → eeg", flush=True) | |
| files4 = sorted(glob.glob(str(ROOT / "omnibrain4d/**/*.npz"), recursive=True)) | |
| by_key = {} | |
| for f in files4: | |
| z = np.load(f, mmap_mode="r") | |
| keys = list(z.keys()) | |
| stem = Path(f).stem | |
| for suf in ("_band", "_eeg", "-band", "-eeg"): | |
| if stem.endswith(suf): | |
| stem = stem[: -len(suf)] | |
| break | |
| if "eeg" in keys: | |
| by_key.setdefault(stem, {})["eeg"] = tuple(z["eeg"].shape) | |
| if "band" in keys: | |
| by_key.setdefault(stem, {})["band"] = tuple(z["band"].shape) | |
| for stem, v in list(by_key.items())[:6]: | |
| if "eeg" in v and "band" in v: | |
| add("omnibrain4d", stem, "omnibrain4d", stem, "derived", | |
| f"eeg{v['eeg']} -> band{v['band']}（带通功率，非切片）", True) | |
| print(f" {stem}: eeg{v['eeg']} band{v['band']}", flush=True) | |
| print(f" 4D 文件 {len(files4)} 个，配对 {len(by_key)} 组", flush=True) | |
| # 依据证据给失败项重新归类，避免一律读成"不是同一记录" | |
| def relabel(r): | |
| if r.relation != "same_record": | |
| return r.relation | |
| if r.verified: | |
| return "same_record" | |
| if str(r.evidence).startswith("missing"): | |
| return "external_source" # 源只在 DANDI，桶内无镜像 | |
| if str(r.evidence).startswith("shape"): | |
| return "resampled_or_reshaped" # 形状不同（如 50→20 ms） | |
| return "unverified" | |
| d = pd.DataFrame(rows) | |
| d["relation"] = d.apply(relabel, axis=1) | |
| d.to_parquet(OUT / "duplicate_map.parquet", index=False) | |
| d.to_csv(OUT / "duplicate_map.csv", index=False) | |
| print(f"\nrows={len(d)}") | |
| print(d.groupby(["relation", "verified"]).size().to_string()) | |
Xet Storage Details
- Size:
- 6.31 kB
- Xet hash:
- 23a61b1d1f5f5afeba37a698b5759d041ad3528dadfeb0776afa3ddc6d3b7544
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.