data: attach pinned training dataset index and safe recovery helpers

CHECKSUMS.sha256

@@ -1,4 +1,4 @@
-1e7edae3c7bea01e80cc0a18228f7ec953c3797a62b14387eabffd5c1dd653f7  README.md
+de97f95cc56c1400755975a453d1da522b15614bf92e298a23e237fe1d9aa14f  README.md
 4e0e37529fbf86ebef6bf7cadec4dda1b762f31ed6dba7815a79732bc6b7e836  checkpoints/caption_grpo/adapter_config.json
 290aacd2808639e3c3e10c81c69acfdd843100b8888ad183a2e03ba6e43f5d11  checkpoints/caption_grpo/adapter_model.safetensors
 62671bc8d430001734a90c33c0059e563388eba42706a5c8603704018035ae8b  checkpoints/caption_grpo/additional_config.json
@@ -77,4 +77,7 @@ c602d4305f59c388105765835b3b7d9cacf2181e8494014fea7f0dc57e728cb8  tools/verify_m
 c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4  LICENSES/Apache-2.0.txt
 4a5b4f13ea4792a8211a69f32d2b460b6d520e57cd23c6301e707c76a6e97a55  LICENSES/MIT-OpenCLIP.txt
 04a70a4302ad5ce449684fc43968bde65420dae210ba0ccb55d9f5bbd908b272  LICENSES/UPSTREAM.md
-f6e8290dcfa126f7172e2e7c3bb8ba031a180367557ae9187ed6c797a11cfe31  bundle_manifest.json
+2569cdcdb30128ffb7d1f178a2669580ead36e532f9f8c65a67f0f25c543caaf  TRAINING_DATA_INDEX.json
+67acb0f58a052ce7fd309e50cdaf473a9880bc31a8d1c7f68212d528088d275c  tools/fetch_training_data.py
+4cd8ba59a93ab631419f29b61153ab8024cc7392f81fca1e0470c400144c8826  tools/restore_legacy_mix6.py
+f495623211474f8cd4f76f07abeb24275e3b743bdeb7e09f5bfde3d54f85bbc3  bundle_manifest.json

README.md

@@ -106,3 +106,23 @@ DiT、T5 的 shard 列表传给一个 `ModelConfig(path=[...])`，再加入 VAE
 Repository: `diffusion-gif/any2ego` (`model`, Public). All listed tensor files are complete and remote hashes verified.
 Source export execution booleans describe the original local export, not current Hub upload status.
 See `LICENSES/UPSTREAM.md` for upstream attribution and component licenses.
+
+## Historical training-data index
+
+`TRAINING_DATA_INDEX.json` is attached to this model download. It identifies
+retained ORIGINAL training JSON/CSV/Parquet, pinned protected snapshot revisions,
+file hashes, original legacy observation backups and separately labeled later data.
+Original restricted annotations remain private; no data-owner credentials are included.
+
+Public registry: https://huggingface.co/datasets/diffusion-gif/training-data-index
+
+```bash
+# For old trajectory checkpoints; inspect other dataset IDs for Any2Ego:
+python tools/fetch_training_data.py --index TRAINING_DATA_INDEX.json --dataset legacy_mix6_sft --output data_download
+# after owner login / dry-run inspection add --execute
+```
+
+A pointer index cannot guarantee that an unbacked original video remains available
+forever. The large Ego-Exo RGB/GT-mask collection is not mirrored here. Launch-time
+content hashes were not recorded for all components; current retained snapshots
+and actual verified path bindings are explicitly distinguished in the index.

bundle_manifest.json

@@ -176,8 +176,8 @@
   "files": [
     {
       "path": "README.md",
-      "bytes": 6575,
-      "sha256": "1e7edae3c7bea01e80cc0a18228f7ec953c3797a62b14387eabffd5c1dd653f7"
+      "bytes": 7610,
+      "sha256": "de97f95cc56c1400755975a453d1da522b15614bf92e298a23e237fe1d9aa14f"
     },
     {
       "path": "checkpoints/caption_grpo/adapter_config.json",
@@ -568,9 +568,24 @@
       "path": "LICENSES/UPSTREAM.md",
       "bytes": 953,
       "sha256": "04a70a4302ad5ce449684fc43968bde65420dae210ba0ccb55d9f5bbd908b272"
+    },
+    {
+      "path": "TRAINING_DATA_INDEX.json",
+      "bytes": 146175,
+      "sha256": "2569cdcdb30128ffb7d1f178a2669580ead36e532f9f8c65a67f0f25c543caaf"
+    },
+    {
+      "path": "tools/fetch_training_data.py",
+      "bytes": 3237,
+      "sha256": "67acb0f58a052ce7fd309e50cdaf473a9880bc31a8d1c7f68212d528088d275c"
+    },
+    {
+      "path": "tools/restore_legacy_mix6.py",
+      "bytes": 2642,
+      "sha256": "4cd8ba59a93ab631419f29b61153ab8024cc7392f81fca1e0470c400144c8826"
     }
   ],
-  "total_file_bytes": 46092494717,
+  "total_file_bytes": 46092647806,
   "largest_file_bytes": 4999831048,
   "native_reader": "safetensors.safe_open(framework=pt, device=cpu), slices only",
   "validation": {
@@ -591,6 +606,13 @@
     "visibility": "public",
     "source_export_manifest_sha256": "f17006bbe222737d8dd9e4cb827bd54f090c59f02762dfef9a9c5928a1caf117",
     "note": "Execution booleans above refer to original local export; README distribution text normalized.",
-    "publication_phase": "complete"
+    "publication_phase": "complete",
+    "training_data_index": {
+      "repo_id": "diffusion-gif/training-data-index",
+      "repo_type": "dataset",
+      "revision": "04182e349e581ce2f5826eb4a7bed2803b2925fc",
+      "path": "TRAINING_DATA_INDEX.json",
+      "source_JSON_sha256": "2569cdcdb30128ffb7d1f178a2669580ead36e532f9f8c65a67f0f25c543caaf"
+    }
   }
 }

tools/fetch_training_data.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+"""Fetch pinned training files from the JSON index; dry-run unless --execute."""
+import argparse,hashlib,json,shutil
+from pathlib import Path,PurePosixPath
+
+
+def safe(value):
+ p=PurePosixPath(value)
+ if p.is_absolute() or '..' in p.parts or not p.parts:raise ValueError('Unsafe relative data path')
+ return Path(*p.parts)
+
+def sha(path):
+ h=hashlib.sha256()
+ with Path(path).open('rb') as f:
+  for b in iter(lambda:f.read(8*1024*1024),b''):h.update(b)
+ return h.hexdigest()
+
+def main():
+ p=argparse.ArgumentParser(description=__doc__)
+ directory=Path(__file__).absolute().parent
+ default_index=directory/'TRAINING_DATA_INDEX.json'
+ if not default_index.exists():default_index=directory.parent/'TRAINING_DATA_INDEX.json'
+ p.add_argument('--index',type=Path,default=default_index)
+ p.add_argument('--dataset',default='legacy_mix6_sft')
+ p.add_argument('--output',type=Path,default=Path('training_data_download'))
+ p.add_argument('--execute',action='store_true');a=p.parse_args()
+ index=json.loads(a.index.read_text());datasets={d['dataset_id']:d for d in index['datasets']}
+ if a.dataset=='all_private':ids={r['artifact_id'] for r in index['private_artifacts']}
+ elif a.dataset in datasets:ids=set(datasets[a.dataset].get('artifact_ids',[]))
+ else:raise ValueError('Unknown dataset ID; inspect index.datasets')
+ rows=[r for r in index['private_artifacts'] if r['artifact_id'] in ids]
+ if not rows:raise ValueError('No private snapshot artifacts for this dataset; inspect public download location')
+ source=index['private_snapshot']
+ if not source['repo_id'].startswith('diffusion-gif/training-data-vault-') or source['repo_type']!='dataset' or not source['private']:raise ValueError('Unexpected private data provider')
+ print(json.dumps({'dataset':a.dataset,'files':len(rows),'bytes':sum(r['bytes'] for r in rows),'source_repo':source['repo_id'],'revision':source['revision'],'login_required':True,'execute':a.execute},indent=2))
+ if not a.execute:return
+ from huggingface_hub import get_token,hf_hub_download
+ token=get_token()
+ if not token:raise RuntimeError('Run hf auth login locally using an account authorized for the private vault. Never put tokens in this file or chat.')
+ a.output.mkdir(parents=True,exist_ok=True)
+ for r in rows:
+  target=a.output/safe(r['path'])
+  if target.exists():
+   if not target.is_file() or target.stat().st_size!=r['bytes'] or sha(target)!=r['sha256']:raise ValueError('Existing user file differs; will not overwrite')
+   continue
+  cached=Path(hf_hub_download(repo_id=source['repo_id'],repo_type='dataset',revision=source['revision'],filename=r['path'],token=token,endpoint='https://huggingface.co'))
+  if cached.stat().st_size!=r['bytes'] or sha(cached)!=r['sha256']:raise ValueError('Downloaded file SHA256 mismatch')
+  target.parent.mkdir(parents=True,exist_ok=True);temporary=target.with_name(target.name+'.copy-partial')
+  shutil.copyfile(cached,temporary)
+  if sha(temporary)!=r['sha256']:raise ValueError('Copied file SHA256 mismatch')
+  temporary.replace(target);print('VERIFIED',r['path'],flush=True)
+ print('Done: all selected frozen files downloaded and SHA256 verified. See restore_legacy_mix6.py for a relocatable SFT working copy.')
+if __name__=='__main__':main()

tools/restore_legacy_mix6.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+"""Restore legacy SFT observations/JSON working copies; never alter frozen files."""
+import argparse,hashlib,json,tarfile
+from pathlib import Path
+
+def main():
+ p=argparse.ArgumentParser(description=__doc__);p.add_argument('--snapshot-root',required=True,type=Path);p.add_argument('--output',required=True,type=Path);p.add_argument('--execute',action='store_true');a=p.parse_args()
+ src=a.snapshot_root/'legacy_mix6';locations={}
+ for line in (src/'observations/locations.private.jsonl').open():
+  r=json.loads(line);key=(r['split'],r['sid'])
+  if key in locations:raise ValueError('Duplicate observation ID')
+  if Path(r['archive']).name!=r['archive'] or Path(r['member']).name!=r['member']:raise ValueError('Unsafe archive/member path')
+  locations[key]=r
+ if not a.execute:
+  print(json.dumps({'records':len(locations),'output':str(a.output),'source_snapshot_modified':False,'execute':False},indent=2));return
+ if a.output.exists() and any(a.output.iterdir()):raise ValueError('Output must be a new/empty directory; will not overwrite user files')
+ a.output.mkdir(parents=True,exist_ok=True);by_archive={}
+ for key,r in locations.items():by_archive.setdefault(r['archive'],{})[r['member']]=(key,r)
+ for name,entries in by_archive.items():
+  with tarfile.open(src/'observations'/name,'r:') as archive:
+   for member in archive:
+    entry=entries.pop(member.name,None)
+    if entry is None or not member.isfile():raise ValueError('Unexpected archive entry')
+    (split,sid),r=entry;data=archive.extractfile(member).read()
+    if len(data)!=r['bytes'] or hashlib.sha256(data).hexdigest()!=r['sha256']:raise ValueError('Original NPZ checksum differs')
+    filename=hashlib.sha256(sid.encode()).hexdigest()[:32]+'.npz'
+    target=a.output/'frames'/split/filename;target.parent.mkdir(parents=True,exist_ok=True)
+    if target.exists():raise ValueError('Duplicate target frame file')
+    target.write_bytes(data)
+  if entries:raise ValueError('Missing observation member')
+ for split in ('train','heldout'):
+  with (src/'sft'/(split+'.jsonl')).open() as inp,(a.output/(split+'.jsonl')).open('w') as out:
+   for line in inp:
+    r=json.loads(line)
+    if (split,r['sid']) not in locations:raise ValueError('Input record lacks backed-up observation')
+    r['frames_root']=str((a.output/'frames'/split).resolve())
+    r['npz']=hashlib.sha256(r['sid'].encode()).hexdigest()[:32]+'.npz'
+    out.write(json.dumps(r,ensure_ascii=False)+'\n')
+ print('Restored exact NPZ bytes and re-based SFT train/heldout JSONL. Only npz/frames_root fields in WORKING copies changed; frozen originals are intact.')
+if __name__=='__main__':main()