data: attach pinned training dataset index and safe recovery helpers

CHECKSUMS.sha256

@@ -45,7 +45,10 @@ d89ef49ce9cd37fbf510158e13c1ef063d9286411c1ec9049932dbe0487143b1  legacy_grpo_st
 33dc3c75269c0d037cb371b8b73116e10080da289911abfbe3648aac3a98f88a  tools/reshard_safetensors.py
 7f54d5b05d644bdc5c8c4a8450e45826ece512c9d2ec391896dde284d5fea552  tests/test_reshard_safetensors.py
 c602d4305f59c388105765835b3b7d9cacf2181e8494014fea7f0dc57e728cb8  tools/verify_model_bundle.py
-9284397ecb8ec79d72f317d1b58b62d3cd202de974f6c74fed3f527b3a40a868  README.md
+d8f10883a186b67cfcc14d88cffc55a1df582b44a315f57ce9a8a498576fbb76  README.md
 c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4  LICENSES/Apache-2.0.txt
 c9b258698bab0fabc1939cc09f0de24992c8b55eb63d4ba21e8803356cdc3320  LICENSES/UPSTREAM.md
-6cf43e61a9c20add3df9453a2d82ea614eccf6a086f4f8dcc3b8e1c415f5f569  bundle_manifest.json
+2569cdcdb30128ffb7d1f178a2669580ead36e532f9f8c65a67f0f25c543caaf  TRAINING_DATA_INDEX.json
+67acb0f58a052ce7fd309e50cdaf473a9880bc31a8d1c7f68212d528088d275c  tools/fetch_training_data.py
+4cd8ba59a93ab631419f29b61153ab8024cc7392f81fca1e0470c400144c8826  tools/restore_legacy_mix6.py
+ca2360e169f3c6898793738c45a537603ce66fddf9546390644b5f62b7fec120  bundle_manifest.json

README.md

@@ -82,3 +82,23 @@ python tools/verify_model_bundle.py --bundle /your/downloaded/trick-prediction
 Repository: `diffusion-gif/trick-prediction` (`model`, Public). All listed tensor files are complete and remote hashes verified.
 Source export execution booleans describe the original local export, not current Hub upload status.
 See `LICENSES/UPSTREAM.md` for upstream attribution and component licenses.
+
+## Historical training-data index
+
+`TRAINING_DATA_INDEX.json` is attached to this model download. It identifies
+retained ORIGINAL training JSON/CSV/Parquet, pinned protected snapshot revisions,
+file hashes, original legacy observation backups and separately labeled later data.
+Original restricted annotations remain private; no data-owner credentials are included.
+
+Public registry: https://huggingface.co/datasets/diffusion-gif/training-data-index
+
+```bash
+# For old trajectory checkpoints; inspect other dataset IDs for Any2Ego:
+python tools/fetch_training_data.py --index TRAINING_DATA_INDEX.json --dataset legacy_mix6_sft --output data_download
+# after owner login / dry-run inspection add --execute
+```
+
+A pointer index cannot guarantee that an unbacked original video remains available
+forever. The large Ego-Exo RGB/GT-mask collection is not mirrored here. Launch-time
+content hashes were not recorded for all components; current retained snapshots
+and actual verified path bindings are explicitly distinguished in the index.

bundle_manifest.json

@@ -260,8 +260,8 @@
     },
     {
       "path": "README.md",
-      "bytes": 4355,
-      "sha256": "9284397ecb8ec79d72f317d1b58b62d3cd202de974f6c74fed3f527b3a40a868"
+      "bytes": 5390,
+      "sha256": "d8f10883a186b67cfcc14d88cffc55a1df582b44a315f57ce9a8a498576fbb76"
     },
     {
       "path": "LICENSES/Apache-2.0.txt",
@@ -272,15 +272,37 @@
       "path": "LICENSES/UPSTREAM.md",
       "bytes": 687,
       "sha256": "c9b258698bab0fabc1939cc09f0de24992c8b55eb63d4ba21e8803356cdc3320"
+    },
+    {
+      "path": "TRAINING_DATA_INDEX.json",
+      "bytes": 146175,
+      "sha256": "2569cdcdb30128ffb7d1f178a2669580ead36e532f9f8c65a67f0f25c543caaf"
+    },
+    {
+      "path": "tools/fetch_training_data.py",
+      "bytes": 3237,
+      "sha256": "67acb0f58a052ce7fd309e50cdaf473a9880bc31a8d1c7f68212d528088d275c"
+    },
+    {
+      "path": "tools/restore_legacy_mix6.py",
+      "bytes": 2642,
+      "sha256": "4cd8ba59a93ab631419f29b61153ab8024cc7392f81fca1e0470c400144c8826"
     }
   ],
-  "total_file_bytes": 109467793631,
+  "total_file_bytes": 109467946720,
   "source_export_identity_sha256": "29f1a102714ddfb0c5a6c6db580ee1a62a27fc46562e6fe783c4b7c2a3a088e5",
   "distribution": {
     "repo_id": "diffusion-gif/trick-prediction",
     "repo_type": "model",
     "visibility": "public",
-    "publication_phase": "complete"
+    "publication_phase": "complete",
+    "training_data_index": {
+      "repo_id": "diffusion-gif/training-data-index",
+      "repo_type": "dataset",
+      "revision": "04182e349e581ce2f5826eb4a7bed2803b2925fc",
+      "path": "TRAINING_DATA_INDEX.json",
+      "source_JSON_sha256": "2569cdcdb30128ffb7d1f178a2669580ead36e532f9f8c65a67f0f25c543caaf"
+    }
   },
   "validation": {
     "source_shard_SHA256_verified_before_upload": true,

tools/fetch_training_data.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+"""Fetch pinned training files from the JSON index; dry-run unless --execute."""
+import argparse,hashlib,json,shutil
+from pathlib import Path,PurePosixPath
+
+
+def safe(value):
+ p=PurePosixPath(value)
+ if p.is_absolute() or '..' in p.parts or not p.parts:raise ValueError('Unsafe relative data path')
+ return Path(*p.parts)
+
+def sha(path):
+ h=hashlib.sha256()
+ with Path(path).open('rb') as f:
+  for b in iter(lambda:f.read(8*1024*1024),b''):h.update(b)
+ return h.hexdigest()
+
+def main():
+ p=argparse.ArgumentParser(description=__doc__)
+ directory=Path(__file__).absolute().parent
+ default_index=directory/'TRAINING_DATA_INDEX.json'
+ if not default_index.exists():default_index=directory.parent/'TRAINING_DATA_INDEX.json'
+ p.add_argument('--index',type=Path,default=default_index)
+ p.add_argument('--dataset',default='legacy_mix6_sft')
+ p.add_argument('--output',type=Path,default=Path('training_data_download'))
+ p.add_argument('--execute',action='store_true');a=p.parse_args()
+ index=json.loads(a.index.read_text());datasets={d['dataset_id']:d for d in index['datasets']}
+ if a.dataset=='all_private':ids={r['artifact_id'] for r in index['private_artifacts']}
+ elif a.dataset in datasets:ids=set(datasets[a.dataset].get('artifact_ids',[]))
+ else:raise ValueError('Unknown dataset ID; inspect index.datasets')
+ rows=[r for r in index['private_artifacts'] if r['artifact_id'] in ids]
+ if not rows:raise ValueError('No private snapshot artifacts for this dataset; inspect public download location')
+ source=index['private_snapshot']
+ if not source['repo_id'].startswith('diffusion-gif/training-data-vault-') or source['repo_type']!='dataset' or not source['private']:raise ValueError('Unexpected private data provider')
+ print(json.dumps({'dataset':a.dataset,'files':len(rows),'bytes':sum(r['bytes'] for r in rows),'source_repo':source['repo_id'],'revision':source['revision'],'login_required':True,'execute':a.execute},indent=2))
+ if not a.execute:return
+ from huggingface_hub import get_token,hf_hub_download
+ token=get_token()
+ if not token:raise RuntimeError('Run hf auth login locally using an account authorized for the private vault. Never put tokens in this file or chat.')
+ a.output.mkdir(parents=True,exist_ok=True)
+ for r in rows:
+  target=a.output/safe(r['path'])
+  if target.exists():
+   if not target.is_file() or target.stat().st_size!=r['bytes'] or sha(target)!=r['sha256']:raise ValueError('Existing user file differs; will not overwrite')
+   continue
+  cached=Path(hf_hub_download(repo_id=source['repo_id'],repo_type='dataset',revision=source['revision'],filename=r['path'],token=token,endpoint='https://huggingface.co'))
+  if cached.stat().st_size!=r['bytes'] or sha(cached)!=r['sha256']:raise ValueError('Downloaded file SHA256 mismatch')
+  target.parent.mkdir(parents=True,exist_ok=True);temporary=target.with_name(target.name+'.copy-partial')
+  shutil.copyfile(cached,temporary)
+  if sha(temporary)!=r['sha256']:raise ValueError('Copied file SHA256 mismatch')
+  temporary.replace(target);print('VERIFIED',r['path'],flush=True)
+ print('Done: all selected frozen files downloaded and SHA256 verified. See restore_legacy_mix6.py for a relocatable SFT working copy.')
+if __name__=='__main__':main()

tools/restore_legacy_mix6.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+"""Restore legacy SFT observations/JSON working copies; never alter frozen files."""
+import argparse,hashlib,json,tarfile
+from pathlib import Path
+
+def main():
+ p=argparse.ArgumentParser(description=__doc__);p.add_argument('--snapshot-root',required=True,type=Path);p.add_argument('--output',required=True,type=Path);p.add_argument('--execute',action='store_true');a=p.parse_args()
+ src=a.snapshot_root/'legacy_mix6';locations={}
+ for line in (src/'observations/locations.private.jsonl').open():
+  r=json.loads(line);key=(r['split'],r['sid'])
+  if key in locations:raise ValueError('Duplicate observation ID')
+  if Path(r['archive']).name!=r['archive'] or Path(r['member']).name!=r['member']:raise ValueError('Unsafe archive/member path')
+  locations[key]=r
+ if not a.execute:
+  print(json.dumps({'records':len(locations),'output':str(a.output),'source_snapshot_modified':False,'execute':False},indent=2));return
+ if a.output.exists() and any(a.output.iterdir()):raise ValueError('Output must be a new/empty directory; will not overwrite user files')
+ a.output.mkdir(parents=True,exist_ok=True);by_archive={}
+ for key,r in locations.items():by_archive.setdefault(r['archive'],{})[r['member']]=(key,r)
+ for name,entries in by_archive.items():
+  with tarfile.open(src/'observations'/name,'r:') as archive:
+   for member in archive:
+    entry=entries.pop(member.name,None)
+    if entry is None or not member.isfile():raise ValueError('Unexpected archive entry')
+    (split,sid),r=entry;data=archive.extractfile(member).read()
+    if len(data)!=r['bytes'] or hashlib.sha256(data).hexdigest()!=r['sha256']:raise ValueError('Original NPZ checksum differs')
+    filename=hashlib.sha256(sid.encode()).hexdigest()[:32]+'.npz'
+    target=a.output/'frames'/split/filename;target.parent.mkdir(parents=True,exist_ok=True)
+    if target.exists():raise ValueError('Duplicate target frame file')
+    target.write_bytes(data)
+  if entries:raise ValueError('Missing observation member')
+ for split in ('train','heldout'):
+  with (src/'sft'/(split+'.jsonl')).open() as inp,(a.output/(split+'.jsonl')).open('w') as out:
+   for line in inp:
+    r=json.loads(line)
+    if (split,r['sid']) not in locations:raise ValueError('Input record lacks backed-up observation')
+    r['frames_root']=str((a.output/'frames'/split).resolve())
+    r['npz']=hashlib.sha256(r['sid'].encode()).hexdigest()[:32]+'.npz'
+    out.write(json.dumps(r,ensure_ascii=False)+'\n')
+ print('Restored exact NPZ bytes and re-based SFT train/heldout JSONL. Only npz/frames_root fields in WORKING copies changed; frozen originals are intact.')
+if __name__=='__main__':main()