Upload fso_kaggle_ingestor.py with huggingface_hub

fso_kaggle_ingestor.py

@@ -0,0 +1,58 @@
+import os
+import json
+import time
+
+class KaggleIngestor:
+    """
+    Law XII Component: The Global Data Ingestor (Kaggle)
+    """
+    def __init__(self, vault, m=256, k=4):
+        self.vault = vault
+        self.m = m
+        self.k = k
+        self.api = None
+
+    def authenticate(self):
+        # Delayed import to ensure environment variables are set
+        from kaggle.api.kaggle_api_extended import KaggleApi
+        try:
+            self.api = KaggleApi()
+            self.api.authenticate()
+            print("\n[✓] KAGGLE AUTHENTICATION SECURED.")
+            return True
+        except Exception as e:
+            print(f"      [-] KAGGLE AUTH ERROR: {str(e)}")
+            return False
+
+    def download_and_shatter_dataset(self, dataset_id, orchestrator):
+        if not self.api:
+            if not self.authenticate(): return
+
+        print(f"\n[*] [KAGGLE]: Consuming Dataset '{dataset_id}'...")
+        try:
+            dataset_name = dataset_id.split('/')[-1]
+            download_path = f"./kaggle_data/{dataset_name}"
+            os.makedirs(download_path, exist_ok=True)
+            self.api.dataset_download_files(dataset_id, path=download_path, unzip=True)
+            
+            total_atoms = 0
+            for root, dirs, files in os.walk(download_path):
+                for file in files:
+                    file_path = os.path.join(root, file)
+                    if os.path.getsize(file_path) > 1024 * 100: continue # Skip large files for demo
+                    with open(file_path, "rb") as f:
+                        content = f.read()
+                    atoms = orchestrator.tgi.ingestor.shatterer.shatter(file, content)
+                    for atom in atoms:
+                        orchestrator.tgi.ingestor.topological_manifold[atom['coord']] = {
+                            "filename": file, "data": atom['data'], "fiber": atom['fiber'], "type": "kaggle_data"
+                        }
+                        total_atoms += 1
+            print(f"\n[✓] KAGGLE INGESTION COMPLETE: {total_atoms} atoms added.")
+            return True
+        except Exception as e:
+            print(f"      [-] KAGGLE INGESTION FAILED: {str(e)}")
+            return False
+
+if __name__ == "__main__":
+    print("Kaggle Ingestor Ready.")