Upload .\scripts\quick_process.py with huggingface_hub

.//scripts//quick_process.py

@@ -0,0 +1,41 @@
+"""Quick processing test."""
+import sys
+import json
+import shutil
+from pathlib import Path
+
+PROJECT_ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(PROJECT_ROOT / "src"))
+from data_collection.data_processor import DataProcessor
+
+raw_dir = str(PROJECT_ROOT / "data/raw_test")
+processed_dir = str(PROJECT_ROOT / "data/processed_test")
+Path(processed_dir).mkdir(parents=True, exist_ok=True)
+
+seen_names = set()
+for jsonl_file in Path(raw_dir).rglob("*.jsonl"):
+    if jsonl_file.parent == Path(raw_dir):
+        continue
+    parent_dir = jsonl_file.parent.name
+    prefixed_name = f"{parent_dir}_{jsonl_file.name}"
+    dest = Path(raw_dir) / prefixed_name
+    if prefixed_name not in seen_names:
+        shutil.copy2(jsonl_file, dest)
+        seen_names.add(prefixed_name)
+        print(f"Copied: {jsonl_file.name} -> {prefixed_name}")
+
+processor = DataProcessor(raw_dir=raw_dir, processed_dir=processed_dir)
+summary = processor.process_all()
+
+print()
+print("=== RESULTS ===")
+for k, v in summary.items():
+    if k != "per_file_stats":
+        print(f"  {k}: {v}")
+print()
+for fs in summary.get("per_file_stats", []):
+    f = fs["file"]
+    l = fs["loaded"]
+    k = fs["kept"]
+    t = fs["tokens"]
+    print(f"  {f}: loaded={l}, kept={k}, tokens={t}")