fix: pin transformers stack and force slow tokenizer by default to avoid fast-tokenizer errors

CELESTIAL_Training_Notebook.ipynb

@@ -1,5 +1,61 @@
 {
   "cells": [
+    {
+      "cell_type": "code",
+      "metadata": {},
+      "execution_count": null,
+      "outputs": [],
+      "source": [
+        "# 🔐 Hugging Face Authentication for Google Colab\n",
+        "try:\n",
+        "    from google.colab import userdata\n",
+        "    import os\n",
+        "    hf_token = userdata.get('HF_TOKEN')\n",
+        "    os.environ['HUGGINGFACE_HUB_TOKEN'] = hf_token\n",
+        "    print('✅ HF token loaded from Colab secrets')\n",
+        "except ImportError:\n",
+        "    print('⚠️ Not running in Colab, skipping token setup')\n",
+        "except Exception as e:\n",
+        "    print(f'⚠️ Could not load HF_TOKEN from Colab secrets: {e}')\n",
+        "    print('💡 Add HF_TOKEN to Colab secrets: Secrets tab → Add new secret → Name: HF_TOKEN')\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "metadata": {},
+      "execution_count": null,
+      "outputs": [],
+      "source": [
+        "# 🔧 Install pinned versions for stable training\n",
+        "!pip install -q transformers==4.46.2 tokenizers==0.20.1\n",
+        "!pip install -q peft==0.14.0 datasets==2.20.0 bitsandbytes==0.43.3 accelerate==0.34.2 huggingface_hub==0.24.6 trl==0.11.4\n",
+        "import os; os.environ['TOKENIZERS_PARALLELISM'] = 'false'\n",
+        "print('✅ Pinned HF stack installed')\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "metadata": {},
+      "execution_count": null,
+      "outputs": [],
+      "source": [
+        "# 🩹 Force safe defaults to avoid fast-tokenizer and remote code import issues\n",
+        "from transformers import AutoTokenizer as _AutoTokenizer, AutoModelForCausalLM as _AutoModelForCausalLM\n",
+        "_orig_tok_from_pretrained = _AutoTokenizer.from_pretrained\n",
+        "def _patched_tok_from_pretrained(*args, **kwargs):\n",
+        "    kwargs.setdefault('use_fast', False)\n",
+        "    kwargs.setdefault('trust_remote_code', False)\n",
+        "    return _orig_tok_from_pretrained(*args, **kwargs)\n",
+        "_AutoTokenizer.from_pretrained = staticmethod(_patched_tok_from_pretrained)\n",
+        "\n",
+        "_orig_model_from_pretrained = _AutoModelForCausalLM.from_pretrained\n",
+        "def _patched_model_from_pretrained(*args, **kwargs):\n",
+        "    kwargs.setdefault('trust_remote_code', False)\n",
+        "    return _orig_model_from_pretrained(*args, **kwargs)\n",
+        "_AutoModelForCausalLM.from_pretrained = staticmethod(_patched_model_from_pretrained)\n",
+        "print('✅ Patched: AutoTokenizer(use_fast=False, trust_remote_code=False) and AutoModel(trust_remote_code=False) by default')\n"
+      ]
+    },
     {
       "cell_type": "code",
       "metadata": {},