fix: pin transformers stack and force slow tokenizer by default to avoid fast-tokenizer errors

CELESTIAL_Training_Notebook.ipynb

@@ -1,5 +1,59 @@
 {
   "cells": [
+    {
+      "cell_type": "code",
+      "metadata": {},
+      "execution_count": null,
+      "outputs": [],
+      "source": [
+        "# 🔐 Hugging Face Authentication for Google Colab\n",
+        "try:\n",
+        "    from google.colab import userdata\n",
+        "    import os\n",
+        "    hf_token = userdata.get('HF_TOKEN')\n",
+        "    os.environ['HUGGINGFACE_HUB_TOKEN'] = hf_token\n",
+        "    print('✅ HF token loaded from Colab secrets')\n",
+        "except ImportError:\n",
+        "    print('⚠️ Not running in Colab, skipping token setup')\n",
+        "except Exception as e:\n",
+        "    print(f'⚠️ Could not load HF_TOKEN from Colab secrets: {e}')\n",
+        "    print('💡 Add HF_TOKEN to Colab secrets: Secrets tab → Add new secret → Name: HF_TOKEN')\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "metadata": {},
+      "execution_count": null,
+      "outputs": [],
+      "source": [
+        "# 🔧 Install compatible versions for stable training\n",
+        "!pip install -q transformers>=4.36.0 tokenizers>=0.15.0\n",
+        "!pip install -q peft>=0.8.0 datasets>=2.16.0 bitsandbytes>=0.42.0 accelerate>=0.26.0 huggingface_hub trl\n",
+        "import os; os.environ['TOKENIZERS_PARALLELISM'] = 'false'\n",
+        "print('✅ Compatible HF stack installed')\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "metadata": {},
+      "execution_count": null,
+      "outputs": [],
+      "source": [
+        "# 🔄 Clear any previous patches and restart imports\n",
+        "import importlib\n",
+        "import sys\n",
+        "\n",
+        "# Clear transformers from cache if it exists\n",
+        "if 'transformers' in sys.modules:\n",
+        "    del sys.modules['transformers']\n",
+        "    print('🧹 Cleared transformers from module cache')\n",
+        "\n",
+        "# Fresh import\n",
+        "from transformers import AutoTokenizer, AutoModelForCausalLM\n",
+        "print('✅ Fresh transformers import - no patches applied')\n",
+        "print('💡 Use explicit parameters: AutoTokenizer.from_pretrained(model, use_fast=False, trust_remote_code=False)')\n"
+      ]
+    },
     {
       "cell_type": "code",
       "metadata": {},