fix: pin transformers stack and force slow tokenizer by default to avoid fast-tokenizer errors

CELESTIAL_Training_Notebook.ipynb

@@ -1,5 +1,63 @@
 {
   "cells": [
+    {
+      "cell_type": "code",
+      "metadata": {},
+      "execution_count": null,
+      "outputs": [],
+      "source": [
+        "# 🔐 Hugging Face Authentication for Google Colab\n",
+        "try:\n",
+        "    from google.colab import userdata\n",
+        "    import os\n",
+        "    hf_token = userdata.get('HF_TOKEN')\n",
+        "    os.environ['HUGGINGFACE_HUB_TOKEN'] = hf_token\n",
+        "    print('✅ HF token loaded from Colab secrets')\n",
+        "except ImportError:\n",
+        "    print('⚠️ Not running in Colab, skipping token setup')\n",
+        "except Exception as e:\n",
+        "    print(f'⚠️ Could not load HF_TOKEN from Colab secrets: {e}')\n",
+        "    print('💡 Add HF_TOKEN to Colab secrets: Secrets tab → Add new secret → Name: HF_TOKEN')\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "metadata": {},
+      "execution_count": null,
+      "outputs": [],
+      "source": [
+        "# 🔧 Install compatible versions for stable training\n",
+        "!pip install -q transformers>=4.36.0 tokenizers>=0.15.0\n",
+        "!pip install -q peft>=0.8.0 datasets>=2.16.0 bitsandbytes>=0.42.0 accelerate>=0.26.0 huggingface_hub trl\n",
+        "import os; os.environ['TOKENIZERS_PARALLELISM'] = 'false'\n",
+        "print('✅ Compatible HF stack installed')\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "metadata": {},
+      "execution_count": null,
+      "outputs": [],
+      "source": [
+        "# 🛡️ Safe loading functions to avoid tokenizer and import errors\n",
+        "from transformers import AutoTokenizer, AutoModelForCausalLM\n",
+        "\n",
+        "def safe_load_tokenizer(model_name, **kwargs):\n",
+        "    \"\"\"Load tokenizer with safe defaults\"\"\"\n",
+        "    kwargs.setdefault('use_fast', False)\n",
+        "    kwargs.setdefault('trust_remote_code', False)\n",
+        "    return AutoTokenizer.from_pretrained(model_name, **kwargs)\n",
+        "\n",
+        "def safe_load_model(model_name, **kwargs):\n",
+        "    \"\"\"Load model with safe defaults\"\"\"\n",
+        "    kwargs.setdefault('trust_remote_code', False)\n",
+        "    return AutoModelForCausalLM.from_pretrained(model_name, **kwargs)\n",
+        "\n",
+        "print('✅ Safe loading functions ready')\n",
+        "print('💡 Use: tokenizer = safe_load_tokenizer(MODEL_NAME)')\n",
+        "print('💡 Use: model = safe_load_model(MODEL_NAME, quantization_config=bnb_config, device_map=\"auto\")')\n"
+      ]
+    },
     {
       "cell_type": "code",
       "metadata": {},