{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# \ud83d\ude80 SimpleAI \u2014 TinyGPT \u52a0\u6cd5\u673a\u5236\u63a2\u9488 (Google Colab \u6781\u901f\u8fd0\u884c\u624b\u518c)\n", "\n", "\u672c Notebook \u63d0\u4f9b\u4e86\u5728 Google Colab \u4e0a\u4f7f\u7528 GPU/CPU \u8bad\u7ec3\u4e0e\u8bc4\u6d4b **additive-rand-transformer** \u7684\u5b8c\u6574\u6d41\u6c34\u7ebf\u3002\n", "\n", "### \u6838\u5fc3\u7279\u6027\uff1a\n", "1. **\u96f6\u989d\u5916\u914d\u7f6e\u76f4\u8fde Hugging Face**\uff1aColab \u4f4d\u4e8e\u6d77\u5916\u7f51\u7edc\uff0c\u76f4\u8fde Hugging Face \u6781\u901f\u4e0b\u8f7d\u4ee3\u7801\u4e0e\u6743\u91cd\uff08\u516c\u5f00\u4ed3\u5e93\u514d Token\uff0c\u79c1\u6709\u4ed3\u5e93\u652f\u6301\u4e00\u952e\u586b\u5165\uff09\u3002\n", "2. **Google Drive \u6df1\u5ea6\u96c6\u6210**\uff1a\u81ea\u52a8\u6302\u8f7d\u5e76\u53cc\u5411\u540c\u6b65\u4ee3\u7801\u3001\u914d\u7f6e\u4e0e\u8bad\u7ec3 Checkpoint (`.pt`)\u3002\n", "3. **JSON \u914d\u7f6e\u7075\u6d3b\u62c9\u8d77**\uff1a\u652f\u6301\u901a\u8fc7 `train.py --config config.json` \u4efb\u610f\u6307\u5b9a\u67b6\u6784\u3001\u8d85\u53c2\u548c\u6570\u636e\u6e90\u3002\n", "4. **\u5b66\u672f\u673a\u5236\u8bca\u65ad\u4e0e INT8 \u91cf\u5316**\uff1aH1 \u8349\u7a3f\u7eb8\u63a2\u9488\u3001\u52a8\u6001\u91cf\u5316\u65e0\u635f\u538b\u7f29\u6bd4\u8bc4\u6d4b\u3001\u4ea4\u4e92\u5f0f REPL\u3002" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## \u6b65\u9aa4 1\uff1a\u6302\u8f7d Google Drive \u5e76\u914d\u7f6e\u5de5\u4f5c\u533a" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from google.colab import drive\n", "import os, sys\n", "\n", "# 1. \u6302\u8f7d Google Drive\uff08\u53ef\u9009\uff0c\u82e5\u4e0d\u9700\u8981\u5b58\u5165 Drive \u4e5f\u53ef\u8df3\u8fc7\u6b64\u6b65\uff09\n", "try:\n", " drive.mount('/content/drive')\n", " DRIVE_WORKSPACE = '/content/drive/MyDrive/simpleAI_workspace'\n", " os.makedirs(f'{DRIVE_WORKSPACE}/checkpoints', exist_ok=True)\n", " os.makedirs(f'{DRIVE_WORKSPACE}/runs', exist_ok=True)\n", " print(f'\u2713 Google Drive \u5de5\u4f5c\u76ee\u5f55\u5c31\u7eea: {DRIVE_WORKSPACE}')\n", "except Exception as e:\n", " print(f'Drive \u6302\u8f7d\u63d0\u793a: {e}\uff0c\u5c06\u4f7f\u7528 Colab \u672c\u5730\u4e34\u65f6\u5b58\u50a8\u3002')\n", " DRIVE_WORKSPACE = None" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## \u6b65\u9aa4 2\uff1a\u73af\u5883\u51c6\u5907\u4e0e Hugging Face \u4ee3\u7801\u4e0b\u8f7d\n", "Colab \u6d77\u5916\u73af\u5883\u76f4\u8fde `huggingface.co`\uff0c\u901f\u5ea6\u901a\u5e38\u53ef\u8fbe 100MB/s+\uff0c**\u516c\u5f00\u4ed3\u5e93\u5b8c\u5168\u4e0d\u9700\u8981\u4efb\u4f55 Token**\u3002" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# 1. \u5b89\u88c5\u6838\u5fc3\u4f9d\u8d56\n", "!pip install -q torch openpyxl huggingface_hub pandas matplotlib\n", "\n", "# 2. \u4ece Hugging Face \u514b\u9686\u6216\u4e0b\u8f7d\u4ee3\u7801\n", "%cd /content\n", "if not os.path.exists('/content/additive-rand-transformer'):\n", " print('\u6b63\u5728\u4ece Hugging Face \u514b\u9686\u4ed3\u5e93...')\n", " !git clone https://huggingface.co/Hana-ame/additive-rand-transformer /content/additive-rand-transformer\n", "\n", "%cd /content/additive-rand-transformer\n", "if '/content/additive-rand-transformer' not in sys.path:\n", " sys.path.insert(0, '/content/additive-rand-transformer')\n", "print('\u2713 \u4ee3\u7801\u4ed3\u5e93\u4e0e Python \u8def\u5f84\u5df2\u5c31\u7eea')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## \u6b65\u9aa4 3\uff1a\u4ece Hugging Face \u6781\u901f\u4e0b\u8f7d\u9884\u8bad\u7ec3\u6743\u91cd (.pt Checkpoints)\n", "> **\u516c\u5f00\u4ed3\u5e93\u514d\u914d\u7f6e**\uff1b\u82e5\u4e3a\u79c1\u6709\u4ed3\u5e93\uff0c\u53ef\u5728\u5de6\u4fa7 Secrets\uff08\ud83d\udd11\u94a5\u5319\u56fe\u6807\uff09\u4e2d\u6dfb\u52a0 `HF_TOKEN` \u6216\u5728\u4e0b\u65b9\u586b\u5165 token\u3002" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import os, torch\n", "from huggingface_hub import hf_hub_download\n", "\n", "# \u5982\u679c\u662f\u79c1\u6709\u4ed3\u5e93\uff0c\u5c1d\u8bd5\u4ece Colab Secrets \u83b7\u53d6 token\uff0c\u516c\u5f00\u4ed3\u5e93\u5219\u4e3a None\n", "hf_token = None\n", "try:\n", " from google.colab import userdata\n", " hf_token = userdata.get('HF_TOKEN')\n", "except Exception:\n", " hf_token = os.environ.get('HF_TOKEN', None)\n", "\n", "REPO_ID = 'Hana-ame/additive-rand-transformer'\n", "CKPT_NAME = 'l4_d128_cot_bias05_final.pt'\n", "os.makedirs('checkpoints', exist_ok=True)\n", "local_path = f'checkpoints/{CKPT_NAME}'\n", "\n", "print(f'\u6b63\u5728\u4ece Hugging Face \u4ed3\u5e93 ({REPO_ID}) \u4e0b\u8f7d\u6743\u91cd: {CKPT_NAME} ...')\n", "try:\n", " downloaded_file = hf_hub_download(\n", " repo_id=REPO_ID,\n", " filename=f'checkpoints/{CKPT_NAME}',\n", " local_dir='.',\n", " token=hf_token\n", " )\n", " size_mb = os.path.getsize(local_path) / (1024 * 1024)\n", " print(f'\u2713 \u6743\u91cd\u4e0b\u8f7d\u6210\u529f: {local_path} ({size_mb:.2f} MB)')\n", " \n", " # \u81ea\u52a8\u5907\u4efd\u81f3 Google Drive\n", " if DRIVE_WORKSPACE:\n", " !cp {local_path} {DRIVE_WORKSPACE}/checkpoints/{CKPT_NAME}\n", " print(f'\u2713 \u5df2\u540c\u6b65\u5907\u4efd\u5230 Google Drive')\n", "except Exception as e:\n", " print(f'\u4e0b\u8f7d\u63d0\u793a: {e}')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## \u6b65\u9aa4 4\uff1a\u751f\u6210\u81ea\u5b9a\u4e49\u5b9e\u9a8c\u914d\u7f6e (`config.json`)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import json\n", "\n", "custom_config = {\n", " 'layers': 4, # \u6a21\u578b\u5c42\u6570 L (1-10)\n", " 'd': 128, # \u5d4c\u5165\u7ef4\u5ea6 d (32-512)\n", " 'heads': 4, # \u6ce8\u610f\u529b\u5934\u6570\n", " 'steps': 4000, # \u8bad\u7ec3\u6b65\u6570 (\u5efa\u8bae 2000-4000)\n", " 'batch_size': 32, # \u6279\u91cf\u5927\u5c0f\n", " 'lr': 3e-4, # \u5b66\u4e60\u7387\n", " 'wd': 0.1, # \u6743\u91cd\u8870\u51cf\n", " 'warmup': 200, # \u9884\u70ed\u6b65\u6570\n", " 'datasource': {\n", " 'type': 'cot', # cot: \u7ad6\u5f0f\u8349\u7a3f\u7eb8 | plain: \u65e0\u4e2d\u95f4\u8fc7\u7a0b\n", " 'max_digits': 4, # \u6700\u5927\u64cd\u4f5c\u6570\u4f4d\u6570 (1-4\u4f4d)\n", " 'bias': 0.5, # 4\u4f4d\u9ad8\u96be\u5ea6\u8fdb\u4f4d\u52a0\u6743 (0.5 \u4e3a\u6700\u4f73\u76f8\u53d8\u70b9)\n", " 'max_spaces': 3, # \u8fd0\u7b97\u7b26\u4e24\u4fa7\u7a7a\u683c\u968f\u673a\u6270\u52a8\n", " 'single': True # \u5355\u6837\u672c\u8bad\u7ec3\uff08\u4e0d\u8de8\u9898\u6253\u5305\uff09\n", " }\n", "}\n", "\n", "with open('config.json', 'w', encoding='utf-8') as f:\n", " json.dump(custom_config, f, indent=2)\n", "\n", "print('\u2713 \u5df2\u751f\u6210 config.json:')\n", "print(json.dumps(custom_config, indent=2))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## \u6b65\u9aa4 5\uff1a\u542f\u52a8\u8bad\u7ec3 (`train.py --config config.json`)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# \u6267\u884c\u8bad\u7ec3\u5e76\u8f93\u51fa\u5b9e\u65f6\u8fdb\u5ea6\u4e0e CoT \u51c6\u786e\u7387\n", "!python -m additive_rand_transformer.train --config config.json" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## \u6b65\u9aa4 6\uff1a\u673a\u5236\u63a2\u9488\u5b66\u672f\u8bca\u65ad (H1 \u8349\u7a3f\u7eb8\u7be1\u6539\u6d4b\u8bd5)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# \u8fd0\u884c H1 \u8349\u7a3f\u7eb8\u7be1\u6539\u654f\u611f\u5ea6\u63a2\u9488\n", "!python -m additive_rand_transformer.explore_h1 || true" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## \u6b65\u9aa4 7\uff1a\u6a21\u578b INT8 \u52a8\u6001\u91cf\u5316\u8bc4\u6d4b (\u538b\u7f29\u6bd4\u4e0e\u7cbe\u5ea6\u9a8c\u8bc1)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# \u8fd0\u884c PyTorch Dynamic INT8 \u91cf\u5316\u57fa\u51c6\n", "!python -m additive_rand_transformer.quantize --checkpoint checkpoints/l4_d128_cot_bias05_final.pt" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## \u6b65\u9aa4 8\uff1a\u4ea4\u4e92\u5f0f\u63a8\u7406\u4f53\u9a8c (REPL)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from additive_rand_transformer.model import TinyGPT, TinyGPTConfig\n", "from additive_rand_transformer.data import BOS, EOS, SP, PLUS, MINUS, _int_to_tokens, decode\n", "\n", "ckpt_file = 'checkpoints/l4_d128_cot_bias05_final.pt'\n", "if not os.path.exists(ckpt_file):\n", " import glob\n", " ckpts = sorted(glob.glob('runs/**/checkpoint*.pt', recursive=True))\n", " if ckpts:\n", " ckpt_file = ckpts[-1]\n", "\n", "if os.path.exists(ckpt_file):\n", " ck = torch.load(ckpt_file, map_location='cpu', weights_only=False)\n", " cfg = TinyGPTConfig(**{k: v for k, v in ck['config'].items() if k in TinyGPTConfig.__dataclass_fields__})\n", " model = TinyGPT(cfg)\n", " model.load_state_dict(ck['model'])\n", " model.eval()\n", " print(f'\u2713 \u6210\u529f\u52a0\u8f7d\u6a21\u578b: {ckpt_file} (L={cfg.n_layer}, d={cfg.n_embd}, {model.num_parameters():,} \u53c2\u6570)')\n", "\n", " def calculate(a, b, op='+'):\n", " op_id = PLUS if op == '+' else MINUS\n", " prompt = [BOS] + _int_to_tokens(a) + [SP, op_id, SP] + _int_to_tokens(b) + [SP]\n", " ids = list(prompt)\n", " with torch.no_grad():\n", " for _ in range(80):\n", " x = torch.tensor([ids], dtype=torch.long)\n", " logits, _ = model(x, None)\n", " nxt = int(logits[0, -1].argmax())\n", " ids.append(nxt)\n", " if nxt == EOS:\n", " break\n", " result_str = decode(ids)\n", " print(f'\u3010\u9898\u76ee\u3011: {a} {op} {b}')\n", " print(f'\u3010\u6a21\u578b\u751f\u6210 (\u542b\u7ad6\u5f0fCoT)\u3011:\\n{result_str}\\n')\n", "\n", " # \u6f14\u793a\u591a\u4f4d\u8ba1\u7b97\n", " calculate(37, 85, '+')\n", " calculate(523, 194, '-')\n", " calculate(1234, 5678, '+')\n", " calculate(9999, 4321, '-')\n", "else:\n", " print('\u672a\u627e\u5230 checkpoint\uff0c\u8bf7\u5148\u8fd0\u884c\u6b65\u9aa4 3 \u6216\u6b65\u9aa4 5\u3002')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## \u6b65\u9aa4 9\uff1a\u5c06\u5168\u90e8\u4ea7\u7269\u5907\u4efd\u81f3 Google Drive" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "if DRIVE_WORKSPACE:\n", " !cp -ru runs/ {DRIVE_WORKSPACE}/runs/ || true\n", " !cp -ru checkpoints/ {DRIVE_WORKSPACE}/checkpoints/ || true\n", " !cp config.json {DRIVE_WORKSPACE}/ || true\n", " print(f'\u2713 \u5168\u90e8\u6743\u91cd\u3001\u914d\u7f6e\u4e0e\u8fd0\u884c\u65e5\u5fd7\u5df2\u6210\u529f\u5f52\u6863\u81f3 Google Drive: {DRIVE_WORKSPACE}')\n", "else:\n", " print('\u672a\u6302\u8f7d Google Drive\uff0c\u4ea7\u7269\u4fdd\u5b58\u5728 Colab \u672c\u5730\u3002')" ] } ], "metadata": { "accelerator": "GPU", "colab": { "toc_visible": true }, "kernelspec": { "display_name": "Python 3", "name": "python3" }, "language_info": { "name": "python" } }, "nbformat": 4, "nbformat_minor": 2 }