Add source: tmt/experiments/03_full_tmt.ipynb

tmt/experiments/03_full_tmt.ipynb

@@ -0,0 +1,58 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": ["# Experiment 03 — Full TMT Training Run\n", "All three innovations active: Mesh Attention + Temporal Decay + Adaptive Depth Routing."]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import torch\n",
+    "from tmt.model.config import TMTConfig\n",
+    "from tmt.training.trainer import TMTTrainer, TrainConfig\n",
+    "from tmt.data.dataset import load_text_dataset\n",
+    "\n",
+    "cfg = TMTConfig(\n",
+    "    vocab_size=50258, d_model=512, n_heads=8, n_layers=12,\n",
+    "    graph_k=8, decay_rate=0.1, exit_threshold=0.85,\n",
+    "    dual_stream=True, memory_anchors=16, ffn_stream_dim=256,\n",
+    ")\n",
+    "print(cfg)\n",
+    "print(f'Device: {\"cuda\" if torch.cuda.is_available() else \"cpu\"}')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "loaders = load_text_dataset('wikitext-2', seq_len=256, batch_size=16)\n",
+    "\n",
+    "train_cfg = TrainConfig(\n",
+    "    total_steps=10_000,\n",
+    "    warmup_steps=500,\n",
+    "    eval_every=500,\n",
+    "    save_every=1000,\n",
+    "    use_wandb=False,   # set True and login to wandb first\n",
+    ")\n",
+    "\n",
+    "trainer = TMTTrainer(cfg, train_cfg, loaders['train'], loaders.get('validation'))\n",
+    "trainer.train()\n",
+    "\n",
+    "tmt_ppl = trainer.evaluate()\n",
+    "print(f'Full TMT perplexity: {tmt_ppl:.2f}')"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {"display_name": "Python 3", "language": "python", "name": "python3"},
+  "language_info": {"name": "python", "version": "3.10.0"}
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}