Sync latest docs/config and dataset adjustments

AMD Cloud Accelerate config.txt

@@ -1,27 +1,16 @@
-root@jupyter-launcher-260401134729-3102:/workspace# python3 -m venv mindi_env
-root@jupyter-launcher-260401134729-3102:/workspace# source mindi_env/bin/activate
-((mindi_env) ) root@jupyter-launcher-260401134729-3102:/workspace# pip install --upgrade pip
-Requirement already satisfied: pip in ./mindi_env/lib/python3.12/site-packages (25.0.1)
-Collecting pip
-  Using cached pip-26.0.1-py3-none-any.whl.metadata (4.7 kB)
-Using cached pip-26.0.1-py3-none-any.whl (1.8 MB)
-Installing collected packages: pip
-  Attempting uninstall: pip
-    Found existing installation: pip 25.0.1
-    Uninstalling pip-25.0.1:
-      Successfully uninstalled pip-25.0.1
-Successfully installed pip-26.0.1
-((mindi_env) ) root@jupyter-launcher-260401134729-3102:/workspace# pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/rocm6.0
+root@jupyter-launcher-260401210247-9205:/workspace# python3 -m venv mindi_env
+root@jupyter-launcher-260401210247-9205:/workspace# source mindi_env/bin/activate
+((mindi_env) ) root@jupyter-launcher-260401210247-9205:/workspace# pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/rocm6.0
 Looking in indexes: https://download.pytorch.org/whl/rocm6.0
 Collecting torch
   Downloading https://download.pytorch.org/whl/rocm6.0/torch-2.4.1%2Brocm6.0-cp312-cp312-linux_x86_64.whl (2363.3 MB)
-     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 2.4/2.4 GB 86.5 MB/s  0:00:06
+     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 2.4/2.4 GB 25.2 MB/s eta 0:00:00
 Collecting torchvision
   Downloading https://download-r2.pytorch.org/whl/rocm6.0/torchvision-0.19.1%2Brocm6.0-cp312-cp312-linux_x86_64.whl (65.8 MB)
-     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 65.8/65.8 MB 248.3 MB/s  0:00:00
+     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 65.8/65.8 MB 230.9 MB/s eta 0:00:00
 Collecting torchaudio
   Downloading https://download-r2.pytorch.org/whl/rocm6.0/torchaudio-2.4.1%2Brocm6.0-cp312-cp312-linux_x86_64.whl (1.7 MB)
-     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.7/1.7 MB 187.5 MB/s  0:00:00
+     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.7/1.7 MB 150.2 MB/s eta 0:00:00
 Collecting filelock (from torch)
   Downloading filelock-3.25.2-py3-none-any.whl.metadata (2.0 kB)
 Collecting typing-extensions>=4.8.0 (from torch)
@@ -38,7 +27,7 @@ Collecting setuptools (from torch)
   Downloading https://download.pytorch.org/whl/setuptools-70.2.0-py3-none-any.whl.metadata (5.8 kB)
 Collecting pytorch-triton-rocm==3.0.0 (from torch)
   Downloading https://download-r2.pytorch.org/whl/pytorch_triton_rocm-3.0.0-cp312-cp312-linux_x86_64.whl (341.7 MB)
-     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 341.7/341.7 MB 180.8 MB/s  0:00:01
+     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 341.7/341.7 MB 109.9 MB/s eta 0:00:00
 Collecting numpy (from torchvision)
   Downloading numpy-2.4.3-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl.metadata (6.6 kB)
 Collecting pillow!=8.3.*,>=5.3.0 (from torchvision)
@@ -48,24 +37,24 @@ Collecting MarkupSafe>=2.0 (from jinja2->torch)
 Collecting mpmath<1.4,>=1.1.0 (from sympy->torch)
   Using cached mpmath-1.3.0-py3-none-any.whl.metadata (8.6 kB)
 Downloading pillow-12.1.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl (7.0 MB)
-   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 7.0/7.0 MB 55.7 MB/s  0:00:00
+   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 7.0/7.0 MB 57.3 MB/s eta 0:00:00
 Downloading https://download.pytorch.org/whl/typing_extensions-4.15.0-py3-none-any.whl (44 kB)
 Downloading filelock-3.25.2-py3-none-any.whl (26 kB)
 Downloading fsspec-2026.2.0-py3-none-any.whl (202 kB)
 Downloading https://download.pytorch.org/whl/jinja2-3.1.6-py3-none-any.whl (134 kB)
-Downloading https://download.pytorch.org/whl/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (22 kB)
 Using cached networkx-3.6.1-py3-none-any.whl (2.1 MB)
 Downloading numpy-2.4.3-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl (16.6 MB)
-   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 16.6/16.6 MB 114.6 MB/s  0:00:00
+   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 16.6/16.6 MB 86.7 MB/s eta 0:00:00
 Downloading https://download.pytorch.org/whl/setuptools-70.2.0-py3-none-any.whl (930 kB)
-   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 930.8/930.8 kB 129.2 MB/s  0:00:00
+   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 930.8/930.8 kB 140.4 MB/s eta 0:00:00
 Using cached sympy-1.14.0-py3-none-any.whl (6.3 MB)
+Downloading https://download.pytorch.org/whl/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (22 kB)
 Using cached mpmath-1.3.0-py3-none-any.whl (536 kB)
 Installing collected packages: mpmath, typing-extensions, sympy, setuptools, pillow, numpy, networkx, MarkupSafe, fsspec, filelock, pytorch-triton-rocm, jinja2, torch, torchvision, torchaudio
 Successfully installed MarkupSafe-3.0.3 filelock-3.25.2 fsspec-2026.2.0 jinja2-3.1.6 mpmath-1.3.0 networkx-3.6.1 numpy-2.4.3 pillow-12.1.1 pytorch-triton-rocm-3.0.0 setuptools-70.2.0 sympy-1.14.0 torch-2.4.1+rocm6.0 torchaudio-2.4.1+rocm6.0 torchvision-0.19.1+rocm6.0 typing-extensions-4.15.0
-((mindi_env) ) root@jupyter-launcher-260401134729-3102:/workspace# python -c "import torch; print(torch.cuda.is_available())"
+((mindi_env) ) root@jupyter-launcher-260401210247-9205:/workspace# python -c "import torch; print(torch.cuda.is_available())"
 True
-((mindi_env) ) root@jupyter-launcher-260401134729-3102:/workspace# pip install transformers datasets accelerate peft bitsandbytes sentencepiece tqdm
+((mindi_env) ) root@jupyter-launcher-260401210247-9205:/workspace# pip install transformers datasets accelerate peft bitsandbytes sentencepiece tqdm
 Collecting transformers
   Downloading transformers-5.4.0-py3-none-any.whl.metadata (32 kB)
 Collecting datasets
@@ -95,23 +84,7 @@ Collecting typer (from transformers)
   Downloading typer-0.24.1-py3-none-any.whl.metadata (16 kB)
 Collecting safetensors>=0.4.3 (from transformers)
   Downloading safetensors-0.7.0-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (4.1 kB)
-Requirement already satisfied: filelock>=3.10.0 in ./mindi_env/lib/python3.12/site-packages (from huggingface-hub<2.0,>=1.5.0->transformers) (3.25.2)
-Requirement already satisfied: fsspec>=2023.5.0 in ./mindi_env/lib/python3.12/site-packages (from huggingface-hub<2.0,>=1.5.0->transformers) (2026.2.0)
-Collecting hf-xet<2.0.0,>=1.4.2 (from huggingface-hub<2.0,>=1.5.0->transformers)
-  Downloading hf_xet-1.4.3-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata (4.9 kB)
-Collecting httpx<1,>=0.23.0 (from huggingface-hub<2.0,>=1.5.0->transformers)
-  Downloading httpx-0.28.1-py3-none-any.whl.metadata (7.1 kB)
-Requirement already satisfied: typing-extensions>=4.1.0 in ./mindi_env/lib/python3.12/site-packages (from huggingface-hub<2.0,>=1.5.0->transformers) (4.15.0)
-Collecting anyio (from httpx<1,>=0.23.0->huggingface-hub<2.0,>=1.5.0->transformers)
-  Downloading anyio-4.13.0-py3-none-any.whl.metadata (4.5 kB)
-Collecting certifi (from httpx<1,>=0.23.0->huggingface-hub<2.0,>=1.5.0->transformers)
-  Downloading certifi-2026.2.25-py3-none-any.whl.metadata (2.5 kB)
-Collecting httpcore==1.* (from httpx<1,>=0.23.0->huggingface-hub<2.0,>=1.5.0->transformers)
-  Downloading httpcore-1.0.9-py3-none-any.whl.metadata (21 kB)
-Collecting idna (from httpx<1,>=0.23.0->huggingface-hub<2.0,>=1.5.0->transformers)
-  Downloading idna-3.11-py3-none-any.whl.metadata (8.4 kB)
-Collecting h11>=0.16 (from httpcore==1.*->httpx<1,>=0.23.0->huggingface-hub<2.0,>=1.5.0->transformers)
-  Downloading h11-0.16.0-py3-none-any.whl.metadata (8.3 kB)
+Requirement already satisfied: filelock in ./mindi_env/lib/python3.12/site-packages (from datasets) (3.25.2)
 Collecting pyarrow>=21.0.0 (from datasets)
   Downloading pyarrow-23.0.1-cp312-cp312-manylinux_2_28_x86_64.whl.metadata (3.1 kB)
 Collecting dill<0.4.2,>=0.3.0 (from datasets)
@@ -120,20 +93,50 @@ Collecting pandas (from datasets)
   Downloading pandas-3.0.2-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl.metadata (79 kB)
 Collecting requests>=2.32.2 (from datasets)
   Downloading requests-2.33.1-py3-none-any.whl.metadata (4.8 kB)
+Collecting httpx<1.0.0 (from datasets)
+  Downloading httpx-0.28.1-py3-none-any.whl.metadata (7.1 kB)
 Collecting xxhash (from datasets)
   Downloading xxhash-3.6.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (13 kB)
 Collecting multiprocess<0.70.20 (from datasets)
   Downloading multiprocess-0.70.19-py312-none-any.whl.metadata (7.5 kB)
-Collecting aiohttp!=4.0.0a0,!=4.0.0a1 (from fsspec[http]<=2026.2.0,>=2023.1.0->datasets)
-  Downloading aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (8.1 kB)
+Requirement already satisfied: fsspec<=2026.2.0,>=2023.1.0 in ./mindi_env/lib/python3.12/site-packages (from fsspec[http]<=2026.2.0,>=2023.1.0->datasets) (2026.2.0)
 Collecting psutil (from accelerate)
   Downloading psutil-7.2.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl.metadata (22 kB)
 Requirement already satisfied: torch>=2.0.0 in ./mindi_env/lib/python3.12/site-packages (from accelerate) (2.4.1+rocm6.0)
+Collecting aiohttp!=4.0.0a0,!=4.0.0a1 (from fsspec[http]<=2026.2.0,>=2023.1.0->datasets)
+  Downloading aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (8.1 kB)
+Collecting anyio (from httpx<1.0.0->datasets)
+  Downloading anyio-4.13.0-py3-none-any.whl.metadata (4.5 kB)
+Collecting certifi (from httpx<1.0.0->datasets)
+  Downloading certifi-2026.2.25-py3-none-any.whl.metadata (2.5 kB)
+Collecting httpcore==1.* (from httpx<1.0.0->datasets)
+  Downloading httpcore-1.0.9-py3-none-any.whl.metadata (21 kB)
+Collecting idna (from httpx<1.0.0->datasets)
+  Downloading idna-3.11-py3-none-any.whl.metadata (8.4 kB)
+Collecting h11>=0.16 (from httpcore==1.*->httpx<1.0.0->datasets)
+  Downloading h11-0.16.0-py3-none-any.whl.metadata (8.3 kB)
+Collecting hf-xet<2.0.0,>=1.4.2 (from huggingface-hub<2.0,>=1.5.0->transformers)
+  Downloading hf_xet-1.4.3-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata (4.9 kB)
+Requirement already satisfied: typing-extensions>=4.1.0 in ./mindi_env/lib/python3.12/site-packages (from huggingface-hub<2.0,>=1.5.0->transformers) (4.15.0)
+Collecting charset_normalizer<4,>=2 (from requests>=2.32.2->datasets)
+  Downloading charset_normalizer-3.4.6-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (40 kB)
+Collecting urllib3<3,>=1.26 (from requests>=2.32.2->datasets)
+  Downloading urllib3-2.6.3-py3-none-any.whl.metadata (6.9 kB)
 Requirement already satisfied: sympy in ./mindi_env/lib/python3.12/site-packages (from torch>=2.0.0->accelerate) (1.14.0)
 Requirement already satisfied: networkx in ./mindi_env/lib/python3.12/site-packages (from torch>=2.0.0->accelerate) (3.6.1)
 Requirement already satisfied: jinja2 in ./mindi_env/lib/python3.12/site-packages (from torch>=2.0.0->accelerate) (3.1.6)
 Requirement already satisfied: setuptools in ./mindi_env/lib/python3.12/site-packages (from torch>=2.0.0->accelerate) (70.2.0)
 Requirement already satisfied: pytorch-triton-rocm==3.0.0 in ./mindi_env/lib/python3.12/site-packages (from torch>=2.0.0->accelerate) (3.0.0)
+Collecting python-dateutil>=2.8.2 (from pandas->datasets)
+  Using cached python_dateutil-2.9.0.post0-py2.py3-none-any.whl.metadata (8.4 kB)
+Collecting click>=8.2.1 (from typer->transformers)
+  Downloading click-8.3.1-py3-none-any.whl.metadata (2.6 kB)
+Collecting shellingham>=1.3.0 (from typer->transformers)
+  Downloading shellingham-1.5.4-py2.py3-none-any.whl.metadata (3.5 kB)
+Collecting rich>=12.3.0 (from typer->transformers)
+  Downloading rich-14.3.3-py3-none-any.whl.metadata (18 kB)
+Collecting annotated-doc>=0.0.2 (from typer->transformers)
+  Downloading annotated_doc-0.0.4-py3-none-any.whl.metadata (6.6 kB)
 Collecting aiohappyeyeballs>=2.5.0 (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<=2026.2.0,>=2023.1.0->datasets)
   Downloading aiohappyeyeballs-2.6.1-py3-none-any.whl.metadata (5.9 kB)
 Collecting aiosignal>=1.4.0 (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<=2026.2.0,>=2023.1.0->datasets)
@@ -148,119 +151,108 @@ Collecting propcache>=0.2.0 (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<=2026
   Downloading propcache-0.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (13 kB)
 Collecting yarl<2.0,>=1.17.0 (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<=2026.2.0,>=2023.1.0->datasets)
   Downloading yarl-1.23.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (79 kB)
-Collecting charset_normalizer<4,>=2 (from requests>=2.32.2->datasets)
-  Downloading charset_normalizer-3.4.6-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (40 kB)
-Collecting urllib3<3,>=1.26 (from requests>=2.32.2->datasets)
-  Downloading urllib3-2.6.3-py3-none-any.whl.metadata (6.9 kB)
-Requirement already satisfied: MarkupSafe>=2.0 in ./mindi_env/lib/python3.12/site-packages (from jinja2->torch>=2.0.0->accelerate) (3.0.3)
-Collecting python-dateutil>=2.8.2 (from pandas->datasets)
-  Using cached python_dateutil-2.9.0.post0-py2.py3-none-any.whl.metadata (8.4 kB)
 Collecting six>=1.5 (from python-dateutil>=2.8.2->pandas->datasets)
   Downloading six-1.17.0-py2.py3-none-any.whl.metadata (1.7 kB)
-Requirement already satisfied: mpmath<1.4,>=1.1.0 in ./mindi_env/lib/python3.12/site-packages (from sympy->torch>=2.0.0->accelerate) (1.3.0)
-Collecting click>=8.2.1 (from typer->transformers)
-  Downloading click-8.3.1-py3-none-any.whl.metadata (2.6 kB)
-Collecting shellingham>=1.3.0 (from typer->transformers)
-  Downloading shellingham-1.5.4-py2.py3-none-any.whl.metadata (3.5 kB)
-Collecting rich>=12.3.0 (from typer->transformers)
-  Downloading rich-14.3.3-py3-none-any.whl.metadata (18 kB)
-Collecting annotated-doc>=0.0.2 (from typer->transformers)
-  Downloading annotated_doc-0.0.4-py3-none-any.whl.metadata (6.6 kB)
 Collecting markdown-it-py>=2.2.0 (from rich>=12.3.0->typer->transformers)
   Downloading markdown_it_py-4.0.0-py3-none-any.whl.metadata (7.3 kB)
 Collecting pygments<3.0.0,>=2.13.0 (from rich>=12.3.0->typer->transformers)
   Downloading pygments-2.20.0-py3-none-any.whl.metadata (2.5 kB)
+Requirement already satisfied: MarkupSafe>=2.0 in ./mindi_env/lib/python3.12/site-packages (from jinja2->torch>=2.0.0->accelerate) (3.0.3)
+Requirement already satisfied: mpmath<1.4,>=1.1.0 in ./mindi_env/lib/python3.12/site-packages (from sympy->torch>=2.0.0->accelerate) (1.3.0)
 Collecting mdurl~=0.1 (from markdown-it-py>=2.2.0->rich>=12.3.0->typer->transformers)
   Downloading mdurl-0.1.2-py3-none-any.whl.metadata (1.6 kB)
 Downloading transformers-5.4.0-py3-none-any.whl (10.1 MB)
-   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 10.1/10.1 MB 55.5 MB/s  0:00:00
-Downloading huggingface_hub-1.8.0-py3-none-any.whl (625 kB)
-   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 625.2/625.2 kB 56.0 MB/s  0:00:00
-Downloading hf_xet-1.4.3-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (4.2 MB)
-   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 4.2/4.2 MB 130.1 MB/s  0:00:00
-Downloading httpx-0.28.1-py3-none-any.whl (73 kB)
-Downloading httpcore-1.0.9-py3-none-any.whl (78 kB)
-Downloading tokenizers-0.22.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.3 MB)
-   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 3.3/3.3 MB 116.5 MB/s  0:00:00
+   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 10.1/10.1 MB 51.7 MB/s eta 0:00:00
 Downloading datasets-4.8.4-py3-none-any.whl (526 kB)
-   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 527.0/527.0 kB 62.6 MB/s  0:00:00
-Downloading dill-0.4.1-py3-none-any.whl (120 kB)
-Downloading multiprocess-0.70.19-py312-none-any.whl (150 kB)
+   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 527.0/527.0 kB 75.6 MB/s eta 0:00:00
 Downloading accelerate-1.13.0-py3-none-any.whl (383 kB)
 Downloading peft-0.18.1-py3-none-any.whl (556 kB)
-   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 557.0/557.0 kB 30.8 MB/s  0:00:00
+   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 557.0/557.0 kB 80.1 MB/s eta 0:00:00
 Downloading bitsandbytes-0.49.2-py3-none-manylinux_2_24_x86_64.whl (60.7 MB)
-   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 60.7/60.7 MB 103.2 MB/s  0:00:00
+   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 60.7/60.7 MB 56.5 MB/s eta 0:00:00
 Downloading sentencepiece-0.2.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl (1.4 MB)
-   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.4/1.4 MB 127.4 MB/s  0:00:00
+   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.4/1.4 MB 2.7 MB/s eta 0:00:00
 Downloading tqdm-4.67.3-py3-none-any.whl (78 kB)
-Downloading aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (1.8 MB)
-   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.8/1.8 MB 208.3 MB/s  0:00:00
-Downloading multidict-6.7.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (256 kB)
-Downloading yarl-1.23.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (100 kB)
-Downloading aiohappyeyeballs-2.6.1-py3-none-any.whl (15 kB)
-Downloading aiosignal-1.4.0-py3-none-any.whl (7.5 kB)
-Downloading attrs-26.1.0-py3-none-any.whl (67 kB)
-Downloading frozenlist-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl (242 kB)
-Downloading h11-0.16.0-py3-none-any.whl (37 kB)
-Downloading idna-3.11-py3-none-any.whl (71 kB)
+Downloading dill-0.4.1-py3-none-any.whl (120 kB)
+Downloading httpx-0.28.1-py3-none-any.whl (73 kB)
+Downloading httpcore-1.0.9-py3-none-any.whl (78 kB)
+Downloading huggingface_hub-1.8.0-py3-none-any.whl (625 kB)
+   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 625.2/625.2 kB 10.3 MB/s eta 0:00:00
+Downloading multiprocess-0.70.19-py312-none-any.whl (150 kB)
 Downloading packaging-26.0-py3-none-any.whl (74 kB)
-Downloading propcache-0.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (221 kB)
 Downloading pyarrow-23.0.1-cp312-cp312-manylinux_2_28_x86_64.whl (47.6 MB)
-   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 47.6/47.6 MB 96.3 MB/s  0:00:00
+   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 47.6/47.6 MB 114.6 MB/s eta 0:00:00
 Downloading pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (807 kB)
-   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 807.9/807.9 kB 102.6 MB/s  0:00:00
+   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 807.9/807.9 kB 105.9 MB/s eta 0:00:00
 Downloading regex-2026.3.32-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (801 kB)
-   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 801.5/801.5 kB 105.5 MB/s  0:00:00
+   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 801.5/801.5 kB 112.8 MB/s eta 0:00:00
 Downloading requests-2.33.1-py3-none-any.whl (64 kB)
-Downloading charset_normalizer-3.4.6-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (207 kB)
-Downloading urllib3-2.6.3-py3-none-any.whl (131 kB)
-Downloading certifi-2026.2.25-py3-none-any.whl (153 kB)
 Downloading safetensors-0.7.0-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (507 kB)
-Downloading anyio-4.13.0-py3-none-any.whl (114 kB)
+Downloading tokenizers-0.22.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.3 MB)
+   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 3.3/3.3 MB 103.2 MB/s eta 0:00:00
 Downloading pandas-3.0.2-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl (10.9 MB)
-   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 10.9/10.9 MB 122.5 MB/s  0:00:00
-Using cached python_dateutil-2.9.0.post0-py2.py3-none-any.whl (229 kB)
-Downloading six-1.17.0-py2.py3-none-any.whl (11 kB)
+   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 10.9/10.9 MB 119.2 MB/s eta 0:00:00
 Downloading psutil-7.2.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl (155 kB)
 Downloading typer-0.24.1-py3-none-any.whl (56 kB)
+Downloading xxhash-3.6.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (193 kB)
+Downloading aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (1.8 MB)
+   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.8/1.8 MB 215.1 MB/s eta 0:00:00
 Downloading annotated_doc-0.0.4-py3-none-any.whl (5.3 kB)
+Downloading certifi-2026.2.25-py3-none-any.whl (153 kB)
+Downloading charset_normalizer-3.4.6-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (207 kB)
 Downloading click-8.3.1-py3-none-any.whl (108 kB)
+Downloading hf_xet-1.4.3-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (4.2 MB)
+   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 4.2/4.2 MB 109.6 MB/s eta 0:00:00
+Downloading idna-3.11-py3-none-any.whl (71 kB)
+Using cached python_dateutil-2.9.0.post0-py2.py3-none-any.whl (229 kB)
 Downloading rich-14.3.3-py3-none-any.whl (310 kB)
-Downloading pygments-2.20.0-py3-none-any.whl (1.2 MB)
-   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.2/1.2 MB 165.6 MB/s  0:00:00
+Downloading shellingham-1.5.4-py2.py3-none-any.whl (9.8 kB)
+Downloading urllib3-2.6.3-py3-none-any.whl (131 kB)
+Downloading anyio-4.13.0-py3-none-any.whl (114 kB)
+Downloading aiohappyeyeballs-2.6.1-py3-none-any.whl (15 kB)
+Downloading aiosignal-1.4.0-py3-none-any.whl (7.5 kB)
+Downloading attrs-26.1.0-py3-none-any.whl (67 kB)
+Downloading frozenlist-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl (242 kB)
+Downloading h11-0.16.0-py3-none-any.whl (37 kB)
 Downloading markdown_it_py-4.0.0-py3-none-any.whl (87 kB)
+Downloading multidict-6.7.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (256 kB)
+Downloading propcache-0.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (221 kB)
+Downloading pygments-2.20.0-py3-none-any.whl (1.2 MB)
+   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.2/1.2 MB 156.9 MB/s eta 0:00:00
+Downloading six-1.17.0-py2.py3-none-any.whl (11 kB)
+Downloading yarl-1.23.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (100 kB)
 Downloading mdurl-0.1.2-py3-none-any.whl (10.0 kB)
-Downloading shellingham-1.5.4-py2.py3-none-any.whl (9.8 kB)
-Downloading xxhash-3.6.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (193 kB)
 Installing collected packages: xxhash, urllib3, tqdm, six, shellingham, sentencepiece, safetensors, regex, pyyaml, pygments, pyarrow, psutil, propcache, packaging, multidict, mdurl, idna, hf-xet, h11, frozenlist, dill, click, charset_normalizer, certifi, attrs, annotated-doc, aiohappyeyeballs, yarl, requests, python-dateutil, multiprocess, markdown-it-py, httpcore, anyio, aiosignal, rich, pandas, httpx, bitsandbytes, aiohttp, typer, huggingface-hub, tokenizers, datasets, accelerate, transformers, peft
 Successfully installed accelerate-1.13.0 aiohappyeyeballs-2.6.1 aiohttp-3.13.5 aiosignal-1.4.0 annotated-doc-0.0.4 anyio-4.13.0 attrs-26.1.0 bitsandbytes-0.49.2 certifi-2026.2.25 charset_normalizer-3.4.6 click-8.3.1 datasets-4.8.4 dill-0.4.1 frozenlist-1.8.0 h11-0.16.0 hf-xet-1.4.3 httpcore-1.0.9 httpx-0.28.1 huggingface-hub-1.8.0 idna-3.11 markdown-it-py-4.0.0 mdurl-0.1.2 multidict-6.7.1 multiprocess-0.70.19 packaging-26.0 pandas-3.0.2 peft-0.18.1 propcache-0.4.1 psutil-7.2.2 pyarrow-23.0.1 pygments-2.20.0 python-dateutil-2.9.0.post0 pyyaml-6.0.3 regex-2026.3.32 requests-2.33.1 rich-14.3.3 safetensors-0.7.0 sentencepiece-0.2.1 shellingham-1.5.4 six-1.17.0 tokenizers-0.22.2 tqdm-4.67.3 transformers-5.4.0 typer-0.24.1 urllib3-2.6.3 xxhash-3.6.0 yarl-1.23.0
-((mindi_env) ) root@jupyter-launcher-260401134729-3102:/workspace# apt update && apt install -y git-lfs
+
+[notice] A new release of pip is available: 25.0.1 -> 26.0.1
+[notice] To update, run: pip install --upgrade pip
+((mindi_env) ) root@jupyter-launcher-260401210247-9205:/workspace# apt update && apt install -y git-lfs
 Get:1 https://repo.radeon.com/amdgpu/7.0/ubuntu jammy InRelease [3183 B]
 Get:2 https://repo.radeon.com/rocm/apt/7.0 jammy InRelease [2603 B]                                                                                
-Get:3 https://repo.radeon.com/amdgpu/7.0/ubuntu jammy/main amd64 Packages [1329 B]                                                                                   
-Get:4 https://repo.radeon.com/rocm/apt/7.0 jammy/main amd64 Packages [82.7 kB]                                                                                                                       
-Get:5 http://security.ubuntu.com/ubuntu jammy-security InRelease [129 kB]                                                               
-Get:6 http://archive.ubuntu.com/ubuntu jammy InRelease [270 kB]                                                   
-Get:7 https://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu jammy InRelease [18.1 kB]                                     
-Get:8 http://security.ubuntu.com/ubuntu jammy-security/multiverse amd64 Packages [62.6 kB]
-Get:9 http://archive.ubuntu.com/ubuntu jammy-updates InRelease [128 kB]      
-Get:10 https://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu jammy/main amd64 Packages [38.9 kB]
-Get:11 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 Packages [1309 kB]
-Get:12 http://archive.ubuntu.com/ubuntu jammy-backports InRelease [127 kB]                 
-Get:13 http://archive.ubuntu.com/ubuntu jammy/main amd64 Packages [1792 kB]                          
-Get:14 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 Packages [6803 kB]
-Get:15 http://archive.ubuntu.com/ubuntu jammy/restricted amd64 Packages [164 kB]            
-Get:16 http://archive.ubuntu.com/ubuntu jammy/universe amd64 Packages [17.5 MB]             
-Get:17 http://security.ubuntu.com/ubuntu jammy-security/main amd64 Packages [3842 kB]        
-Get:18 http://archive.ubuntu.com/ubuntu jammy/multiverse amd64 Packages [266 kB]              
-Get:19 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages [4217 kB]
-Get:20 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 Packages [7125 kB]
-Get:21 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse amd64 Packages [70.9 kB]
-Get:22 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 Packages [1621 kB]
-Get:23 http://archive.ubuntu.com/ubuntu jammy-backports/universe amd64 Packages [35.6 kB]
-Get:24 http://archive.ubuntu.com/ubuntu jammy-backports/main amd64 Packages [84.0 kB]
-Fetched 45.7 MB in 3s (16.7 MB/s)                          
+Get:3 https://repo.radeon.com/amdgpu/7.0/ubuntu jammy/main amd64 Packages [1329 B]                                                     
+Get:4 http://archive.ubuntu.com/ubuntu jammy InRelease [270 kB]                                                                                                        
+Get:5 https://repo.radeon.com/rocm/apt/7.0 jammy/main amd64 Packages [82.7 kB]                                                                  
+Get:6 http://security.ubuntu.com/ubuntu jammy-security InRelease [129 kB]                                                                          
+Get:7 http://archive.ubuntu.com/ubuntu jammy-updates InRelease [128 kB]                                                   
+Get:8 http://archive.ubuntu.com/ubuntu jammy-backports InRelease [127 kB]                
+Get:9 http://archive.ubuntu.com/ubuntu jammy/main amd64 Packages [1792 kB]                
+Get:10 http://archive.ubuntu.com/ubuntu jammy/universe amd64 Packages [17.5 MB]          
+Get:11 https://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu jammy InRelease [18.1 kB]                          
+Get:12 http://archive.ubuntu.com/ubuntu jammy/restricted amd64 Packages [164 kB]
+Get:13 http://archive.ubuntu.com/ubuntu jammy/multiverse amd64 Packages [266 kB] 
+Get:14 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages [4219 kB]
+Get:15 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 Packages [1622 kB]
+Get:16 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 Packages [7125 kB]              
+Get:17 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 Packages [6803 kB]             
+Get:18 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse amd64 Packages [70.9 kB]                          
+Get:19 http://archive.ubuntu.com/ubuntu jammy-backports/universe amd64 Packages [35.6 kB]                          
+Get:20 http://archive.ubuntu.com/ubuntu jammy-backports/main amd64 Packages [84.0 kB]                              
+Get:21 https://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu jammy/main amd64 Packages [38.9 kB]                  
+Get:22 http://security.ubuntu.com/ubuntu jammy-security/multiverse amd64 Packages [62.6 kB]  
+Get:23 http://security.ubuntu.com/ubuntu jammy-security/main amd64 Packages [3844 kB]
+Get:24 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 Packages [1310 kB]
+Fetched 45.7 MB in 2s (23.2 MB/s)                           
 Reading package lists... Done
 Building dependency tree... Done
 Reading state information... Done
@@ -276,24 +268,24 @@ The following NEW packages will be installed:
 Need to get 3544 kB of archives.
 After this operation, 10.5 MB of additional disk space will be used.
 Get:1 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 git-lfs amd64 3.0.2-1ubuntu0.3 [3544 kB]
-Fetched 3544 kB in 1s (2611 kB/s)  
+Fetched 3544 kB in 1s (2561 kB/s)  
 debconf: delaying package configuration, since apt-utils is not installed
 Selecting previously unselected package git-lfs.
 (Reading database ... 58520 files and directories currently installed.)
 Preparing to unpack .../git-lfs_3.0.2-1ubuntu0.3_amd64.deb ...
 Unpacking git-lfs (3.0.2-1ubuntu0.3) ...
 Setting up git-lfs (3.0.2-1ubuntu0.3) ...
-((mindi_env) ) root@jupyter-launcher-260401134729-3102:/workspace# git clone https://huggingface.co/Mindigenous/mindi-backup
+((mindi_env) ) root@jupyter-launcher-260401210247-9205:/workspace# git clone https://huggingface.co/Mindigenous/mindi-backup
 Cloning into 'mindi-backup'...
-remote: Enumerating objects: 286, done.
-remote: Counting objects: 100% (283/283), done.
-remote: Compressing objects: 100% (266/266), done.
-remote: Total 286 (delta 15), reused 217 (delta 6), pack-reused 3 (from 1)
-Receiving objects: 100% (286/286), 151.83 KiB | 9.49 MiB/s, done.
-Resolving deltas: 100% (15/15), done.
-Filtering content: 100% (85/85), 9.38 GiB | 533.08 MiB/s, done.
-((mindi_env) ) root@jupyter-launcher-260401134729-3102:/workspace# cd mindi-backup/
-((mindi_env) ) root@jupyter-launcher-260401134729-3102:/workspace/mindi-backup# accelerate config
+remote: Enumerating objects: 304, done.
+remote: Counting objects: 100% (301/301), done.
+remote: Compressing objects: 100% (283/283), done.
+remote: Total 304 (delta 21), reused 216 (delta 6), pack-reused 3 (from 1)
+Receiving objects: 100% (304/304), 159.79 KiB | 26.63 MiB/s, done.
+Resolving deltas: 100% (21/21), done.
+Filtering content: 100% (86/86), 11.12 GiB | 181.12 MiB/s, done.
+((mindi_env) ) root@jupyter-launcher-260401210247-9205:/workspace# cd mindi-backup/
+((mindi_env) ) root@jupyter-launcher-260401210247-9205:/workspace/mindi-backup# accelerate config
 ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------In which compute environment are you running?
 This machine                                                                                                                                                                                                                  
 ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------Which type of machine are you using?                                                                                                                                                                                          
@@ -306,28 +298,106 @@ Would you like to enable numa efficiency? (Currently only supported on NVIDIA ha
 ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------Do you wish to use mixed precision?                                                                                                                                                                                           
 bf16                                                                                                                                                                                                                          
 accelerate configuration saved at /root/.cache/huggingface/accelerate/default_config.yaml                                                                                                                                     
-((mindi_env) ) root@jupyter-launcher-260401134729-3102:/workspace/mindi-backup# python -c "import torch; print(torch.cuda.is_available(), torch.cuda.device_count())"
-True 1
-((mindi_env) ) root@jupyter-launcher-260401134729-3102:/workspace/mindi-backup# accelerate launch train.py \
-  --data_path data/final/train.jsonl \
-  --output_dir checkpoints \
-  --batch_size 4 \
-  --gradient_accumulation 16 \
-  --learning_rate 2e-5 \
-  --num_train_steps 60000 \
-  --save_steps 2000 \
-  --logging_steps 50
-usage: train.py [-h] [--no-resume]
-train.py: error: unrecognized arguments: --data_path data/final/train.jsonl --output_dir checkpoints --batch_size 4 --gradient_accumulation 16 --learning_rate 2e-5 --num_train_steps 60000 --save_steps 2000 --logging_steps 50
+((mindi_env) ) root@jupyter-launcher-260401210247-9205:/workspace/mindi-backup# accelerate launch train.py --no-resume                                                                                                        
+2026-04-01 21:12:28 | WARNING | Primary model path /workspace/mindi-backup/model is missing HF files. Falling back to /workspace/mindi-backup/hf_release/MINDI-1.0-420M                                                       
+2026-04-01 21:12:28 | INFO | Loading model from hf_release/MINDI-1.0-420M                                                                                                                                                     
+Loading weights: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 186/186 [00:00<00:00, 1542.64it/s]
+MindiForCausalLM LOAD REPORT from: hf_release/MINDI-1.0-420M
+Key            | Status  | Details
+---------------+---------+--------
+lm_head.weight | MISSING |        
+
+Notes:
+- MISSING:      those params were newly initialized because missing from the checkpoint. Consider training on your downstream task.
+bitsandbytes library load error: Configured ROCm binary not found at /workspace/mindi_env/lib/python3.12/site-packages/bitsandbytes/libbitsandbytes_rocm60.so
+Traceback (most recent call last):
+  File "/workspace/mindi_env/lib/python3.12/site-packages/bitsandbytes/cextension.py", line 320, in <module>
+    lib = get_native_library()
+          ^^^^^^^^^^^^^^^^^^^^
+  File "/workspace/mindi_env/lib/python3.12/site-packages/bitsandbytes/cextension.py", line 288, in get_native_library
+    raise RuntimeError(f"Configured {BNB_BACKEND} binary not found at {cuda_binary_path}")
+RuntimeError: Configured ROCm binary not found at /workspace/mindi_env/lib/python3.12/site-packages/bitsandbytes/libbitsandbytes_rocm60.so
+trainable params: 7,630,848 || all params: 489,165,696 || trainable%: 1.5600
+2026-04-01 21:12:39 | INFO | Starting training...
+2026-04-01 21:12:39 | WARNING | No valid checkpoint → starting fresh training
+  0%|                                                                                                                                                                                 | 29/156250 [01:27<130:16:46,  3.00s/it]^CTraceback (most recent call last):
+  File "/workspace/mindi-backup/train.py", line 243, in <module>
+    train(resume=not args.no_resume)
+  File "/workspace/mindi-backup/train.py", line 227, in train
+    safe_train(trainer, str(PATHS.checkpoint_dir), logger)
+  File "/workspace/mindi-backup/train.py", line 173, in safe_train
+    trainer.train()
+  File "/workspace/mindi_env/lib/python3.12/site-packages/transformers/trainer.py", line 1424, in train
+    return inner_training_loop(
+           ^^^^^^^^^^^^^^^^^^^^
+  File "/workspace/mindi_env/lib/python3.12/site-packages/transformers/trainer.py", line 1506, in _inner_training_loop
+    self._run_epoch(
+  File "/workspace/mindi_env/lib/python3.12/site-packages/transformers/trainer.py", line 1734, in _run_epoch
+    tr_loss_step = self.training_step(model, inputs, num_items_in_batch)
+                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/workspace/mindi_env/lib/python3.12/site-packages/transformers/trainer.py", line 1934, in training_step
+    self.accelerator.backward(loss, **kwargs)
+  File "/workspace/mindi_env/lib/python3.12/site-packages/accelerate/accelerator.py", line 2834, in backward
+    self.scaler.scale(loss).backward(**kwargs)
+  File "/workspace/mindi_env/lib/python3.12/site-packages/torch/_tensor.py", line 521, in backward
+    torch.autograd.backward(
+  File "/workspace/mindi_env/lib/python3.12/site-packages/torch/autograd/__init__.py", line 289, in backward
+    _engine_run_backward(
+  File "/workspace/mindi_env/lib/python3.12/site-packages/torch/autograd/graph.py", line 769, in _engine_run_backward
+    return Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+KeyboardInterrupt
+  0%|                                                                                                                                                                                 | 29/156250 [01:28<132:03:53,  3.04s/it]
 Traceback (most recent call last):
-  File "/workspace/mindi_env/bin/accelerate", line 6, in <module>
+  File "/workspace/mindi_env/bin/accelerate", line 8, in <module>
     sys.exit(main())
              ^^^^^^
   File "/workspace/mindi_env/lib/python3.12/site-packages/accelerate/commands/accelerate_cli.py", line 50, in main
     args.func(args)
   File "/workspace/mindi_env/lib/python3.12/site-packages/accelerate/commands/launch.py", line 1405, in launch_command
     simple_launcher(args)
-  File "/workspace/mindi_env/lib/python3.12/site-packages/accelerate/commands/launch.py", line 993, in simple_launcher
-    raise subprocess.CalledProcessError(returncode=process.returncode, cmd=cmd)
-subprocess.CalledProcessError: Command '['/workspace/mindi_env/bin/python3', 'train.py', '--data_path', 'data/final/train.jsonl', '--output_dir', 'checkpoints', '--batch_size', '4', '--gradient_accumulation', '16', '--learning_rate', '2e-5', '--num_train_steps', '60000', '--save_steps', '2000', '--logging_steps', '50']' returned non-zero exit status 2.
-((mindi_env) ) root@jupyter-launcher-260401134729-3102:/workspace/mindi-backup# 
+  File "/workspace/mindi_env/lib/python3.12/site-packages/accelerate/commands/launch.py", line 990, in simple_launcher
+    process.wait()
+  File "/usr/lib/python3.12/subprocess.py", line 1264, in wait
+    return self._wait(timeout=timeout)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/lib/python3.12/subprocess.py", line 2053, in _wait
+    (pid, sts) = self._try_wait(0)
+                 ^^^^^^^^^^^^^^^^^
+  File "/usr/lib/python3.12/subprocess.py", line 2011, in _try_wait
+    (pid, sts) = os.waitpid(self.pid, wait_flags)
+                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+KeyboardInterrupt
+
+((mindi_env) ) root@jupyter-launcher-260401210247-9205:/workspace/mindi-backup# pip uninstall bitsandbytes -y
+Found existing installation: bitsandbytes 0.49.2
+Uninstalling bitsandbytes-0.49.2:
+  Successfully uninstalled bitsandbytes-0.49.2
+((mindi_env) ) root@jupyter-launcher-260401210247-9205:/workspace/mindi-backup# accelerate launch train.py --no-resume
+2026-04-01 21:16:01 | WARNING | Primary model path /workspace/mindi-backup/model is missing HF files. Falling back to /workspace/mindi-backup/hf_release/MINDI-1.0-420M
+2026-04-01 21:16:01 | INFO | Loading model from hf_release/MINDI-1.0-420M
+Loading weights: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 186/186 [00:00<00:00, 1271.99it/s]
+MindiForCausalLM LOAD REPORT from: hf_release/MINDI-1.0-420M
+Key            | Status  | Details
+---------------+---------+--------
+lm_head.weight | MISSING |        
+
+Notes:
+- MISSING:      those params were newly initialized because missing from the checkpoint. Consider training on your downstream task.
+trainable params: 7,630,848 || all params: 489,165,696 || trainable%: 1.5600
+2026-04-01 21:16:11 | INFO | Starting training...
+2026-04-01 21:16:11 | WARNING | No valid checkpoint → starting fresh training
+{'loss': '167.2', 'grad_norm': '2.187', 'learning_rate': '1.999e-05', 'epoch': '0.0016'}                                                                                                                                      
+{'loss': '126.3', 'grad_norm': '0.9016', 'learning_rate': '1.999e-05', 'epoch': '0.0032'}                                                                                                                                     
+{'loss': '105.4', 'grad_norm': '0.3636', 'learning_rate': '1.998e-05', 'epoch': '0.0048'}                                                                                                                                     
+{'loss': '101.1', 'grad_norm': '0.2803', 'learning_rate': '1.997e-05', 'epoch': '0.0064'}                                                                                                                                     
+  0%|▎                                                                                                                                                                               | 223/156250 [11:11<130:24:40,  3.01s/it]
+
+
+python3 -m venv mindi_env
+source mindi_env/bin/activate
+pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/rocm6.0
+python -c "import torch; print(torch.cuda.is_available())"
+pip install transformers datasets accelerate peft sentencepiece tqdm
+apt update && apt install -y git-lfs
+git clone https://huggingface.co/Mindigenous/mindi-backup

Auto Backup HF script.txt

@@ -0,0 +1,12 @@
+cd /workspace/mindi-backup
+
+while true; do
+  for f in backups/*.tar.gz; do
+    if [ ! -f "$f.uploaded" ]; then
+      echo "Uploading $f"
+      hf upload mindi-backups "$f"
+      touch "$f.uploaded"
+    fi
+  done
+  sleep 300
+done

Full pipeline setup for AMC jupyter notebook.txt

@@ -0,0 +1,82 @@
+ok tell me from start after I reopened my jupyter notebook what to its important to setup an seprate env for it or not
+
+Good — let’s reset cleanly and do this proper production-style setup so you don’t face issues again.
+
+I’ll give you exact steps from scratch after restart 👇
+
+🚀 0. DO YOU NEED NEW ENV?
+✔ YES — ALWAYS create fresh venv after restart
+Why:
+✔ Jupyter sessions are not persistent
+✔ Packages may break / mismatch
+✔ Cleaner + safer setup
+🚀 1. GO TO WORKSPACE
+cd /workspace
+🚀 2. CREATE NEW ENV
+python3 -m venv mindi_env
+source mindi_env/bin/activate
+🚀 3. INSTALL PYTORCH (AMD ROCm)
+pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/rocm6.0
+🚀 4. VERIFY GPU
+python -c "import torch; print(torch.cuda.is_available())"
+
+👉 Must print:
+
+True
+🚀 5. INSTALL REQUIRED LIBS
+pip install transformers datasets accelerate peft sentencepiece tqdm
+
+⚠️ DO NOT install:
+
+❌ bitsandbytes
+🚀 6. INSTALL GIT LFS (IMPORTANT)
+apt update && apt install -y git-lfs
+🚀 7. CLONE YOUR REPO
+git clone https://huggingface.co/Mindigenous/mindi-backup
+cd mindi-backup
+🚀 8. LOGIN HF (FOR BACKUPS)
+hf auth login
+🚀 9. DOWNLOAD LATEST BACKUP
+
+👉 Replace with your latest step:
+
+hf download mindi-backups backup_step750.tar.gz
+🚀 10. EXTRACT BACKUP
+tar -xzf backup_step750.tar.gz
+🚀 11. VERIFY CHECKPOINT
+ls output/checkpoints
+
+👉 Should show:
+
+checkpoint-750
+🚀 12. CONFIGURE ACCELERATE
+accelerate config
+
+Use:
+
+✔ No distributed
+✔ GPU
+✔ bf16
+✔ No DeepSpeed
+🚀 13. START TRAINING (IMPORTANT)
+accelerate launch train.py
+
+❌ DO NOT use:
+
+--no-resume
+🧠 RESULT
+✔ Training resumes from last checkpoint
+✔ No data loss
+✔ Continues from step ~750
+
+# download properly
+hf download Mindigenous/mindi-backups backup_step500.tar.gz --local-dir .
+
+# extract
+tar -xzf backup_step500.tar.gz
+
+# verify
+ls output/checkpoints
+
+# resume
+accelerate launch train.py

data/final/old train jsonl/train.jsonl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:0f78cdcc0171535ad8ac533beccaec3aab78ef94870055bdf7dd5c798d629aef
-size 1867627125

dataset.py

@@ -7,11 +7,10 @@ from config import PATHS, TRAINING_CONFIG
 from utils import read_jsonl
 
 
-def format_prompt(instruction: str, input_text: str, output_text: str) -> str:
+def format_prompt(instruction: str, response: str) -> str:
     return (
-        f"### Instruction:\n{instruction}\n"
-        f"### Input:\n{input_text}\n"
-        f"### Response:\n{output_text}"
+        f"### Instruction:\n{instruction}\n\n"
+        f"### Response:\n{response}"
     )
 
 
@@ -29,13 +28,17 @@ class LocalJsonlInstructionDataset(Dataset):
 
     def __getitem__(self, idx: int) -> Dict[str, torch.Tensor]:
         sample = self.samples[idx]
-        text = format_prompt(
-            instruction=sample["instruction"],
-            input_text="",
-            output_text=sample["response"],
-        )
+
+        instruction = sample["instruction"]
+        response = sample["response"]
+
+        # 🔥 Build prompt (without response first)
+        prompt = f"### Instruction:\n{instruction}\n\n### Response:\n"
+        full_text = prompt + response
+
+        # Tokenize full text
         encoded = self.tokenizer(
-            text,
+            full_text,
             truncation=True,
             max_length=self.max_length,
             padding="max_length",
@@ -46,11 +49,23 @@ class LocalJsonlInstructionDataset(Dataset):
         attention_mask = encoded["attention_mask"].squeeze(0)
 
         labels = input_ids.clone()
+
+        # 🔥 Mask instruction part (ONLY train on response)
+        prompt_ids = self.tokenizer(
+            prompt,
+            truncation=True,
+            max_length=self.max_length,
+            return_tensors="pt",
+        )["input_ids"].squeeze(0)
+
+        prompt_len = min(len(prompt_ids), self.max_length)
+        labels[:prompt_len] = -100
+
+        # Mask padding
         labels[attention_mask == 0] = -100
 
         return {
             "input_ids": input_ids,
             "attention_mask": attention_mask,
             "labels": labels,
-        }
-
+        }