Sync latest dataset/config updates and logs

AMD Cloud Accelerate config.txt

@@ -0,0 +1,333 @@
+root@jupyter-launcher-260401134729-3102:/workspace# python3 -m venv mindi_env
+root@jupyter-launcher-260401134729-3102:/workspace# source mindi_env/bin/activate
+((mindi_env) ) root@jupyter-launcher-260401134729-3102:/workspace# pip install --upgrade pip
+Requirement already satisfied: pip in ./mindi_env/lib/python3.12/site-packages (25.0.1)
+Collecting pip
+  Using cached pip-26.0.1-py3-none-any.whl.metadata (4.7 kB)
+Using cached pip-26.0.1-py3-none-any.whl (1.8 MB)
+Installing collected packages: pip
+  Attempting uninstall: pip
+    Found existing installation: pip 25.0.1
+    Uninstalling pip-25.0.1:
+      Successfully uninstalled pip-25.0.1
+Successfully installed pip-26.0.1
+((mindi_env) ) root@jupyter-launcher-260401134729-3102:/workspace# pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/rocm6.0
+Looking in indexes: https://download.pytorch.org/whl/rocm6.0
+Collecting torch
+  Downloading https://download.pytorch.org/whl/rocm6.0/torch-2.4.1%2Brocm6.0-cp312-cp312-linux_x86_64.whl (2363.3 MB)
+     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 2.4/2.4 GB 86.5 MB/s  0:00:06
+Collecting torchvision
+  Downloading https://download-r2.pytorch.org/whl/rocm6.0/torchvision-0.19.1%2Brocm6.0-cp312-cp312-linux_x86_64.whl (65.8 MB)
+     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 65.8/65.8 MB 248.3 MB/s  0:00:00
+Collecting torchaudio
+  Downloading https://download-r2.pytorch.org/whl/rocm6.0/torchaudio-2.4.1%2Brocm6.0-cp312-cp312-linux_x86_64.whl (1.7 MB)
+     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.7/1.7 MB 187.5 MB/s  0:00:00
+Collecting filelock (from torch)
+  Downloading filelock-3.25.2-py3-none-any.whl.metadata (2.0 kB)
+Collecting typing-extensions>=4.8.0 (from torch)
+  Downloading https://download.pytorch.org/whl/typing_extensions-4.15.0-py3-none-any.whl.metadata (3.3 kB)
+Collecting sympy (from torch)
+  Using cached sympy-1.14.0-py3-none-any.whl.metadata (12 kB)
+Collecting networkx (from torch)
+  Using cached networkx-3.6.1-py3-none-any.whl.metadata (6.8 kB)
+Collecting jinja2 (from torch)
+  Downloading https://download.pytorch.org/whl/jinja2-3.1.6-py3-none-any.whl.metadata (2.9 kB)
+Collecting fsspec (from torch)
+  Downloading fsspec-2026.2.0-py3-none-any.whl.metadata (10 kB)
+Collecting setuptools (from torch)
+  Downloading https://download.pytorch.org/whl/setuptools-70.2.0-py3-none-any.whl.metadata (5.8 kB)
+Collecting pytorch-triton-rocm==3.0.0 (from torch)
+  Downloading https://download-r2.pytorch.org/whl/pytorch_triton_rocm-3.0.0-cp312-cp312-linux_x86_64.whl (341.7 MB)
+     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 341.7/341.7 MB 180.8 MB/s  0:00:01
+Collecting numpy (from torchvision)
+  Downloading numpy-2.4.3-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl.metadata (6.6 kB)
+Collecting pillow!=8.3.*,>=5.3.0 (from torchvision)
+  Downloading pillow-12.1.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl.metadata (8.8 kB)
+Collecting MarkupSafe>=2.0 (from jinja2->torch)
+  Downloading https://download.pytorch.org/whl/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (2.7 kB)
+Collecting mpmath<1.4,>=1.1.0 (from sympy->torch)
+  Using cached mpmath-1.3.0-py3-none-any.whl.metadata (8.6 kB)
+Downloading pillow-12.1.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl (7.0 MB)
+   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 7.0/7.0 MB 55.7 MB/s  0:00:00
+Downloading https://download.pytorch.org/whl/typing_extensions-4.15.0-py3-none-any.whl (44 kB)
+Downloading filelock-3.25.2-py3-none-any.whl (26 kB)
+Downloading fsspec-2026.2.0-py3-none-any.whl (202 kB)
+Downloading https://download.pytorch.org/whl/jinja2-3.1.6-py3-none-any.whl (134 kB)
+Downloading https://download.pytorch.org/whl/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (22 kB)
+Using cached networkx-3.6.1-py3-none-any.whl (2.1 MB)
+Downloading numpy-2.4.3-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl (16.6 MB)
+   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 16.6/16.6 MB 114.6 MB/s  0:00:00
+Downloading https://download.pytorch.org/whl/setuptools-70.2.0-py3-none-any.whl (930 kB)
+   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 930.8/930.8 kB 129.2 MB/s  0:00:00
+Using cached sympy-1.14.0-py3-none-any.whl (6.3 MB)
+Using cached mpmath-1.3.0-py3-none-any.whl (536 kB)
+Installing collected packages: mpmath, typing-extensions, sympy, setuptools, pillow, numpy, networkx, MarkupSafe, fsspec, filelock, pytorch-triton-rocm, jinja2, torch, torchvision, torchaudio
+Successfully installed MarkupSafe-3.0.3 filelock-3.25.2 fsspec-2026.2.0 jinja2-3.1.6 mpmath-1.3.0 networkx-3.6.1 numpy-2.4.3 pillow-12.1.1 pytorch-triton-rocm-3.0.0 setuptools-70.2.0 sympy-1.14.0 torch-2.4.1+rocm6.0 torchaudio-2.4.1+rocm6.0 torchvision-0.19.1+rocm6.0 typing-extensions-4.15.0
+((mindi_env) ) root@jupyter-launcher-260401134729-3102:/workspace# python -c "import torch; print(torch.cuda.is_available())"
+True
+((mindi_env) ) root@jupyter-launcher-260401134729-3102:/workspace# pip install transformers datasets accelerate peft bitsandbytes sentencepiece tqdm
+Collecting transformers
+  Downloading transformers-5.4.0-py3-none-any.whl.metadata (32 kB)
+Collecting datasets
+  Downloading datasets-4.8.4-py3-none-any.whl.metadata (19 kB)
+Collecting accelerate
+  Downloading accelerate-1.13.0-py3-none-any.whl.metadata (19 kB)
+Collecting peft
+  Downloading peft-0.18.1-py3-none-any.whl.metadata (14 kB)
+Collecting bitsandbytes
+  Downloading bitsandbytes-0.49.2-py3-none-manylinux_2_24_x86_64.whl.metadata (10 kB)
+Collecting sentencepiece
+  Downloading sentencepiece-0.2.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl.metadata (10 kB)
+Collecting tqdm
+  Downloading tqdm-4.67.3-py3-none-any.whl.metadata (57 kB)
+Collecting huggingface-hub<2.0,>=1.5.0 (from transformers)
+  Downloading huggingface_hub-1.8.0-py3-none-any.whl.metadata (13 kB)
+Requirement already satisfied: numpy>=1.17 in ./mindi_env/lib/python3.12/site-packages (from transformers) (2.4.3)
+Collecting packaging>=20.0 (from transformers)
+  Downloading packaging-26.0-py3-none-any.whl.metadata (3.3 kB)
+Collecting pyyaml>=5.1 (from transformers)
+  Downloading pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (2.4 kB)
+Collecting regex>=2025.10.22 (from transformers)
+  Downloading regex-2026.3.32-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (40 kB)
+Collecting tokenizers<=0.23.0,>=0.22.0 (from transformers)
+  Downloading tokenizers-0.22.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (7.3 kB)
+Collecting typer (from transformers)
+  Downloading typer-0.24.1-py3-none-any.whl.metadata (16 kB)
+Collecting safetensors>=0.4.3 (from transformers)
+  Downloading safetensors-0.7.0-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (4.1 kB)
+Requirement already satisfied: filelock>=3.10.0 in ./mindi_env/lib/python3.12/site-packages (from huggingface-hub<2.0,>=1.5.0->transformers) (3.25.2)
+Requirement already satisfied: fsspec>=2023.5.0 in ./mindi_env/lib/python3.12/site-packages (from huggingface-hub<2.0,>=1.5.0->transformers) (2026.2.0)
+Collecting hf-xet<2.0.0,>=1.4.2 (from huggingface-hub<2.0,>=1.5.0->transformers)
+  Downloading hf_xet-1.4.3-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata (4.9 kB)
+Collecting httpx<1,>=0.23.0 (from huggingface-hub<2.0,>=1.5.0->transformers)
+  Downloading httpx-0.28.1-py3-none-any.whl.metadata (7.1 kB)
+Requirement already satisfied: typing-extensions>=4.1.0 in ./mindi_env/lib/python3.12/site-packages (from huggingface-hub<2.0,>=1.5.0->transformers) (4.15.0)
+Collecting anyio (from httpx<1,>=0.23.0->huggingface-hub<2.0,>=1.5.0->transformers)
+  Downloading anyio-4.13.0-py3-none-any.whl.metadata (4.5 kB)
+Collecting certifi (from httpx<1,>=0.23.0->huggingface-hub<2.0,>=1.5.0->transformers)
+  Downloading certifi-2026.2.25-py3-none-any.whl.metadata (2.5 kB)
+Collecting httpcore==1.* (from httpx<1,>=0.23.0->huggingface-hub<2.0,>=1.5.0->transformers)
+  Downloading httpcore-1.0.9-py3-none-any.whl.metadata (21 kB)
+Collecting idna (from httpx<1,>=0.23.0->huggingface-hub<2.0,>=1.5.0->transformers)
+  Downloading idna-3.11-py3-none-any.whl.metadata (8.4 kB)
+Collecting h11>=0.16 (from httpcore==1.*->httpx<1,>=0.23.0->huggingface-hub<2.0,>=1.5.0->transformers)
+  Downloading h11-0.16.0-py3-none-any.whl.metadata (8.3 kB)
+Collecting pyarrow>=21.0.0 (from datasets)
+  Downloading pyarrow-23.0.1-cp312-cp312-manylinux_2_28_x86_64.whl.metadata (3.1 kB)
+Collecting dill<0.4.2,>=0.3.0 (from datasets)
+  Downloading dill-0.4.1-py3-none-any.whl.metadata (10 kB)
+Collecting pandas (from datasets)
+  Downloading pandas-3.0.2-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl.metadata (79 kB)
+Collecting requests>=2.32.2 (from datasets)
+  Downloading requests-2.33.1-py3-none-any.whl.metadata (4.8 kB)
+Collecting xxhash (from datasets)
+  Downloading xxhash-3.6.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (13 kB)
+Collecting multiprocess<0.70.20 (from datasets)
+  Downloading multiprocess-0.70.19-py312-none-any.whl.metadata (7.5 kB)
+Collecting aiohttp!=4.0.0a0,!=4.0.0a1 (from fsspec[http]<=2026.2.0,>=2023.1.0->datasets)
+  Downloading aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (8.1 kB)
+Collecting psutil (from accelerate)
+  Downloading psutil-7.2.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl.metadata (22 kB)
+Requirement already satisfied: torch>=2.0.0 in ./mindi_env/lib/python3.12/site-packages (from accelerate) (2.4.1+rocm6.0)
+Requirement already satisfied: sympy in ./mindi_env/lib/python3.12/site-packages (from torch>=2.0.0->accelerate) (1.14.0)
+Requirement already satisfied: networkx in ./mindi_env/lib/python3.12/site-packages (from torch>=2.0.0->accelerate) (3.6.1)
+Requirement already satisfied: jinja2 in ./mindi_env/lib/python3.12/site-packages (from torch>=2.0.0->accelerate) (3.1.6)
+Requirement already satisfied: setuptools in ./mindi_env/lib/python3.12/site-packages (from torch>=2.0.0->accelerate) (70.2.0)
+Requirement already satisfied: pytorch-triton-rocm==3.0.0 in ./mindi_env/lib/python3.12/site-packages (from torch>=2.0.0->accelerate) (3.0.0)
+Collecting aiohappyeyeballs>=2.5.0 (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<=2026.2.0,>=2023.1.0->datasets)
+  Downloading aiohappyeyeballs-2.6.1-py3-none-any.whl.metadata (5.9 kB)
+Collecting aiosignal>=1.4.0 (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<=2026.2.0,>=2023.1.0->datasets)
+  Downloading aiosignal-1.4.0-py3-none-any.whl.metadata (3.7 kB)
+Collecting attrs>=17.3.0 (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<=2026.2.0,>=2023.1.0->datasets)
+  Downloading attrs-26.1.0-py3-none-any.whl.metadata (8.8 kB)
+Collecting frozenlist>=1.1.1 (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<=2026.2.0,>=2023.1.0->datasets)
+  Downloading frozenlist-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl.metadata (20 kB)
+Collecting multidict<7.0,>=4.5 (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<=2026.2.0,>=2023.1.0->datasets)
+  Downloading multidict-6.7.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (5.3 kB)
+Collecting propcache>=0.2.0 (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<=2026.2.0,>=2023.1.0->datasets)
+  Downloading propcache-0.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (13 kB)
+Collecting yarl<2.0,>=1.17.0 (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<=2026.2.0,>=2023.1.0->datasets)
+  Downloading yarl-1.23.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (79 kB)
+Collecting charset_normalizer<4,>=2 (from requests>=2.32.2->datasets)
+  Downloading charset_normalizer-3.4.6-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (40 kB)
+Collecting urllib3<3,>=1.26 (from requests>=2.32.2->datasets)
+  Downloading urllib3-2.6.3-py3-none-any.whl.metadata (6.9 kB)
+Requirement already satisfied: MarkupSafe>=2.0 in ./mindi_env/lib/python3.12/site-packages (from jinja2->torch>=2.0.0->accelerate) (3.0.3)
+Collecting python-dateutil>=2.8.2 (from pandas->datasets)
+  Using cached python_dateutil-2.9.0.post0-py2.py3-none-any.whl.metadata (8.4 kB)
+Collecting six>=1.5 (from python-dateutil>=2.8.2->pandas->datasets)
+  Downloading six-1.17.0-py2.py3-none-any.whl.metadata (1.7 kB)
+Requirement already satisfied: mpmath<1.4,>=1.1.0 in ./mindi_env/lib/python3.12/site-packages (from sympy->torch>=2.0.0->accelerate) (1.3.0)
+Collecting click>=8.2.1 (from typer->transformers)
+  Downloading click-8.3.1-py3-none-any.whl.metadata (2.6 kB)
+Collecting shellingham>=1.3.0 (from typer->transformers)
+  Downloading shellingham-1.5.4-py2.py3-none-any.whl.metadata (3.5 kB)
+Collecting rich>=12.3.0 (from typer->transformers)
+  Downloading rich-14.3.3-py3-none-any.whl.metadata (18 kB)
+Collecting annotated-doc>=0.0.2 (from typer->transformers)
+  Downloading annotated_doc-0.0.4-py3-none-any.whl.metadata (6.6 kB)
+Collecting markdown-it-py>=2.2.0 (from rich>=12.3.0->typer->transformers)
+  Downloading markdown_it_py-4.0.0-py3-none-any.whl.metadata (7.3 kB)
+Collecting pygments<3.0.0,>=2.13.0 (from rich>=12.3.0->typer->transformers)
+  Downloading pygments-2.20.0-py3-none-any.whl.metadata (2.5 kB)
+Collecting mdurl~=0.1 (from markdown-it-py>=2.2.0->rich>=12.3.0->typer->transformers)
+  Downloading mdurl-0.1.2-py3-none-any.whl.metadata (1.6 kB)
+Downloading transformers-5.4.0-py3-none-any.whl (10.1 MB)
+   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 10.1/10.1 MB 55.5 MB/s  0:00:00
+Downloading huggingface_hub-1.8.0-py3-none-any.whl (625 kB)
+   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 625.2/625.2 kB 56.0 MB/s  0:00:00
+Downloading hf_xet-1.4.3-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (4.2 MB)
+   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 4.2/4.2 MB 130.1 MB/s  0:00:00
+Downloading httpx-0.28.1-py3-none-any.whl (73 kB)
+Downloading httpcore-1.0.9-py3-none-any.whl (78 kB)
+Downloading tokenizers-0.22.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.3 MB)
+   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 3.3/3.3 MB 116.5 MB/s  0:00:00
+Downloading datasets-4.8.4-py3-none-any.whl (526 kB)
+   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 527.0/527.0 kB 62.6 MB/s  0:00:00
+Downloading dill-0.4.1-py3-none-any.whl (120 kB)
+Downloading multiprocess-0.70.19-py312-none-any.whl (150 kB)
+Downloading accelerate-1.13.0-py3-none-any.whl (383 kB)
+Downloading peft-0.18.1-py3-none-any.whl (556 kB)
+   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 557.0/557.0 kB 30.8 MB/s  0:00:00
+Downloading bitsandbytes-0.49.2-py3-none-manylinux_2_24_x86_64.whl (60.7 MB)
+   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 60.7/60.7 MB 103.2 MB/s  0:00:00
+Downloading sentencepiece-0.2.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl (1.4 MB)
+   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.4/1.4 MB 127.4 MB/s  0:00:00
+Downloading tqdm-4.67.3-py3-none-any.whl (78 kB)
+Downloading aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (1.8 MB)
+   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.8/1.8 MB 208.3 MB/s  0:00:00
+Downloading multidict-6.7.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (256 kB)
+Downloading yarl-1.23.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (100 kB)
+Downloading aiohappyeyeballs-2.6.1-py3-none-any.whl (15 kB)
+Downloading aiosignal-1.4.0-py3-none-any.whl (7.5 kB)
+Downloading attrs-26.1.0-py3-none-any.whl (67 kB)
+Downloading frozenlist-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl (242 kB)
+Downloading h11-0.16.0-py3-none-any.whl (37 kB)
+Downloading idna-3.11-py3-none-any.whl (71 kB)
+Downloading packaging-26.0-py3-none-any.whl (74 kB)
+Downloading propcache-0.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (221 kB)
+Downloading pyarrow-23.0.1-cp312-cp312-manylinux_2_28_x86_64.whl (47.6 MB)
+   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 47.6/47.6 MB 96.3 MB/s  0:00:00
+Downloading pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (807 kB)
+   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 807.9/807.9 kB 102.6 MB/s  0:00:00
+Downloading regex-2026.3.32-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (801 kB)
+   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 801.5/801.5 kB 105.5 MB/s  0:00:00
+Downloading requests-2.33.1-py3-none-any.whl (64 kB)
+Downloading charset_normalizer-3.4.6-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (207 kB)
+Downloading urllib3-2.6.3-py3-none-any.whl (131 kB)
+Downloading certifi-2026.2.25-py3-none-any.whl (153 kB)
+Downloading safetensors-0.7.0-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (507 kB)
+Downloading anyio-4.13.0-py3-none-any.whl (114 kB)
+Downloading pandas-3.0.2-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl (10.9 MB)
+   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 10.9/10.9 MB 122.5 MB/s  0:00:00
+Using cached python_dateutil-2.9.0.post0-py2.py3-none-any.whl (229 kB)
+Downloading six-1.17.0-py2.py3-none-any.whl (11 kB)
+Downloading psutil-7.2.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl (155 kB)
+Downloading typer-0.24.1-py3-none-any.whl (56 kB)
+Downloading annotated_doc-0.0.4-py3-none-any.whl (5.3 kB)
+Downloading click-8.3.1-py3-none-any.whl (108 kB)
+Downloading rich-14.3.3-py3-none-any.whl (310 kB)
+Downloading pygments-2.20.0-py3-none-any.whl (1.2 MB)
+   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.2/1.2 MB 165.6 MB/s  0:00:00
+Downloading markdown_it_py-4.0.0-py3-none-any.whl (87 kB)
+Downloading mdurl-0.1.2-py3-none-any.whl (10.0 kB)
+Downloading shellingham-1.5.4-py2.py3-none-any.whl (9.8 kB)
+Downloading xxhash-3.6.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (193 kB)
+Installing collected packages: xxhash, urllib3, tqdm, six, shellingham, sentencepiece, safetensors, regex, pyyaml, pygments, pyarrow, psutil, propcache, packaging, multidict, mdurl, idna, hf-xet, h11, frozenlist, dill, click, charset_normalizer, certifi, attrs, annotated-doc, aiohappyeyeballs, yarl, requests, python-dateutil, multiprocess, markdown-it-py, httpcore, anyio, aiosignal, rich, pandas, httpx, bitsandbytes, aiohttp, typer, huggingface-hub, tokenizers, datasets, accelerate, transformers, peft
+Successfully installed accelerate-1.13.0 aiohappyeyeballs-2.6.1 aiohttp-3.13.5 aiosignal-1.4.0 annotated-doc-0.0.4 anyio-4.13.0 attrs-26.1.0 bitsandbytes-0.49.2 certifi-2026.2.25 charset_normalizer-3.4.6 click-8.3.1 datasets-4.8.4 dill-0.4.1 frozenlist-1.8.0 h11-0.16.0 hf-xet-1.4.3 httpcore-1.0.9 httpx-0.28.1 huggingface-hub-1.8.0 idna-3.11 markdown-it-py-4.0.0 mdurl-0.1.2 multidict-6.7.1 multiprocess-0.70.19 packaging-26.0 pandas-3.0.2 peft-0.18.1 propcache-0.4.1 psutil-7.2.2 pyarrow-23.0.1 pygments-2.20.0 python-dateutil-2.9.0.post0 pyyaml-6.0.3 regex-2026.3.32 requests-2.33.1 rich-14.3.3 safetensors-0.7.0 sentencepiece-0.2.1 shellingham-1.5.4 six-1.17.0 tokenizers-0.22.2 tqdm-4.67.3 transformers-5.4.0 typer-0.24.1 urllib3-2.6.3 xxhash-3.6.0 yarl-1.23.0
+((mindi_env) ) root@jupyter-launcher-260401134729-3102:/workspace# apt update && apt install -y git-lfs
+Get:1 https://repo.radeon.com/amdgpu/7.0/ubuntu jammy InRelease [3183 B]
+Get:2 https://repo.radeon.com/rocm/apt/7.0 jammy InRelease [2603 B]                                                                                
+Get:3 https://repo.radeon.com/amdgpu/7.0/ubuntu jammy/main amd64 Packages [1329 B]                                                                                   
+Get:4 https://repo.radeon.com/rocm/apt/7.0 jammy/main amd64 Packages [82.7 kB]                                                                                                                       
+Get:5 http://security.ubuntu.com/ubuntu jammy-security InRelease [129 kB]                                                               
+Get:6 http://archive.ubuntu.com/ubuntu jammy InRelease [270 kB]                                                   
+Get:7 https://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu jammy InRelease [18.1 kB]                                     
+Get:8 http://security.ubuntu.com/ubuntu jammy-security/multiverse amd64 Packages [62.6 kB]
+Get:9 http://archive.ubuntu.com/ubuntu jammy-updates InRelease [128 kB]      
+Get:10 https://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu jammy/main amd64 Packages [38.9 kB]
+Get:11 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 Packages [1309 kB]
+Get:12 http://archive.ubuntu.com/ubuntu jammy-backports InRelease [127 kB]                 
+Get:13 http://archive.ubuntu.com/ubuntu jammy/main amd64 Packages [1792 kB]                          
+Get:14 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 Packages [6803 kB]
+Get:15 http://archive.ubuntu.com/ubuntu jammy/restricted amd64 Packages [164 kB]            
+Get:16 http://archive.ubuntu.com/ubuntu jammy/universe amd64 Packages [17.5 MB]             
+Get:17 http://security.ubuntu.com/ubuntu jammy-security/main amd64 Packages [3842 kB]        
+Get:18 http://archive.ubuntu.com/ubuntu jammy/multiverse amd64 Packages [266 kB]              
+Get:19 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages [4217 kB]
+Get:20 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 Packages [7125 kB]
+Get:21 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse amd64 Packages [70.9 kB]
+Get:22 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 Packages [1621 kB]
+Get:23 http://archive.ubuntu.com/ubuntu jammy-backports/universe amd64 Packages [35.6 kB]
+Get:24 http://archive.ubuntu.com/ubuntu jammy-backports/main amd64 Packages [84.0 kB]
+Fetched 45.7 MB in 3s (16.7 MB/s)                          
+Reading package lists... Done
+Building dependency tree... Done
+Reading state information... Done
+107 packages can be upgraded. Run 'apt list --upgradable' to see them.
+W: https://repo.radeon.com/amdgpu/7.0/ubuntu/dists/jammy/InRelease: Key is stored in legacy trusted.gpg keyring (/etc/apt/trusted.gpg), see the DEPRECATION section in apt-key(8) for details.
+W: https://repo.radeon.com/rocm/apt/7.0/dists/jammy/InRelease: Key is stored in legacy trusted.gpg keyring (/etc/apt/trusted.gpg), see the DEPRECATION section in apt-key(8) for details.
+Reading package lists... Done
+Building dependency tree... Done
+Reading state information... Done
+The following NEW packages will be installed:
+  git-lfs
+0 upgraded, 1 newly installed, 0 to remove and 107 not upgraded.
+Need to get 3544 kB of archives.
+After this operation, 10.5 MB of additional disk space will be used.
+Get:1 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 git-lfs amd64 3.0.2-1ubuntu0.3 [3544 kB]
+Fetched 3544 kB in 1s (2611 kB/s)  
+debconf: delaying package configuration, since apt-utils is not installed
+Selecting previously unselected package git-lfs.
+(Reading database ... 58520 files and directories currently installed.)
+Preparing to unpack .../git-lfs_3.0.2-1ubuntu0.3_amd64.deb ...
+Unpacking git-lfs (3.0.2-1ubuntu0.3) ...
+Setting up git-lfs (3.0.2-1ubuntu0.3) ...
+((mindi_env) ) root@jupyter-launcher-260401134729-3102:/workspace# git clone https://huggingface.co/Mindigenous/mindi-backup
+Cloning into 'mindi-backup'...
+remote: Enumerating objects: 286, done.
+remote: Counting objects: 100% (283/283), done.
+remote: Compressing objects: 100% (266/266), done.
+remote: Total 286 (delta 15), reused 217 (delta 6), pack-reused 3 (from 1)
+Receiving objects: 100% (286/286), 151.83 KiB | 9.49 MiB/s, done.
+Resolving deltas: 100% (15/15), done.
+Filtering content: 100% (85/85), 9.38 GiB | 533.08 MiB/s, done.
+((mindi_env) ) root@jupyter-launcher-260401134729-3102:/workspace# cd mindi-backup/
+((mindi_env) ) root@jupyter-launcher-260401134729-3102:/workspace/mindi-backup# accelerate config
+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------In which compute environment are you running?
+This machine                                                                                                                                                                                                                  
+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------Which type of machine are you using?                                                                                                                                                                                          
+No distributed training                                                                                                                                                                                                       
+Do you want to run your training on CPU only (even if a GPU / Apple Silicon / Ascend NPU device is available)? [yes/NO]:no                                                                                                    
+Do you wish to optimize your script with torch dynamo?[yes/NO]:no                                                                                                                                                             
+Do you want to use DeepSpeed? [yes/NO]: no                                                                                                                                                                                    
+What GPU(s) (by id) should be used for training on this machine as a comma-separated list? [all]:                                                                                                                             
+Would you like to enable numa efficiency? (Currently only supported on NVIDIA hardware). [yes/NO]: no                                                                                                                         
+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------Do you wish to use mixed precision?                                                                                                                                                                                           
+bf16                                                                                                                                                                                                                          
+accelerate configuration saved at /root/.cache/huggingface/accelerate/default_config.yaml                                                                                                                                     
+((mindi_env) ) root@jupyter-launcher-260401134729-3102:/workspace/mindi-backup# python -c "import torch; print(torch.cuda.is_available(), torch.cuda.device_count())"
+True 1
+((mindi_env) ) root@jupyter-launcher-260401134729-3102:/workspace/mindi-backup# accelerate launch train.py \
+  --data_path data/final/train.jsonl \
+  --output_dir checkpoints \
+  --batch_size 4 \
+  --gradient_accumulation 16 \
+  --learning_rate 2e-5 \
+  --num_train_steps 60000 \
+  --save_steps 2000 \
+  --logging_steps 50
+usage: train.py [-h] [--no-resume]
+train.py: error: unrecognized arguments: --data_path data/final/train.jsonl --output_dir checkpoints --batch_size 4 --gradient_accumulation 16 --learning_rate 2e-5 --num_train_steps 60000 --save_steps 2000 --logging_steps 50
+Traceback (most recent call last):
+  File "/workspace/mindi_env/bin/accelerate", line 6, in <module>
+    sys.exit(main())
+             ^^^^^^
+  File "/workspace/mindi_env/lib/python3.12/site-packages/accelerate/commands/accelerate_cli.py", line 50, in main
+    args.func(args)
+  File "/workspace/mindi_env/lib/python3.12/site-packages/accelerate/commands/launch.py", line 1405, in launch_command
+    simple_launcher(args)
+  File "/workspace/mindi_env/lib/python3.12/site-packages/accelerate/commands/launch.py", line 993, in simple_launcher
+    raise subprocess.CalledProcessError(returncode=process.returncode, cmd=cmd)
+subprocess.CalledProcessError: Command '['/workspace/mindi_env/bin/python3', 'train.py', '--data_path', 'data/final/train.jsonl', '--output_dir', 'checkpoints', '--batch_size', '4', '--gradient_accumulation', '16', '--learning_rate', '2e-5', '--num_train_steps', '60000', '--save_steps', '2000', '--logging_steps', '50']' returned non-zero exit status 2.
+((mindi_env) ) root@jupyter-launcher-260401134729-3102:/workspace/mindi-backup# 

config.py

@@ -10,7 +10,7 @@ class Paths:
     output_dir: Path = Path("./output")
     logs_dir: Path = Path("./logs")
 
-    train_jsonl: Path = Path("./data/train.jsonl")
+    train_jsonl: Path = Path("./data/final/train.jsonl")
     dataset_cache_dir: Path = Path("./data/cache")
     raw_dataset_dir: Path = Path("./data/cache/raw")
     checkpoint_dir: Path = Path("./output/checkpoints")
@@ -30,9 +30,9 @@ class DataConfig:
 @dataclass(frozen=True)
 class TrainingConfig:
     num_train_epochs: int = 5
-    per_device_train_batch_size: int = 1
-    gradient_accumulation_steps: int = 8
-    learning_rate: float = 1e-5
+    per_device_train_batch_size: int = 2
+    gradient_accumulation_steps: int = 16
+    learning_rate: float = 2e-5
     max_length: int = 1024
     save_steps: int = 250
     logging_steps: int = 20

data/final/_rebalance_tmp/instruction.jsonl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:77fd726ec9b3b9135edc4b22c251c760ea444060507315d09e0156a9ad08cff2
-size 359523113
+oid sha256:9d2ca5ea336cf6221a35c593aff2e53fdd90843ad447a53e38e007dea39987a7
+size 329198633

data/final/_rebalance_tmp/problem.jsonl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dea7bb001629f0e7b72363ea4ffc8da89b21cab0322c80faea3bd4352d2d28cd
-size 392283637
+oid sha256:741903a7f222a7c4d889dab9d8452af3bdaf893ec7e3e2b49e62e6d63443b1c4
+size 394655786

data/final/_rebalance_tmp/rebalance_seen.sqlite

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:27065d4515ff3d381d784058a61a997517d89621147f71316b4517997a7567a0
-size 85028864
+oid sha256:2cb112c95f94d5d233248778914011656d5fad065b9f84e3f52efbace1698104
+size 82251776

data/final/_rebalance_tmp/structured.jsonl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:83898474a15e20eba8f8fd4b4163c381d69b87374c4d82b3e289898da0b9f2fc
-size 283810066
+oid sha256:cc526cfd678a7d97925dc643f99c383f1c593c7dd60ea706f3ed25870d9b4cf4
+size 283750640

data/final/dedupe_hashes.sqlite

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:89622c37e7ff270b6745693d3dbd63a25a7937bf7e5261a01d4676568207d7ea
-size 84996096
+oid sha256:f9963e9da830061a8206460f22ac2bf1d8655c58f6321683fe805b05d45a7b70
+size 82325504

data/final/old train jsonl/train.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f78cdcc0171535ad8ac533beccaec3aab78ef94870055bdf7dd5c798d629aef
+size 1867627125

data/final/train.jsonl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0f78cdcc0171535ad8ac533beccaec3aab78ef94870055bdf7dd5c798d629aef
-size 1867627125
+oid sha256:cbd441aec7d166d52d0c961a0b83ddce734be2fe7abd422d2050fabf265348d1
+size 1907053927

dataset.py

@@ -31,8 +31,8 @@ class LocalJsonlInstructionDataset(Dataset):
         sample = self.samples[idx]
         text = format_prompt(
             instruction=sample["instruction"],
-            input_text=sample["input"],
-            output_text=sample["output"],
+            input_text="",
+            output_text=sample["response"],
         )
         encoded = self.tokenizer(
             text,

dataset_cleaner.py

@@ -15,6 +15,7 @@ CODE_PATTERN = re.compile(
     r"(\bdef\b|\bclass\b|\bimport\b|\breturn\b|=>|function\s+\w+|public\s+class|#include|```)",
     re.IGNORECASE,
 )
+MIN_CODE_SIGNAL_RE = re.compile(r"(\bdef\s+|\bclass\s+|\bimport\s+|=|\breturn\s+|\bfor\s+|\bif\s+)")
 EXPLANATION_PATTERN = re.compile(
     r"\b(explain|because|algorithm|steps|approach|complexity|solution)\b", re.IGNORECASE
 )
@@ -22,6 +23,15 @@ PROBLEM_PROMPT_RE = re.compile(
     r"\b(solve|given|find|compute|return|input|output|problem|algorithm|task|challenge)\b",
     re.IGNORECASE,
 )
+YAML_FRONTMATTER_RE = re.compile(r"^\s*---\s*\n.*?\n---\s*", re.DOTALL)
+FENCE_LINE_RE = re.compile(r"^\s*```(?:[a-zA-Z0-9_+-]+)?\s*$")
+CODEBLOCK_DIRECTIVE_RE = re.compile(r"^\s*code-block::\s*\w*\s*$", re.IGNORECASE)
+CLI_NOISE_LINE_RE = re.compile(
+    r"^\s*(\[[^\]]+\]\s*)?(INFO|WARNING|ERROR|DEBUG|TRACE)\b|"
+    r"^\s*(PS\s+[A-Za-z]:\\|[A-Za-z]:\\[^>]*>)|"
+    r"^\s*(Traceback \(most recent call last\):|File \".*\", line \d+|Exception:|RuntimeError:|ValueError:|TypeError:)",
+    re.IGNORECASE,
+)
 
 
 def estimate_tokens(text: str) -> int:
@@ -40,6 +50,41 @@ def normalize_text(text: str) -> str:
     return "\n".join(lines).strip()
 
 
+def clean_response_text(response: str) -> str:
+    text = normalize_text(response)
+    if not text:
+        return ""
+
+    # Remove YAML front matter and markdown wrappers.
+    text = YAML_FRONTMATTER_RE.sub("", text)
+    kept_lines: List[str] = []
+    for line in text.split("\n"):
+        if FENCE_LINE_RE.match(line):
+            continue
+        if CODEBLOCK_DIRECTIVE_RE.match(line):
+            continue
+        if CLI_NOISE_LINE_RE.match(line):
+            continue
+        kept_lines.append(line)
+
+    text = "\n".join(kept_lines)
+    text = text.replace("```python", "").replace("```py", "").replace("```", "")
+
+    # Normalize indentation and drop leading/trailing blank lines.
+    normalized_lines = [ln.rstrip() for ln in text.split("\n")]
+    while normalized_lines and not normalized_lines[0].strip():
+        normalized_lines.pop(0)
+    while normalized_lines and not normalized_lines[-1].strip():
+        normalized_lines.pop()
+    if not normalized_lines:
+        return ""
+
+    # Keep indentation consistent: convert tabs to 4 spaces.
+    normalized_lines = [ln.replace("\t", "    ") for ln in normalized_lines]
+    text = "\n".join(normalized_lines)
+    return text
+
+
 def _ascii_ratio(text: str) -> float:
     if not text:
         return 1.0
@@ -74,7 +119,7 @@ def clean_record(
     max_tokens: int = 2048,
 ) -> Optional[Dict[str, str]]:
     instruction = normalize_text(record.get("instruction", ""))
-    response = normalize_text(record.get("response", ""))
+    response = clean_response_text(record.get("response", ""))
     source = normalize_text(record.get("_source", "unknown"))
     category = normalize_text(record.get("_category", ""))
     if not category:
@@ -100,8 +145,12 @@ def clean_record(
 
     if not instruction or not response:
         return None
+    if len(response) < 40:
+        return None
     if _ascii_ratio(instruction + response) < 0.85:
         return None
+    if not MIN_CODE_SIGNAL_RE.search(response):
+        return None
     if not _response_is_valid(response):
         return None
     if category == "problem":

logs/data_fetch.log

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:250881bf6b7747176c7432a40e84fb3dc4eeca6f9a1a75378ee7e3ccdf662fbf
-size 44778
+oid sha256:3b551005e4457068bfd214b0191e50752db3f180e8c4636a1f936525663b0b02
+size 51055