b200 megatron env manifest, freeze, prebuilt mamba/causal-conv1d CUDA extensions

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+b200/overlays/causal_conv1d_cuda.cpython-312-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+b200/overlays/selective_scan_cuda.cpython-312-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text

b200/README.md

@@ -0,0 +1,22 @@
+# Cluster environment backups
+
+Per-cluster runtimes for Qwen3.5 / Nemotron post-training. Public on purpose:
+these are recipes and build artifacts, no credentials.
+
+Each cluster directory holds `manifest.json` (python, torch, CUDA, driver, GPU,
+`torch.cuda.get_arch_list()`), `pip-freeze-*.txt`, the build scripts, and any
+expensively-built CUDA extensions under `overlays/`.
+
+## Arch compatibility, measured
+
+b200 is compute capability 10.0, b300 is 10.3. Neither cluster's torch lists
+`sm_103`, yet a torch built for `sm_100` runs correctly on a B300: matmul,
+transformer-engine Linear and causal_conv1d CUDA ops were all verified on a
+B300 SXM6. Blackwell `sm_103` is binary compatible with `sm_100` cubins, so
+these artifacts are portable across b200 and b300.
+
+## Overlay rule
+
+Install overlays with `pip install --target ... --no-deps` and prepend to
+PYTHONPATH. Without `--no-deps`, pip drags PyPI torch and triton into the
+overlay and silently shadows the cluster runtime.

b200/build_mamba_overlay.sbatch

@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+#SBATCH --job-name=mamba-overlay
+#SBATCH --partition=main
+#SBATCH --nodes=1
+#SBATCH --ntasks-per-node=1
+#SBATCH --gres=gpu:1
+#SBATCH --cpus-per-task=32
+#SBATCH --mem=256G
+#SBATCH --time=04:00:00
+#SBATCH --output=/NHNHOME/WORKSPACE/wbl-workspace/agentic-rl-260803/logs/%x-%j.log
+#SBATCH --error=/NHNHOME/WORKSPACE/wbl-workspace/agentic-rl-260803/logs/%x-%j.err
+# Build causal-conv1d and mamba-ssm for the B200 megatron-bridge runtime.
+#
+# Nemotron-3-Nano is a hybrid Mamba2/MoE model, and megatron-core's MambaMixer
+# raises ImportError at construction without mamba-ssm, so this is a hard
+# dependency for that architecture alone. Qwen models do not need it.
+#
+# Installed with --target into an overlay directory rather than into the shared
+# venv, which an active SFT job is running out of. Consumers prepend the overlay
+# to PYTHONPATH, exactly like the triton override.
+#
+# No prebuilt wheel exists for torch 2.10 + CUDA 13.1 + cp312, so both packages
+# are compiled from source; FORCE_BUILD stops setup.py from fetching a
+# mismatched release wheel and failing later at import.
+set -euo pipefail
+
+ROOT=${ROOT:-/NHNHOME/WORKSPACE/wbl-workspace/agentic-rl-260803}
+PYTHON_BIN=${PYTHON_BIN:-/NHNHOME/WORKSPACE/wbl-workspace/mk/sft_megatron_bridge/env/.venv/bin/python}
+OVERLAY=${OVERLAY:-${ROOT}/env/mamba-overlay}
+# B200 is sm_100. Building every architecture would multiply compile time.
+export TORCH_CUDA_ARCH_LIST=${TORCH_CUDA_ARCH_LIST:-10.0}
+export MAX_JOBS=${MAX_JOBS:-16}
+export CAUSAL_CONV1D_FORCE_BUILD=TRUE
+export MAMBA_FORCE_BUILD=TRUE
+export CUDA_HOME=${CUDA_HOME:-/usr/local/cuda}
+export PATH="${CUDA_HOME}/bin:${PATH}"
+
+[[ -x ${PYTHON_BIN} ]] || { echo "missing python ${PYTHON_BIN}" >&2; exit 2; }
+command -v nvcc >/dev/null || { echo "nvcc not on PATH" >&2; exit 2; }
+rm -rf "${OVERLAY}"
+mkdir -p "${OVERLAY}"
+
+echo "building into ${OVERLAY} arch=${TORCH_CUDA_ARCH_LIST} nvcc=$(nvcc --version | tail -1)"
+
+# --no-build-isolation so the build compiles against the installed NGC torch.
+# --no-deps because the overlay is prepended to PYTHONPATH: letting pip resolve
+# dependencies here pulls PyPI torch (2.13) and triton 3.7.1 into the overlay,
+# which would shadow torch 2.10.0a0+nv25.12 and re-introduce exactly the triton
+# this runtime already has to override. Every real runtime dependency
+# (torch, einops, transformers, ninja, packaging) is present in the venv.
+for pkg in causal-conv1d mamba-ssm; do
+  echo "=== ${pkg} $(date -Is)"
+  "${PYTHON_BIN}" -m pip install --no-cache-dir --no-build-isolation --no-deps \
+    --target "${OVERLAY}" --upgrade "${pkg}"
+done
+
+# The overlay must contain the two extensions and nothing that shadows the
+# runtime. A stray torch/triton here is worse than a missing build.
+for forbidden in torch triton nvidia; do
+  if [[ -e ${OVERLAY}/${forbidden} ]]; then
+    echo "overlay contains ${forbidden}; it would shadow the runtime" >&2
+    exit 2
+  fi
+done
+
+# A successful pip exit does not mean the CUDA extension imports.
+PYTHONPATH="${OVERLAY}:${PYTHONPATH:-}" "${PYTHON_BIN}" - <<'PYCHECK'
+import causal_conv1d, mamba_ssm
+from mamba_ssm.ops.triton.selective_state_update import selective_state_update  # noqa: F401
+from causal_conv1d import causal_conv1d_fn  # noqa: F401
+print(f"import ok causal_conv1d={causal_conv1d.__version__} mamba_ssm={mamba_ssm.__version__}")
+PYCHECK
+echo "mamba overlay ready at ${OVERLAY}"

b200/manifest.json

@@ -0,0 +1,23 @@
+{
+  "cluster": "b200",
+  "python": "3.12.3",
+  "torch": "2.10.0a0+b4e4ee81d3.nv25.12",
+  "torch_cuda": "13.1",
+  "arch_list": [
+    "sm_75",
+    "sm_80",
+    "sm_86",
+    "sm_90",
+    "sm_100",
+    "sm_120",
+    "compute_120"
+  ],
+  "driver": "580.95.05",
+  "gpu": "NVIDIA B200",
+  "megatron_venv": "/NHNHOME/WORKSPACE/wbl-workspace/mk/sft_megatron_bridge/env/.venv",
+  "note": "venv uses include-system-site-packages; torch lives in /usr/local/lib/python3.12/dist-packages",
+  "overlays": {
+    "triton-override": "symlinks to system pytorch-triton 3.5.1+nv25.12",
+    "mamba-overlay": "causal_conv1d 1.6.2.post1 + mamba_ssm 2.3.2.post1, TORCH_CUDA_ARCH_LIST=10.0"
+  }
+}

b200/overlays/causal_conv1d_cuda.cpython-312-x86_64-linux-gnu.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:34638d5ecd6ad203268cf1101bff7aca6c89f7f59936fbc71f6b104796490495
+size 263955008

b200/overlays/selective_scan_cuda.cpython-312-x86_64-linux-gnu.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b0c7538e3fb583a9e38691a030d5430ed71111f7a1dcc0e451a5d15bbcae838
+size 352830088

b200/pip-freeze-megatron.txt

@@ -0,0 +1,358 @@
+absl-py==2.5.0
+accelerate==1.14.0
+aiohappyeyeballs==2.6.1
+aiohttp==3.13.2
+aiosignal==1.4.0
+annotated-doc==0.0.4
+annotated-types==0.7.0
+antlr4-python3-runtime==4.9.3
+anyio==4.14.2
+apex @ file:///opt/pytorch/apex/dist/apex-0.1-cp312-cp312-linux_x86_64.whl#sha256=c6b3c445cf3dc8a24feac6070374927729abb4dcf4d27f2df9eacc9256e9575b
+apturl==0.5.2
+argon2-cffi==25.1.0
+argon2-cffi-bindings==25.1.0
+arrow==1.4.0
+asttokens==3.0.1
+astunparse==1.6.3
+async-lru==2.0.5
+attrs==26.1.0
+audioread==3.1.0
+ayatana-settings==23.11.14
+babel==2.17.0
+bcrypt==3.2.2
+beautifulsoup4==4.14.3
+black==25.12.0
+bleach==6.3.0
+blinker==1.7.0
+Brlapi==0.8.5
+Brotli==1.1.0
+build==1.3.0
+certifi==2026.6.17
+cffi==2.0.0
+chardet==5.2.0
+charset-normalizer==3.4.9
+click==8.4.2
+cloud-init==25.2
+cmake==3.31.6
+colorama==0.4.6
+comm==0.2.3
+configobj==5.0.8
+contourpy==1.3.3
+cryptography==41.0.7
+cuda-bindings==13.3.1
+cuda-pathfinder==1.5.6
+cuda-toolkit==13.0.3.0
+cupshelpers==1.0
+cycler==0.12.1
+Cython==3.2.3
+datasets==4.4.1
+dbus-python==1.3.2
+debugpy==1.8.19
+decorator==5.2.1
+defer==1.0.6
+defusedxml==0.7.1
+deja-dup-caja==0.0.9
+dill==0.4.0
+distro==1.9.0
+distro-info==1.7+build1
+dllist==2.0.0
+dm-tree==0.1.9
+docstring_parser==0.17.0
+duplicity==2.1.4
+einops==0.8.2
+execnet==2.1.2
+executing==2.2.1
+expecttest==0.3.0
+fasteners==0.18
+fastjsonschema==2.21.2
+filelock==3.31.1
+fla-core==0.5.1
+flash-linear-attention==0.5.1
+flash_attn @ file:///opt/transfer/flash_attn-2.7.4.post1%2B25.12-40345937-cp312-cp312-linux_x86_64.whl#sha256=6bea1da1398e405b0e92cf109d3acd739c401928c8595a377810867f3a69ebb6
+folder-color-caja==0.0.86
+folder-color-common==0.0.86
+fonttools==4.61.1
+fqdn==1.5.1
+frozenlist==1.8.0
+fsspec==2026.6.0
+gast==0.6.0
+gpg==1.18.0
+grpcio==1.82.1
+grpcio-tools==1.82.1
+h11==0.16.0
+hf-xet==1.5.2
+httpcore==1.0.9
+httplib2==0.20.4
+httpx==0.28.1
+huggingface_hub==1.24.0
+hydra-core==1.3.2
+hypothesis==6.130.8
+idna==3.18
+importlib_metadata==8.7.0
+iniconfig==2.3.0
+intel-openmp==2021.4.0
+ipykernel==7.1.0
+ipython==9.8.0
+ipython_pygments_lexers==1.1.1
+isoduration==20.11.0
+isort==7.0.0
+jedi==0.19.2
+Jinja2==3.1.6
+joblib==1.5.3
+json5==0.12.1
+jsonpatch==1.32
+jsonpointer==3.0.0
+jsonschema==4.26.0
+jsonschema-specifications==2025.9.1
+jupyter-events==0.12.0
+jupyter-lsp==2.3.0
+jupyter_client==8.7.0
+jupyter_core==5.9.1
+jupyter_server==2.17.0
+jupyter_server_terminals==0.5.3
+jupyterlab==4.5.1
+jupyterlab_code_formatter==3.0.2
+jupyterlab_pygments==0.3.0
+jupyterlab_server==2.28.0
+jupyterlab_tensorboard_pro @ file:///builder/jupyterlab_tensorboard_pro-4.0.0-py2.py3-none-any.whl#sha256=f6a3aac9a33e4253412508bc3a237dcb7cfc751badc0ffc6cab28c0b3a8b44da
+jupytext==1.18.1
+kiwisolver==1.4.9
+language-selector==0.1
+lark==1.3.1
+launchpadlib==1.11.0
+lazr.restfulclient==0.14.6
+lazr.uri==1.0.6
+lazy_loader==0.4
+librosa==0.11.0
+lightning-thunder @ file:///opt/pytorch/lightning-thunder
+lightning-utilities==0.15.2
+lintrunner==0.12.7
+llvmlite==0.46.0
+looseversion==1.3.0
+louis==3.29.0
+Magnus==1.0.3
+makefun==1.16.0
+Mako==1.3.2.dev0
+Markdown==3.10.2
+markdown-it-py==4.2.0
+MarkupSafe==3.0.3
+mate-hud==22.10.3
+mate-menu==22.4.2
+mate-tweak==22.10.0
+matplotlib==3.10.8
+matplotlib-inline==0.2.1
+mdit-py-plugins==0.5.0
+mdurl==0.1.2
+megatron-bridge==0.5.0
+megatron-core==0.18.0
+mistral_common==1.11.6
+mistune==3.1.4
+mkl==2021.1.1
+mkl-devel==2021.1.1
+mkl-include==2021.1.1
+ml_dtypes==0.5.4
+mock==5.2.0
+monotonic==1.6
+mpi4py==4.1.1
+mpmath==1.3.0
+msgpack==1.1.2
+multidict==6.7.0
+multiprocess==0.70.18
+mutagen==1.46.0
+mypy_extensions==1.1.0
+nbclient==0.10.2
+nbconvert==7.16.6
+nbformat==5.10.4
+nest-asyncio==1.6.0
+networkx==3.6.1
+ninja==1.13.0
+notebook==7.5.1
+notebook_shim==0.2.4
+numba==0.63.1
+numpy==2.3.5
+nv-one-logger-core==2.3.1
+nv-one-logger-training-telemetry==2.3.1
+nvfuser @ file:///opt/pytorch/fuser/python
+nvidia-cublas==13.1.1.3
+nvidia-cuda-cupti==13.0.85
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-runtime-cu13==0.0.0a0
+nvidia-cudnn-cu13==9.20.0.48
+nvidia-cudnn-frontend @ file:///opt/pytorch/lightning-thunder/tmp/cudnn_frontend
+nvidia-cufft==12.0.0.61
+nvidia-cufile==1.15.1.6
+nvidia-curand==10.4.0.35
+nvidia-cusolver==12.0.4.66
+nvidia-cusparse==12.6.3.3
+nvidia-cusparselt-cu13==0.8.1
+nvidia-dali-cuda130==1.52.0
+nvidia-ml-py==13.610.43
+nvidia-modelopt==0.39.0
+nvidia-nccl-cu13==2.29.7
+nvidia-nvcomp-cu13==5.0.0.6
+nvidia-nvimgcodec-cu13==0.6.1.37
+nvidia-nvjitlink==13.3.33
+nvidia-nvjpeg==13.0.2.28
+nvidia-nvjpeg2k-cu13==0.9.1.47
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvtiff-cu13==0.6.0.78
+nvidia-nvtx==13.0.85
+nvidia-resiliency-ext==0.6.0
+nvtx==0.2.14
+oauthlib==3.2.2
+olefile==0.46
+omegaconf==2.3.1
+onboard==1.4.1
+onnx @ file:///opt/pytorch/pytorch/third_party/onnx
+onnx-ir==0.1.12
+onnxscript==0.5.7
+opt_einsum==3.4.0
+optree==0.18.0
+orjson==3.11.9
+overrides==7.7.0
+packaging==26.2
+pandas==2.3.3
+pandocfilters==1.5.1
+paramiko==2.12.0
+parso==0.8.5
+pathspec==0.12.1
+peft==0.19.1
+pexpect==4.9.0
+pillow==12.3.0
+platformdirs==4.10.1
+pluggy==1.6.0
+polygraphy==0.49.26
+pooch==1.8.2
+prometheus_client==0.23.1
+prompt_toolkit==3.0.52
+propcache==0.4.1
+protobuf==7.35.1
+psutil==7.2.2
+ptyprocess==0.7.0
+PuLP==3.3.0
+pulsemixer==1.5.1
+pure_eval==0.2.3
+pyarrow==25.0.0
+pyasyncore==1.0.2
+pybind11==3.0.1
+pybind11-global==3.0.1
+pycairo==1.25.1
+pycocotools @ git+https://github.com/nvidia/cocoapi.git@08350a157b41f0755a536441b5abec89ab934c5b#subdirectory=PythonAPI
+pycountry==26.2.16
+pycparser==2.23
+pycryptodomex==3.20.0
+pycups==2.0.1
+pydantic==2.13.4
+pydantic-extra-types==2.11.1
+pydantic_core==2.46.4
+Pygments==2.20.0
+PyGObject==3.48.2
+pyinotify==0.9.6
+PyJWT==2.7.0
+PyNaCl==1.5.0
+pyparsing==3.2.5
+pyproject_hooks==1.2.0
+pyrsistent==0.20.0
+pyserial==3.5
+pytest==8.1.1
+pytest-flakefinder==1.1.0
+pytest-rerunfailures==16.1
+pytest-shard==0.1.2
+pytest-xdist==3.8.0
+python-apt==2.7.7+ubuntu5.1
+python-dateutil==2.9.0.post0
+python-debian==0.1.49+ubuntu2
+python-json-logger==4.0.0
+python-xlib==0.33
+python_hostlist==2.3.0
+pytokens==0.3.0
+pytorch-triton @ file:///tmp/dist/pytorch_triton-3.5.1%2Bgitbfeb0668.nv25.12-cp312-cp312-linux_x86_64.whl#sha256=5b91915ef04aa474fc46b0b2523c18c298052d104ada0b6d3c91daeba8bda919
+pytz==2025.2
+pyxdg==0.28
+PyYAML==6.0.3
+pyzmq==27.1.0
+referencing==0.37.0
+regex==2026.7.19
+requests==2.34.2
+rfc3339-validator==0.1.4
+rfc3986-validator==0.1.1
+rfc3987-syntax==1.1.0
+rich==15.0.0
+rpds-py==2026.6.3
+safetensors==0.8.0
+scikit-learn==1.8.0
+scipy==1.16.3
+Send2Trash==1.8.3
+sentry-sdk==2.66.0
+setproctitle==1.3.3
+setuptools==83.0.0
+shellingham==1.5.4
+six==1.17.0
+sortedcontainers==2.4.0
+soundfile==0.13.1
+soupsieve==2.8
+soxr==1.0.0
+ssh-import-id==5.11
+stack-data==0.6.3
+StrEnum==0.4.15
+sympy==1.14.0
+systemd-python==235
+tabulate==0.9.0
+tbb==2021.13.1
+tensorboard==2.21.0
+tensorboard-data-server==0.7.2
+tensorrt @ file:///workspace/TensorRT-10.14.1.48/python/tensorrt-10.14.1.48-cp312-none-linux_x86_64.whl#sha256=a5cf85f323da63a902f66869f5fbf07079bd1ca4e397b8810c1bf7cb4b19de76
+terminado==0.18.1
+threadpoolctl==3.6.0
+tiktoken==0.13.0
+tinycss2==1.4.0
+tokenizers==0.22.2
+toml==0.10.2
+tomli==2.3.0
+torch @ file:///opt/transfer/torch-2.10.0a0%2Bb4e4ee81d3.nv25.12-40345937-cp312-cp312-linux_x86_64.whl#sha256=f4951a7100ee9e38e9ee497b31524f71597a8f16edfaee66735a03bda61e4af7
+torch_tensorrt @ file:///opt/pytorch/torch_tensorrt/dist/torch_tensorrt-2.10.0a0-cp312-cp312-linux_x86_64.whl#sha256=d065f2cc3c14e76494635909a64b91c4c5ed6cfbdde9337eabab0a91da88150e
+torchao @ file:///opt/pytorch/ao
+torchdata==0.11.0
+torchprofile==0.0.4
+torchtitan @ file:///opt/pytorch/torchtitan
+torchvision @ file:///opt/transfer/torchvision-0.25.0a0%2Bca221243-40345937-cp312-cp312-manylinux_2_28_x86_64.whl#sha256=9c7f2a12ea750859c1433d20e4b79570d22c04164dd74f2224c8d7cd9fecc49d
+tornado==6.5.4
+tqdm==4.69.0
+traitlets==5.14.3
+transformer_engine @ file:///workspace/TransformerEngine
+transformers==5.8.1
+triton==3.7.1
+triton_kernels @ file:///tmp/dist/triton_kernels-1.0.0%2Bgitbfeb0668.nv25.12-py3-none-any.whl#sha256=f79455e18d1992e2f1ec665551c90f4b3fdf6f1253810da8fa029735716b499d
+typeguard==4.4.4
+typer==0.27.0
+typer-slim==0.20.0
+typing-inspection==0.4.2
+typing_extensions==4.16.0
+tyro==1.0.2
+tzdata==2025.3
+ubuntu-drivers-common==0.0.0
+ubuntu-pro-client==8001
+ufw==0.36.2
+unattended-upgrades==0.1
+uri-template==1.3.0
+urllib3==2.7.0
+uv==0.9.18
+wadllib==1.3.6
+wandb==0.28.1
+wcwidth==0.2.14
+webcolors==25.10.0
+webencodings==0.5.1
+websocket-client==1.9.0
+websockets==10.4
+Werkzeug==3.1.8
+wheel==0.45.1
+wrapt==2.0.1
+xdg==5
+xdoctest==1.0.2
+xkit==0.0.0
+xxhash==3.6.0
+yarl==1.22.0
+yt-dlp==2024.4.9
+zipp==3.23.0
+zstandard==0.25.0