push files

.gitignore

@@ -0,0 +1 @@
+wandb/

arguments.json

@@ -0,0 +1,64 @@
+{
+    "model_name_or_path": "PKU-Alignment/alpaca-7b-reproduced",
+    "max_length": 512,
+    "trust_remote_code": true,
+    "train_datasets": [
+        [
+            "PKU-SafeRLHF/train",
+            {
+                "proportion": 1.0
+            }
+        ]
+    ],
+    "eval_datasets": [
+        [
+            "PKU-SafeRLHF/test",
+            {
+                "proportion": 1.0
+            }
+        ]
+    ],
+    "loss_type": "sequence-wise",
+    "epochs": 2,
+    "per_device_train_batch_size": 16,
+    "per_device_eval_batch_size": 16,
+    "gradient_accumulation_steps": 1,
+    "gradient_checkpointing": true,
+    "regularization": 0.001,
+    "normalize_score_during_training": false,
+    "normalizer_type": "ExponentialMovingAverage",
+    "normalizer_momentum": 0.9,
+    "lr": 2e-05,
+    "lr_scheduler_type": "cosine",
+    "lr_warmup_ratio": 0.03,
+    "weight_decay": 0.1,
+    "seed": 42,
+    "fp16": false,
+    "bf16": true,
+    "tf32": true,
+    "eval_strategy": "epoch",
+    "eval_interval": 1000000,
+    "need_eval": true,
+    "eval_split_ratio": null,
+    "output_dir": "/home/alison.cs13/safe-rlhf/output/rm",
+    "log_type": "wandb",
+    "log_dir": "/home/alison.cs13/safe-rlhf/output/rm",
+    "log_project": "Safe-RLHF-RM",
+    "log_run_name": "reward-2024-09-20-20-02-06",
+    "save_16bit": false,
+    "save_interval": 1000000,
+    "local_rank": 0,
+    "zero_stage": 3,
+    "offload": "none",
+    "deepspeed": false,
+    "deepspeed_config": null,
+    "deepscale": false,
+    "deepscale_config": null,
+    "global_rank": 0,
+    "device": {
+        "type": "torch.device",
+        "repr": "device(type='cuda', index=0)"
+    },
+    "num_update_steps_per_epoch": 1155,
+    "total_training_steps": 2310
+}

arguments.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f2bf53fef59890fe1c372ca6b3c18033f5b84b1a00383e676f705d3762dc271
+size 1236

config.json

@@ -0,0 +1,42 @@
+{
+  "_name_or_path": "PKU-Alignment/alpaca-7b-reproduced",
+  "architectures": [
+    "LlamaForScore"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "do_normalize": false,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 2048,
+  "mean": [
+    0.12345914542675018
+  ],
+  "mlp_bias": false,
+  "model_type": "llama",
+  "momentum": 0.9,
+  "normalizer_type": "ExponentialMovingAverage",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "pad_token_id": 32000,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "score_bias": true,
+  "score_dim": 1,
+  "score_type": "reward",
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.44.1",
+  "use_cache": true,
+  "var": [
+    14.092216491699219
+  ],
+  "vocab_size": 32001
+}

environ.txt

@@ -0,0 +1,245 @@
+BASH_FUNC__module_raw%%=() {  unset _mlshdbg;
+ if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then
+ case "$-" in 
+ *v*x*)
+ set +vx;
+ _mlshdbg='vx'
+ ;;
+ *v*)
+ set +v;
+ _mlshdbg='v'
+ ;;
+ *x*)
+ set +x;
+ _mlshdbg='x'
+ ;;
+ *)
+ _mlshdbg=''
+ ;;
+ esac;
+ fi;
+ unset _mlre _mlIFS;
+ if [ -n "${IFS+x}" ]; then
+ _mlIFS=$IFS;
+ fi;
+ IFS=' ';
+ for _mlv in ${MODULES_RUN_QUARANTINE:-};
+ do
+ if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then
+ if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then
+ _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' ";
+ fi;
+ _mlrv="MODULES_RUNENV_${_mlv}";
+ _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' ";
+ fi;
+ done;
+ if [ -n "${_mlre:-}" ]; then
+ eval `eval ${_mlre} /usr/bin/tclsh /cm/local/apps/environment-modules/4.5.3/libexec/modulecmd.tcl bash '"$@"'`;
+ else
+ eval `/usr/bin/tclsh /cm/local/apps/environment-modules/4.5.3/libexec/modulecmd.tcl bash "$@"`;
+ fi;
+ _mlstatus=$?;
+ if [ -n "${_mlIFS+x}" ]; then
+ IFS=$_mlIFS;
+ else
+ unset IFS;
+ fi;
+ unset _mlre _mlv _mlrv _mlIFS;
+ if [ -n "${_mlshdbg:-}" ]; then
+ set -$_mlshdbg;
+ fi;
+ unset _mlshdbg;
+ return $_mlstatus
+}
+BASH_FUNC_ml%%=() {  module ml "$@"
+}
+BASH_FUNC_module%%=() {  _module_raw "$@" 2>&1
+}
+BASH_FUNC_switchml%%=() {  typeset swfound=1;
+ if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then
+ typeset swname='main';
+ if [ -e /cm/local/apps/environment-modules/4.5.3/libexec/modulecmd.tcl ]; then
+ typeset swfound=0;
+ unset MODULES_USE_COMPAT_VERSION;
+ fi;
+ else
+ typeset swname='compatibility';
+ if [ -e /cm/local/apps/environment-modules/4.5.3/libexec/modulecmd-compat ]; then
+ typeset swfound=0;
+ MODULES_USE_COMPAT_VERSION=1;
+ export MODULES_USE_COMPAT_VERSION;
+ fi;
+ fi;
+ if [ $swfound -eq 0 ]; then
+ echo "Switching to Modules $swname version";
+ source /cm/local/apps/environment-modules/4.5.3/init/bash;
+ else
+ echo "Cannot switch to Modules $swname version, command not found";
+ return 1;
+ fi
+}
+CC=/cm/shared/apps/hpc_sdk/2024_241/Linux_x86_64/24.1/compilers/bin/nvc
+CMD_WLM_CLUSTER_NAME=slurm
+CONDA_DEFAULT_ENV=safe-rlhf-12-4
+CONDA_EXE=/cm/shared/apps/anaconda/2024.02/bin/conda
+CONDA_PREFIX=/home/alison.cs13/.conda/envs/safe-rlhf-12-4
+CONDA_PREFIX_1=/cm/shared/apps/anaconda/2024.02
+CONDA_PROMPT_MODIFIER=(safe-rlhf-12-4) 
+CONDA_PYTHON_EXE=/cm/shared/apps/anaconda/2024.02/bin/python
+CONDA_SHLVL=2
+CPATH=/cm/shared/apps/anaconda/2024.02/include:/cm/shared/apps/hpc_sdk/2024_241/Linux_x86_64/24.1/compilers/extras/qd/include/qd:/cm/shared/apps/hpc_sdk/2024_241/Linux_x86_64/24.1/comm_libs/nvshmem/include:/cm/shared/apps/hpc_sdk/2024_241/Linux_x86_64/24.1/comm_libs/nccl/include:/cm/shared/apps/hpc_sdk/2024_241/Linux_x86_64/24.1/math_libs/include:/cm/shared/apps/slurm/current/include
+CPATH_modshare=/cm/shared/apps/hpc_sdk/2024_241/Linux_x86_64/24.1/comm_libs/nvshmem/include:1:/cm/shared/apps/anaconda/2024.02/include:1:/cm/shared/apps/hpc_sdk/2024_241/Linux_x86_64/24.1/math_libs/include:1:/cm/shared/apps/hpc_sdk/2024_241/Linux_x86_64/24.1/compilers/extras/qd/include/qd:1:/cm/shared/apps/hpc_sdk/2024_241/Linux_x86_64/24.1/comm_libs/nccl/include:1:/cm/shared/apps/slurm/current/include:1
+CPP=cpp
+CROSS_RANK=0
+CROSS_SIZE=1
+CUDA_MODULE_LOADING=LAZY
+CUDA_VISIBLE_DEVICES=0,1,2,3
+CXX=g++
+C_INCLUDE_PATH=/cm/shared/apps/anaconda/2024.02/include
+C_INCLUDE_PATH_modshare=/cm/shared/apps/anaconda/2024.02/include:1
+DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/1048/bus
+ENABLE_LMOD=0
+ENVIRONMENT=BATCH
+F77=/cm/shared/apps/hpc_sdk/2024_241/Linux_x86_64/24.1/compilers/bin/nvfortran
+F90=/cm/shared/apps/hpc_sdk/2024_241/Linux_x86_64/24.1/compilers/bin/nvfortran
+FC=/cm/shared/apps/hpc_sdk/2024_241/Linux_x86_64/24.1/compilers/bin/nvfortran
+GPU_DEVICE_ORDINAL=0,1,2,3
+HOME=/home/alison.cs13
+HOSTNAME=DGX-CN01
+INCLUDE=/cm/shared/apps/anaconda/2024.02/include
+INCLUDE_modshare=/cm/shared/apps/anaconda/2024.02/include:1
+LANG=en_US.UTF-8
+LD_LIBRARY_PATH=/cm/shared/apps/anaconda/2024.02/lib:/cm/local/apps/gcc/13.1.0/lib:/cm/local/apps/gcc/13.1.0/lib64:/cm/local/apps/gcc/13.1.0/lib32:/cm/shared/apps/hpc_sdk/2024_241/Linux_x86_64/24.1/comm_libs/nvshmem/lib:/cm/shared/apps/hpc_sdk/2024_241/Linux_x86_64/24.1/comm_libs/nccl/lib:/cm/shared/apps/hpc_sdk/2024_241/Linux_x86_64/24.1/math_libs/lib64:/cm/shared/apps/hpc_sdk/2024_241/Linux_x86_64/24.1/compilers/lib:/cm/shared/apps/hpc_sdk/2024_241/Linux_x86_64/24.1/compilers/extras/qd/lib:/cm/shared/apps/hpc_sdk/2024_241/Linux_x86_64/24.1/cuda/extras/CUPTI/lib64:/cm/shared/apps/hpc_sdk/2024_241/Linux_x86_64/24.1/cuda/lib64:/cm/shared/apps/slurm/current/lib64/slurm:/cm/shared/apps/slurm/current/lib64
+LD_LIBRARY_PATH_modshare=/cm/local/apps/gcc/13.1.0/lib64:1:/cm/shared/apps/hpc_sdk/2024_241/Linux_x86_64/24.1/comm_libs/nvshmem/lib:1:/cm/shared/apps/anaconda/2024.02/lib:1:/cm/shared/apps/hpc_sdk/2024_241/Linux_x86_64/24.1/comm_libs/nccl/lib:1:/cm/local/apps/gcc/13.1.0/lib32:1:/cm/shared/apps/hpc_sdk/2024_241/Linux_x86_64/24.1/math_libs/lib64:1:/cm/shared/apps/hpc_sdk/2024_241/Linux_x86_64/24.1/compilers/lib:1:/cm/local/apps/gcc/13.1.0/lib:1:/cm/shared/apps/hpc_sdk/2024_241/Linux_x86_64/24.1/compilers/extras/qd/lib:1:/cm/shared/apps/slurm/current/lib64/slurm:1:/cm/shared/apps/slurm/current/lib64:1:/cm/shared/apps/hpc_sdk/2024_241/Linux_x86_64/24.1/cuda/lib64:1:/cm/shared/apps/hpc_sdk/2024_241/Linux_x86_64/24.1/cuda/extras/CUPTI/lib64:1
+LD_RUN_PATH=/cm/shared/apps/anaconda/2024.02/lib
+LD_RUN_PATH_modshare=/cm/shared/apps/anaconda/2024.02/lib:1
+LESSCLOSE=/usr/bin/lesspipe %s %s
+LESSOPEN=| /usr/bin/lesspipe %s
+LIBRARY_PATH=/cm/shared/apps/slurm/current/lib64/slurm:/cm/shared/apps/slurm/current/lib64
+LIBRARY_PATH_modshare=/cm/shared/apps/slurm/current/lib64:1:/cm/shared/apps/slurm/current/lib64/slurm:1
+LOADEDMODULES=slurm/slurm/23.02.4:nvidia-hpc/2024_241:nvhpc/24.1:gcc/13.1.0:anaconda/2024.02
+LOADEDMODULES_modshare=anaconda/2024.02:1:nvidia-hpc/2024_241:1:gcc/13.1.0:1:slurm/slurm/23.02.4:1:nvhpc/24.1:1
+LOCAL_RANK=0
+LOCAL_SIZE=4
+LOGLEVEL=WARNING
+LOGNAME=alison.cs13
+LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=00:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.webp=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=00;36:*.au=00;36:*.flac=00;36:*.m4a=00;36:*.mid=00;36:*.midi=00;36:*.mka=00;36:*.mp3=00;36:*.mpc=00;36:*.ogg=00;36:*.ra=00;36:*.wav=00;36:*.oga=00;36:*.opus=00;36:*.spx=00;36:*.xspf=00;36:
+MANPATH=/cm/shared/apps/anaconda/2024.02/share/man:/cm/shared/apps/hpc_sdk/2024_241/Linux_x86_64/24.1/compilers/man:/cm/shared/apps/slurm/current/man:/cm/local/apps/cm-setup/man:/cm/local/apps/cm-setup/share/man:/cm/local/apps/cluster-tools/man:/cm/local/apps/cmd/man:/usr/local/man:/usr/local/share/man:/usr/share/man:/cm/local/apps/environment-modules/4.5.3/share/man:/opt/dell/srvadmin/share/man:/cm/local/apps/environment-modules/current/share/man
+MANPATH_modshare=/cm/shared/apps/anaconda/2024.02/share/man:1:/usr/local/share/man:1:/opt/dell/srvadmin/share/man:1:/cm/local/apps/cm-setup/man:1:/usr/local/man:1:/cm/shared/apps/slurm/current/man:1:/cm/shared/apps/hpc_sdk/2024_241/Linux_x86_64/24.1/compilers/man:1:/cm/local/apps/cmd/man:1:/cm/local/apps/environment-modules/4.5.3/share/man:1:/cm/local/apps/environment-modules/current/share/man:1:/cm/local/apps/cm-setup/share/man:1:/cm/local/apps/cluster-tools/man:1:/usr/share/man:1
+MASTER_ADDR=127.0.0.1
+MASTER_PORT=24823
+MODULEPATH=/cm/shared/apps/hpc_sdk/2024_241/Linux_x86_64/24.1/comm_libs/12.3/hpcx/latest/modulefiles:/cm/local/modulefiles:/cm/shared/modulefiles:/cm/shared/apps/hpc_sdk/2024_241/modulefiles
+MODULEPATH_modshare=/cm/shared/apps/hpc_sdk/2024_241/modulefiles:1:/cm/local/modulefiles:1:/cm/shared/modulefiles:1:/cm/shared/apps/hpc_sdk/2024_241/Linux_x86_64/24.1/comm_libs/12.3/hpcx/latest/modulefiles:1
+MODULESHOME=/cm/local/apps/environment-modules/4.5.3
+MODULES_AUTO_HANDLING=1
+MODULES_CMD=/cm/local/apps/environment-modules/4.5.3/libexec/modulecmd.tcl
+MODULES_LMCONFLICT=nvhpc/24.1&nvhpc&nvhpc-nompi&nvhpc-byo-compiler&nvhpc-hpcx&nvhpc-openmpi3
+MODULES_LMCONFLICT_modshare=nvhpc/24.1&nvhpc&nvhpc-nompi&nvhpc-byo-compiler&nvhpc-hpcx&nvhpc-openmpi3:1
+MODULES_SET_SHELL_STARTUP=0
+MOTD_SHOWN=pam
+NVHPC=/cm/shared/apps/hpc_sdk/2024_241
+NVHPC_ROOT=/cm/shared/apps/hpc_sdk/2024_241/Linux_x86_64/24.1
+OLDPWD=/home/alison.cs13/safe-rlhf
+PATH=/home/alison.cs13/.conda/envs/safe-rlhf-12-4/bin:/cm/local/apps/gcc/13.1.0/bin:/cm/shared/apps/hpc_sdk/2024_241/Linux_x86_64/24.1/compilers/extras/qd/bin:/cm/shared/apps/hpc_sdk/2024_241/Linux_x86_64/24.1/comm_libs/mpi/bin:/cm/shared/apps/hpc_sdk/2024_241/Linux_x86_64/24.1/compilers/bin:/cm/shared/apps/hpc_sdk/2024_241/Linux_x86_64/24.1/cuda/bin:/cm/shared/apps/slurm/current/sbin:/cm/shared/apps/slurm/current/bin:/cm/shared/apps/anaconda/2024.02/bin:/cm/shared/apps/anaconda/2024.02/condabin:/cm/local/apps/cm-setup/bin:/cm/local/apps/cluster-tools/bin:/cm/local/apps/cmd/sbin:/cm/local/apps/cmd/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/cm/local/apps/environment-modules/4.5.3/bin:/opt/dell/srvadmin/sbin:/sbin:/usr/sbin:/cm/local/apps/environment-modules/4.5.3/bin
+PATH_modshare=/opt/dell/srvadmin/sbin:1:/usr/bin:1:/cm/local/apps/cm-setup/bin:1:/cm/shared/apps/hpc_sdk/2024_241/Linux_x86_64/24.1/comm_libs/mpi/bin:1:/cm/local/apps/cmd/sbin:1:/cm/shared/apps/anaconda/2024.02/bin:1:/usr/local/bin:1:/cm/shared/apps/slurm/current/bin:1:/cm/shared/apps/hpc_sdk/2024_241/Linux_x86_64/24.1/cuda/bin:1:/cm/shared/apps/hpc_sdk/2024_241/Linux_x86_64/24.1/compilers/bin:1:/cm/local/apps/cmd/bin:1:/cm/shared/apps/slurm/current/sbin:1:/cm/shared/apps/anaconda/2024.02/condabin:1:/cm/shared/apps/hpc_sdk/2024_241/Linux_x86_64/24.1/compilers/extras/qd/bin:1:/cm/local/apps/gcc/13.1.0/bin:1:/bin:1:/snap/bin:1:/home/alison.cs13/.conda/envs/safe-rlhf-12-4/bin:1:/cm/local/apps/cluster-tools/bin:1:/sbin:1:/usr/sbin:1:/usr/games:1:/cm/local/apps/environment-modules/4.5.3/bin:1:/usr/local/sbin:1:/usr/local/games:1
+PMIX_BFROP_BUFFER_TYPE=PMIX_BFROP_BUFFER_NON_DESC
+PMIX_DSTORE_21_BASE_PATH=/cm/local/apps/slurm/var/spool/pmix.4692.0//pmix_dstor_ds21_1501636
+PMIX_DSTORE_ESH_BASE_PATH=/cm/local/apps/slurm/var/spool/pmix.4692.0//pmix_dstor_ds12_1501636
+PMIX_GDS_MODULE=ds21,ds12,hash
+PMIX_HOSTNAME=DGX-CN01
+PMIX_NAMESPACE=slurm.pmix.4692.0
+PMIX_RANK=0
+PMIX_SECURITY_MODE=native
+PMIX_SERVER_TMPDIR=/cm/local/apps/slurm/var/spool/pmix.4692.0/
+PMIX_SERVER_URI2=pmix-server.1501636;tcp4://127.0.0.1:58961
+PMIX_SERVER_URI21=pmix-server.1501636;tcp4://127.0.0.1:58961
+PMIX_SERVER_URI3=pmix-server.1501636;tcp4://127.0.0.1:58961
+PMIX_SERVER_URI4=pmix-server.1501636;tcp4://127.0.0.1:58961
+PMIX_SERVER_URI41=pmix-server.1501636;tcp4://127.0.0.1:58961
+PMIX_SYSTEM_TMPDIR=/tmp
+PMIX_VERSION=4.1.1
+PWD=/home/alison.cs13/safe-rlhf/scripts
+PYTHONHASHSEED=42
+PYTHONPATH=/home/alison.cs13/safe-rlhf
+RANK=0
+ROCR_VISIBLE_DEVICES=0,1,2,3
+SHELL=/bin/bash
+SHLVL=3
+SLURMD_DEBUG=2
+SLURMD_NODENAME=DGX-CN01
+SLURM_CLUSTER_NAME=slurm
+SLURM_CONF=/cm/shared/apps/slurm/var/etc/slurm/slurm.conf
+SLURM_CPUS_ON_NODE=8
+SLURM_CPUS_PER_TASK=8
+SLURM_DISTRIBUTION=cyclic
+SLURM_GPUS_ON_NODE=4
+SLURM_GTIDS=0
+SLURM_JOBID=4692
+SLURM_JOB_ACCOUNT=at0816
+SLURM_JOB_CPUS_PER_NODE=8
+SLURM_JOB_END_TIME=1726855212
+SLURM_JOB_GID=1026
+SLURM_JOB_GPUS=3,5,6,7
+SLURM_JOB_ID=4692
+SLURM_JOB_NAME=Train_original_rm
+SLURM_JOB_NODELIST=DGX-CN01
+SLURM_JOB_NUM_NODES=1
+SLURM_JOB_PARTITION=defq
+SLURM_JOB_QOS=normal
+SLURM_JOB_START_TIME=1726833612
+SLURM_JOB_UID=1048
+SLURM_JOB_USER=alison.cs13
+SLURM_LAUNCH_NODE_IPADDR=10.141.0.1
+SLURM_LOCALID=0
+SLURM_MEM_PER_NODE=204800
+SLURM_MPI_TYPE=pmix
+SLURM_NNODES=1
+SLURM_NODEID=0
+SLURM_NODELIST=DGX-CN01
+SLURM_NPROCS=1
+SLURM_NTASKS=1
+SLURM_NTASKS_PER_NODE=1
+SLURM_PMIXP_ABORT_AGENT_PORT=45281
+SLURM_PMIX_MAPPING_SERV=(vector,(0,1,1))
+SLURM_PRIO_PROCESS=0
+SLURM_PROCID=0
+SLURM_SRUN_COMM_HOST=10.141.0.1
+SLURM_SRUN_COMM_PORT=33951
+SLURM_STEPID=0
+SLURM_STEP_GPUS=3,5,6,7
+SLURM_STEP_ID=0
+SLURM_STEP_LAUNCHER_PORT=33951
+SLURM_STEP_NODELIST=DGX-CN01
+SLURM_STEP_NUM_NODES=1
+SLURM_STEP_NUM_TASKS=1
+SLURM_STEP_TASKS_PER_NODE=1
+SLURM_SUBMIT_DIR=/home/alison.cs13/safe-rlhf/scripts
+SLURM_SUBMIT_HOST=headnode
+SLURM_TASKS_PER_NODE=1
+SLURM_TASK_PID=1501656
+SLURM_TOPOLOGY_ADDR=DGX-CN01
+SLURM_TOPOLOGY_ADDR_PATTERN=node
+SLURM_UMASK=0022
+SLURM_WORKING_CLUSTER=slurm:headnode:6817:9984:109
+SRUN_DEBUG=3
+SSH_CLIENT=140.113.15.75 51020 22
+SSH_CONNECTION=140.113.15.75 51020 140.113.9.253 22
+SSH_TTY=/dev/pts/10
+TERM=xterm-256color
+TMPDIR=/tmp
+TORCH_CUDA_ARCH_LIST=9.0
+USER=alison.cs13
+WANDB_MODE=offline
+WANDB_SERVICE=2-1502872-tcp-localhost-59305
+WORLD_SIZE=4
+XDG_DATA_DIRS=/usr/share/gnome:/usr/local/share:/usr/share:/var/lib/snapd/desktop
+XDG_RUNTIME_DIR=/run/user/1048
+XDG_SESSION_CLASS=user
+XDG_SESSION_ID=375863
+XDG_SESSION_TYPE=tty
+ZE_AFFINITY_MASK=0,1,2,3
+_=/home/alison.cs13/.conda/envs/safe-rlhf-12-4/bin/deepspeed
+_CE_CONDA=
+_CE_M=
+_LMFILES_=/cm/local/modulefiles/slurm/slurm/23.02.4:/cm/shared/modulefiles/nvidia-hpc/2024_241:/cm/shared/apps/hpc_sdk/2024_241/modulefiles/nvhpc/24.1:/cm/local/modulefiles/gcc/13.1.0:/cm/shared/modulefiles/anaconda/2024.02
+_LMFILES__modshare=/cm/shared/modulefiles/anaconda/2024.02:1:/cm/shared/modulefiles/nvidia-hpc/2024_241:1:/cm/local/modulefiles/gcc/13.1.0:1:/cm/shared/apps/hpc_sdk/2024_241/modulefiles/nvhpc/24.1:1:/cm/local/modulefiles/slurm/slurm/23.02.4:1

latest

@@ -0,0 +1 @@
+global_step2310

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cea964375058bd71d8a6bd0df4a839453bff4df6746132714d4a4da20d8a72bc
+size 26429505838

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

stdout.log

@@ -0,0 +1,7 @@
+The token has not been saved to the git credentials helper. Pass `add_to_git_credential=True` in this function directly or `--add-to-git-credential` if using via `huggingface-cli` if you want to set the git credential as well.
+Token is valid (permission: read).
+Your token has been saved to /home/alison/.cache/huggingface/token
+Login successful
+[2024-10-15 02:04:31,019] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2024-10-15 02:04:33,253] [WARNING] [runner.py:212:fetch_hostfile] Unable to find hostfile, will proceed with training with local resources only.
+[2024-10-15 02:04:33,253] [INFO] [runner.py:585:main] cmd = /home/alison/anaconda3/envs/safe-rlhf-12.4/bin/python -u -m deepspeed.launcher.launch --world_info=eyJsb2NhbGhvc3QiOiBbMF19 --master_addr=127.0.0.1 --master_port=29959 --module --enable_each_rank_log=None safe_rlhf.values.reward --train_datasets PKU-SafeRLHF/train --eval_datasets PKU-SafeRLHF/test --model_name_or_path PKU-Alignment/alpaca-7b-reproduced --max_length 512 --trust_remote_code True --loss_type sequence-wise --epochs 2 --per_device_train_batch_size 16 --per_device_eval_batch_size 16 --gradient_accumulation_steps 1 --gradient_checkpointing --regularization 0.001 --normalize_score_during_training False --normalizer_type ExponentialMovingAverage --normalizer_momentum 0.9 --learning_rate 2e-5 --lr_scheduler_type cosine --lr_warmup_ratio 0.03 --weight_decay 0.1 --seed 42 --need_eval --eval_strategy epoch --output_dir /home/alison/safe-rlhf/output/rm --log_type wandb --log_project Safe-RLHF-RM --zero_stage 3 --offload none --bf16 True --tf32 True

tokenizer_config.json

@@ -0,0 +1,51 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32000": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 512,
+  "pad_token": "<pad>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

zero_to_fp32.py

@@ -0,0 +1,604 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example: python zero_to_fp32.py . pytorch_model.bin
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+                                            FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+                                            FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
+
+
+@dataclass
+class zero_model_state:
+    buffers: dict()
+    param_shapes: dict()
+    shared_params: list
+    ds_version: int
+    frozen_param_shapes: dict()
+    frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+    return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+    '''
+    alist.sort(key=natural_keys) sorts in human order
+    http://nedbatchelder.com/blog/200712/human_sorting.html
+    (See Toothy's implementation in the comments)
+    '''
+    return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+    if not os.path.isdir(checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+    # there should be only one file
+    if zero_stage <= 2:
+        file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+    elif zero_stage == 3:
+        file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+    if not os.path.exists(file):
+        raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+    return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+    # XXX: need to test that this simple glob rule works for multi-node setup too
+    ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+    if len(ckpt_files) == 0:
+        raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+    return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def parse_model_states(files):
+    zero_model_states = []
+    for file in files:
+        state_dict = torch.load(file, map_location=device)
+
+        if BUFFER_NAMES not in state_dict:
+            raise ValueError(f"{file} is not a model state checkpoint")
+        buffer_names = state_dict[BUFFER_NAMES]
+        if debug:
+            print("Found buffers:", buffer_names)
+
+        # recover just the buffers while restoring them to fp32 if they were saved in fp16
+        buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+        param_shapes = state_dict[PARAM_SHAPES]
+
+        # collect parameters that are included in param_shapes
+        param_names = []
+        for s in param_shapes:
+            for name in s.keys():
+                param_names.append(name)
+
+        # update with frozen parameters
+        frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+        if frozen_param_shapes is not None:
+            if debug:
+                print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+            param_names += list(frozen_param_shapes.keys())
+
+        # handle shared params
+        shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+        ds_version = state_dict.get(DS_VERSION, None)
+
+        frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+        z_model_state = zero_model_state(buffers=buffers,
+                                         param_shapes=param_shapes,
+                                         shared_params=shared_params,
+                                         ds_version=ds_version,
+                                         frozen_param_shapes=frozen_param_shapes,
+                                         frozen_param_fragments=frozen_param_fragments)
+        zero_model_states.append(z_model_state)
+
+    return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+
+    total_files = len(files)
+    state_dicts = []
+    for f in files:
+        state_dict = torch.load(f, map_location=device)
+        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+        # and also handle the case where it was already removed by another helper script
+        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+        state_dicts.append(state_dict)
+
+    if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]:
+        raise ValueError(f"{files[0]} is not a zero checkpoint")
+    zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+    world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+    # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+    # parameters can be different from data parallelism for non-expert parameters. So we can just
+    # use the max of the partition_count to get the dp world_size.
+
+    if type(world_size) is list:
+        world_size = max(world_size)
+
+    if world_size != total_files:
+        raise ValueError(
+            f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+            "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+        )
+
+    # the groups are named differently in each stage
+    if zero_stage <= 2:
+        fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+    elif zero_stage == 3:
+        fp32_groups_key = FP32_FLAT_GROUPS
+    else:
+        raise ValueError(f"unknown zero stage {zero_stage}")
+
+    if zero_stage <= 2:
+        fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+    elif zero_stage == 3:
+        # if there is more than one param group, there will be multiple flattened tensors - one
+        # flattened tensor per group - for simplicity merge them into a single tensor
+        #
+        # XXX: could make the script more memory efficient for when there are multiple groups - it
+        # will require matching the sub-lists of param_shapes for each param group flattened tensor
+
+        fp32_flat_groups = [
+            torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts))
+        ]
+
+    return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+    """
+    Returns fp32 state_dict reconstructed from ds checkpoint
+
+    Args:
+        - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+    """
+    print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+    optim_files = get_optim_files(ds_checkpoint_dir)
+    zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+    print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+    model_files = get_model_state_files(ds_checkpoint_dir)
+
+    zero_model_states = parse_model_states(model_files)
+    print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+    if zero_stage <= 2:
+        return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+    elif zero_stage == 3:
+        return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+    frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+    if debug:
+        num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+        print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        state_dict[name] = frozen_param_fragments[name]
+
+        if debug:
+            print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+    attr = getattr(obj, fn, None)
+    return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+
+    # Reconstruction protocol:
+    #
+    # XXX: document this
+
+    if debug:
+        for i in range(world_size):
+            for j in range(len(fp32_flat_groups[0])):
+                print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+    # XXX: memory usage doubles here (zero2)
+    num_param_groups = len(fp32_flat_groups[0])
+    merged_single_partition_of_fp32_groups = []
+    for i in range(num_param_groups):
+        merged_partitions = [sd[i] for sd in fp32_flat_groups]
+        full_single_fp32_vector = torch.cat(merged_partitions, 0)
+        merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+    avail_numel = sum(
+        [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+    if debug:
+        wanted_params = sum([len(shapes) for shapes in param_shapes])
+        wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+        # not asserting if there is a mismatch due to possible padding
+        print(f"Have {avail_numel} numels to process.")
+        print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    total_numel = 0
+    total_params = 0
+    for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+        offset = 0
+        avail_numel = full_single_fp32_vector.numel()
+        for name, shape in shapes.items():
+
+            unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+            total_numel += unpartitioned_numel
+            total_params += 1
+
+            if debug:
+                print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+            state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+            offset += unpartitioned_numel
+
+        # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+        # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+        # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+        # live optimizer object, so we are checking that the numbers are within the right range
+        align_to = 2 * world_size
+
+        def zero2_align(x):
+            return align_to * math.ceil(x / align_to)
+
+        if debug:
+            print(f"original offset={offset}, avail_numel={avail_numel}")
+
+        offset = zero2_align(offset)
+        avail_numel = zero2_align(avail_numel)
+
+        if debug:
+            print(f"aligned  offset={offset}, avail_numel={avail_numel}")
+
+        # Sanity check
+        if offset != avail_numel:
+            raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+    _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+    remainder = unpartitioned_numel % world_size
+    padding_numel = (world_size - remainder) if remainder else 0
+    partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+    return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    if debug:
+        for i in range(world_size):
+            num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+            print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in zero_model_states[0].frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+        state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+    avail_numel = fp32_flat_groups[0].numel() * world_size
+    # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+    # param, re-consolidating each param, while dealing with padding if any
+
+    # merge list of dicts, preserving order
+    param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+    if debug:
+        for i in range(world_size):
+            print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+        wanted_params = len(param_shapes)
+        wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+        # not asserting if there is a mismatch due to possible padding
+        avail_numel = fp32_flat_groups[0].numel() * world_size
+        print(f"Trainable params: Have {avail_numel} numels to process.")
+        print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    offset = 0
+    total_numel = 0
+    total_params = 0
+    for name, shape in param_shapes.items():
+
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+        total_params += 1
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+        # XXX: memory usage doubles here
+        state_dict[name] = torch.cat(
+            tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)),
+            0).narrow(0, 0, unpartitioned_numel).view(shape)
+        offset += partitioned_numel
+
+    offset *= world_size
+
+    # Sanity check
+    if offset != avail_numel:
+        raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+    _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+    ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+    via a model hub.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+
+    Returns:
+        - pytorch ``state_dict``
+
+    Note: this approach may not work if your application doesn't have sufficient free CPU memory and
+    you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+    the checkpoint.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        # do the training and checkpoint saving
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+        model = model.cpu() # move to cpu
+        model.load_state_dict(state_dict)
+        # submit to model hub or save the model to share with others
+
+    In this example the ``model`` will no longer be usable in the deepspeed context of the same
+    application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+    """
+    if tag is None:
+        latest_path = os.path.join(checkpoint_dir, 'latest')
+        if os.path.isfile(latest_path):
+            with open(latest_path, 'r') as fd:
+                tag = fd.read().strip()
+        else:
+            raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+    ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+    if not os.path.isdir(ds_checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+    return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+    loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+    """
+
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag, exclude_frozen_parameters)
+    print(f"Saving fp32 state dict to {output_file}")
+    torch.save(state_dict, output_file)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+    """
+    1. Put the provided model to cpu
+    2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+    3. Load it into the provided model
+
+    Args:
+        - ``model``: the model object to update
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+    Returns:
+        - ``model`: modified model
+
+    Make sure you have plenty of CPU memory available before you call this function. If you don't
+    have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+    conveniently placed for you in the checkpoint folder.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+        model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+        # submit to model hub or save the model to share with others
+
+    Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+    of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    """
+    logger.info(f"Extracting fp32 weights")
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+    logger.info(f"Overwriting model with fp32 weights")
+    model = model.cpu()
+    model.load_state_dict(state_dict, strict=False)
+
+    return model
+
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("checkpoint_dir",
+                        type=str,
+                        help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+    parser.add_argument(
+        "output_file",
+        type=str,
+        help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)")
+    parser.add_argument("-t",
+                        "--tag",
+                        type=str,
+                        default=None,
+                        help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+    parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+    parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+    args = parser.parse_args()
+
+    debug = args.debug
+
+    convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+                                               args.output_file,
+                                               tag=args.tag,
+                                               exclude_frozen_parameters=args.exclude_frozen_parameters)