Upload folder using huggingface_hub

.gitignore

@@ -0,0 +1,2 @@
+wandb/
+script.sh

arguments.json

@@ -0,0 +1,93 @@
+{
+    "actor_model_name_or_path": "AlisonWen/safe-rlhf-sft",
+    "reward_model_name_or_path": "AlisonWen/rm-seed-42-targeted-flipped-0.5",
+    "reward_critic_model_name_or_path": "AlisonWen/rm-seed-42-targeted-flipped-0.5",
+    "max_length": 512,
+    "trust_remote_code": true,
+    "train_datasets": [
+        [
+            "PKU-SafeRLHF/train",
+            {
+                "proportion": 1.0
+            }
+        ]
+    ],
+    "ptx_datasets": [
+        [
+            "alpaca",
+            {
+                "proportion": 1.0
+            }
+        ]
+    ],
+    "eval_datasets": null,
+    "kl_coeff": 0.02,
+    "clip_range_ratio": 0.2,
+    "clip_range_score": 50.0,
+    "clip_range_value": 5.0,
+    "ptx_coeff": 16.0,
+    "epochs": 1,
+    "update_iters": 2,
+    "per_device_prompt_batch_size": 16,
+    "per_device_train_batch_size": 16,
+    "per_device_eval_batch_size": 16,
+    "gradient_accumulation_steps": 2,
+    "actor_lr": 1e-05,
+    "actor_weight_decay": 0.01,
+    "actor_lr_scheduler_type": "cosine",
+    "actor_lr_warmup_ratio": 0.03,
+    "actor_gradient_checkpointing": true,
+    "critic_lr": 5e-06,
+    "critic_weight_decay": 0.0,
+    "critic_lr_scheduler_type": "constant",
+    "critic_lr_warmup_ratio": 0.03,
+    "critic_gradient_checkpointing": true,
+    "normalize_reward": false,
+    "seed": 100,
+    "fp16": false,
+    "bf16": true,
+    "tf32": true,
+    "pth": 0.5,
+    "pth_dynamic_adjust": null,
+    "pth_meta_lr": 0.0,
+    "meta_pth_scale": 10.0,
+    "meta_pth_update_interval": 10,
+    "linear_pth": null,
+    "linear_pth_init": 0.3,
+    "linear_pth_end": 1.0,
+    "random_reward_poisoning": null,
+    "attack_prob": null,
+    "attack_strength": null,
+    "uniform_random_poisoning": null,
+    "malicious_babble_attack": null,
+    "dynamic_pth_rate": 0.0,
+    "pth_batch_size": 8,
+    "temperature": 1.0,
+    "top_p": 1.0,
+    "num_return_sequences": 1,
+    "repetition_penalty": 1.0,
+    "eval_strategy": "epoch",
+    "eval_interval": 1000000,
+    "need_eval": false,
+    "eval_split_ratio": null,
+    "output_dir": "/mnt/home/alison.cs13/RobustRLHF/output/ppo-robust-0.5-resample-large-batch-random-select-voted-flipped",
+    "log_type": "wandb",
+    "log_dir": "/mnt/home/alison.cs13/RobustRLHF/output/ppo-robust-0.5-resample-large-batch-random-select-voted-flipped",
+    "log_project": "Safe-RLHF-PPO",
+    "log_run_name": "ppo-robust-0.5-resample-large-batch-voted-flipped-step-604",
+    "save_16bit": false,
+    "save_interval": 1000000,
+    "local_rank": 0,
+    "zero_stage": 3,
+    "offload": "none",
+    "deepspeed": false,
+    "deepspeed_config": null,
+    "deepscale": false,
+    "deepscale_config": null,
+    "global_rank": 0,
+    "device": {
+        "type": "torch.device",
+        "repr": "device(type='cuda', index=0)"
+    },
+    "total_training_steps": 1208
+}

arguments.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e617aa8ecb05af101b8b2ec74dc4ce12b8df3fadee5127f4f2f1d9f899a0188e
+size 2117

environ.txt

@@ -0,0 +1,236 @@
+BASH_FUNC_ml%%=() {  module ml "$@"
+}
+BASH_FUNC_module%%=() {  unset _mlshdbg;
+ if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then
+ case "$-" in 
+ *v*x*)
+ set +vx;
+ _mlshdbg='vx'
+ ;;
+ *v*)
+ set +v;
+ _mlshdbg='v'
+ ;;
+ *x*)
+ set +x;
+ _mlshdbg='x'
+ ;;
+ *)
+ _mlshdbg=''
+ ;;
+ esac;
+ fi;
+ unset _mlre _mlIFS;
+ if [ -n "${IFS+x}" ]; then
+ _mlIFS=$IFS;
+ fi;
+ IFS=' ';
+ for _mlv in ${MODULES_RUN_QUARANTINE:-};
+ do
+ if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then
+ if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then
+ _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' ";
+ fi;
+ _mlrv="MODULES_RUNENV_${_mlv}";
+ _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' ";
+ fi;
+ done;
+ if [ -n "${_mlre:-}" ]; then
+ eval `eval ${_mlre} /usr/bin/tclsh /cm/local/apps/environment-modules/4.5.3/libexec/modulecmd.tcl bash '"$@"'`;
+ else
+ eval `/usr/bin/tclsh /cm/local/apps/environment-modules/4.5.3/libexec/modulecmd.tcl bash "$@"`;
+ fi;
+ _mlstatus=$?;
+ if [ -n "${_mlIFS+x}" ]; then
+ IFS=$_mlIFS;
+ else
+ unset IFS;
+ fi;
+ unset _mlre _mlv _mlrv _mlIFS;
+ if [ -n "${_mlshdbg:-}" ]; then
+ set -$_mlshdbg;
+ fi;
+ unset _mlshdbg;
+ return $_mlstatus
+}
+BASH_FUNC_switchml%%=() {  typeset swfound=1;
+ if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then
+ typeset swname='main';
+ if [ -e /cm/local/apps/environment-modules/4.5.3/libexec/modulecmd.tcl ]; then
+ typeset swfound=0;
+ unset MODULES_USE_COMPAT_VERSION;
+ fi;
+ else
+ typeset swname='compatibility';
+ if [ -e /cm/local/apps/environment-modules/4.5.3/libexec/modulecmd-compat ]; then
+ typeset swfound=0;
+ MODULES_USE_COMPAT_VERSION=1;
+ export MODULES_USE_COMPAT_VERSION;
+ fi;
+ fi;
+ if [ $swfound -eq 0 ]; then
+ echo "Switching to Modules $swname version";
+ source /cm/local/apps/environment-modules/4.5.3/init/bash;
+ else
+ echo "Cannot switch to Modules $swname version, command not found";
+ return 1;
+ fi
+}
+BROWSER=/home/alison.cs13/.vscode-server/cli/servers/Stable-4437686ffebaf200fa4a6e6e67f735f3edf24ada/server/bin/helpers/browser.sh
+CC=/usr/bin/gcc
+CMD_WLM_CLUSTER_NAME=slurm
+COLORTERM=truecolor
+CONDA_DEFAULT_ENV=safe-rlhf-12.4
+CONDA_EXE=/mnt/home/alison.cs13/anaconda3/bin/conda
+CONDA_PREFIX=/mnt/home/alison.cs13/anaconda3/envs/safe-rlhf-12.4
+CONDA_PREFIX_1=/mnt/home/alison.cs13/anaconda3
+CONDA_PROMPT_MODIFIER=(safe-rlhf-12.4) 
+CONDA_PYTHON_EXE=/mnt/home/alison.cs13/anaconda3/bin/python
+CONDA_SHLVL=2
+CPATH=/cm/shared/apps/slurm/current/include
+CPATH_modshare=/cm/shared/apps/slurm/current/include:1
+CROSS_RANK=0
+CROSS_SIZE=1
+CUDA_MODULE_LOADING=LAZY
+CUDA_VISIBLE_DEVICES=0,1,2,3
+CXX=/usr/bin/g++
+DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/1109/bus
+ENABLE_LMOD=0
+ENVIRONMENT=BATCH
+GIT_ASKPASS=/home/alison.cs13/.vscode-server/cli/servers/Stable-4437686ffebaf200fa4a6e6e67f735f3edf24ada/server/extensions/git/dist/askpass.sh
+GPU_DEVICE_ORDINAL=0,1,2,3
+HOME=/home/alison.cs13
+HOSTNAME=cnode4-002
+LANG=en_US.UTF-8
+LD_LIBRARY_PATH=/cm/shared/apps/slurm/current/lib64/slurm:/cm/shared/apps/slurm/current/lib64:/cm/local/apps/gcc/13.1.0/lib:/cm/local/apps/gcc/13.1.0/lib64:/cm/local/apps/gcc/13.1.0/lib32
+LD_LIBRARY_PATH_modshare=/cm/shared/apps/slurm/current/lib64:1:/cm/local/apps/gcc/13.1.0/lib64:1:/cm/shared/apps/slurm/current/lib64/slurm:1:/cm/local/apps/gcc/13.1.0/lib:1:/cm/local/apps/gcc/13.1.0/lib32:1
+LESSCLOSE=/usr/bin/lesspipe %s %s
+LESSOPEN=| /usr/bin/lesspipe %s
+LIBRARY_PATH=/cm/shared/apps/slurm/current/lib64/slurm:/cm/shared/apps/slurm/current/lib64
+LIBRARY_PATH_modshare=/cm/shared/apps/slurm/current/lib64:1:/cm/shared/apps/slurm/current/lib64/slurm:1
+LOADEDMODULES=gcc/13.1.0:slurm/slurm/23.02.8
+LOADEDMODULES_modshare=gcc/13.1.0:1:slurm/slurm/23.02.8:1
+LOCAL_RANK=0
+LOCAL_SIZE=4
+LOGLEVEL=WARNING
+LOGNAME=alison.cs13
+LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=00:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.webp=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=00;36:*.au=00;36:*.flac=00;36:*.m4a=00;36:*.mid=00;36:*.midi=00;36:*.mka=00;36:*.mp3=00;36:*.mpc=00;36:*.ogg=00;36:*.ra=00;36:*.wav=00;36:*.oga=00;36:*.opus=00;36:*.spx=00;36:*.xspf=00;36:
+MANPATH=/cm/shared/apps/slurm/current/man:/cm/local/apps/environment-modules/4.5.3/share/man:/usr/local/man:/usr/local/share/man:/usr/share/man:/cm/local/apps/environment-modules/current/share/man
+MANPATH_modshare=/usr/local/share/man:1:/cm/local/apps/environment-modules/current/share/man:1:/cm/local/apps/environment-modules/4.5.3/share/man:1:/usr/local/man:1:/usr/share/man:1:/cm/shared/apps/slurm/current/man:1
+MASTER_ADDR=127.0.0.1
+MASTER_PORT=26638
+MODULEPATH=/cm/local/modulefiles:/cm/shared/modulefiles
+MODULESHOME=/cm/local/apps/environment-modules/4.5.3
+MODULES_CMD=/cm/local/apps/environment-modules/4.5.3/libexec/modulecmd.tcl
+MODULES_SET_SHELL_STARTUP=0
+MOTD_SHOWN=pam
+NVCC_FLAGS=--ccbin=/usr/bin/g++
+OLDPWD=/mnt/home/alison.cs13/RobustRLHF
+PATH=/cm/shared/apps/slurm/current/sbin:/cm/shared/apps/slurm/current/bin:/mnt/home/alison.cs13/anaconda3/envs/safe-rlhf-12.4/bin:/mnt/home/alison.cs13/anaconda3/condabin:/cm/local/apps/gcc/13.1.0/bin:/home/alison.cs13/.vscode-server/cli/servers/Stable-4437686ffebaf200fa4a6e6e67f735f3edf24ada/server/bin/remote-cli:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/sbin:/usr/sbin:/cm/local/apps/environment-modules/4.5.3/bin
+PATH_modshare=/home/alison.cs13/.vscode-server/cli/servers/Stable-4437686ffebaf200fa4a6e6e67f735f3edf24ada/server/bin/remote-cli:1:/usr/bin:1:/usr/local/bin:1:/cm/shared/apps/slurm/current/bin:1:/cm/shared/apps/slurm/current/sbin:1:/mnt/home/alison.cs13/anaconda3/condabin:1:/cm/local/apps/gcc/13.1.0/bin:1:/bin:1:/snap/bin:1:/mnt/home/alison.cs13/anaconda3/bin:1:/sbin:1:/usr/sbin:1:/usr/games:1:/cm/local/apps/environment-modules/4.5.3/bin:1:/usr/local/sbin:1:/usr/local/games:1
+PMIX_BFROP_BUFFER_TYPE=PMIX_BFROP_BUFFER_NON_DESC
+PMIX_GDS_MODULE=hash
+PMIX_HOSTNAME=cnode4-002
+PMIX_NAMESPACE=slurm.pmix.1046.0
+PMIX_RANK=0
+PMIX_SECURITY_MODE=none
+PMIX_SERVER_TMPDIR=/cm/local/apps/slurm/var/spool/pmix.1046.0/
+PMIX_SERVER_URI2=pmix-server.3992859;tcp4://127.0.0.1:52873
+PMIX_SERVER_URI21=pmix-server.3992859;tcp4://127.0.0.1:52873
+PMIX_SERVER_URI3=pmix-server.3992859;tcp4://127.0.0.1:52873
+PMIX_SERVER_URI4=pmix-server.3992859;tcp4://127.0.0.1:52873
+PMIX_SERVER_URI41=pmix-server.3992859;tcp4://127.0.0.1:52873
+PMIX_SYSTEM_TMPDIR=/var/empty
+PMIX_VERSION=4.1.3
+PWD=/mnt/home/alison.cs13/RobustRLHF/scripts
+PYTHONHASHSEED=100
+PYTHONPATH=/mnt/home/alison.cs13/RobustRLHF
+RANK=0
+ROCR_VISIBLE_DEVICES=0,1,2,3
+SHELL=/bin/bash
+SHLVL=5
+SLURMD_DEBUG=2
+SLURMD_NODENAME=cnode4-002
+SLURM_CLUSTER_NAME=slurm
+SLURM_CONF=/cm/shared/apps/slurm/var/etc/slurm/slurm.conf
+SLURM_CPUS_ON_NODE=8
+SLURM_CPUS_PER_TASK=8
+SLURM_DISTRIBUTION=cyclic
+SLURM_GPUS_ON_NODE=4
+SLURM_GTIDS=0
+SLURM_JOBID=1046
+SLURM_JOB_CPUS_PER_NODE=8
+SLURM_JOB_END_TIME=1748625845
+SLURM_JOB_GID=1109
+SLURM_JOB_GPUS=0,1,2,3
+SLURM_JOB_ID=1046
+SLURM_JOB_NAME=Train-PPO-robust-0.5-resample-large-batch-voted-flipped-voted-flipped
+SLURM_JOB_NODELIST=cnode4-002
+SLURM_JOB_NUM_NODES=1
+SLURM_JOB_PARTITION=p1
+SLURM_JOB_QOS=normal
+SLURM_JOB_START_TIME=1748496245
+SLURM_JOB_UID=1109
+SLURM_JOB_USER=alison.cs13
+SLURM_LAUNCH_NODE_IPADDR=10.225.1.228
+SLURM_LOCALID=0
+SLURM_MEM_PER_NODE=204800
+SLURM_MPI_TYPE=pmix
+SLURM_NNODES=1
+SLURM_NODEID=0
+SLURM_NODELIST=cnode4-002
+SLURM_NPROCS=1
+SLURM_NTASKS=1
+SLURM_NTASKS_PER_NODE=1
+SLURM_PMIXP_ABORT_AGENT_PORT=37393
+SLURM_PMIX_MAPPING_SERV=(vector,(0,1,1))
+SLURM_PRIO_PROCESS=0
+SLURM_PROCID=0
+SLURM_SRUN_COMM_HOST=10.225.1.228
+SLURM_SRUN_COMM_PORT=37925
+SLURM_STEPID=0
+SLURM_STEP_GPUS=0,1,2,3
+SLURM_STEP_ID=0
+SLURM_STEP_LAUNCHER_PORT=37925
+SLURM_STEP_NODELIST=cnode4-002
+SLURM_STEP_NUM_NODES=1
+SLURM_STEP_NUM_TASKS=1
+SLURM_STEP_TASKS_PER_NODE=1
+SLURM_SUBMIT_DIR=/mnt/home/alison.cs13/RobustRLHF/scripts
+SLURM_SUBMIT_HOST=Tunnel4-TRDC-02
+SLURM_TASKS_PER_NODE=1
+SLURM_TASK_PID=3992869
+SLURM_TOPOLOGY_ADDR=cnode4-002
+SLURM_TOPOLOGY_ADDR_PATTERN=node
+SLURM_UMASK=0002
+SLURM_WORKING_CLUSTER=slurm:Tunnel4-TRDC-02:6817:9984:109
+SRUN_DEBUG=3
+SSH_CLIENT=127.0.0.1 58318 22
+SSH_CONNECTION=127.0.0.1 58318 127.0.0.1 22
+SSL_CERT_DIR=/usr/lib/ssl/certs
+SSL_CERT_FILE=/usr/lib/ssl/certs/ca-certificates.crt
+TERM=xterm-256color
+TERM_PROGRAM=vscode
+TERM_PROGRAM_VERSION=1.99.0
+TMPDIR=/tmp
+TORCH_CUDA_ARCH_LIST=9.0
+USER=alison.cs13
+VSCODE_GIT_ASKPASS_EXTRA_ARGS=
+VSCODE_GIT_ASKPASS_MAIN=/home/alison.cs13/.vscode-server/cli/servers/Stable-4437686ffebaf200fa4a6e6e67f735f3edf24ada/server/extensions/git/dist/askpass-main.js
+VSCODE_GIT_ASKPASS_NODE=/home/alison.cs13/.vscode-server/cli/servers/Stable-4437686ffebaf200fa4a6e6e67f735f3edf24ada/server/node
+VSCODE_GIT_IPC_HANDLE=/run/user/1109/vscode-git-a784703e7c.sock
+VSCODE_IPC_HOOK_CLI=/run/user/1109/vscode-ipc-f85ac3c8-43d6-4f74-9af5-5e756b3860de.sock
+WANDB_MODE=online
+WANDB_SERVICE=2-3993654-tcp-localhost-41861
+WORLD_SIZE=4
+XDG_DATA_DIRS=/usr/local/share:/usr/share:/var/lib/snapd/desktop
+XDG_RUNTIME_DIR=/run/user/1109
+XDG_SESSION_CLASS=user
+XDG_SESSION_ID=28663
+XDG_SESSION_TYPE=tty
+ZE_AFFINITY_MASK=0,1,2,3
+_=/mnt/home/alison.cs13/anaconda3/envs/safe-rlhf-12.4/bin/deepspeed
+_CE_CONDA=
+_CE_M=
+_LMFILES_=/cm/local/modulefiles/gcc/13.1.0:/cm/local/modulefiles/slurm/slurm/23.02.8
+_LMFILES__modshare=/cm/local/modulefiles/gcc/13.1.0:1:/cm/local/modulefiles/slurm/slurm/23.02.8:1