diff --git "a/logs/test-5b-158049.out" "b/logs/test-5b-158049.out" new file mode 100644--- /dev/null +++ "b/logs/test-5b-158049.out" @@ -0,0 +1,10601 @@ ++ source /admin/home/loubna/.bashrc +++ HISTCONTROL=ignoreboth +++ shopt -s histappend +++ HISTSIZE=1000 +++ HISTFILESIZE=2000 +++ shopt -s checkwinsize +++ '[' -x /usr/bin/lesspipe ']' ++++ SHELL=/bin/sh ++++ lesspipe +++ eval 'export LESSOPEN="| /usr/bin/lesspipe %s"; +export LESSCLOSE="/usr/bin/lesspipe %s %s";' ++++ export 'LESSOPEN=| /usr/bin/lesspipe %s' ++++ LESSOPEN='| /usr/bin/lesspipe %s' ++++ export 'LESSCLOSE=/usr/bin/lesspipe %s %s' ++++ LESSCLOSE='/usr/bin/lesspipe %s %s' +++ '[' -z '' ']' +++ '[' -r /etc/debian_chroot ']' +++ case "$TERM" in +++ color_prompt=yes +++ '[' -n '' ']' +++ '[' yes = yes ']' +++ PS1='${debian_chroot:+($debian_chroot)}\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ ' +++ unset color_prompt force_color_prompt +++ case "$TERM" in +++ PS1='\[\e]0;\u@\h: \w\a\]${debian_chroot:+($debian_chroot)}\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ ' +++ '[' -x /usr/bin/dircolors ']' +++ test -r /admin/home/loubna/.dircolors ++++ dircolors -b +++ eval 'LS_COLORS='\''rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=00:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=00;36:*.au=00;36:*.flac=00;36:*.m4a=00;36:*.mid=00;36:*.midi=00;36:*.mka=00;36:*.mp3=00;36:*.mpc=00;36:*.ogg=00;36:*.ra=00;36:*.wav=00;36:*.oga=00;36:*.opus=00;36:*.spx=00;36:*.xspf=00;36:'\''; +export LS_COLORS' ++++ LS_COLORS='rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=00:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=00;36:*.au=00;36:*.flac=00;36:*.m4a=00;36:*.mid=00;36:*.midi=00;36:*.mka=00;36:*.mp3=00;36:*.mpc=00;36:*.ogg=00;36:*.ra=00;36:*.wav=00;36:*.oga=00;36:*.opus=00;36:*.spx=00;36:*.xspf=00;36:' ++++ export LS_COLORS +++ alias 'ls=ls --color=auto' +++ alias 'grep=grep --color=auto' +++ alias 'fgrep=fgrep --color=auto' +++ alias 'egrep=egrep --color=auto' +++ alias 'll=ls -alF' +++ alias 'la=ls -A' +++ alias 'l=ls -CF' +++ alias 'alert=notify-send --urgency=low -i "$([ $? = 0 ] && echo terminal || echo error)" "$(history|tail -n1|sed -e '\''s/^\s*[0-9]\+\s*//;s/[;&|]\s*alert$//'\'')"' +++ '[' -f /admin/home/loubna/.bash_aliases ']' +++ shopt -oq posix +++ '[' -f /usr/share/bash-completion/bash_completion ']' +++ . /usr/share/bash-completion/bash_completion ++++ BASH_COMPLETION_VERSINFO=(2 10) ++++ [[ ehxB == *v* ]] ++++ BASH_COMPLETION_ORIGINAL_V_VALUE=+v ++++ [[ -n '' ]] ++++ set +v ++++ _blacklist_glob='@(acroread.sh)' ++++ shopt -s extglob progcomp ++++ complete -u groups slay w sux ++++ complete -A stopped -P '"%' -S '"' bg ++++ complete -j -P '"%' -S '"' fg jobs disown ++++ complete -v readonly unset ++++ complete -A setopt set ++++ complete -A shopt shopt ++++ complete -A helptopic help ++++ complete -a unalias ++++ complete -c command type which ++++ complete -b builtin ++++ [[ linux-gnu == *@(solaris|aix)* ]] ++++ [[ linux-gnu == *@(solaris|aix)* ]] ++++ [[ linux-gnu == *@(solaris|aix)* ]] ++++ _backup_glob='@(#*#|*@(~|.@(bak|orig|rej|swp|dpkg*|rpm@(orig|new|save))))' ++++ complete -F _service service ++++ _sysvdirs ++++ sysvdirs=() ++++ [[ -d /etc/rc.d/init.d ]] ++++ [[ -d /etc/init.d ]] ++++ sysvdirs+=(/etc/init.d) ++++ [[ -f /etc/slackware-version ]] ++++ return 0 ++++ for svcdir in "${sysvdirs[@]}" ++++ for svc in $svcdir/!($_backup_glob) ++++ [[ -x /etc/init.d/acpid ]] ++++ complete -F _service /etc/init.d/acpid ++++ for svc in $svcdir/!($_backup_glob) ++++ [[ -x /etc/init.d/apparmor ]] ++++ complete -F _service /etc/init.d/apparmor ++++ for svc in $svcdir/!($_backup_glob) ++++ [[ -x /etc/init.d/apport ]] ++++ complete -F _service /etc/init.d/apport ++++ for svc in $svcdir/!($_backup_glob) ++++ [[ -x /etc/init.d/atd ]] ++++ complete -F _service /etc/init.d/atd ++++ for svc in $svcdir/!($_backup_glob) ++++ [[ -x /etc/init.d/chrony ]] ++++ complete -F _service /etc/init.d/chrony ++++ for svc in $svcdir/!($_backup_glob) ++++ [[ -x /etc/init.d/console-setup.sh ]] ++++ complete -F _service /etc/init.d/console-setup.sh ++++ for svc in $svcdir/!($_backup_glob) ++++ [[ -x /etc/init.d/cron ]] ++++ complete -F _service /etc/init.d/cron ++++ for svc in $svcdir/!($_backup_glob) ++++ [[ -x /etc/init.d/cryptdisks ]] ++++ complete -F _service /etc/init.d/cryptdisks ++++ for svc in $svcdir/!($_backup_glob) ++++ [[ -x /etc/init.d/cryptdisks-early ]] ++++ complete -F _service /etc/init.d/cryptdisks-early ++++ for svc in $svcdir/!($_backup_glob) ++++ [[ -x /etc/init.d/datadog-agent ]] ++++ complete -F _service /etc/init.d/datadog-agent ++++ for svc in $svcdir/!($_backup_glob) ++++ [[ -x /etc/init.d/datadog-agent-process ]] ++++ complete -F _service /etc/init.d/datadog-agent-process ++++ for svc in $svcdir/!($_backup_glob) ++++ [[ -x /etc/init.d/datadog-agent-security ]] ++++ complete -F _service /etc/init.d/datadog-agent-security ++++ for svc in $svcdir/!($_backup_glob) ++++ [[ -x /etc/init.d/datadog-agent-trace ]] ++++ complete -F _service /etc/init.d/datadog-agent-trace ++++ for svc in $svcdir/!($_backup_glob) ++++ [[ -x /etc/init.d/dbus ]] ++++ complete -F _service /etc/init.d/dbus ++++ for svc in $svcdir/!($_backup_glob) ++++ [[ -x /etc/init.d/docker ]] ++++ complete -F _service /etc/init.d/docker ++++ for svc in $svcdir/!($_backup_glob) ++++ [[ -x /etc/init.d/gdrdrv ]] ++++ complete -F _service /etc/init.d/gdrdrv ++++ for svc in $svcdir/!($_backup_glob) ++++ [[ -x /etc/init.d/grub-common ]] ++++ complete -F _service /etc/init.d/grub-common ++++ for svc in $svcdir/!($_backup_glob) ++++ [[ -x /etc/init.d/hwclock.sh ]] ++++ complete -F _service /etc/init.d/hwclock.sh ++++ for svc in $svcdir/!($_backup_glob) ++++ [[ -x /etc/init.d/irqbalance ]] ++++ complete -F _service /etc/init.d/irqbalance ++++ for svc in $svcdir/!($_backup_glob) ++++ [[ -x /etc/init.d/iscsid ]] ++++ complete -F _service /etc/init.d/iscsid ++++ for svc in $svcdir/!($_backup_glob) ++++ [[ -x /etc/init.d/iwpmd ]] ++++ complete -F _service /etc/init.d/iwpmd ++++ for svc in $svcdir/!($_backup_glob) ++++ [[ -x /etc/init.d/keyboard-setup.sh ]] ++++ complete -F _service /etc/init.d/keyboard-setup.sh ++++ for svc in $svcdir/!($_backup_glob) ++++ [[ -x /etc/init.d/kmod ]] ++++ complete -F _service /etc/init.d/kmod ++++ for svc in $svcdir/!($_backup_glob) ++++ [[ -x /etc/init.d/lvm2 ]] ++++ complete -F _service /etc/init.d/lvm2 ++++ for svc in $svcdir/!($_backup_glob) ++++ [[ -x /etc/init.d/lvm2-lvmpolld ]] ++++ complete -F _service /etc/init.d/lvm2-lvmpolld ++++ for svc in $svcdir/!($_backup_glob) ++++ [[ -x /etc/init.d/multipath-tools ]] ++++ complete -F _service /etc/init.d/multipath-tools ++++ for svc in $svcdir/!($_backup_glob) ++++ [[ -x /etc/init.d/munge ]] ++++ complete -F _service /etc/init.d/munge ++++ for svc in $svcdir/!($_backup_glob) ++++ [[ -x /etc/init.d/nfs-common ]] ++++ complete -F _service /etc/init.d/nfs-common ++++ for svc in $svcdir/!($_backup_glob) ++++ [[ -x /etc/init.d/open-iscsi ]] ++++ complete -F _service /etc/init.d/open-iscsi ++++ for svc in $svcdir/!($_backup_glob) ++++ [[ -x /etc/init.d/open-vm-tools ]] ++++ complete -F _service /etc/init.d/open-vm-tools ++++ for svc in $svcdir/!($_backup_glob) ++++ [[ -x /etc/init.d/plymouth ]] ++++ complete -F _service /etc/init.d/plymouth ++++ for svc in $svcdir/!($_backup_glob) ++++ [[ -x /etc/init.d/plymouth-log ]] ++++ complete -F _service /etc/init.d/plymouth-log ++++ for svc in $svcdir/!($_backup_glob) ++++ [[ -x /etc/init.d/procps ]] ++++ complete -F _service /etc/init.d/procps ++++ for svc in $svcdir/!($_backup_glob) ++++ [[ -x /etc/init.d/rpcbind ]] ++++ complete -F _service /etc/init.d/rpcbind ++++ for svc in $svcdir/!($_backup_glob) ++++ [[ -x /etc/init.d/rsync ]] ++++ complete -F _service /etc/init.d/rsync ++++ for svc in $svcdir/!($_backup_glob) ++++ [[ -x /etc/init.d/rsyslog ]] ++++ complete -F _service /etc/init.d/rsyslog ++++ for svc in $svcdir/!($_backup_glob) ++++ [[ -x /etc/init.d/screen-cleanup ]] ++++ complete -F _service /etc/init.d/screen-cleanup ++++ for svc in $svcdir/!($_backup_glob) ++++ [[ -x /etc/init.d/sendmail ]] ++++ complete -F _service /etc/init.d/sendmail ++++ for svc in $svcdir/!($_backup_glob) ++++ [[ -x /etc/init.d/ssh ]] ++++ complete -F _service /etc/init.d/ssh ++++ for svc in $svcdir/!($_backup_glob) ++++ [[ -x /etc/init.d/sysstat ]] ++++ complete -F _service /etc/init.d/sysstat ++++ for svc in $svcdir/!($_backup_glob) ++++ [[ -x /etc/init.d/udev ]] ++++ complete -F _service /etc/init.d/udev ++++ for svc in $svcdir/!($_backup_glob) ++++ [[ -x /etc/init.d/ufw ]] ++++ complete -F _service /etc/init.d/ufw ++++ for svc in $svcdir/!($_backup_glob) ++++ [[ -x /etc/init.d/unattended-upgrades ]] ++++ complete -F _service /etc/init.d/unattended-upgrades ++++ for svc in $svcdir/!($_backup_glob) ++++ [[ -x /etc/init.d/uuidd ]] ++++ complete -F _service /etc/init.d/uuidd ++++ for svc in $svcdir/!($_backup_glob) ++++ [[ -x /etc/init.d/x11-common ]] ++++ complete -F _service /etc/init.d/x11-common ++++ unset svc svcdir sysvdirs ++++ [[ linux-gnu == *freebsd* ]] ++++ shopt -u hostcomplete ++++ complete -F _user_at_host talk ytalk finger ++++ complete -F _known_hosts traceroute traceroute6 fping fping6 telnet rsh rlogin ftp dig mtr ssh-installkeys showmount ++++ shopt -q cdable_vars ++++ complete -F _cd -o nospace cd pushd ++++ complete -F _command aoss command do else eval exec ltrace nice nohup padsp then time tsocks vsound xargs ++++ complete -F _root_command fakeroot gksu gksudo kdesudo really ++++ complete -F _longopt a2ps awk base64 bash bc bison cat chroot colordiff cp csplit cut date df diff dir du enscript env expand fmt fold gperf grep grub head irb ld ldd less ln ls m4 md5sum mkdir mkfifo mknod mv netstat nl nm objcopy objdump od paste pr ptx readelf rm rmdir sed seq shasum sha1sum sha224sum sha256sum sha384sum sha512sum shar sort split strip sum tac tail tee texindex touch tr uname unexpand uniq units vdir wc who ++++ [[ 5 -gt 4 ]] ++++ declare -Ag _xspecs ++++ _install_xspec '!*.?(t)bz?(2)' bunzip2 bzcat pbunzip2 pbzcat lbunzip2 lbzcat ++++ local 'xspec=!*.?(t)bz?(2)' cmd ++++ shift ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.?(t)bz?(2)' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.?(t)bz?(2)' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.?(t)bz?(2)' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.?(t)bz?(2)' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.?(t)bz?(2)' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.?(t)bz?(2)' ++++ _install_xspec '!*.@(zip|[egjswx]ar|exe|pk3|wsz|zargo|xpi|s[tx][cdiw]|sx[gm]|o[dt][tspgfc]|od[bm]|oxt|epub|apk|aab|ipa|do[ct][xm]|p[op]t[mx]|xl[st][xm]|pyz|whl)' unzip zipinfo ++++ local 'xspec=!*.@(zip|[egjswx]ar|exe|pk3|wsz|zargo|xpi|s[tx][cdiw]|sx[gm]|o[dt][tspgfc]|od[bm]|oxt|epub|apk|aab|ipa|do[ct][xm]|p[op]t[mx]|xl[st][xm]|pyz|whl)' cmd ++++ shift ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(zip|[egjswx]ar|exe|pk3|wsz|zargo|xpi|s[tx][cdiw]|sx[gm]|o[dt][tspgfc]|od[bm]|oxt|epub|apk|aab|ipa|do[ct][xm]|p[op]t[mx]|xl[st][xm]|pyz|whl)' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(zip|[egjswx]ar|exe|pk3|wsz|zargo|xpi|s[tx][cdiw]|sx[gm]|o[dt][tspgfc]|od[bm]|oxt|epub|apk|aab|ipa|do[ct][xm]|p[op]t[mx]|xl[st][xm]|pyz|whl)' ++++ _install_xspec '*.Z' compress znew ++++ local 'xspec=*.Z' cmd ++++ shift ++++ for cmd in "$@" ++++ _xspecs[$cmd]='*.Z' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='*.Z' ++++ _install_xspec '!*.@(Z|[gGd]z|t[ag]z)' gunzip zcat ++++ local 'xspec=!*.@(Z|[gGd]z|t[ag]z)' cmd ++++ shift ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(Z|[gGd]z|t[ag]z)' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(Z|[gGd]z|t[ag]z)' ++++ _install_xspec '!*.@(Z|[gGdz]z|t[ag]z)' unpigz ++++ local 'xspec=!*.@(Z|[gGdz]z|t[ag]z)' cmd ++++ shift ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(Z|[gGdz]z|t[ag]z)' ++++ _install_xspec '!*.Z' uncompress ++++ local 'xspec=!*.Z' cmd ++++ shift ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.Z' ++++ _install_xspec '!*.@(tlz|lzma)' lzcat lzegrep lzfgrep lzgrep lzless lzmore unlzma ++++ local 'xspec=!*.@(tlz|lzma)' cmd ++++ shift ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(tlz|lzma)' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(tlz|lzma)' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(tlz|lzma)' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(tlz|lzma)' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(tlz|lzma)' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(tlz|lzma)' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(tlz|lzma)' ++++ _install_xspec '!*.@(?(t)xz|tlz|lzma)' unxz xzcat ++++ local 'xspec=!*.@(?(t)xz|tlz|lzma)' cmd ++++ shift ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(?(t)xz|tlz|lzma)' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(?(t)xz|tlz|lzma)' ++++ _install_xspec '!*.lrz' lrunzip ++++ local 'xspec=!*.lrz' cmd ++++ shift ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.lrz' ++++ _install_xspec '!*.@(gif|jp?(e)g|miff|tif?(f)|pn[gm]|p[bgp]m|bmp|xpm|ico|xwd|tga|pcx)' ee ++++ local 'xspec=!*.@(gif|jp?(e)g|miff|tif?(f)|pn[gm]|p[bgp]m|bmp|xpm|ico|xwd|tga|pcx)' cmd ++++ shift ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(gif|jp?(e)g|miff|tif?(f)|pn[gm]|p[bgp]m|bmp|xpm|ico|xwd|tga|pcx)' ++++ _install_xspec '!*.@(gif|jp?(e)g|tif?(f)|png|p[bgp]m|bmp|x[bp]m|rle|rgb|pcx|fits|pm|svg)' qiv ++++ local 'xspec=!*.@(gif|jp?(e)g|tif?(f)|png|p[bgp]m|bmp|x[bp]m|rle|rgb|pcx|fits|pm|svg)' cmd ++++ shift ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(gif|jp?(e)g|tif?(f)|png|p[bgp]m|bmp|x[bp]m|rle|rgb|pcx|fits|pm|svg)' ++++ _install_xspec '!*.@(gif|jp?(e)g?(2)|j2[ck]|jp[2f]|tif?(f)|png|p[bgp]m|bmp|x[bp]m|rle|rgb|pcx|fits|pm|?(e)ps)' xv ++++ local 'xspec=!*.@(gif|jp?(e)g?(2)|j2[ck]|jp[2f]|tif?(f)|png|p[bgp]m|bmp|x[bp]m|rle|rgb|pcx|fits|pm|?(e)ps)' cmd ++++ shift ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(gif|jp?(e)g?(2)|j2[ck]|jp[2f]|tif?(f)|png|p[bgp]m|bmp|x[bp]m|rle|rgb|pcx|fits|pm|?(e)ps)' ++++ _install_xspec '!*.@(@(?(e)ps|?(E)PS|pdf|PDF)?(.gz|.GZ|.bz2|.BZ2|.Z))' gv ggv kghostview ++++ local 'xspec=!*.@(@(?(e)ps|?(E)PS|pdf|PDF)?(.gz|.GZ|.bz2|.BZ2|.Z))' cmd ++++ shift ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(@(?(e)ps|?(E)PS|pdf|PDF)?(.gz|.GZ|.bz2|.BZ2|.Z))' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(@(?(e)ps|?(E)PS|pdf|PDF)?(.gz|.GZ|.bz2|.BZ2|.Z))' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(@(?(e)ps|?(E)PS|pdf|PDF)?(.gz|.GZ|.bz2|.BZ2|.Z))' ++++ _install_xspec '!*.@(dvi|DVI)?(.@(gz|Z|bz2))' xdvi kdvi ++++ local 'xspec=!*.@(dvi|DVI)?(.@(gz|Z|bz2))' cmd ++++ shift ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(dvi|DVI)?(.@(gz|Z|bz2))' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(dvi|DVI)?(.@(gz|Z|bz2))' ++++ _install_xspec '!*.dvi' dvips dviselect dvitype dvipdf advi dvipdfm dvipdfmx ++++ local 'xspec=!*.dvi' cmd ++++ shift ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.dvi' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.dvi' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.dvi' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.dvi' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.dvi' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.dvi' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.dvi' ++++ _install_xspec '!*.[pf]df' acroread gpdf ++++ local 'xspec=!*.[pf]df' cmd ++++ shift ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.[pf]df' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.[pf]df' ++++ _install_xspec '!*.@(pdf|fdf)?(.@(gz|GZ|bz2|BZ2|Z))' xpdf ++++ local 'xspec=!*.@(pdf|fdf)?(.@(gz|GZ|bz2|BZ2|Z))' cmd ++++ shift ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(pdf|fdf)?(.@(gz|GZ|bz2|BZ2|Z))' ++++ _install_xspec '!*.@(?(e)ps|pdf)' kpdf ++++ local 'xspec=!*.@(?(e)ps|pdf)' cmd ++++ shift ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(?(e)ps|pdf)' ++++ _install_xspec '!*.@(okular|@(?(e|x)ps|?(E|X)PS|[pf]df|[PF]DF|dvi|DVI|cb[rz]|CB[RZ]|djv?(u)|DJV?(U)|dvi|DVI|gif|jp?(e)g|miff|tif?(f)|pn[gm]|p[bgp]m|bmp|xpm|ico|xwd|tga|pcx|GIF|JP?(E)G|MIFF|TIF?(F)|PN[GM]|P[BGP]M|BMP|XPM|ICO|XWD|TGA|PCX|epub|EPUB|odt|ODT|fb?(2)|FB?(2)|mobi|MOBI|g3|G3|chm|CHM)?(.?(gz|GZ|bz2|BZ2|xz|XZ)))' okular ++++ local 'xspec=!*.@(okular|@(?(e|x)ps|?(E|X)PS|[pf]df|[PF]DF|dvi|DVI|cb[rz]|CB[RZ]|djv?(u)|DJV?(U)|dvi|DVI|gif|jp?(e)g|miff|tif?(f)|pn[gm]|p[bgp]m|bmp|xpm|ico|xwd|tga|pcx|GIF|JP?(E)G|MIFF|TIF?(F)|PN[GM]|P[BGP]M|BMP|XPM|ICO|XWD|TGA|PCX|epub|EPUB|odt|ODT|fb?(2)|FB?(2)|mobi|MOBI|g3|G3|chm|CHM)?(.?(gz|GZ|bz2|BZ2|xz|XZ)))' cmd ++++ shift ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(okular|@(?(e|x)ps|?(E|X)PS|[pf]df|[PF]DF|dvi|DVI|cb[rz]|CB[RZ]|djv?(u)|DJV?(U)|dvi|DVI|gif|jp?(e)g|miff|tif?(f)|pn[gm]|p[bgp]m|bmp|xpm|ico|xwd|tga|pcx|GIF|JP?(E)G|MIFF|TIF?(F)|PN[GM]|P[BGP]M|BMP|XPM|ICO|XWD|TGA|PCX|epub|EPUB|odt|ODT|fb?(2)|FB?(2)|mobi|MOBI|g3|G3|chm|CHM)?(.?(gz|GZ|bz2|BZ2|xz|XZ)))' ++++ _install_xspec '!*.pdf' epdfview pdfunite ++++ local 'xspec=!*.pdf' cmd ++++ shift ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.pdf' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.pdf' ++++ _install_xspec '!*.@(cb[rz7t]|djv?(u)|?(e)ps|pdf)' zathura ++++ local 'xspec=!*.@(cb[rz7t]|djv?(u)|?(e)ps|pdf)' cmd ++++ shift ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(cb[rz7t]|djv?(u)|?(e)ps|pdf)' ++++ _install_xspec '!*.@(?(e)ps|pdf)' ps2pdf ps2pdf12 ps2pdf13 ps2pdf14 ps2pdfwr ++++ local 'xspec=!*.@(?(e)ps|pdf)' cmd ++++ shift ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(?(e)ps|pdf)' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(?(e)ps|pdf)' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(?(e)ps|pdf)' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(?(e)ps|pdf)' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(?(e)ps|pdf)' ++++ _install_xspec '!*.texi*' makeinfo texi2html ++++ local 'xspec=!*.texi*' cmd ++++ shift ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.texi*' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.texi*' ++++ _install_xspec '!*.@(?(la)tex|texi|dtx|ins|ltx|dbj)' tex latex slitex jadetex pdfjadetex pdftex pdflatex texi2dvi xetex xelatex luatex lualatex ++++ local 'xspec=!*.@(?(la)tex|texi|dtx|ins|ltx|dbj)' cmd ++++ shift ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(?(la)tex|texi|dtx|ins|ltx|dbj)' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(?(la)tex|texi|dtx|ins|ltx|dbj)' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(?(la)tex|texi|dtx|ins|ltx|dbj)' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(?(la)tex|texi|dtx|ins|ltx|dbj)' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(?(la)tex|texi|dtx|ins|ltx|dbj)' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(?(la)tex|texi|dtx|ins|ltx|dbj)' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(?(la)tex|texi|dtx|ins|ltx|dbj)' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(?(la)tex|texi|dtx|ins|ltx|dbj)' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(?(la)tex|texi|dtx|ins|ltx|dbj)' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(?(la)tex|texi|dtx|ins|ltx|dbj)' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(?(la)tex|texi|dtx|ins|ltx|dbj)' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(?(la)tex|texi|dtx|ins|ltx|dbj)' ++++ _install_xspec '!*.mp3' mpg123 mpg321 madplay ++++ local 'xspec=!*.mp3' cmd ++++ shift ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.mp3' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.mp3' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.mp3' ++++ _install_xspec '!*@(.@(mp?(e)g|MP?(E)G|wm[av]|WM[AV]|avi|AVI|asf|vob|VOB|bin|dat|divx|DIVX|vcd|ps|pes|fli|flv|FLV|fxm|FXM|viv|rm|ram|yuv|mov|MOV|qt|QT|web[am]|WEB[AM]|mp[234]|MP[234]|m?(p)4[av]|M?(P)4[AV]|mkv|MKV|og[agmvx]|OG[AGMVX]|t[ps]|T[PS]|m2t?(s)|M2T?(S)|mts|MTS|wav|WAV|flac|FLAC|asx|ASX|mng|MNG|srt|m[eo]d|M[EO]D|s[3t]m|S[3T]M|it|IT|xm|XM)|+([0-9]).@(vdr|VDR))?(.@(crdownload|part))' xine aaxine fbxine ++++ local 'xspec=!*@(.@(mp?(e)g|MP?(E)G|wm[av]|WM[AV]|avi|AVI|asf|vob|VOB|bin|dat|divx|DIVX|vcd|ps|pes|fli|flv|FLV|fxm|FXM|viv|rm|ram|yuv|mov|MOV|qt|QT|web[am]|WEB[AM]|mp[234]|MP[234]|m?(p)4[av]|M?(P)4[AV]|mkv|MKV|og[agmvx]|OG[AGMVX]|t[ps]|T[PS]|m2t?(s)|M2T?(S)|mts|MTS|wav|WAV|flac|FLAC|asx|ASX|mng|MNG|srt|m[eo]d|M[EO]D|s[3t]m|S[3T]M|it|IT|xm|XM)|+([0-9]).@(vdr|VDR))?(.@(crdownload|part))' cmd ++++ shift ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*@(.@(mp?(e)g|MP?(E)G|wm[av]|WM[AV]|avi|AVI|asf|vob|VOB|bin|dat|divx|DIVX|vcd|ps|pes|fli|flv|FLV|fxm|FXM|viv|rm|ram|yuv|mov|MOV|qt|QT|web[am]|WEB[AM]|mp[234]|MP[234]|m?(p)4[av]|M?(P)4[AV]|mkv|MKV|og[agmvx]|OG[AGMVX]|t[ps]|T[PS]|m2t?(s)|M2T?(S)|mts|MTS|wav|WAV|flac|FLAC|asx|ASX|mng|MNG|srt|m[eo]d|M[EO]D|s[3t]m|S[3T]M|it|IT|xm|XM)|+([0-9]).@(vdr|VDR))?(.@(crdownload|part))' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*@(.@(mp?(e)g|MP?(E)G|wm[av]|WM[AV]|avi|AVI|asf|vob|VOB|bin|dat|divx|DIVX|vcd|ps|pes|fli|flv|FLV|fxm|FXM|viv|rm|ram|yuv|mov|MOV|qt|QT|web[am]|WEB[AM]|mp[234]|MP[234]|m?(p)4[av]|M?(P)4[AV]|mkv|MKV|og[agmvx]|OG[AGMVX]|t[ps]|T[PS]|m2t?(s)|M2T?(S)|mts|MTS|wav|WAV|flac|FLAC|asx|ASX|mng|MNG|srt|m[eo]d|M[EO]D|s[3t]m|S[3T]M|it|IT|xm|XM)|+([0-9]).@(vdr|VDR))?(.@(crdownload|part))' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*@(.@(mp?(e)g|MP?(E)G|wm[av]|WM[AV]|avi|AVI|asf|vob|VOB|bin|dat|divx|DIVX|vcd|ps|pes|fli|flv|FLV|fxm|FXM|viv|rm|ram|yuv|mov|MOV|qt|QT|web[am]|WEB[AM]|mp[234]|MP[234]|m?(p)4[av]|M?(P)4[AV]|mkv|MKV|og[agmvx]|OG[AGMVX]|t[ps]|T[PS]|m2t?(s)|M2T?(S)|mts|MTS|wav|WAV|flac|FLAC|asx|ASX|mng|MNG|srt|m[eo]d|M[EO]D|s[3t]m|S[3T]M|it|IT|xm|XM)|+([0-9]).@(vdr|VDR))?(.@(crdownload|part))' ++++ _install_xspec '!*@(.@(mp?(e)g|MP?(E)G|wm[av]|WM[AV]|avi|AVI|asf|vob|VOB|bin|dat|divx|DIVX|vcd|ps|pes|fli|flv|FLV|fxm|FXM|viv|rm|ram|yuv|mov|MOV|qt|QT|web[am]|WEB[AM]|mp[234]|MP[234]|m?(p)4[av]|M?(P)4[AV]|mkv|MKV|og[agmvx]|OG[AGMVX]|t[ps]|T[PS]|m2t?(s)|M2T?(S)|mts|MTS|wav|WAV|flac|FLAC|asx|ASX|mng|MNG|srt|m[eo]d|M[EO]D|s[3t]m|S[3T]M|it|IT|xm|XM|iso|ISO)|+([0-9]).@(vdr|VDR))?(.@(crdownload|part))' kaffeine dragon ++++ local 'xspec=!*@(.@(mp?(e)g|MP?(E)G|wm[av]|WM[AV]|avi|AVI|asf|vob|VOB|bin|dat|divx|DIVX|vcd|ps|pes|fli|flv|FLV|fxm|FXM|viv|rm|ram|yuv|mov|MOV|qt|QT|web[am]|WEB[AM]|mp[234]|MP[234]|m?(p)4[av]|M?(P)4[AV]|mkv|MKV|og[agmvx]|OG[AGMVX]|t[ps]|T[PS]|m2t?(s)|M2T?(S)|mts|MTS|wav|WAV|flac|FLAC|asx|ASX|mng|MNG|srt|m[eo]d|M[EO]D|s[3t]m|S[3T]M|it|IT|xm|XM|iso|ISO)|+([0-9]).@(vdr|VDR))?(.@(crdownload|part))' cmd ++++ shift ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*@(.@(mp?(e)g|MP?(E)G|wm[av]|WM[AV]|avi|AVI|asf|vob|VOB|bin|dat|divx|DIVX|vcd|ps|pes|fli|flv|FLV|fxm|FXM|viv|rm|ram|yuv|mov|MOV|qt|QT|web[am]|WEB[AM]|mp[234]|MP[234]|m?(p)4[av]|M?(P)4[AV]|mkv|MKV|og[agmvx]|OG[AGMVX]|t[ps]|T[PS]|m2t?(s)|M2T?(S)|mts|MTS|wav|WAV|flac|FLAC|asx|ASX|mng|MNG|srt|m[eo]d|M[EO]D|s[3t]m|S[3T]M|it|IT|xm|XM|iso|ISO)|+([0-9]).@(vdr|VDR))?(.@(crdownload|part))' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*@(.@(mp?(e)g|MP?(E)G|wm[av]|WM[AV]|avi|AVI|asf|vob|VOB|bin|dat|divx|DIVX|vcd|ps|pes|fli|flv|FLV|fxm|FXM|viv|rm|ram|yuv|mov|MOV|qt|QT|web[am]|WEB[AM]|mp[234]|MP[234]|m?(p)4[av]|M?(P)4[AV]|mkv|MKV|og[agmvx]|OG[AGMVX]|t[ps]|T[PS]|m2t?(s)|M2T?(S)|mts|MTS|wav|WAV|flac|FLAC|asx|ASX|mng|MNG|srt|m[eo]d|M[EO]D|s[3t]m|S[3T]M|it|IT|xm|XM|iso|ISO)|+([0-9]).@(vdr|VDR))?(.@(crdownload|part))' ++++ _install_xspec '!*.@(avi|asf|wmv)' aviplay ++++ local 'xspec=!*.@(avi|asf|wmv)' cmd ++++ shift ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(avi|asf|wmv)' ++++ _install_xspec '!*.@(rm?(j)|ra?(m)|smi?(l))' realplay ++++ local 'xspec=!*.@(rm?(j)|ra?(m)|smi?(l))' cmd ++++ shift ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(rm?(j)|ra?(m)|smi?(l))' ++++ _install_xspec '!*.@(mpg|mpeg|avi|mov|qt)' xanim ++++ local 'xspec=!*.@(mpg|mpeg|avi|mov|qt)' cmd ++++ shift ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(mpg|mpeg|avi|mov|qt)' ++++ _install_xspec '!*.@(og[ag]|m3u|flac|spx)' ogg123 ++++ local 'xspec=!*.@(og[ag]|m3u|flac|spx)' cmd ++++ shift ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(og[ag]|m3u|flac|spx)' ++++ _install_xspec '!*.@(mp3|og[ag]|pls|m3u)' gqmpeg freeamp ++++ local 'xspec=!*.@(mp3|og[ag]|pls|m3u)' cmd ++++ shift ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(mp3|og[ag]|pls|m3u)' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(mp3|og[ag]|pls|m3u)' ++++ _install_xspec '!*.fig' xfig ++++ local 'xspec=!*.fig' cmd ++++ shift ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.fig' ++++ _install_xspec '!*.@(mid?(i)|cmf)' playmidi ++++ local 'xspec=!*.@(mid?(i)|cmf)' cmd ++++ shift ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(mid?(i)|cmf)' ++++ _install_xspec '!*.@(mid?(i)|rmi|rcp|[gr]36|g18|mod|xm|it|x3m|s[3t]m|kar)' timidity ++++ local 'xspec=!*.@(mid?(i)|rmi|rcp|[gr]36|g18|mod|xm|it|x3m|s[3t]m|kar)' cmd ++++ shift ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(mid?(i)|rmi|rcp|[gr]36|g18|mod|xm|it|x3m|s[3t]m|kar)' ++++ _install_xspec '!*.@(669|abc|am[fs]|d[bs]m|dmf|far|it|mdl|m[eo]d|mid?(i)|mt[2m]|oct|okt?(a)|p[st]m|s[3t]m|ult|umx|wav|xm)' modplugplay modplug123 ++++ local 'xspec=!*.@(669|abc|am[fs]|d[bs]m|dmf|far|it|mdl|m[eo]d|mid?(i)|mt[2m]|oct|okt?(a)|p[st]m|s[3t]m|ult|umx|wav|xm)' cmd ++++ shift ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(669|abc|am[fs]|d[bs]m|dmf|far|it|mdl|m[eo]d|mid?(i)|mt[2m]|oct|okt?(a)|p[st]m|s[3t]m|ult|umx|wav|xm)' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(669|abc|am[fs]|d[bs]m|dmf|far|it|mdl|m[eo]d|mid?(i)|mt[2m]|oct|okt?(a)|p[st]m|s[3t]m|ult|umx|wav|xm)' ++++ _install_xspec '*.@([ao]|so|so.!(conf|*/*)|[rs]pm|gif|jp?(e)g|mp3|mp?(e)g|avi|asf|ogg|class)' vi vim gvim rvim view rview rgvim rgview gview emacs xemacs sxemacs kate kwrite ++++ local 'xspec=*.@([ao]|so|so.!(conf|*/*)|[rs]pm|gif|jp?(e)g|mp3|mp?(e)g|avi|asf|ogg|class)' cmd ++++ shift ++++ for cmd in "$@" ++++ _xspecs[$cmd]='*.@([ao]|so|so.!(conf|*/*)|[rs]pm|gif|jp?(e)g|mp3|mp?(e)g|avi|asf|ogg|class)' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='*.@([ao]|so|so.!(conf|*/*)|[rs]pm|gif|jp?(e)g|mp3|mp?(e)g|avi|asf|ogg|class)' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='*.@([ao]|so|so.!(conf|*/*)|[rs]pm|gif|jp?(e)g|mp3|mp?(e)g|avi|asf|ogg|class)' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='*.@([ao]|so|so.!(conf|*/*)|[rs]pm|gif|jp?(e)g|mp3|mp?(e)g|avi|asf|ogg|class)' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='*.@([ao]|so|so.!(conf|*/*)|[rs]pm|gif|jp?(e)g|mp3|mp?(e)g|avi|asf|ogg|class)' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='*.@([ao]|so|so.!(conf|*/*)|[rs]pm|gif|jp?(e)g|mp3|mp?(e)g|avi|asf|ogg|class)' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='*.@([ao]|so|so.!(conf|*/*)|[rs]pm|gif|jp?(e)g|mp3|mp?(e)g|avi|asf|ogg|class)' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='*.@([ao]|so|so.!(conf|*/*)|[rs]pm|gif|jp?(e)g|mp3|mp?(e)g|avi|asf|ogg|class)' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='*.@([ao]|so|so.!(conf|*/*)|[rs]pm|gif|jp?(e)g|mp3|mp?(e)g|avi|asf|ogg|class)' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='*.@([ao]|so|so.!(conf|*/*)|[rs]pm|gif|jp?(e)g|mp3|mp?(e)g|avi|asf|ogg|class)' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='*.@([ao]|so|so.!(conf|*/*)|[rs]pm|gif|jp?(e)g|mp3|mp?(e)g|avi|asf|ogg|class)' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='*.@([ao]|so|so.!(conf|*/*)|[rs]pm|gif|jp?(e)g|mp3|mp?(e)g|avi|asf|ogg|class)' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='*.@([ao]|so|so.!(conf|*/*)|[rs]pm|gif|jp?(e)g|mp3|mp?(e)g|avi|asf|ogg|class)' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='*.@([ao]|so|so.!(conf|*/*)|[rs]pm|gif|jp?(e)g|mp3|mp?(e)g|avi|asf|ogg|class)' ++++ _install_xspec '!*.@(zip|z|gz|tgz)' bzme ++++ local 'xspec=!*.@(zip|z|gz|tgz)' cmd ++++ shift ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(zip|z|gz|tgz)' ++++ _install_xspec '!*.@(?([xX]|[sS])[hH][tT][mM]?([lL]))' netscape mozilla lynx galeon dillo elinks amaya epiphany ++++ local 'xspec=!*.@(?([xX]|[sS])[hH][tT][mM]?([lL]))' cmd ++++ shift ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(?([xX]|[sS])[hH][tT][mM]?([lL]))' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(?([xX]|[sS])[hH][tT][mM]?([lL]))' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(?([xX]|[sS])[hH][tT][mM]?([lL]))' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(?([xX]|[sS])[hH][tT][mM]?([lL]))' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(?([xX]|[sS])[hH][tT][mM]?([lL]))' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(?([xX]|[sS])[hH][tT][mM]?([lL]))' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(?([xX]|[sS])[hH][tT][mM]?([lL]))' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(?([xX]|[sS])[hH][tT][mM]?([lL]))' ++++ _install_xspec '!*.@(sxw|stw|sxg|sgl|doc?([mx])|dot?([mx])|rtf|txt|htm|html|?(f)odt|ott|odm|pdf)' oowriter lowriter ++++ local 'xspec=!*.@(sxw|stw|sxg|sgl|doc?([mx])|dot?([mx])|rtf|txt|htm|html|?(f)odt|ott|odm|pdf)' cmd ++++ shift ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(sxw|stw|sxg|sgl|doc?([mx])|dot?([mx])|rtf|txt|htm|html|?(f)odt|ott|odm|pdf)' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(sxw|stw|sxg|sgl|doc?([mx])|dot?([mx])|rtf|txt|htm|html|?(f)odt|ott|odm|pdf)' ++++ _install_xspec '!*.@(sxi|sti|pps?(x)|ppt?([mx])|pot?([mx])|?(f)odp|otp)' ooimpress loimpress ++++ local 'xspec=!*.@(sxi|sti|pps?(x)|ppt?([mx])|pot?([mx])|?(f)odp|otp)' cmd ++++ shift ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(sxi|sti|pps?(x)|ppt?([mx])|pot?([mx])|?(f)odp|otp)' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(sxi|sti|pps?(x)|ppt?([mx])|pot?([mx])|?(f)odp|otp)' ++++ _install_xspec '!*.@(sxc|stc|xls?([bmx])|xlw|xlt?([mx])|[ct]sv|?(f)ods|ots)' oocalc localc ++++ local 'xspec=!*.@(sxc|stc|xls?([bmx])|xlw|xlt?([mx])|[ct]sv|?(f)ods|ots)' cmd ++++ shift ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(sxc|stc|xls?([bmx])|xlw|xlt?([mx])|[ct]sv|?(f)ods|ots)' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(sxc|stc|xls?([bmx])|xlw|xlt?([mx])|[ct]sv|?(f)ods|ots)' ++++ _install_xspec '!*.@(sxd|std|sda|sdd|?(f)odg|otg)' oodraw lodraw ++++ local 'xspec=!*.@(sxd|std|sda|sdd|?(f)odg|otg)' cmd ++++ shift ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(sxd|std|sda|sdd|?(f)odg|otg)' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(sxd|std|sda|sdd|?(f)odg|otg)' ++++ _install_xspec '!*.@(sxm|smf|mml|odf)' oomath lomath ++++ local 'xspec=!*.@(sxm|smf|mml|odf)' cmd ++++ shift ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(sxm|smf|mml|odf)' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(sxm|smf|mml|odf)' ++++ _install_xspec '!*.odb' oobase lobase ++++ local 'xspec=!*.odb' cmd ++++ shift ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.odb' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.odb' ++++ _install_xspec '!*.[rs]pm' rpm2cpio ++++ local 'xspec=!*.[rs]pm' cmd ++++ shift ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.[rs]pm' ++++ _install_xspec '!*.aux' bibtex ++++ local 'xspec=!*.aux' cmd ++++ shift ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.aux' ++++ _install_xspec '!*.po' poedit gtranslator kbabel lokalize ++++ local 'xspec=!*.po' cmd ++++ shift ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.po' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.po' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.po' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.po' ++++ _install_xspec '!*.@([Pp][Rr][Gg]|[Cc][Ll][Pp])' harbour gharbour hbpp ++++ local 'xspec=!*.@([Pp][Rr][Gg]|[Cc][Ll][Pp])' cmd ++++ shift ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@([Pp][Rr][Gg]|[Cc][Ll][Pp])' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@([Pp][Rr][Gg]|[Cc][Ll][Pp])' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@([Pp][Rr][Gg]|[Cc][Ll][Pp])' ++++ _install_xspec '!*.[Hh][Rr][Bb]' hbrun ++++ local 'xspec=!*.[Hh][Rr][Bb]' cmd ++++ shift ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.[Hh][Rr][Bb]' ++++ _install_xspec '!*.ly' lilypond ly2dvi ++++ local 'xspec=!*.ly' cmd ++++ shift ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.ly' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.ly' ++++ _install_xspec '!*.@(dif?(f)|?(d)patch)?(.@([gx]z|bz2|lzma))' cdiff ++++ local 'xspec=!*.@(dif?(f)|?(d)patch)?(.@([gx]z|bz2|lzma))' cmd ++++ shift ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(dif?(f)|?(d)patch)?(.@([gx]z|bz2|lzma))' ++++ _install_xspec '!@(*.@(ks|jks|jceks|p12|pfx|bks|ubr|gkr|cer|crt|cert|p7b|pkipath|pem|p10|csr|crl)|cacerts)' portecle ++++ local 'xspec=!@(*.@(ks|jks|jceks|p12|pfx|bks|ubr|gkr|cer|crt|cert|p7b|pkipath|pem|p10|csr|crl)|cacerts)' cmd ++++ shift ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!@(*.@(ks|jks|jceks|p12|pfx|bks|ubr|gkr|cer|crt|cert|p7b|pkipath|pem|p10|csr|crl)|cacerts)' ++++ _install_xspec '!*.@(mp[234c]|og[ag]|@(fl|a)ac|m4[abp]|spx|tta|w?(a)v|wma|aif?(f)|asf|ape)' kid3 kid3-qt ++++ local 'xspec=!*.@(mp[234c]|og[ag]|@(fl|a)ac|m4[abp]|spx|tta|w?(a)v|wma|aif?(f)|asf|ape)' cmd ++++ shift ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(mp[234c]|og[ag]|@(fl|a)ac|m4[abp]|spx|tta|w?(a)v|wma|aif?(f)|asf|ape)' ++++ for cmd in "$@" ++++ _xspecs[$cmd]='!*.@(mp[234c]|og[ag]|@(fl|a)ac|m4[abp]|spx|tta|w?(a)v|wma|aif?(f)|asf|ape)' ++++ unset -f _install_xspec ++++ complete -F _minimal '' ++++ complete -D -F _completion_loader ++++ compat_dir=/etc/bash_completion.d ++++ [[ -d /etc/bash_completion.d ]] ++++ [[ -r /etc/bash_completion.d ]] ++++ [[ -x /etc/bash_completion.d ]] ++++ for i in "$compat_dir"/* ++++ [[ apport_completion != @(@(#*#|*@(~|.@(bak|orig|rej|swp|dpkg*|rpm@(orig|new|save))))|Makefile*|@(acroread.sh)) ]] ++++ [[ -f /etc/bash_completion.d/apport_completion ]] ++++ [[ -r /etc/bash_completion.d/apport_completion ]] ++++ . /etc/bash_completion.d/apport_completion +++++ complete -F _apport-bug -o filenames -o dirnames ubuntu-bug +++++ complete -F _apport-bug -o filenames -o dirnames apport-bug +++++ complete -F _apport-cli -o filenames -o dirnames apport-cli +++++ complete -F _apport-unpack -o filenames -o dirnames apport-unpack +++++ complete -F _apport-collect apport-collect ++++ for i in "$compat_dir"/* ++++ [[ git-prompt != @(@(#*#|*@(~|.@(bak|orig|rej|swp|dpkg*|rpm@(orig|new|save))))|Makefile*|@(acroread.sh)) ]] ++++ [[ -f /etc/bash_completion.d/git-prompt ]] ++++ [[ -r /etc/bash_completion.d/git-prompt ]] ++++ . /etc/bash_completion.d/git-prompt +++++ [[ -e /usr/lib/git-core/git-sh-prompt ]] +++++ . /usr/lib/git-core/git-sh-prompt ++++++ __git_printf_supports_v= ++++++ printf -v __git_printf_supports_v -- %s yes ++++ unset compat_dir i _blacklist_glob ++++ user_completion=/admin/home/loubna/.bash_completion ++++ [[ /usr/share/bash-completion/bash_completion != /admin/home/loubna/.bash_completion ]] ++++ [[ -r /admin/home/loubna/.bash_completion ]] ++++ unset user_completion ++++ unset -f have ++++ unset have ++++ set +v ++++ unset BASH_COMPLETION_ORIGINAL_V_VALUE +++ export PATH=/opt/slurm/bin:/opt/slurm/sbin:/fsx/loubna/google-cloud-sdk/bin:/usr/local/cuda-11.6/bin:/fsx/loubna/miniconda3/envs/megatron/bin:/fsx/loubna/miniconda3/condabin:/opt/slurm/bin:/opt/slurm/sbin:/admin/home/loubna/.vscode-server/bin/d045a5eda657f4d7b676dedbfa7aab8207f8a075/bin/remote-cli:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/admin/home/loubna/.local/bin +++ PATH=/opt/slurm/bin:/opt/slurm/sbin:/fsx/loubna/google-cloud-sdk/bin:/usr/local/cuda-11.6/bin:/fsx/loubna/miniconda3/envs/megatron/bin:/fsx/loubna/miniconda3/condabin:/opt/slurm/bin:/opt/slurm/sbin:/admin/home/loubna/.vscode-server/bin/d045a5eda657f4d7b676dedbfa7aab8207f8a075/bin/remote-cli:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/admin/home/loubna/.local/bin ++++ /fsx/loubna/miniconda3/bin/conda shell.bash hook +++ __conda_setup='export CONDA_EXE='\''/fsx/loubna/miniconda3/bin/conda'\'' +export _CE_M='\'''\'' +export _CE_CONDA='\'''\'' +export CONDA_PYTHON_EXE='\''/fsx/loubna/miniconda3/bin/python'\'' + +# Copyright (C) 2012 Anaconda, Inc +# SPDX-License-Identifier: BSD-3-Clause + +__conda_exe() ( + "$CONDA_EXE" $_CE_M $_CE_CONDA "$@" +) + +__conda_hashr() { + if [ -n "${ZSH_VERSION:+x}" ]; then + \rehash + elif [ -n "${POSH_VERSION:+x}" ]; then + : # pass + else + \hash -r + fi +} + +__conda_activate() { + if [ -n "${CONDA_PS1_BACKUP:+x}" ]; then + # Handle transition from shell activated with conda <= 4.3 to a subsequent activation + # after conda updated to >= 4.4. See issue #6173. + PS1="$CONDA_PS1_BACKUP" + \unset CONDA_PS1_BACKUP + fi + \local ask_conda + ask_conda="$(PS1="${PS1:-}" __conda_exe shell.posix "$@")" || \return + \eval "$ask_conda" + __conda_hashr +} + +__conda_reactivate() { + \local ask_conda + ask_conda="$(PS1="${PS1:-}" __conda_exe shell.posix reactivate)" || \return + \eval "$ask_conda" + __conda_hashr +} + +conda() { + \local cmd="${1-__missing__}" + case "$cmd" in + activate|deactivate) + __conda_activate "$@" + ;; + install|update|upgrade|remove|uninstall) + __conda_exe "$@" || \return + __conda_reactivate + ;; + *) + __conda_exe "$@" + ;; + esac +} + +if [ -z "${CONDA_SHLVL+x}" ]; then + \export CONDA_SHLVL=0 + # In dev-mode CONDA_EXE is python.exe and on Windows + # it is in a different relative location to condabin. + if [ -n "${_CE_CONDA:+x}" ] && [ -n "${WINDIR+x}" ]; then + PATH="$(\dirname "$CONDA_EXE")/condabin${PATH:+":${PATH}"}" + else + PATH="$(\dirname "$(\dirname "$CONDA_EXE")")/condabin${PATH:+":${PATH}"}" + fi + \export PATH + + # We'\''re not allowing PS1 to be unbound. It must at least be set. + # However, we'\''re not exporting it, which can cause problems when starting a second shell + # via a first shell (i.e. starting zsh from bash). + if [ -z "${PS1+x}" ]; then + PS1= + fi +fi + +conda activate base' +++ '[' 0 -eq 0 ']' +++ eval 'export CONDA_EXE='\''/fsx/loubna/miniconda3/bin/conda'\'' +export _CE_M='\'''\'' +export _CE_CONDA='\'''\'' +export CONDA_PYTHON_EXE='\''/fsx/loubna/miniconda3/bin/python'\'' + +# Copyright (C) 2012 Anaconda, Inc +# SPDX-License-Identifier: BSD-3-Clause + +__conda_exe() ( + "$CONDA_EXE" $_CE_M $_CE_CONDA "$@" +) + +__conda_hashr() { + if [ -n "${ZSH_VERSION:+x}" ]; then + \rehash + elif [ -n "${POSH_VERSION:+x}" ]; then + : # pass + else + \hash -r + fi +} + +__conda_activate() { + if [ -n "${CONDA_PS1_BACKUP:+x}" ]; then + # Handle transition from shell activated with conda <= 4.3 to a subsequent activation + # after conda updated to >= 4.4. See issue #6173. + PS1="$CONDA_PS1_BACKUP" + \unset CONDA_PS1_BACKUP + fi + \local ask_conda + ask_conda="$(PS1="${PS1:-}" __conda_exe shell.posix "$@")" || \return + \eval "$ask_conda" + __conda_hashr +} + +__conda_reactivate() { + \local ask_conda + ask_conda="$(PS1="${PS1:-}" __conda_exe shell.posix reactivate)" || \return + \eval "$ask_conda" + __conda_hashr +} + +conda() { + \local cmd="${1-__missing__}" + case "$cmd" in + activate|deactivate) + __conda_activate "$@" + ;; + install|update|upgrade|remove|uninstall) + __conda_exe "$@" || \return + __conda_reactivate + ;; + *) + __conda_exe "$@" + ;; + esac +} + +if [ -z "${CONDA_SHLVL+x}" ]; then + \export CONDA_SHLVL=0 + # In dev-mode CONDA_EXE is python.exe and on Windows + # it is in a different relative location to condabin. + if [ -n "${_CE_CONDA:+x}" ] && [ -n "${WINDIR+x}" ]; then + PATH="$(\dirname "$CONDA_EXE")/condabin${PATH:+":${PATH}"}" + else + PATH="$(\dirname "$(\dirname "$CONDA_EXE")")/condabin${PATH:+":${PATH}"}" + fi + \export PATH + + # We'\''re not allowing PS1 to be unbound. It must at least be set. + # However, we'\''re not exporting it, which can cause problems when starting a second shell + # via a first shell (i.e. starting zsh from bash). + if [ -z "${PS1+x}" ]; then + PS1= + fi +fi + +conda activate base' ++++ export CONDA_EXE=/fsx/loubna/miniconda3/bin/conda ++++ CONDA_EXE=/fsx/loubna/miniconda3/bin/conda ++++ export _CE_M= ++++ _CE_M= ++++ export _CE_CONDA= ++++ _CE_CONDA= ++++ export CONDA_PYTHON_EXE=/fsx/loubna/miniconda3/bin/python ++++ CONDA_PYTHON_EXE=/fsx/loubna/miniconda3/bin/python ++++ '[' -z x ']' ++++ conda activate base ++++ local cmd=activate ++++ case "$cmd" in ++++ __conda_activate activate base ++++ '[' -n '' ']' ++++ local ask_conda +++++ PS1='\[\e]0;\u@\h: \w\a\]${debian_chroot:+($debian_chroot)}\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ ' +++++ __conda_exe shell.posix activate base +++++ /fsx/loubna/miniconda3/bin/conda shell.posix activate base ++++ ask_conda='PS1='\''(base) \[\e]0;\u@\h: \w\a\]${debian_chroot:+($debian_chroot)}\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ '\'' +export PATH='\''/opt/slurm/bin:/opt/slurm/sbin:/fsx/loubna/google-cloud-sdk/bin:/usr/local/cuda-11.6/bin:/fsx/loubna/miniconda3/bin:/fsx/loubna/miniconda3/condabin:/opt/slurm/bin:/opt/slurm/sbin:/admin/home/loubna/.vscode-server/bin/d045a5eda657f4d7b676dedbfa7aab8207f8a075/bin/remote-cli:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/admin/home/loubna/.local/bin'\'' +export CONDA_PREFIX='\''/fsx/loubna/miniconda3'\'' +export CONDA_SHLVL='\''3'\'' +export CONDA_DEFAULT_ENV='\''base'\'' +export CONDA_PROMPT_MODIFIER='\''(base) '\'' +export CONDA_PREFIX_2='\''/fsx/loubna/miniconda3/envs/megatron'\'' +export CONDA_EXE='\''/fsx/loubna/miniconda3/bin/conda'\'' +export _CE_M='\'''\'' +export _CE_CONDA='\'''\'' +export CONDA_PYTHON_EXE='\''/fsx/loubna/miniconda3/bin/python'\''' ++++ eval 'PS1='\''(base) \[\e]0;\u@\h: \w\a\]${debian_chroot:+($debian_chroot)}\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ '\'' +export PATH='\''/opt/slurm/bin:/opt/slurm/sbin:/fsx/loubna/google-cloud-sdk/bin:/usr/local/cuda-11.6/bin:/fsx/loubna/miniconda3/bin:/fsx/loubna/miniconda3/condabin:/opt/slurm/bin:/opt/slurm/sbin:/admin/home/loubna/.vscode-server/bin/d045a5eda657f4d7b676dedbfa7aab8207f8a075/bin/remote-cli:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/admin/home/loubna/.local/bin'\'' +export CONDA_PREFIX='\''/fsx/loubna/miniconda3'\'' +export CONDA_SHLVL='\''3'\'' +export CONDA_DEFAULT_ENV='\''base'\'' +export CONDA_PROMPT_MODIFIER='\''(base) '\'' +export CONDA_PREFIX_2='\''/fsx/loubna/miniconda3/envs/megatron'\'' +export CONDA_EXE='\''/fsx/loubna/miniconda3/bin/conda'\'' +export _CE_M='\'''\'' +export _CE_CONDA='\'''\'' +export CONDA_PYTHON_EXE='\''/fsx/loubna/miniconda3/bin/python'\''' +++++ PS1='(base) \[\e]0;\u@\h: \w\a\]${debian_chroot:+($debian_chroot)}\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ ' +++++ export PATH=/opt/slurm/bin:/opt/slurm/sbin:/fsx/loubna/google-cloud-sdk/bin:/usr/local/cuda-11.6/bin:/fsx/loubna/miniconda3/bin:/fsx/loubna/miniconda3/condabin:/opt/slurm/bin:/opt/slurm/sbin:/admin/home/loubna/.vscode-server/bin/d045a5eda657f4d7b676dedbfa7aab8207f8a075/bin/remote-cli:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/admin/home/loubna/.local/bin +++++ PATH=/opt/slurm/bin:/opt/slurm/sbin:/fsx/loubna/google-cloud-sdk/bin:/usr/local/cuda-11.6/bin:/fsx/loubna/miniconda3/bin:/fsx/loubna/miniconda3/condabin:/opt/slurm/bin:/opt/slurm/sbin:/admin/home/loubna/.vscode-server/bin/d045a5eda657f4d7b676dedbfa7aab8207f8a075/bin/remote-cli:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/admin/home/loubna/.local/bin +++++ export CONDA_PREFIX=/fsx/loubna/miniconda3 +++++ CONDA_PREFIX=/fsx/loubna/miniconda3 +++++ export CONDA_SHLVL=3 +++++ CONDA_SHLVL=3 +++++ export CONDA_DEFAULT_ENV=base +++++ CONDA_DEFAULT_ENV=base +++++ export 'CONDA_PROMPT_MODIFIER=(base) ' +++++ CONDA_PROMPT_MODIFIER='(base) ' +++++ export CONDA_PREFIX_2=/fsx/loubna/miniconda3/envs/megatron +++++ CONDA_PREFIX_2=/fsx/loubna/miniconda3/envs/megatron +++++ export CONDA_EXE=/fsx/loubna/miniconda3/bin/conda +++++ CONDA_EXE=/fsx/loubna/miniconda3/bin/conda +++++ export _CE_M= +++++ _CE_M= +++++ export _CE_CONDA= +++++ _CE_CONDA= +++++ export CONDA_PYTHON_EXE=/fsx/loubna/miniconda3/bin/python +++++ CONDA_PYTHON_EXE=/fsx/loubna/miniconda3/bin/python ++++ __conda_hashr ++++ '[' -n '' ']' ++++ '[' -n '' ']' ++++ hash -r +++ unset __conda_setup +++ export WANDB_CACHE_DIR=/fsx/loubna/.tmp/wandb +++ WANDB_CACHE_DIR=/fsx/loubna/.tmp/wandb +++ export TMPDIR=/fsx/loubna/.tmp +++ TMPDIR=/fsx/loubna/.tmp +++ export HUGGINGFACE_HUB_CACHE=/fsx/loubna/.cache +++ HUGGINGFACE_HUB_CACHE=/fsx/loubna/.cache +++ export HF_DATASETS_CACHE=/fsx/loubna/.cache +++ HF_DATASETS_CACHE=/fsx/loubna/.cache +++ export PATH=/usr/local/cuda-11.6/bin:/opt/slurm/bin:/opt/slurm/sbin:/fsx/loubna/google-cloud-sdk/bin:/usr/local/cuda-11.6/bin:/fsx/loubna/miniconda3/bin:/fsx/loubna/miniconda3/condabin:/opt/slurm/bin:/opt/slurm/sbin:/admin/home/loubna/.vscode-server/bin/d045a5eda657f4d7b676dedbfa7aab8207f8a075/bin/remote-cli:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/admin/home/loubna/.local/bin +++ PATH=/usr/local/cuda-11.6/bin:/opt/slurm/bin:/opt/slurm/sbin:/fsx/loubna/google-cloud-sdk/bin:/usr/local/cuda-11.6/bin:/fsx/loubna/miniconda3/bin:/fsx/loubna/miniconda3/condabin:/opt/slurm/bin:/opt/slurm/sbin:/admin/home/loubna/.vscode-server/bin/d045a5eda657f4d7b676dedbfa7aab8207f8a075/bin/remote-cli:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/admin/home/loubna/.local/bin +++ export LD_LIBRARY_PATH=/usr/local/cuda-11.6/lib64:/usr/local/cuda-11.6/lib64:/opt/amazon/efa/lib:/opt/amazon/openmpi/lib:/usr/local/cuda/efa/lib:/usr/local/lib:/usr/lib:/opt/amazon/efa/lib:/opt/amazon/openmpi/lib:/usr/local/cuda/efa/lib:/usr/local/lib:/usr/lib: +++ LD_LIBRARY_PATH=/usr/local/cuda-11.6/lib64:/usr/local/cuda-11.6/lib64:/opt/amazon/efa/lib:/opt/amazon/openmpi/lib:/usr/local/cuda/efa/lib:/usr/local/lib:/usr/lib:/opt/amazon/efa/lib:/opt/amazon/openmpi/lib:/usr/local/cuda/efa/lib:/usr/local/lib:/usr/lib: +++ PATH=/usr/local/cuda-11.6/bin:/opt/slurm/bin:/opt/slurm/sbin:/fsx/loubna/google-cloud-sdk/bin:/usr/local/cuda-11.6/bin:/fsx/loubna/miniconda3/bin:/fsx/loubna/miniconda3/condabin:/opt/slurm/bin:/opt/slurm/sbin:/admin/home/loubna/.vscode-server/bin/d045a5eda657f4d7b676dedbfa7aab8207f8a075/bin/remote-cli:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/admin/home/loubna/.local/bin:/admin/home/loubna/.local/bin +++ '[' -f /fsx/loubna/google-cloud-sdk/path.bash.inc ']' +++ . /fsx/loubna/google-cloud-sdk/path.bash.inc +++++ command readlink /fsx/loubna/google-cloud-sdk/path.bash.inc +++++ readlink /fsx/loubna/google-cloud-sdk/path.bash.inc ++++ script_link= ++++ script_link=/fsx/loubna/google-cloud-sdk/path.bash.inc ++++ apparent_sdk_dir=/fsx/loubna/google-cloud-sdk ++++ '[' /fsx/loubna/google-cloud-sdk == /fsx/loubna/google-cloud-sdk/path.bash.inc ']' +++++ command cd -P /fsx/loubna/google-cloud-sdk +++++ cd -P /fsx/loubna/google-cloud-sdk +++++ command pwd -P +++++ pwd -P ++++ sdk_dir=/fsx/loubna/google-cloud-sdk ++++ bin_path=/fsx/loubna/google-cloud-sdk/bin ++++ [[ :/usr/local/cuda-11.6/bin:/opt/slurm/bin:/opt/slurm/sbin:/fsx/loubna/google-cloud-sdk/bin:/usr/local/cuda-11.6/bin:/fsx/loubna/miniconda3/bin:/fsx/loubna/miniconda3/condabin:/opt/slurm/bin:/opt/slurm/sbin:/admin/home/loubna/.vscode-server/bin/d045a5eda657f4d7b676dedbfa7aab8207f8a075/bin/remote-cli:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/admin/home/loubna/.local/bin:/admin/home/loubna/.local/bin: != *\:\/\f\s\x\/\l\o\u\b\n\a\/\g\o\o\g\l\e\-\c\l\o\u\d\-\s\d\k\/\b\i\n\:* ]] +++ '[' -f /fsx/loubna/google-cloud-sdk/completion.bash.inc ']' +++ . /fsx/loubna/google-cloud-sdk/completion.bash.inc ++++ complete -o nospace -F _python_argcomplete gcloud ++++ unset bq_COMMANDS ++++ complete -F _bq_completer bq ++++ complete -o nospace -F _python_argcomplete gsutil ++ conda activate megatron ++ local cmd=activate ++ case "$cmd" in ++ __conda_activate activate megatron ++ '[' -n '' ']' ++ local ask_conda +++ PS1='(base) \[\e]0;\u@\h: \w\a\]${debian_chroot:+($debian_chroot)}\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ ' +++ __conda_exe shell.posix activate megatron +++ /fsx/loubna/miniconda3/bin/conda shell.posix activate megatron ++ ask_conda='PS1='\''(megatron) \[\e]0;\u@\h: \w\a\]${debian_chroot:+($debian_chroot)}\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ '\'' +export PATH='\''/usr/local/cuda-11.6/bin:/opt/slurm/bin:/opt/slurm/sbin:/fsx/loubna/google-cloud-sdk/bin:/usr/local/cuda-11.6/bin:/fsx/loubna/miniconda3/envs/megatron/bin:/fsx/loubna/miniconda3/condabin:/opt/slurm/bin:/opt/slurm/sbin:/admin/home/loubna/.vscode-server/bin/d045a5eda657f4d7b676dedbfa7aab8207f8a075/bin/remote-cli:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/admin/home/loubna/.local/bin:/admin/home/loubna/.local/bin'\'' +export CONDA_PREFIX='\''/fsx/loubna/miniconda3/envs/megatron'\'' +export CONDA_SHLVL='\''4'\'' +export CONDA_DEFAULT_ENV='\''megatron'\'' +export CONDA_PROMPT_MODIFIER='\''(megatron) '\'' +export CONDA_PREFIX_3='\''/fsx/loubna/miniconda3'\'' +export CONDA_EXE='\''/fsx/loubna/miniconda3/bin/conda'\'' +export _CE_M='\'''\'' +export _CE_CONDA='\'''\'' +export CONDA_PYTHON_EXE='\''/fsx/loubna/miniconda3/bin/python'\''' ++ eval 'PS1='\''(megatron) \[\e]0;\u@\h: \w\a\]${debian_chroot:+($debian_chroot)}\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ '\'' +export PATH='\''/usr/local/cuda-11.6/bin:/opt/slurm/bin:/opt/slurm/sbin:/fsx/loubna/google-cloud-sdk/bin:/usr/local/cuda-11.6/bin:/fsx/loubna/miniconda3/envs/megatron/bin:/fsx/loubna/miniconda3/condabin:/opt/slurm/bin:/opt/slurm/sbin:/admin/home/loubna/.vscode-server/bin/d045a5eda657f4d7b676dedbfa7aab8207f8a075/bin/remote-cli:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/admin/home/loubna/.local/bin:/admin/home/loubna/.local/bin'\'' +export CONDA_PREFIX='\''/fsx/loubna/miniconda3/envs/megatron'\'' +export CONDA_SHLVL='\''4'\'' +export CONDA_DEFAULT_ENV='\''megatron'\'' +export CONDA_PROMPT_MODIFIER='\''(megatron) '\'' +export CONDA_PREFIX_3='\''/fsx/loubna/miniconda3'\'' +export CONDA_EXE='\''/fsx/loubna/miniconda3/bin/conda'\'' +export _CE_M='\'''\'' +export _CE_CONDA='\'''\'' +export CONDA_PYTHON_EXE='\''/fsx/loubna/miniconda3/bin/python'\''' +++ PS1='(megatron) \[\e]0;\u@\h: \w\a\]${debian_chroot:+($debian_chroot)}\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ ' +++ export PATH=/usr/local/cuda-11.6/bin:/opt/slurm/bin:/opt/slurm/sbin:/fsx/loubna/google-cloud-sdk/bin:/usr/local/cuda-11.6/bin:/fsx/loubna/miniconda3/envs/megatron/bin:/fsx/loubna/miniconda3/condabin:/opt/slurm/bin:/opt/slurm/sbin:/admin/home/loubna/.vscode-server/bin/d045a5eda657f4d7b676dedbfa7aab8207f8a075/bin/remote-cli:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/admin/home/loubna/.local/bin:/admin/home/loubna/.local/bin +++ PATH=/usr/local/cuda-11.6/bin:/opt/slurm/bin:/opt/slurm/sbin:/fsx/loubna/google-cloud-sdk/bin:/usr/local/cuda-11.6/bin:/fsx/loubna/miniconda3/envs/megatron/bin:/fsx/loubna/miniconda3/condabin:/opt/slurm/bin:/opt/slurm/sbin:/admin/home/loubna/.vscode-server/bin/d045a5eda657f4d7b676dedbfa7aab8207f8a075/bin/remote-cli:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/admin/home/loubna/.local/bin:/admin/home/loubna/.local/bin +++ export CONDA_PREFIX=/fsx/loubna/miniconda3/envs/megatron +++ CONDA_PREFIX=/fsx/loubna/miniconda3/envs/megatron +++ export CONDA_SHLVL=4 +++ CONDA_SHLVL=4 +++ export CONDA_DEFAULT_ENV=megatron +++ CONDA_DEFAULT_ENV=megatron +++ export 'CONDA_PROMPT_MODIFIER=(megatron) ' +++ CONDA_PROMPT_MODIFIER='(megatron) ' +++ export CONDA_PREFIX_3=/fsx/loubna/miniconda3 +++ CONDA_PREFIX_3=/fsx/loubna/miniconda3 +++ export CONDA_EXE=/fsx/loubna/miniconda3/bin/conda +++ CONDA_EXE=/fsx/loubna/miniconda3/bin/conda +++ export _CE_M= +++ _CE_M= +++ export _CE_CONDA= +++ _CE_CONDA= +++ export CONDA_PYTHON_EXE=/fsx/loubna/miniconda3/bin/python +++ CONDA_PYTHON_EXE=/fsx/loubna/miniconda3/bin/python ++ __conda_hashr ++ '[' -n '' ']' ++ '[' -n '' ']' ++ hash -r +++ date ++ echo 'START TIME: Thu Jun 15 12:21:02 UTC 2023' +START TIME: Thu Jun 15 12:21:02 UTC 2023 ++ BRRR_REPO=/fsx/loubna/code/fork/brrr ++ SCRIPT_REPO=/fsx/loubna/code/fork/brrr/examples/gpt2_mqa ++ pushd /fsx/loubna/code/fork/brrr/examples/gpt2_mqa +/fsx/loubna/code/fork/brrr/examples/gpt2_mqa /fsx/loubna/code/fork/brrr ++ GPUS_PER_NODE=8 ++ NNODES=16 +++ head -n 1 +++ scontrol show hostnames 'ip-26-0-147-[141,187,189,193,204,233,245,247],ip-26-0-148-[55,93,115,151,170,193,245],ip-26-0-149-1' ++ MASTER_ADDR=ip-26-0-147-141 ++ MASTER_PORT=6000 ++ CONFIG_FILE=/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/configs/config_5b_explicit_model_conf.yaml ++ export USE_FAST=1 ++ USE_FAST=1 ++ export CUDA_DEVICE_MAX_CONNECTIONS=1 ++ CUDA_DEVICE_MAX_CONNECTIONS=1 ++ CMD=' main.py --config-file /fsx/loubna/code/fork/brrr/examples/gpt2_mqa/configs/config_5b_explicit_model_conf.yaml + ' ++ export 'LAUNCHER=python -u -m torch.distributed.run --nproc_per_node 8 --nnodes 16 --rdzv_endpoint ip-26-0-147-141:6000 --rdzv_backend c10d --max_restarts 0 --tee 3 ' ++ LAUNCHER='python -u -m torch.distributed.run --nproc_per_node 8 --nnodes 16 --rdzv_endpoint ip-26-0-147-141:6000 --rdzv_backend c10d --max_restarts 0 --tee 3 ' ++ echo main.py --config-file /fsx/loubna/code/fork/brrr/examples/gpt2_mqa/configs/config_5b_explicit_model_conf.yaml +main.py --config-file /fsx/loubna/code/fork/brrr/examples/gpt2_mqa/configs/config_5b_explicit_model_conf.yaml ++ export NCCL_ASYNC_ERROR_HANDLING=1 ++ NCCL_ASYNC_ERROR_HANDLING=1 ++ export NCCL_PROTO=simple ++ NCCL_PROTO=simple ++ export RDMAV_FORK_SAFE=1 ++ RDMAV_FORK_SAFE=1 ++ export FI_EFA_FORK_SAFE=1 ++ FI_EFA_FORK_SAFE=1 ++ export FI_EFA_USE_DEVICE_RDMA=1 ++ FI_EFA_USE_DEVICE_RDMA=1 ++ export FI_PROVIDER=efa ++ FI_PROVIDER=efa ++ export FI_LOG_LEVEL=1 ++ FI_LOG_LEVEL=1 ++ export NCCL_IB_DISABLE=1 ++ NCCL_IB_DISABLE=1 ++ export NCCL_SOCKET_IFNAME=ens ++ NCCL_SOCKET_IFNAME=ens ++ SRUN_ARGS=' --wait=60 --kill-on-bad-exit=1 ' ++ SLURM_JOB_ID=158049 ++ srun --wait=60 --kill-on-bad-exit=1 --jobid 158049 -u bash -c 'python -u -m torch.distributed.run --nproc_per_node 8 --nnodes 16 --rdzv_endpoint ip-26-0-147-141:6000 --rdzv_backend c10d --max_restarts 0 --tee 3 --node_rank $SLURM_PROCID --role $SLURMD_NODENAME: main.py --config-file /fsx/loubna/code/fork/brrr/examples/gpt2_mqa/configs/config_5b_explicit_model_conf.yaml + ' +WARNING:__main__: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +WARNING:__main__: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +WARNING:__main__: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +WARNING:__main__: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +WARNING:__main__: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +WARNING:__main__: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +WARNING:__main__: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +WARNING:__main__: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +WARNING:__main__: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +WARNING:__main__: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +WARNING:__main__: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +WARNING:__main__: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +WARNING:__main__: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +WARNING:__main__: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +WARNING:__main__: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +WARNING:__main__: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: Config(general=GeneralArgs(name='brrr_example_gpt2_mqa_starcoder_5b_2', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: kill_switch_path=PosixPath('/fsx/loubna/br4-experiments/kill_loubna_starcoder'), +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: ignore_sanity_checks=True), +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: profile=None, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: checkpoints=CheckpointsArgs(checkpoints_path=PosixPath('/fsx/loubna/br4-experiments/checkpoints'), +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: checkpoint_interval=2500, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: load_from_specific_checkpoint=None, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: upload_s3_path=None, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: upload_s3_num_workers=None), +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: parallelism=ParallelismArgs(dp=32, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: pp=1, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: tp=4, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: pp_engine=, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: tp_mode=, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: recompute_granularity=None, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: tp_column_linear_async_communication=True), +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: model=ModelArgs(hf_gpt2_model_name='loubnabnl/starcoder-5b', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: hidden_size=3584, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: num_attention_heads=28, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: n_inner=14336, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: n_layer=40, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: max_position_embeddings=8192, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: vocab_size=49152, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: layer_norm_epsilon=1e-05, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: scale_attn_weights=True, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: activation_function='gelu', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: attention_softmax_in_fp32=True, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: resid_pdrop=0.1, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: attn_pdrop=0.1, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: embd_pdrop=0.1, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: initializer_range=0.02, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: pad_key_length=True, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: scale_attention_softmax_in_fp32=True, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: make_vocab_size_divisible_by=128, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: dtype=torch.bfloat16, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: init_method=RandomInit(std=0.01275), +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: seed=42), +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: logging=LoggingArgs(log_level='info', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: log_level_replica='info', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: iteration_step_info_interval=1, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: tensorboard_logger=TensorboardLoggerConfig(tensorboard_dir=PosixPath('/fsx/loubna/br4-experiments/tensorboard'))), +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: tokens=TokensArgs(sequence_length=8192, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: train_steps=29652, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: micro_batch_size=2, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: batch_accumulation_per_replica=8, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: val_check_interval=2500, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: limit_val_batches=2, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: limit_test_batches=0), +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: optimizer=OptimizerArgs(zero_stage=1, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: weight_decay=0.1, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: clip_grad=1.0, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: accumulate_grad_in_fp32=True, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: adam_eps=1e-08, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: adam_beta1=0.9, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: adam_beta2=0.95, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: learning_rate=0.0003), +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: learning_rate_scheduler=LRSchedulerArgs(lr_warmup_steps=2000, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: lr_warmup_style='linear', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: lr_decay_style='cosine', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: lr_decay_steps=29652, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: min_decay_lr=3e-05), +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: data=DataArgs(seed=1234, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: num_loading_workers=2, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: dataset=PretrainNemoArgs(data_prefix=[3.0, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/css/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 0.01, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/prolog/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 53.89, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 1.78, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/fortran/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 0.85, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/solidity/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 5.68, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/kotlin/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 0.01, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-agda/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 1.31, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/julia/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 0.98, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java-server-pages/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 0.08, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/isabelle/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 0.03, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/idris/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 0.09, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lean/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 1.12, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/powershell/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 23.78, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/go/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 0.7, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/erlang/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 0.61, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/f-sharp/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 0.26, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ada/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 1.68, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/pascal/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 2.23, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/perl/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 0.3, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/r/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 0.31, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/protocol-buffer/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 0.45, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cmake/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 0.12, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sas/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 6.81, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ruby/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 9.11, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rust/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 0.06, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rmarkdown/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 44.66, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c-sharp/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 0.58, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/smalltalk/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 2.23, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/haskell/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 0.01, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/maple/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 1.25, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/mathematica/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 1.03, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ocaml/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 1.31, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/makefile/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 2.87, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lua/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 0.01, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-coffeescript/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 0.05, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-haskell/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 3.32, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/restructuredtext/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 0.03, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/racket/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 0.19, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/standard-ml/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 0.39, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/systemverilog/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 5.2, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tex/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 0.02, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/awk/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 1.56, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/assembly/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 0.01, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/alloy/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 0.07, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/agda/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 0.41, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/emacs-lisp/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 3.66, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dart/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 0.56, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cuda/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 0.03, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/bluespec/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 0.001, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/augeas/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 0.23, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/batchfile/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 0.02, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcsh/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 0.01, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stan/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 4.69, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scala/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 0.35, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcl/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 0.33, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stata/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 0.01, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/applescript/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 3.09, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/shell/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 0.46, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/clojure/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 0.2, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scheme/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 0.05, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/antlr/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 0.04, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sparql/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 11.09, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sql/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 0.4, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/glsl/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 0.3, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elm/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 0.42, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dockerfile/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 48.92, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cpp/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 0.64, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/coffeescript/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 1.4, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/common-lisp/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 0.71, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elixir/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 0.91, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/groovy/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 29.36, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/html/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 86.94, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 64.71, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/javascript/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 74.93, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/markdown/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 60.89, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/php/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 60.4, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/python/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 26.52, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/typescript/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 0.001, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/verilog/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 1.42, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/visual-basic/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 0.94, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/vhdl/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 0.01, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/thrift/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 0.0002, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/matlab/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 0.11, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yacc/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 0.18, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/zig/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 0.05, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/xslt/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 1.0, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/json/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 1.0, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yaml/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 54.4, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_issues/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 32.0, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_commits/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 7.12, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_scripts/gpt2-preprocessed_content_document', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: 6.0, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_structured/gpt2-preprocessed_content_document'], +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: index_mapping_dir=None, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: splits_string='0.969,0.999,1', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: skip_warmup=True, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: dataloader_type='single', +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: validation_drop_last=True, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: eod_mask_loss=False, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: no_seqlen_plus_one_input_tokens=False, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: pad_samples_to_global_batch_size=False))) +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: GPTBigCodeConfig { +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: "activation_function": "gelu", +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: "architectures": [ +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: "GPTBigCodeForCausalLM" +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: ], +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: "attention_softmax_in_fp32": true, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: "attention_type": 1, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: "attn_pdrop": 0.1, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: "bos_token_id": 0, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: "embd_pdrop": 0.1, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: "eos_token_id": 0, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: "inference_runner": 0, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: "initializer_range": 0.02, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: "layer_norm_epsilon": 1e-05, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: "max_batch_size": null, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: "max_sequence_length": null, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: "model_type": "gpt_bigcode", +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: "multi_query": true, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: "n_embd": 3584, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: "n_head": 28, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: "n_inner": 14336, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: "n_layer": 40, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: "n_positions": 8192, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: "pad_key_length": true, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: "pre_allocate_kv_cache": false, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: "resid_pdrop": 0.1, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: "scale_attention_softmax_in_fp32": true, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: "scale_attn_weights": true, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: "summary_activation": null, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: "summary_first_dropout": 0.1, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: "summary_proj_to_labels": true, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: "summary_type": "cls_index", +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: "summary_use_proj": true, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: "torch_dtype": "float32", +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: "transformers_version": "4.27.0.dev0", +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: "use_cache": true, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: "validate_runner_input": true, +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: "vocab_size": 49152 +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: } +[ip-26-0-147-141:0]:06/15/2023 12:21:15 [INFO|DP=0|PP=0|TP=0]: +[ip-26-0-147-141:0]:06/15/2023 12:21:22 [INFO|DP=0|PP=0|TP=0]: Number of parameters: 1373642752 (2620.02MB). Expecting peak 4*param_size=10480.06MB with grads and Adam optim states (w/o memory optims) +[ip-26-0-147-141:0]:06/15/2023 12:21:22 [INFO|DP=0|PP=0|TP=0]: [After model building] Memory usage: 2620.03MB. Peak reserved memory: 3066.00MB +[ip-26-0-147-141:3]:06/15/2023 12:21:22 [INFO|DP=0|PP=0|TP=3]: Number of parameters: 1373356032 (2619.47MB). Expecting peak 4*param_size=10477.88MB with grads and Adam optim states (w/o memory optims) +[ip-26-0-147-141:1]:06/15/2023 12:21:22 [INFO|DP=0|PP=0|TP=1]: Number of parameters: 1373356032 (2619.47MB). Expecting peak 4*param_size=10477.88MB with grads and Adam optim states (w/o memory optims) +[ip-26-0-147-141:3]:06/15/2023 12:21:22 [INFO|DP=0|PP=0|TP=3]: [After model building] Memory usage: 2619.49MB. Peak reserved memory: 3064.00MB +[ip-26-0-147-141:1]:06/15/2023 12:21:22 [INFO|DP=0|PP=0|TP=1]: [After model building] Memory usage: 2619.49MB. Peak reserved memory: 3064.00MB +[ip-26-0-147-141:2]:06/15/2023 12:21:22 [INFO|DP=0|PP=0|TP=2]: Number of parameters: 1373356032 (2619.47MB). Expecting peak 4*param_size=10477.88MB with grads and Adam optim states (w/o memory optims) +[ip-26-0-147-141:2]:06/15/2023 12:21:22 [INFO|DP=0|PP=0|TP=2]: [After model building] Memory usage: 2619.49MB. Peak reserved memory: 3064.00MB +[ip-26-0-148-151:3]:06/15/2023 12:21:26 [INFO|DP=18|PP=0|TP=3]: Global rank: 75/128 | PP: 0/1 | DP: 18/32 | TP: 3/4 +[ip-26-0-148-93:3]:06/15/2023 12:21:26 [INFO|DP=28|PP=0|TP=3]: Global rank: 115/128 | PP: 0/1 | DP: 28/32 | TP: 3/4 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: Global rank: 0/128 | PP: 0/1 | DP: 0/32 | TP: 0/4 +[ip-26-0-147-141:3]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=3]: Global rank: 3/128 | PP: 0/1 | DP: 0/32 | TP: 3/4 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: Using Nemo Dataloader +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: Building GPT datasets. +[ip-26-0-147-141:2]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=2]: Global rank: 2/128 | PP: 0/1 | DP: 0/32 | TP: 2/4 +[ip-26-0-147-141:1]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=1]: Global rank: 1/128 | PP: 0/1 | DP: 0/32 | TP: 1/4 +[ip-26-0-147-141:4]:06/15/2023 12:21:26 [INFO|DP=1|PP=0|TP=0]: Global rank: 4/128 | PP: 0/1 | DP: 1/32 | TP: 0/4 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.004817 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: number of documents: 2721616 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 888559) total of 888559 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: document indices in [888559, 1804629) total of 916070 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: document indices in [1804629, 2721616) total of 916987 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/css/gpt2-preprocessed_content_document_train_indexmap_59682ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/css/gpt2-preprocessed_content_document_train_indexmap_59682ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/css/gpt2-preprocessed_content_document_train_indexmap_59682ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.011 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 192826 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/css/gpt2-preprocessed_content_document_valid_indexmap_49ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/css/gpt2-preprocessed_content_document_valid_indexmap_49ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/css/gpt2-preprocessed_content_document_valid_indexmap_49ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 197784 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-149-1:7]:06/15/2023 12:21:26 [INFO|DP=31|PP=0|TP=3]: Global rank: 127/128 | PP: 0/1 | DP: 31/32 | TP: 3/4 +[ip-26-0-148-245:1]:06/15/2023 12:21:26 [INFO|DP=24|PP=0|TP=1]: Global rank: 97/128 | PP: 0/1 | DP: 24/32 | TP: 1/4 +[ip-26-0-148-245:4]:06/15/2023 12:21:26 [INFO|DP=25|PP=0|TP=0]: Global rank: 100/128 | PP: 0/1 | DP: 25/32 | TP: 0/4 +[ip-26-0-149-1:6]:06/15/2023 12:21:26 [INFO|DP=31|PP=0|TP=2]: Global rank: 126/128 | PP: 0/1 | DP: 31/32 | TP: 2/4 +[ip-26-0-149-1:3]:06/15/2023 12:21:26 [INFO|DP=30|PP=0|TP=3]: Global rank: 123/128 | PP: 0/1 | DP: 30/32 | TP: 3/4 +[ip-26-0-149-1:1]:06/15/2023 12:21:26 [INFO|DP=30|PP=0|TP=1]: Global rank: 121/128 | PP: 0/1 | DP: 30/32 | TP: 1/4 +[ip-26-0-149-1:0]:06/15/2023 12:21:26 [INFO|DP=30|PP=0|TP=0]: Global rank: 120/128 | PP: 0/1 | DP: 30/32 | TP: 0/4 +[ip-26-0-149-1:4]:06/15/2023 12:21:26 [INFO|DP=31|PP=0|TP=0]: Global rank: 124/128 | PP: 0/1 | DP: 31/32 | TP: 0/4 +[ip-26-0-148-151:5]:06/15/2023 12:21:26 [INFO|DP=19|PP=0|TP=1]: Global rank: 77/128 | PP: 0/1 | DP: 19/32 | TP: 1/4 +[ip-26-0-148-151:6]:06/15/2023 12:21:26 [INFO|DP=19|PP=0|TP=2]: Global rank: 78/128 | PP: 0/1 | DP: 19/32 | TP: 2/4 +[ip-26-0-149-1:5]:06/15/2023 12:21:26 [INFO|DP=31|PP=0|TP=1]: Global rank: 125/128 | PP: 0/1 | DP: 31/32 | TP: 1/4 +[ip-26-0-149-1:2]:06/15/2023 12:21:26 [INFO|DP=30|PP=0|TP=2]: Global rank: 122/128 | PP: 0/1 | DP: 30/32 | TP: 2/4 +[ip-26-0-147-247:3]:06/15/2023 12:21:26 [INFO|DP=14|PP=0|TP=3]: Global rank: 59/128 | PP: 0/1 | DP: 14/32 | TP: 3/4 +[ip-26-0-147-247:0]:06/15/2023 12:21:26 [INFO|DP=14|PP=0|TP=0]: Global rank: 56/128 | PP: 0/1 | DP: 14/32 | TP: 0/4 +[ip-26-0-147-247:1]:06/15/2023 12:21:26 [INFO|DP=14|PP=0|TP=1]: Global rank: 57/128 | PP: 0/1 | DP: 14/32 | TP: 1/4 +[ip-26-0-148-151:1]:06/15/2023 12:21:26 [INFO|DP=18|PP=0|TP=1]: Global rank: 73/128 | PP: 0/1 | DP: 18/32 | TP: 1/4 +[ip-26-0-148-151:2]:06/15/2023 12:21:26 [INFO|DP=18|PP=0|TP=2]: Global rank: 74/128 | PP: 0/1 | DP: 18/32 | TP: 2/4 +[ip-26-0-147-187:0]:06/15/2023 12:21:26 [INFO|DP=2|PP=0|TP=0]: Global rank: 8/128 | PP: 0/1 | DP: 2/32 | TP: 0/4 +[ip-26-0-148-151:4]:06/15/2023 12:21:26 [INFO|DP=19|PP=0|TP=0]: Global rank: 76/128 | PP: 0/1 | DP: 19/32 | TP: 0/4 +[ip-26-0-147-233:7]:06/15/2023 12:21:26 [INFO|DP=11|PP=0|TP=3]: Global rank: 47/128 | PP: 0/1 | DP: 11/32 | TP: 3/4 +[ip-26-0-147-233:3]:06/15/2023 12:21:26 [INFO|DP=10|PP=0|TP=3]: Global rank: 43/128 | PP: 0/1 | DP: 10/32 | TP: 3/4 +[ip-26-0-147-233:1]:06/15/2023 12:21:26 [INFO|DP=10|PP=0|TP=1]: Global rank: 41/128 | PP: 0/1 | DP: 10/32 | TP: 1/4 +[ip-26-0-148-151:7]:06/15/2023 12:21:26 [INFO|DP=19|PP=0|TP=3]: Global rank: 79/128 | PP: 0/1 | DP: 19/32 | TP: 3/4 +[ip-26-0-147-233:5]:06/15/2023 12:21:26 [INFO|DP=11|PP=0|TP=1]: Global rank: 45/128 | PP: 0/1 | DP: 11/32 | TP: 1/4 +[ip-26-0-147-187:4]:06/15/2023 12:21:26 [INFO|DP=3|PP=0|TP=0]: Global rank: 12/128 | PP: 0/1 | DP: 3/32 | TP: 0/4 +[ip-26-0-148-170:5]:06/15/2023 12:21:26 [INFO|DP=21|PP=0|TP=1]: Global rank: 85/128 | PP: 0/1 | DP: 21/32 | TP: 1/4 +[ip-26-0-147-247:5]:06/15/2023 12:21:26 [INFO|DP=15|PP=0|TP=1]: Global rank: 61/128 | PP: 0/1 | DP: 15/32 | TP: 1/4 +[ip-26-0-148-151:0]:06/15/2023 12:21:26 [INFO|DP=18|PP=0|TP=0]: Global rank: 72/128 | PP: 0/1 | DP: 18/32 | TP: 0/4 +[ip-26-0-147-233:4]:06/15/2023 12:21:26 [INFO|DP=11|PP=0|TP=0]: Global rank: 44/128 | PP: 0/1 | DP: 11/32 | TP: 0/4 +[ip-26-0-148-245:0]:06/15/2023 12:21:26 [INFO|DP=24|PP=0|TP=0]: Global rank: 96/128 | PP: 0/1 | DP: 24/32 | TP: 0/4 +[ip-26-0-148-170:0]:06/15/2023 12:21:26 [INFO|DP=20|PP=0|TP=0]: Global rank: 80/128 | PP: 0/1 | DP: 20/32 | TP: 0/4 +[ip-26-0-148-170:1]:06/15/2023 12:21:26 [INFO|DP=20|PP=0|TP=1]: Global rank: 81/128 | PP: 0/1 | DP: 20/32 | TP: 1/4 +[ip-26-0-147-247:2]:06/15/2023 12:21:26 [INFO|DP=14|PP=0|TP=2]: Global rank: 58/128 | PP: 0/1 | DP: 14/32 | TP: 2/4 +[ip-26-0-147-247:7]:06/15/2023 12:21:26 [INFO|DP=15|PP=0|TP=3]: Global rank: 63/128 | PP: 0/1 | DP: 15/32 | TP: 3/4 +[ip-26-0-148-115:4]:06/15/2023 12:21:26 [INFO|DP=17|PP=0|TP=0]: Global rank: 68/128 | PP: 0/1 | DP: 17/32 | TP: 0/4 +[ip-26-0-147-247:4]:06/15/2023 12:21:26 [INFO|DP=15|PP=0|TP=0]: Global rank: 60/128 | PP: 0/1 | DP: 15/32 | TP: 0/4 +[ip-26-0-148-115:6]:06/15/2023 12:21:26 [INFO|DP=17|PP=0|TP=2]: Global rank: 70/128 | PP: 0/1 | DP: 17/32 | TP: 2/4 +[ip-26-0-148-93:5]:06/15/2023 12:21:26 [INFO|DP=29|PP=0|TP=1]: Global rank: 117/128 | PP: 0/1 | DP: 29/32 | TP: 1/4 +[ip-26-0-148-115:5]:06/15/2023 12:21:26 [INFO|DP=17|PP=0|TP=1]: Global rank: 69/128 | PP: 0/1 | DP: 17/32 | TP: 1/4 +[ip-26-0-147-247:6]:06/15/2023 12:21:26 [INFO|DP=15|PP=0|TP=2]: Global rank: 62/128 | PP: 0/1 | DP: 15/32 | TP: 2/4 +[ip-26-0-148-170:4]:06/15/2023 12:21:26 [INFO|DP=21|PP=0|TP=0]: Global rank: 84/128 | PP: 0/1 | DP: 21/32 | TP: 0/4 +[ip-26-0-148-170:7]:06/15/2023 12:21:26 [INFO|DP=21|PP=0|TP=3]: Global rank: 87/128 | PP: 0/1 | DP: 21/32 | TP: 3/4 +[ip-26-0-148-170:6]:06/15/2023 12:21:26 [INFO|DP=21|PP=0|TP=2]: Global rank: 86/128 | PP: 0/1 | DP: 21/32 | TP: 2/4 +[ip-26-0-148-245:2]:06/15/2023 12:21:26 [INFO|DP=24|PP=0|TP=2]: Global rank: 98/128 | PP: 0/1 | DP: 24/32 | TP: 2/4 +[ip-26-0-148-170:2]:06/15/2023 12:21:26 [INFO|DP=20|PP=0|TP=2]: Global rank: 82/128 | PP: 0/1 | DP: 20/32 | TP: 2/4 +[ip-26-0-148-93:2]:06/15/2023 12:21:26 [INFO|DP=28|PP=0|TP=2]: Global rank: 114/128 | PP: 0/1 | DP: 28/32 | TP: 2/4 +[ip-26-0-148-245:5]:06/15/2023 12:21:26 [INFO|DP=25|PP=0|TP=1]: Global rank: 101/128 | PP: 0/1 | DP: 25/32 | TP: 1/4 +[ip-26-0-148-170:3]:06/15/2023 12:21:26 [INFO|DP=20|PP=0|TP=3]: Global rank: 83/128 | PP: 0/1 | DP: 20/32 | TP: 3/4 +[ip-26-0-148-245:3]:06/15/2023 12:21:26 [INFO|DP=24|PP=0|TP=3]: Global rank: 99/128 | PP: 0/1 | DP: 24/32 | TP: 3/4 +[ip-26-0-148-93:4]:06/15/2023 12:21:26 [INFO|DP=29|PP=0|TP=0]: Global rank: 116/128 | PP: 0/1 | DP: 29/32 | TP: 0/4 +[ip-26-0-148-245:6]:06/15/2023 12:21:26 [INFO|DP=25|PP=0|TP=2]: Global rank: 102/128 | PP: 0/1 | DP: 25/32 | TP: 2/4 +[ip-26-0-148-93:0]:06/15/2023 12:21:26 [INFO|DP=28|PP=0|TP=0]: Global rank: 112/128 | PP: 0/1 | DP: 28/32 | TP: 0/4 +[ip-26-0-147-204:3]:06/15/2023 12:21:26 [INFO|DP=8|PP=0|TP=3]: Global rank: 35/128 | PP: 0/1 | DP: 8/32 | TP: 3/4 +[ip-26-0-147-204:6]:06/15/2023 12:21:26 [INFO|DP=9|PP=0|TP=2]: Global rank: 38/128 | PP: 0/1 | DP: 9/32 | TP: 2/4 +[ip-26-0-148-93:7]:06/15/2023 12:21:26 [INFO|DP=29|PP=0|TP=3]: Global rank: 119/128 | PP: 0/1 | DP: 29/32 | TP: 3/4 +[ip-26-0-148-245:7]:06/15/2023 12:21:26 [INFO|DP=25|PP=0|TP=3]: Global rank: 103/128 | PP: 0/1 | DP: 25/32 | TP: 3/4 +[ip-26-0-147-141:5]:06/15/2023 12:21:26 [INFO|DP=1|PP=0|TP=1]: Global rank: 5/128 | PP: 0/1 | DP: 1/32 | TP: 1/4 +[ip-26-0-148-93:1]:06/15/2023 12:21:26 [INFO|DP=28|PP=0|TP=1]: Global rank: 113/128 | PP: 0/1 | DP: 28/32 | TP: 1/4 +[ip-26-0-147-187:6]:06/15/2023 12:21:26 [INFO|DP=3|PP=0|TP=2]: Global rank: 14/128 | PP: 0/1 | DP: 3/32 | TP: 2/4 +[ip-26-0-148-93:6]:06/15/2023 12:21:26 [INFO|DP=29|PP=0|TP=2]: Global rank: 118/128 | PP: 0/1 | DP: 29/32 | TP: 2/4 +[ip-26-0-147-187:5]:06/15/2023 12:21:26 [INFO|DP=3|PP=0|TP=1]: Global rank: 13/128 | PP: 0/1 | DP: 3/32 | TP: 1/4 +[ip-26-0-147-187:7]:06/15/2023 12:21:26 [INFO|DP=3|PP=0|TP=3]: Global rank: 15/128 | PP: 0/1 | DP: 3/32 | TP: 3/4 +[ip-26-0-148-115:0]:06/15/2023 12:21:26 [INFO|DP=16|PP=0|TP=0]: Global rank: 64/128 | PP: 0/1 | DP: 16/32 | TP: 0/4 +[ip-26-0-147-204:5]:06/15/2023 12:21:26 [INFO|DP=9|PP=0|TP=1]: Global rank: 37/128 | PP: 0/1 | DP: 9/32 | TP: 1/4 +[ip-26-0-148-115:2]:06/15/2023 12:21:26 [INFO|DP=16|PP=0|TP=2]: Global rank: 66/128 | PP: 0/1 | DP: 16/32 | TP: 2/4 +[ip-26-0-147-233:2]:06/15/2023 12:21:26 [INFO|DP=10|PP=0|TP=2]: Global rank: 42/128 | PP: 0/1 | DP: 10/32 | TP: 2/4 +[ip-26-0-147-204:7]:06/15/2023 12:21:26 [INFO|DP=9|PP=0|TP=3]: Global rank: 39/128 | PP: 0/1 | DP: 9/32 | TP: 3/4 +[ip-26-0-147-233:0]:06/15/2023 12:21:26 [INFO|DP=10|PP=0|TP=0]: Global rank: 40/128 | PP: 0/1 | DP: 10/32 | TP: 0/4 +[ip-26-0-148-115:1]:06/15/2023 12:21:26 [INFO|DP=16|PP=0|TP=1]: Global rank: 65/128 | PP: 0/1 | DP: 16/32 | TP: 1/4 +[ip-26-0-147-187:1]:06/15/2023 12:21:26 [INFO|DP=2|PP=0|TP=1]: Global rank: 9/128 | PP: 0/1 | DP: 2/32 | TP: 1/4 +[ip-26-0-147-204:2]:06/15/2023 12:21:26 [INFO|DP=8|PP=0|TP=2]: Global rank: 34/128 | PP: 0/1 | DP: 8/32 | TP: 2/4 +[ip-26-0-147-204:0]:06/15/2023 12:21:26 [INFO|DP=8|PP=0|TP=0]: Global rank: 32/128 | PP: 0/1 | DP: 8/32 | TP: 0/4 +[ip-26-0-147-204:1]:06/15/2023 12:21:26 [INFO|DP=8|PP=0|TP=1]: Global rank: 33/128 | PP: 0/1 | DP: 8/32 | TP: 1/4 +[ip-26-0-147-204:4]:06/15/2023 12:21:26 [INFO|DP=9|PP=0|TP=0]: Global rank: 36/128 | PP: 0/1 | DP: 9/32 | TP: 0/4 +[ip-26-0-148-55:2]:06/15/2023 12:21:26 [INFO|DP=26|PP=0|TP=2]: Global rank: 106/128 | PP: 0/1 | DP: 26/32 | TP: 2/4 +[ip-26-0-148-55:6]:06/15/2023 12:21:26 [INFO|DP=27|PP=0|TP=2]: Global rank: 110/128 | PP: 0/1 | DP: 27/32 | TP: 2/4 +[ip-26-0-148-55:1]:06/15/2023 12:21:26 [INFO|DP=26|PP=0|TP=1]: Global rank: 105/128 | PP: 0/1 | DP: 26/32 | TP: 1/4 +[ip-26-0-147-187:3]:06/15/2023 12:21:26 [INFO|DP=2|PP=0|TP=3]: Global rank: 11/128 | PP: 0/1 | DP: 2/32 | TP: 3/4 +[ip-26-0-148-55:7]:06/15/2023 12:21:26 [INFO|DP=27|PP=0|TP=3]: Global rank: 111/128 | PP: 0/1 | DP: 27/32 | TP: 3/4 +[ip-26-0-148-55:5]:06/15/2023 12:21:26 [INFO|DP=27|PP=0|TP=1]: Global rank: 109/128 | PP: 0/1 | DP: 27/32 | TP: 1/4 +[ip-26-0-148-115:7]:06/15/2023 12:21:26 [INFO|DP=17|PP=0|TP=3]: Global rank: 71/128 | PP: 0/1 | DP: 17/32 | TP: 3/4 +[ip-26-0-148-115:3]:06/15/2023 12:21:26 [INFO|DP=16|PP=0|TP=3]: Global rank: 67/128 | PP: 0/1 | DP: 16/32 | TP: 3/4 +[ip-26-0-148-55:0]:06/15/2023 12:21:26 [INFO|DP=26|PP=0|TP=0]: Global rank: 104/128 | PP: 0/1 | DP: 26/32 | TP: 0/4 +[ip-26-0-147-187:2]:06/15/2023 12:21:26 [INFO|DP=2|PP=0|TP=2]: Global rank: 10/128 | PP: 0/1 | DP: 2/32 | TP: 2/4 +[ip-26-0-147-233:6]:06/15/2023 12:21:26 [INFO|DP=11|PP=0|TP=2]: Global rank: 46/128 | PP: 0/1 | DP: 11/32 | TP: 2/4 +[ip-26-0-147-189:0]:06/15/2023 12:21:26 [INFO|DP=4|PP=0|TP=0]: Global rank: 16/128 | PP: 0/1 | DP: 4/32 | TP: 0/4 +[ip-26-0-147-189:6]:06/15/2023 12:21:26 [INFO|DP=5|PP=0|TP=2]: Global rank: 22/128 | PP: 0/1 | DP: 5/32 | TP: 2/4 +[ip-26-0-147-189:3]:06/15/2023 12:21:26 [INFO|DP=4|PP=0|TP=3]: Global rank: 19/128 | PP: 0/1 | DP: 4/32 | TP: 3/4 +[ip-26-0-147-189:5]:06/15/2023 12:21:26 [INFO|DP=5|PP=0|TP=1]: Global rank: 21/128 | PP: 0/1 | DP: 5/32 | TP: 1/4 +[ip-26-0-148-55:4]:06/15/2023 12:21:26 [INFO|DP=27|PP=0|TP=0]: Global rank: 108/128 | PP: 0/1 | DP: 27/32 | TP: 0/4 +[ip-26-0-147-189:7]:06/15/2023 12:21:26 [INFO|DP=5|PP=0|TP=3]: Global rank: 23/128 | PP: 0/1 | DP: 5/32 | TP: 3/4 +[ip-26-0-147-189:4]:06/15/2023 12:21:26 [INFO|DP=5|PP=0|TP=0]: Global rank: 20/128 | PP: 0/1 | DP: 5/32 | TP: 0/4 +[ip-26-0-147-189:2]:06/15/2023 12:21:26 [INFO|DP=4|PP=0|TP=2]: Global rank: 18/128 | PP: 0/1 | DP: 4/32 | TP: 2/4 +[ip-26-0-147-189:1]:06/15/2023 12:21:26 [INFO|DP=4|PP=0|TP=1]: Global rank: 17/128 | PP: 0/1 | DP: 4/32 | TP: 1/4 +[ip-26-0-147-245:0]:06/15/2023 12:21:26 [INFO|DP=12|PP=0|TP=0]: Global rank: 48/128 | PP: 0/1 | DP: 12/32 | TP: 0/4 +[ip-26-0-148-55:3]:06/15/2023 12:21:26 [INFO|DP=26|PP=0|TP=3]: Global rank: 107/128 | PP: 0/1 | DP: 26/32 | TP: 3/4 +[ip-26-0-147-245:1]:06/15/2023 12:21:26 [INFO|DP=12|PP=0|TP=1]: Global rank: 49/128 | PP: 0/1 | DP: 12/32 | TP: 1/4 +[ip-26-0-147-245:6]:06/15/2023 12:21:26 [INFO|DP=13|PP=0|TP=2]: Global rank: 54/128 | PP: 0/1 | DP: 13/32 | TP: 2/4 +[ip-26-0-148-193:1]:06/15/2023 12:21:26 [INFO|DP=22|PP=0|TP=1]: Global rank: 89/128 | PP: 0/1 | DP: 22/32 | TP: 1/4 +[ip-26-0-147-245:3]:06/15/2023 12:21:26 [INFO|DP=12|PP=0|TP=3]: Global rank: 51/128 | PP: 0/1 | DP: 12/32 | TP: 3/4 +[ip-26-0-147-245:2]:06/15/2023 12:21:26 [INFO|DP=12|PP=0|TP=2]: Global rank: 50/128 | PP: 0/1 | DP: 12/32 | TP: 2/4 +[ip-26-0-147-245:5]:06/15/2023 12:21:26 [INFO|DP=13|PP=0|TP=1]: Global rank: 53/128 | PP: 0/1 | DP: 13/32 | TP: 1/4 +[ip-26-0-148-193:6]:06/15/2023 12:21:26 [INFO|DP=23|PP=0|TP=2]: Global rank: 94/128 | PP: 0/1 | DP: 23/32 | TP: 2/4 +[ip-26-0-148-193:4]:06/15/2023 12:21:26 [INFO|DP=23|PP=0|TP=0]: Global rank: 92/128 | PP: 0/1 | DP: 23/32 | TP: 0/4 +[ip-26-0-148-193:0]:06/15/2023 12:21:26 [INFO|DP=22|PP=0|TP=0]: Global rank: 88/128 | PP: 0/1 | DP: 22/32 | TP: 0/4 +[ip-26-0-148-193:7]:06/15/2023 12:21:26 [INFO|DP=23|PP=0|TP=3]: Global rank: 95/128 | PP: 0/1 | DP: 23/32 | TP: 3/4 +[ip-26-0-148-193:2]:06/15/2023 12:21:26 [INFO|DP=22|PP=0|TP=2]: Global rank: 90/128 | PP: 0/1 | DP: 22/32 | TP: 2/4 +[ip-26-0-148-193:3]:06/15/2023 12:21:26 [INFO|DP=22|PP=0|TP=3]: Global rank: 91/128 | PP: 0/1 | DP: 22/32 | TP: 3/4 +[ip-26-0-147-245:7]:06/15/2023 12:21:26 [INFO|DP=13|PP=0|TP=3]: Global rank: 55/128 | PP: 0/1 | DP: 13/32 | TP: 3/4 +[ip-26-0-147-245:4]:06/15/2023 12:21:26 [INFO|DP=13|PP=0|TP=0]: Global rank: 52/128 | PP: 0/1 | DP: 13/32 | TP: 0/4 +[ip-26-0-148-193:5]:06/15/2023 12:21:26 [INFO|DP=23|PP=0|TP=1]: Global rank: 93/128 | PP: 0/1 | DP: 23/32 | TP: 1/4 +[ip-26-0-147-193:0]:06/15/2023 12:21:26 [INFO|DP=6|PP=0|TP=0]: Global rank: 24/128 | PP: 0/1 | DP: 6/32 | TP: 0/4 +[ip-26-0-147-193:6]:06/15/2023 12:21:26 [INFO|DP=7|PP=0|TP=2]: Global rank: 30/128 | PP: 0/1 | DP: 7/32 | TP: 2/4 +[ip-26-0-147-193:3]:06/15/2023 12:21:26 [INFO|DP=6|PP=0|TP=3]: Global rank: 27/128 | PP: 0/1 | DP: 6/32 | TP: 3/4 +[ip-26-0-147-193:2]:06/15/2023 12:21:26 [INFO|DP=6|PP=0|TP=2]: Global rank: 26/128 | PP: 0/1 | DP: 6/32 | TP: 2/4 +[ip-26-0-147-193:4]:06/15/2023 12:21:26 [INFO|DP=7|PP=0|TP=0]: Global rank: 28/128 | PP: 0/1 | DP: 7/32 | TP: 0/4 +[ip-26-0-147-193:1]:06/15/2023 12:21:26 [INFO|DP=6|PP=0|TP=1]: Global rank: 25/128 | PP: 0/1 | DP: 6/32 | TP: 1/4 +[ip-26-0-147-193:7]:06/15/2023 12:21:26 [INFO|DP=7|PP=0|TP=3]: Global rank: 31/128 | PP: 0/1 | DP: 7/32 | TP: 3/4 +[ip-26-0-147-193:5]:06/15/2023 12:21:26 [INFO|DP=7|PP=0|TP=1]: Global rank: 29/128 | PP: 0/1 | DP: 7/32 | TP: 1/4 +[ip-26-0-147-141:7]:06/15/2023 12:21:26 [INFO|DP=1|PP=0|TP=3]: Global rank: 7/128 | PP: 0/1 | DP: 1/32 | TP: 3/4 +[ip-26-0-147-141:6]:06/15/2023 12:21:26 [INFO|DP=1|PP=0|TP=2]: Global rank: 6/128 | PP: 0/1 | DP: 1/32 | TP: 2/4 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/css/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/css/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/css/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 197931 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.003793 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: number of documents: 968 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 316) total of 316 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: document indices in [316, 642) total of 326 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: document indices in [642, 968) total of 326 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/prolog/gpt2-preprocessed_content_document_train_indexmap_199ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/prolog/gpt2-preprocessed_content_document_train_indexmap_199ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/prolog/gpt2-preprocessed_content_document_train_indexmap_199ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 336 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/prolog/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/prolog/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/prolog/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 135 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/prolog/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/prolog/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/prolog/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 157 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.001673 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: number of documents: 8536791 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 2787113) total of 2787113 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: document indices in [2787113, 5660514) total of 2873401 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: document indices in [5660514, 8536791) total of 2876277 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c/gpt2-preprocessed_content_document_train_indexmap_1072087ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c/gpt2-preprocessed_content_document_train_indexmap_1072087ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c/gpt2-preprocessed_content_document_train_indexmap_1072087ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1619255 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c/gpt2-preprocessed_content_document_valid_indexmap_868ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c/gpt2-preprocessed_content_document_valid_indexmap_868ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c/gpt2-preprocessed_content_document_valid_indexmap_868ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 833054 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 843540 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.002066 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: number of documents: 158792 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 51843) total of 51843 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: document indices in [51843, 105291) total of 53448 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: document indices in [105291, 158792) total of 53501 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/fortran/gpt2-preprocessed_content_document_train_indexmap_35412ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/fortran/gpt2-preprocessed_content_document_train_indexmap_35412ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/fortran/gpt2-preprocessed_content_document_train_indexmap_35412ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 53749 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/fortran/gpt2-preprocessed_content_document_valid_indexmap_29ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/fortran/gpt2-preprocessed_content_document_valid_indexmap_29ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/fortran/gpt2-preprocessed_content_document_valid_indexmap_29ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 27838 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/fortran/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/fortran/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/fortran/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 27546 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.001546 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: number of documents: 153194 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 50015) total of 50015 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: document indices in [50015, 101579) total of 51564 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: document indices in [101579, 153194) total of 51615 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/solidity/gpt2-preprocessed_content_document_train_indexmap_16910ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/solidity/gpt2-preprocessed_content_document_train_indexmap_16910ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/solidity/gpt2-preprocessed_content_document_train_indexmap_16910ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 22706 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/solidity/gpt2-preprocessed_content_document_valid_indexmap_14ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/solidity/gpt2-preprocessed_content_document_valid_indexmap_14ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/solidity/gpt2-preprocessed_content_document_valid_indexmap_14ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 11373 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/solidity/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/solidity/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/solidity/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 12126 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.005520 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: number of documents: 2239354 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 731110) total of 731110 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: document indices in [731110, 1484855) total of 753745 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: document indices in [1484855, 2239354) total of 754499 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/kotlin/gpt2-preprocessed_content_document_train_indexmap_112998ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/kotlin/gpt2-preprocessed_content_document_train_indexmap_112998ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/kotlin/gpt2-preprocessed_content_document_train_indexmap_112998ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 115558 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/kotlin/gpt2-preprocessed_content_document_valid_indexmap_92ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/kotlin/gpt2-preprocessed_content_document_valid_indexmap_92ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/kotlin/gpt2-preprocessed_content_document_valid_indexmap_92ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 59022 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/kotlin/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/kotlin/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/kotlin/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 59162 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.006306 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: number of documents: 523 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 171) total of 171 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: document indices in [171, 347) total of 176 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: document indices in [347, 523) total of 176 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-agda/gpt2-preprocessed_content_document_train_indexmap_199ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-agda/gpt2-preprocessed_content_document_train_indexmap_199ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-agda/gpt2-preprocessed_content_document_train_indexmap_199ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 255 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 3 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-agda/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-agda/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-agda/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 82 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-agda/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-agda/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-agda/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 75 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.001288 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: number of documents: 295364 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 96431) total of 96431 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: document indices in [96431, 195848) total of 99417 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: document indices in [195848, 295364) total of 99516 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/julia/gpt2-preprocessed_content_document_train_indexmap_26062ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/julia/gpt2-preprocessed_content_document_train_indexmap_26062ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/julia/gpt2-preprocessed_content_document_train_indexmap_26062ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 38582 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/julia/gpt2-preprocessed_content_document_valid_indexmap_22ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/julia/gpt2-preprocessed_content_document_valid_indexmap_22ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/julia/gpt2-preprocessed_content_document_valid_indexmap_22ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 19483 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/julia/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/julia/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/julia/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 19724 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.001931 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: number of documents: 210816 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 68827) total of 68827 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: document indices in [68827, 139786) total of 70959 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: document indices in [139786, 210816) total of 71030 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java-server-pages/gpt2-preprocessed_content_document_train_indexmap_19497ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java-server-pages/gpt2-preprocessed_content_document_train_indexmap_19497ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java-server-pages/gpt2-preprocessed_content_document_train_indexmap_19497ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 23179 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java-server-pages/gpt2-preprocessed_content_document_valid_indexmap_16ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java-server-pages/gpt2-preprocessed_content_document_valid_indexmap_16ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java-server-pages/gpt2-preprocessed_content_document_valid_indexmap_16ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 11792 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java-server-pages/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java-server-pages/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java-server-pages/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 11881 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.001231 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: number of documents: 5001 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 1633) total of 1633 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: document indices in [1633, 3316) total of 1683 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: document indices in [3316, 5001) total of 1685 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/isabelle/gpt2-preprocessed_content_document_train_indexmap_1592ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/isabelle/gpt2-preprocessed_content_document_train_indexmap_1592ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/isabelle/gpt2-preprocessed_content_document_train_indexmap_1592ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 2565 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/isabelle/gpt2-preprocessed_content_document_valid_indexmap_2ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/isabelle/gpt2-preprocessed_content_document_valid_indexmap_2ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/isabelle/gpt2-preprocessed_content_document_valid_indexmap_2ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1255 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/isabelle/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/isabelle/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/isabelle/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.011 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1256 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.003090 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: number of documents: 8042 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 2625) total of 2625 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: document indices in [2625, 5332) total of 2707 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: document indices in [5332, 8042) total of 2710 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/idris/gpt2-preprocessed_content_document_train_indexmap_597ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/idris/gpt2-preprocessed_content_document_train_indexmap_597ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/idris/gpt2-preprocessed_content_document_train_indexmap_597ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 797 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/idris/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/idris/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/idris/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 397 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/idris/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/idris/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/idris/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 395 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.002827 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: number of documents: 16870 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 5508) total of 5508 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: document indices in [5508, 11186) total of 5678 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: document indices in [11186, 16870) total of 5684 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lean/gpt2-preprocessed_content_document_train_indexmap_1791ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lean/gpt2-preprocessed_content_document_train_indexmap_1791ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lean/gpt2-preprocessed_content_document_train_indexmap_1791ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 3142 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lean/gpt2-preprocessed_content_document_valid_indexmap_2ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lean/gpt2-preprocessed_content_document_valid_indexmap_2ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lean/gpt2-preprocessed_content_document_valid_indexmap_2ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.042 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1568 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lean/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lean/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lean/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1522 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.001988 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: number of documents: 267627 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 87375) total of 87375 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: document indices in [87375, 177456) total of 90081 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: document indices in [177456, 267627) total of 90171 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/powershell/gpt2-preprocessed_content_document_train_indexmap_22282ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/powershell/gpt2-preprocessed_content_document_train_indexmap_22282ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/powershell/gpt2-preprocessed_content_document_train_indexmap_22282ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 22581 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/powershell/gpt2-preprocessed_content_document_valid_indexmap_19ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/powershell/gpt2-preprocessed_content_document_valid_indexmap_19ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/powershell/gpt2-preprocessed_content_document_valid_indexmap_19ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 12030 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/powershell/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/powershell/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/powershell/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 11686 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.002234 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: number of documents: 4700526 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 1534640) total of 1534640 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: document indices in [1534640, 3116791) total of 1582151 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: document indices in [3116791, 4700526) total of 1583735 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/go/gpt2-preprocessed_content_document_train_indexmap_473079ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/go/gpt2-preprocessed_content_document_train_indexmap_473079ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/go/gpt2-preprocessed_content_document_train_indexmap_473079ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 679828 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/go/gpt2-preprocessed_content_document_valid_indexmap_383ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/go/gpt2-preprocessed_content_document_valid_indexmap_383ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/go/gpt2-preprocessed_content_document_valid_indexmap_383ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 350499 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/go/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/go/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/go/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 349999 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.003933 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: number of documents: 98447 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 32142) total of 32142 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: document indices in [32142, 65278) total of 33136 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: document indices in [65278, 98447) total of 33169 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/erlang/gpt2-preprocessed_content_document_train_indexmap_13926ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/erlang/gpt2-preprocessed_content_document_train_indexmap_13926ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/erlang/gpt2-preprocessed_content_document_train_indexmap_13926ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 17794 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/erlang/gpt2-preprocessed_content_document_valid_indexmap_12ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/erlang/gpt2-preprocessed_content_document_valid_indexmap_12ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/erlang/gpt2-preprocessed_content_document_valid_indexmap_12ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 9455 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/erlang/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/erlang/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/erlang/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 9199 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.002829 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: number of documents: 124066 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 40506) total of 40506 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: document indices in [40506, 82265) total of 41759 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: document indices in [82265, 124066) total of 41801 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/f-sharp/gpt2-preprocessed_content_document_train_indexmap_12136ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/f-sharp/gpt2-preprocessed_content_document_train_indexmap_12136ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/f-sharp/gpt2-preprocessed_content_document_train_indexmap_12136ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 12957 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/f-sharp/gpt2-preprocessed_content_document_valid_indexmap_10ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/f-sharp/gpt2-preprocessed_content_document_valid_indexmap_10ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/f-sharp/gpt2-preprocessed_content_document_valid_indexmap_10ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.011 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 6943 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/f-sharp/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/f-sharp/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/f-sharp/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 6521 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.003764 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: number of documents: 30934 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 10099) total of 10099 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: document indices in [10099, 20511) total of 10412 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: document indices in [20511, 30934) total of 10423 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ada/gpt2-preprocessed_content_document_train_indexmap_5173ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ada/gpt2-preprocessed_content_document_train_indexmap_5173ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ada/gpt2-preprocessed_content_document_train_indexmap_5173ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 5597 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ada/gpt2-preprocessed_content_document_valid_indexmap_5ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ada/gpt2-preprocessed_content_document_valid_indexmap_5ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ada/gpt2-preprocessed_content_document_valid_indexmap_5ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 2939 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ada/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ada/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ada/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 2818 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.003087 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: number of documents: 110981 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 36233) total of 36233 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: document indices in [36233, 73588) total of 37355 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: document indices in [73588, 110981) total of 37393 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/pascal/gpt2-preprocessed_content_document_train_indexmap_33422ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/pascal/gpt2-preprocessed_content_document_train_indexmap_33422ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/pascal/gpt2-preprocessed_content_document_train_indexmap_33422ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 53606 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/pascal/gpt2-preprocessed_content_document_valid_indexmap_28ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/pascal/gpt2-preprocessed_content_document_valid_indexmap_28ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/pascal/gpt2-preprocessed_content_document_valid_indexmap_28ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 29260 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/pascal/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/pascal/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/pascal/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 27777 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.003123 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: number of documents: 365491 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 119326) total of 119326 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: document indices in [119326, 242347) total of 123021 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: document indices in [242347, 365491) total of 123144 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/perl/gpt2-preprocessed_content_document_train_indexmap_44364ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/perl/gpt2-preprocessed_content_document_train_indexmap_44364ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/perl/gpt2-preprocessed_content_document_train_indexmap_44364ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 64332 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/perl/gpt2-preprocessed_content_document_valid_indexmap_36ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/perl/gpt2-preprocessed_content_document_valid_indexmap_36ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/perl/gpt2-preprocessed_content_document_valid_indexmap_36ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 33791 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/perl/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/perl/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/perl/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 33144 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.002089 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: number of documents: 39042 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 12747) total of 12747 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: document indices in [12747, 25888) total of 13141 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: document indices in [25888, 39042) total of 13154 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/r/gpt2-preprocessed_content_document_train_indexmap_5969ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/r/gpt2-preprocessed_content_document_train_indexmap_5969ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/r/gpt2-preprocessed_content_document_train_indexmap_5969ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 8357 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/r/gpt2-preprocessed_content_document_valid_indexmap_5ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/r/gpt2-preprocessed_content_document_valid_indexmap_5ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/r/gpt2-preprocessed_content_document_valid_indexmap_5ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 4310 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/r/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/r/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/r/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 4206 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.002168 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: number of documents: 97167 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 31724) total of 31724 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: document indices in [31724, 64429) total of 32705 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: document indices in [64429, 97167) total of 32738 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/protocol-buffer/gpt2-preprocessed_content_document_train_indexmap_6168ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/protocol-buffer/gpt2-preprocessed_content_document_train_indexmap_6168ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/protocol-buffer/gpt2-preprocessed_content_document_train_indexmap_6168ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 7976 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/protocol-buffer/gpt2-preprocessed_content_document_valid_indexmap_5ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/protocol-buffer/gpt2-preprocessed_content_document_valid_indexmap_5ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/protocol-buffer/gpt2-preprocessed_content_document_valid_indexmap_5ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.011 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 4021 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/protocol-buffer/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/protocol-buffer/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/protocol-buffer/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 4218 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.003626 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: number of documents: 186375 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 60848) total of 60848 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: document indices in [60848, 123580) total of 62732 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: document indices in [123580, 186375) total of 62795 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cmake/gpt2-preprocessed_content_document_train_indexmap_8953ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cmake/gpt2-preprocessed_content_document_train_indexmap_8953ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cmake/gpt2-preprocessed_content_document_train_indexmap_8953ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.011 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 12190 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cmake/gpt2-preprocessed_content_document_valid_indexmap_8ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cmake/gpt2-preprocessed_content_document_valid_indexmap_8ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cmake/gpt2-preprocessed_content_document_valid_indexmap_8ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 6104 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cmake/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cmake/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cmake/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 6249 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.001561 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: number of documents: 9226 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 3013) total of 3013 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: document indices in [3013, 6118) total of 3105 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: document indices in [6118, 9226) total of 3108 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sas/gpt2-preprocessed_content_document_train_indexmap_2388ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sas/gpt2-preprocessed_content_document_train_indexmap_2388ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sas/gpt2-preprocessed_content_document_train_indexmap_2388ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 4469 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sas/gpt2-preprocessed_content_document_valid_indexmap_2ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sas/gpt2-preprocessed_content_document_valid_indexmap_2ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sas/gpt2-preprocessed_content_document_valid_indexmap_2ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1942 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sas/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sas/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sas/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 2227 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.002272 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: number of documents: 3390320 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 1106880) total of 1106880 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: document indices in [1106880, 2248029) total of 1141149 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: document indices in [2248029, 3390320) total of 1142291 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ruby/gpt2-preprocessed_content_document_train_indexmap_135479ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ruby/gpt2-preprocessed_content_document_train_indexmap_135479ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ruby/gpt2-preprocessed_content_document_train_indexmap_135479ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 161899 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ruby/gpt2-preprocessed_content_document_valid_indexmap_110ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ruby/gpt2-preprocessed_content_document_valid_indexmap_110ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ruby/gpt2-preprocessed_content_document_valid_indexmap_110ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 81430 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ruby/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ruby/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ruby/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 82121 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.001870 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: number of documents: 1380468 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 450699) total of 450699 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: document indices in [450699, 915351) total of 464652 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: document indices in [915351, 1380468) total of 465117 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rust/gpt2-preprocessed_content_document_train_indexmap_181235ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rust/gpt2-preprocessed_content_document_train_indexmap_181235ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rust/gpt2-preprocessed_content_document_train_indexmap_181235ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 214187 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rust/gpt2-preprocessed_content_document_valid_indexmap_147ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rust/gpt2-preprocessed_content_document_valid_indexmap_147ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rust/gpt2-preprocessed_content_document_valid_indexmap_147ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 110422 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rust/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rust/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rust/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 110680 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.004217 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: number of documents: 5386 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 1758) total of 1758 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: document indices in [1758, 3571) total of 1813 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: document indices in [3571, 5386) total of 1815 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rmarkdown/gpt2-preprocessed_content_document_train_indexmap_1194ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rmarkdown/gpt2-preprocessed_content_document_train_indexmap_1194ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rmarkdown/gpt2-preprocessed_content_document_train_indexmap_1194ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1584 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rmarkdown/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rmarkdown/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rmarkdown/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 816 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rmarkdown/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rmarkdown/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rmarkdown/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 776 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.001454 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: number of documents: 10801285 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 3526430) total of 3526430 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: document indices in [3526430, 7162038) total of 3635608 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: document indices in [7162038, 10801285) total of 3639247 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c-sharp/gpt2-preprocessed_content_document_train_indexmap_888466ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c-sharp/gpt2-preprocessed_content_document_train_indexmap_888466ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c-sharp/gpt2-preprocessed_content_document_train_indexmap_888466ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1254025 +[ip-26-0-147-141:0]:06/15/2023 12:21:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 3 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c-sharp/gpt2-preprocessed_content_document_valid_indexmap_720ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c-sharp/gpt2-preprocessed_content_document_valid_indexmap_720ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c-sharp/gpt2-preprocessed_content_document_valid_indexmap_720ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 430672 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c-sharp/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c-sharp/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c-sharp/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 429969 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.003251 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: number of documents: 587748 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 191890) total of 191890 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [191890, 389720) total of 197830 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [389720, 587748) total of 198028 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/smalltalk/gpt2-preprocessed_content_document_train_indexmap_11539ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/smalltalk/gpt2-preprocessed_content_document_train_indexmap_11539ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/smalltalk/gpt2-preprocessed_content_document_train_indexmap_11539ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 15786 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/smalltalk/gpt2-preprocessed_content_document_valid_indexmap_10ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/smalltalk/gpt2-preprocessed_content_document_valid_indexmap_10ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/smalltalk/gpt2-preprocessed_content_document_valid_indexmap_10ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 7945 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/smalltalk/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/smalltalk/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/smalltalk/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 8334 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.002932 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: number of documents: 541454 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 176775) total of 176775 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [176775, 359023) total of 182248 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [359023, 541454) total of 182431 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/haskell/gpt2-preprocessed_content_document_train_indexmap_44364ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/haskell/gpt2-preprocessed_content_document_train_indexmap_44364ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/haskell/gpt2-preprocessed_content_document_train_indexmap_44364ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 52419 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/haskell/gpt2-preprocessed_content_document_valid_indexmap_36ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/haskell/gpt2-preprocessed_content_document_valid_indexmap_36ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/haskell/gpt2-preprocessed_content_document_valid_indexmap_36ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 26827 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/haskell/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/haskell/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/haskell/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 26582 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.000497 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: number of documents: 1152 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 376) total of 376 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [376, 764) total of 388 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [764, 1152) total of 388 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/maple/gpt2-preprocessed_content_document_train_indexmap_199ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/maple/gpt2-preprocessed_content_document_train_indexmap_199ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/maple/gpt2-preprocessed_content_document_train_indexmap_199ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 209 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 3 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/maple/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/maple/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/maple/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 71 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/maple/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/maple/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/maple/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 58 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.000612 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: number of documents: 22653 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 7396) total of 7396 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [7396, 15021) total of 7625 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [15021, 22653) total of 7632 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/mathematica/gpt2-preprocessed_content_document_train_indexmap_24868ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/mathematica/gpt2-preprocessed_content_document_train_indexmap_24868ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/mathematica/gpt2-preprocessed_content_document_train_indexmap_24868ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 39578 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/mathematica/gpt2-preprocessed_content_document_valid_indexmap_21ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/mathematica/gpt2-preprocessed_content_document_valid_indexmap_21ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/mathematica/gpt2-preprocessed_content_document_valid_indexmap_21ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 21128 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/mathematica/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/mathematica/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/mathematica/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 21473 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.002305 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: number of documents: 158356 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 51701) total of 51701 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [51701, 105002) total of 53301 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [105002, 158356) total of 53354 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ocaml/gpt2-preprocessed_content_document_train_indexmap_20491ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ocaml/gpt2-preprocessed_content_document_train_indexmap_20491ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ocaml/gpt2-preprocessed_content_document_train_indexmap_20491ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 27268 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ocaml/gpt2-preprocessed_content_document_valid_indexmap_17ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ocaml/gpt2-preprocessed_content_document_valid_indexmap_17ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ocaml/gpt2-preprocessed_content_document_valid_indexmap_17ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 13487 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ocaml/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ocaml/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ocaml/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 13675 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.003232 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: number of documents: 657349 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 214613) total of 214613 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [214613, 435870) total of 221257 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [435870, 657349) total of 221479 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/makefile/gpt2-preprocessed_content_document_train_indexmap_26062ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/makefile/gpt2-preprocessed_content_document_train_indexmap_26062ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/makefile/gpt2-preprocessed_content_document_train_indexmap_26062ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 39676 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/makefile/gpt2-preprocessed_content_document_valid_indexmap_22ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/makefile/gpt2-preprocessed_content_document_valid_indexmap_22ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/makefile/gpt2-preprocessed_content_document_valid_indexmap_22ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 20452 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/makefile/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/makefile/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/makefile/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 20653 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.003805 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: number of documents: 549459 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 179388) total of 179388 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [179388, 364331) total of 184943 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [364331, 549459) total of 185128 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lua/gpt2-preprocessed_content_document_train_indexmap_57096ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lua/gpt2-preprocessed_content_document_train_indexmap_57096ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lua/gpt2-preprocessed_content_document_train_indexmap_57096ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 81936 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lua/gpt2-preprocessed_content_document_valid_indexmap_47ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lua/gpt2-preprocessed_content_document_valid_indexmap_47ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lua/gpt2-preprocessed_content_document_valid_indexmap_47ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 41600 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lua/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lua/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lua/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 42207 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.001941 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: number of documents: 1133 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 370) total of 370 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [370, 751) total of 381 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [751, 1133) total of 382 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-coffeescript/gpt2-preprocessed_content_document_train_indexmap_199ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-coffeescript/gpt2-preprocessed_content_document_train_indexmap_199ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-coffeescript/gpt2-preprocessed_content_document_train_indexmap_199ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 207 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 4 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-coffeescript/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-coffeescript/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-coffeescript/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 53 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-coffeescript/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-coffeescript/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-coffeescript/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 49 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.002468 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: number of documents: 6104 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 1992) total of 1992 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [1992, 4047) total of 2055 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [4047, 6104) total of 2057 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-haskell/gpt2-preprocessed_content_document_train_indexmap_995ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-haskell/gpt2-preprocessed_content_document_train_indexmap_995ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-haskell/gpt2-preprocessed_content_document_train_indexmap_995ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1341 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-haskell/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-haskell/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-haskell/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 696 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-haskell/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-haskell/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-haskell/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 660 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.003418 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: number of documents: 896880 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 292816) total of 292816 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [292816, 594697) total of 301881 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [594697, 896880) total of 302183 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/restructuredtext/gpt2-preprocessed_content_document_train_indexmap_66049ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/restructuredtext/gpt2-preprocessed_content_document_train_indexmap_66049ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/restructuredtext/gpt2-preprocessed_content_document_train_indexmap_66049ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 84311 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/restructuredtext/gpt2-preprocessed_content_document_valid_indexmap_54ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/restructuredtext/gpt2-preprocessed_content_document_valid_indexmap_54ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/restructuredtext/gpt2-preprocessed_content_document_valid_indexmap_54ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 42767 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/restructuredtext/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/restructuredtext/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/restructuredtext/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 42618 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.000572 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: number of documents: 3688 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 1204) total of 1204 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [1204, 2445) total of 1241 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [2445, 3688) total of 1243 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/racket/gpt2-preprocessed_content_document_train_indexmap_597ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/racket/gpt2-preprocessed_content_document_train_indexmap_597ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/racket/gpt2-preprocessed_content_document_train_indexmap_597ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 891 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 3 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/racket/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/racket/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/racket/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 347 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/racket/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/racket/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/racket/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 301 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.000576 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: number of documents: 19630 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 6409) total of 6409 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [6409, 13016) total of 6607 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [13016, 19630) total of 6614 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/standard-ml/gpt2-preprocessed_content_document_train_indexmap_3780ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/standard-ml/gpt2-preprocessed_content_document_train_indexmap_3780ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/standard-ml/gpt2-preprocessed_content_document_train_indexmap_3780ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 5494 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/standard-ml/gpt2-preprocessed_content_document_valid_indexmap_4ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/standard-ml/gpt2-preprocessed_content_document_valid_indexmap_4ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/standard-ml/gpt2-preprocessed_content_document_valid_indexmap_4ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 2812 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/standard-ml/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/standard-ml/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/standard-ml/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 2578 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.003384 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: number of documents: 46270 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 15106) total of 15106 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [15106, 30680) total of 15574 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [30680, 46270) total of 15590 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/systemverilog/gpt2-preprocessed_content_document_train_indexmap_7759ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/systemverilog/gpt2-preprocessed_content_document_train_indexmap_7759ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/systemverilog/gpt2-preprocessed_content_document_train_indexmap_7759ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 11740 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/systemverilog/gpt2-preprocessed_content_document_valid_indexmap_7ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/systemverilog/gpt2-preprocessed_content_document_valid_indexmap_7ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/systemverilog/gpt2-preprocessed_content_document_valid_indexmap_7ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 6106 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/systemverilog/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/systemverilog/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/systemverilog/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 6324 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.003550 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: number of documents: 522778 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 170678) total of 170678 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [170678, 346640) total of 175962 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [346640, 522778) total of 176138 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tex/gpt2-preprocessed_content_document_train_indexmap_103449ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tex/gpt2-preprocessed_content_document_train_indexmap_103449ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tex/gpt2-preprocessed_content_document_train_indexmap_103449ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 149597 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tex/gpt2-preprocessed_content_document_valid_indexmap_84ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tex/gpt2-preprocessed_content_document_valid_indexmap_84ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tex/gpt2-preprocessed_content_document_valid_indexmap_84ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 78658 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tex/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tex/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tex/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 77461 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.003439 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: number of documents: 10289 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 3359) total of 3359 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [3359, 6822) total of 3463 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [6822, 10289) total of 3467 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/awk/gpt2-preprocessed_content_document_train_indexmap_398ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/awk/gpt2-preprocessed_content_document_train_indexmap_398ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/awk/gpt2-preprocessed_content_document_train_indexmap_398ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 623 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/awk/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/awk/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/awk/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 377 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/awk/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/awk/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/awk/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 312 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.002055 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: number of documents: 247919 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 80941) total of 80941 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [80941, 164388) total of 83447 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [164388, 247919) total of 83531 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/assembly/gpt2-preprocessed_content_document_train_indexmap_31035ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/assembly/gpt2-preprocessed_content_document_train_indexmap_31035ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/assembly/gpt2-preprocessed_content_document_train_indexmap_31035ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 32911 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/assembly/gpt2-preprocessed_content_document_valid_indexmap_26ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/assembly/gpt2-preprocessed_content_document_valid_indexmap_26ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/assembly/gpt2-preprocessed_content_document_valid_indexmap_26ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 31936 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/assembly/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/assembly/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/assembly/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 32667 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.005008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: number of documents: 5368 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 1752) total of 1752 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [1752, 3559) total of 1807 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [3559, 5368) total of 1809 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/alloy/gpt2-preprocessed_content_document_train_indexmap_199ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/alloy/gpt2-preprocessed_content_document_train_indexmap_199ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/alloy/gpt2-preprocessed_content_document_train_indexmap_199ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 253 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/alloy/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/alloy/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/alloy/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.012 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 137 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/alloy/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/alloy/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/alloy/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 117 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.003975 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: number of documents: 17554 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 5731) total of 5731 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [5731, 11640) total of 5909 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [11640, 17554) total of 5914 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/agda/gpt2-preprocessed_content_document_train_indexmap_1393ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/agda/gpt2-preprocessed_content_document_train_indexmap_1393ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/agda/gpt2-preprocessed_content_document_train_indexmap_1393ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 2751 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/agda/gpt2-preprocessed_content_document_valid_indexmap_2ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/agda/gpt2-preprocessed_content_document_valid_indexmap_2ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/agda/gpt2-preprocessed_content_document_valid_indexmap_2ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.011 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1328 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/agda/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/agda/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/agda/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1277 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.003052 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: number of documents: 52838 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 17250) total of 17250 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [17250, 35035) total of 17785 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: document indices in [35035, 52838) total of 17803 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/emacs-lisp/gpt2-preprocessed_content_document_train_indexmap_8157ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/emacs-lisp/gpt2-preprocessed_content_document_train_indexmap_8157ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/emacs-lisp/gpt2-preprocessed_content_document_train_indexmap_8157ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 10038 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/emacs-lisp/gpt2-preprocessed_content_document_valid_indexmap_7ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/emacs-lisp/gpt2-preprocessed_content_document_valid_indexmap_7ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/emacs-lisp/gpt2-preprocessed_content_document_valid_indexmap_7ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 5153 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/emacs-lisp/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/emacs-lisp/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/emacs-lisp/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 5279 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.001215 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: number of documents: 928415 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 303112) total of 303112 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [303112, 615607) total of 312495 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [615607, 928415) total of 312808 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dart/gpt2-preprocessed_content_document_train_indexmap_72812ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dart/gpt2-preprocessed_content_document_train_indexmap_72812ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dart/gpt2-preprocessed_content_document_train_indexmap_72812ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 74412 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dart/gpt2-preprocessed_content_document_valid_indexmap_59ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dart/gpt2-preprocessed_content_document_valid_indexmap_59ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dart/gpt2-preprocessed_content_document_valid_indexmap_59ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 38815 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dart/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dart/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dart/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 38423 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.003638 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: number of documents: 58151 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 18985) total of 18985 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [18985, 38558) total of 19573 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [38558, 58151) total of 19593 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cuda/gpt2-preprocessed_content_document_train_indexmap_11141ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cuda/gpt2-preprocessed_content_document_train_indexmap_11141ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cuda/gpt2-preprocessed_content_document_train_indexmap_11141ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 15246 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cuda/gpt2-preprocessed_content_document_valid_indexmap_10ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cuda/gpt2-preprocessed_content_document_valid_indexmap_10ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cuda/gpt2-preprocessed_content_document_valid_indexmap_10ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 7892 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cuda/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cuda/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cuda/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 7791 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.002030 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: number of documents: 5928 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 1936) total of 1936 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [1936, 3931) total of 1995 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [3931, 5928) total of 1997 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/bluespec/gpt2-preprocessed_content_document_train_indexmap_597ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/bluespec/gpt2-preprocessed_content_document_train_indexmap_597ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/bluespec/gpt2-preprocessed_content_document_train_indexmap_597ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 801 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/bluespec/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/bluespec/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/bluespec/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.003 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 370 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/bluespec/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/bluespec/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/bluespec/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 510 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.001606 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: number of documents: 180 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 58) total of 58 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [58, 119) total of 61 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [119, 180) total of 61 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/augeas/gpt2-preprocessed_content_document_train_indexmap_20ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/augeas/gpt2-preprocessed_content_document_train_indexmap_20ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/augeas/gpt2-preprocessed_content_document_train_indexmap_20ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 21 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 4 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/augeas/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/augeas/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/augeas/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 9 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/augeas/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/augeas/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/augeas/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 9 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.001278 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: number of documents: 239568 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 78215) total of 78215 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [78215, 158851) total of 80636 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [158851, 239568) total of 80717 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/batchfile/gpt2-preprocessed_content_document_train_indexmap_4576ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/batchfile/gpt2-preprocessed_content_document_train_indexmap_4576ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/batchfile/gpt2-preprocessed_content_document_train_indexmap_4576ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.011 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 7142 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/batchfile/gpt2-preprocessed_content_document_valid_indexmap_4ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/batchfile/gpt2-preprocessed_content_document_valid_indexmap_4ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/batchfile/gpt2-preprocessed_content_document_valid_indexmap_4ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 4062 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/batchfile/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/batchfile/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/batchfile/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.012 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 4032 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.001633 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: number of documents: 4806 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 1569) total of 1569 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [1569, 3187) total of 1618 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [3187, 4806) total of 1619 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcsh/gpt2-preprocessed_content_document_train_indexmap_398ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcsh/gpt2-preprocessed_content_document_train_indexmap_398ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcsh/gpt2-preprocessed_content_document_train_indexmap_398ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 524 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcsh/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcsh/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcsh/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 254 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcsh/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcsh/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcsh/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 284 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.001472 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: number of documents: 5429 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 1773) total of 1773 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [1773, 3600) total of 1827 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [3600, 5429) total of 1829 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stan/gpt2-preprocessed_content_document_train_indexmap_199ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stan/gpt2-preprocessed_content_document_train_indexmap_199ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stan/gpt2-preprocessed_content_document_train_indexmap_199ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.011 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 395 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stan/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stan/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stan/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 238 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stan/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stan/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stan/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 215 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.002406 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: number of documents: 1355788 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 442641) total of 442641 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [442641, 898986) total of 456345 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [898986, 1355788) total of 456802 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scala/gpt2-preprocessed_content_document_train_indexmap_93303ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scala/gpt2-preprocessed_content_document_train_indexmap_93303ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scala/gpt2-preprocessed_content_document_train_indexmap_93303ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 103367 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scala/gpt2-preprocessed_content_document_valid_indexmap_76ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scala/gpt2-preprocessed_content_document_valid_indexmap_76ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scala/gpt2-preprocessed_content_document_valid_indexmap_76ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 53519 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scala/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scala/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scala/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 53423 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.000803 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: number of documents: 49335 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 16107) total of 16107 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [16107, 32713) total of 16606 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [32713, 49335) total of 16622 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcl/gpt2-preprocessed_content_document_train_indexmap_6963ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcl/gpt2-preprocessed_content_document_train_indexmap_6963ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcl/gpt2-preprocessed_content_document_train_indexmap_6963ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 9597 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcl/gpt2-preprocessed_content_document_valid_indexmap_6ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcl/gpt2-preprocessed_content_document_valid_indexmap_6ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcl/gpt2-preprocessed_content_document_valid_indexmap_6ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 5123 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcl/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcl/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcl/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 5055 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.000833 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: number of documents: 24208 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 7904) total of 7904 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [7904, 16052) total of 8148 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [16052, 24208) total of 8156 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stata/gpt2-preprocessed_content_document_train_indexmap_6566ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stata/gpt2-preprocessed_content_document_train_indexmap_6566ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stata/gpt2-preprocessed_content_document_train_indexmap_6566ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 7506 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stata/gpt2-preprocessed_content_document_valid_indexmap_6ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stata/gpt2-preprocessed_content_document_valid_indexmap_6ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stata/gpt2-preprocessed_content_document_valid_indexmap_6ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 9094 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stata/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stata/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stata/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 9893 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.000572 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: number of documents: 4737 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 1547) total of 1547 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [1547, 3141) total of 1594 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [3141, 4737) total of 1596 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/applescript/gpt2-preprocessed_content_document_train_indexmap_199ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/applescript/gpt2-preprocessed_content_document_train_indexmap_199ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/applescript/gpt2-preprocessed_content_document_train_indexmap_199ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 221 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/applescript/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/applescript/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/applescript/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 103 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/applescript/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/applescript/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/applescript/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 102 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.003286 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: number of documents: 2206327 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 720327) total of 720327 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [720327, 1462955) total of 742628 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [1462955, 2206327) total of 743372 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/shell/gpt2-preprocessed_content_document_train_indexmap_61473ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/shell/gpt2-preprocessed_content_document_train_indexmap_61473ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/shell/gpt2-preprocessed_content_document_train_indexmap_61473ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 86412 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/shell/gpt2-preprocessed_content_document_valid_indexmap_50ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/shell/gpt2-preprocessed_content_document_valid_indexmap_50ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/shell/gpt2-preprocessed_content_document_valid_indexmap_50ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 44088 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/shell/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/shell/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/shell/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 44638 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.002559 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: number of documents: 125163 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 40863) total of 40863 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [40863, 82992) total of 42129 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [82992, 125163) total of 42171 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/clojure/gpt2-preprocessed_content_document_train_indexmap_9152ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/clojure/gpt2-preprocessed_content_document_train_indexmap_9152ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/clojure/gpt2-preprocessed_content_document_train_indexmap_9152ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.042 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 10752 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/clojure/gpt2-preprocessed_content_document_valid_indexmap_8ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/clojure/gpt2-preprocessed_content_document_valid_indexmap_8ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/clojure/gpt2-preprocessed_content_document_valid_indexmap_8ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.012 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 5432 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/clojure/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/clojure/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/clojure/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 5607 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.000679 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: number of documents: 41890 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 13676) total of 13676 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [13676, 27776) total of 14100 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [27776, 41890) total of 14114 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scheme/gpt2-preprocessed_content_document_train_indexmap_3979ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scheme/gpt2-preprocessed_content_document_train_indexmap_3979ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scheme/gpt2-preprocessed_content_document_train_indexmap_3979ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 5078 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scheme/gpt2-preprocessed_content_document_valid_indexmap_4ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scheme/gpt2-preprocessed_content_document_valid_indexmap_4ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scheme/gpt2-preprocessed_content_document_valid_indexmap_4ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 2583 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scheme/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scheme/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scheme/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 2873 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.002409 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: number of documents: 7917 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 2585) total of 2585 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [2585, 5250) total of 2665 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [5250, 7917) total of 2667 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/antlr/gpt2-preprocessed_content_document_train_indexmap_995ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/antlr/gpt2-preprocessed_content_document_train_indexmap_995ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/antlr/gpt2-preprocessed_content_document_train_indexmap_995ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1503 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/antlr/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/antlr/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/antlr/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 692 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/antlr/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/antlr/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/antlr/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 720 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.001859 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: number of documents: 13716 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 4478) total of 4478 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [4478, 9095) total of 4617 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [9095, 13716) total of 4621 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sparql/gpt2-preprocessed_content_document_train_indexmap_796ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sparql/gpt2-preprocessed_content_document_train_indexmap_796ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sparql/gpt2-preprocessed_content_document_train_indexmap_796ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1284 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sparql/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sparql/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sparql/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.011 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 678 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sparql/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sparql/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sparql/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.005 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 622 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.001160 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: number of documents: 975420 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 318457) total of 318457 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [318457, 646774) total of 328317 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [646774, 975420) total of 328646 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sql/gpt2-preprocessed_content_document_train_indexmap_220625ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sql/gpt2-preprocessed_content_document_train_indexmap_220625ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sql/gpt2-preprocessed_content_document_train_indexmap_220625ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 220816 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sql/gpt2-preprocessed_content_document_valid_indexmap_179ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sql/gpt2-preprocessed_content_document_valid_indexmap_179ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sql/gpt2-preprocessed_content_document_valid_indexmap_179ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 220204 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sql/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sql/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sql/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 223151 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.002860 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: number of documents: 167701 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 54751) total of 54751 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [54751, 111198) total of 56447 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [111198, 167701) total of 56503 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/glsl/gpt2-preprocessed_content_document_train_indexmap_7958ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/glsl/gpt2-preprocessed_content_document_train_indexmap_7958ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/glsl/gpt2-preprocessed_content_document_train_indexmap_7958ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 13241 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/glsl/gpt2-preprocessed_content_document_valid_indexmap_7ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/glsl/gpt2-preprocessed_content_document_valid_indexmap_7ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/glsl/gpt2-preprocessed_content_document_valid_indexmap_7ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 7937 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/glsl/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/glsl/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/glsl/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 6885 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.004448 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: number of documents: 62033 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 20252) total of 20252 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [20252, 41132) total of 20880 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [41132, 62033) total of 20901 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elm/gpt2-preprocessed_content_document_train_indexmap_5969ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elm/gpt2-preprocessed_content_document_train_indexmap_5969ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elm/gpt2-preprocessed_content_document_train_indexmap_5969ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 6169 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elm/gpt2-preprocessed_content_document_valid_indexmap_5ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elm/gpt2-preprocessed_content_document_valid_indexmap_5ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elm/gpt2-preprocessed_content_document_valid_indexmap_5ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 3168 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elm/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elm/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elm/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 2971 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.002376 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: number of documents: 571506 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 186587) total of 186587 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [186587, 378950) total of 192363 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [378950, 571506) total of 192556 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dockerfile/gpt2-preprocessed_content_document_train_indexmap_8356ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dockerfile/gpt2-preprocessed_content_document_train_indexmap_8356ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dockerfile/gpt2-preprocessed_content_document_train_indexmap_8356ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 11744 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dockerfile/gpt2-preprocessed_content_document_valid_indexmap_7ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dockerfile/gpt2-preprocessed_content_document_valid_indexmap_7ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dockerfile/gpt2-preprocessed_content_document_valid_indexmap_7ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 6044 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dockerfile/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dockerfile/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dockerfile/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 6002 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.001623 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: number of documents: 6353527 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 2074315) total of 2074315 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [2074315, 4212851) total of 2138536 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [4212851, 6353527) total of 2140676 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cpp/gpt2-preprocessed_content_document_train_indexmap_973214ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cpp/gpt2-preprocessed_content_document_train_indexmap_973214ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cpp/gpt2-preprocessed_content_document_train_indexmap_973214ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1288594 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cpp/gpt2-preprocessed_content_document_valid_indexmap_788ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cpp/gpt2-preprocessed_content_document_valid_indexmap_788ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cpp/gpt2-preprocessed_content_document_valid_indexmap_788ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 662447 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cpp/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cpp/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cpp/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.011 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 667385 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.001648 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: number of documents: 226209 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 73853) total of 73853 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [73853, 149993) total of 76140 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: document indices in [149993, 226209) total of 76216 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/coffeescript/gpt2-preprocessed_content_document_train_indexmap_12733ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/coffeescript/gpt2-preprocessed_content_document_train_indexmap_12733ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/coffeescript/gpt2-preprocessed_content_document_train_indexmap_12733ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 14695 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/coffeescript/gpt2-preprocessed_content_document_valid_indexmap_11ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/coffeescript/gpt2-preprocessed_content_document_valid_indexmap_11ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/coffeescript/gpt2-preprocessed_content_document_valid_indexmap_11ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 7544 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/coffeescript/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/coffeescript/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/coffeescript/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 7711 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:29 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.003495 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: number of documents: 98733 +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 32234) total of 32234 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: document indices in [32234, 65467) total of 33233 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: document indices in [65467, 98733) total of 33266 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/common-lisp/gpt2-preprocessed_content_document_train_indexmap_27852ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/common-lisp/gpt2-preprocessed_content_document_train_indexmap_27852ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/common-lisp/gpt2-preprocessed_content_document_train_indexmap_27852ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: total number of samples: 37898 +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/common-lisp/gpt2-preprocessed_content_document_valid_indexmap_23ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/common-lisp/gpt2-preprocessed_content_document_valid_indexmap_23ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/common-lisp/gpt2-preprocessed_content_document_valid_indexmap_23ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: total number of samples: 20792 +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/common-lisp/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/common-lisp/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/common-lisp/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: total number of samples: 20468 +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.001991 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: number of documents: 281016 +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 91747) total of 91747 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: document indices in [91747, 186334) total of 94587 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: document indices in [186334, 281016) total of 94682 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elixir/gpt2-preprocessed_content_document_train_indexmap_14125ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elixir/gpt2-preprocessed_content_document_train_indexmap_14125ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elixir/gpt2-preprocessed_content_document_train_indexmap_14125ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: total number of samples: 17928 +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elixir/gpt2-preprocessed_content_document_valid_indexmap_12ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elixir/gpt2-preprocessed_content_document_valid_indexmap_12ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elixir/gpt2-preprocessed_content_document_valid_indexmap_12ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: total number of samples: 8898 +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elixir/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elixir/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elixir/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.011 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: total number of samples: 8929 +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.004328 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: number of documents: 250834 +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 81893) total of 81893 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: document indices in [81893, 166321) total of 84428 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: document indices in [166321, 250834) total of 84513 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/groovy/gpt2-preprocessed_content_document_train_indexmap_18104ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/groovy/gpt2-preprocessed_content_document_train_indexmap_18104ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/groovy/gpt2-preprocessed_content_document_train_indexmap_18104ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: total number of samples: 18411 +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/groovy/gpt2-preprocessed_content_document_valid_indexmap_15ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/groovy/gpt2-preprocessed_content_document_valid_indexmap_15ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/groovy/gpt2-preprocessed_content_document_valid_indexmap_15ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: total number of samples: 9369 +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/groovy/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/groovy/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/groovy/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: total number of samples: 9429 +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.002383 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: number of documents: 3299965 +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 1077381) total of 1077381 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: document indices in [1077381, 2188117) total of 1110736 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: document indices in [2188117, 3299965) total of 1111848 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/html/gpt2-preprocessed_content_document_train_indexmap_584088ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/html/gpt2-preprocessed_content_document_train_indexmap_584088ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/html/gpt2-preprocessed_content_document_train_indexmap_584088ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: total number of samples: 783002 +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/html/gpt2-preprocessed_content_document_valid_indexmap_473ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/html/gpt2-preprocessed_content_document_valid_indexmap_473ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/html/gpt2-preprocessed_content_document_valid_indexmap_473ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: total number of samples: 404400 +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/html/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/html/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/html/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: total number of samples: 405099 +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.002219 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: number of documents: 20071773 +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 6553082) total of 6553082 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: document indices in [6553082, 13309046) total of 6755964 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: document indices in [13309046, 20071773) total of 6762727 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java/gpt2-preprocessed_content_document_train_indexmap_1729583ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java/gpt2-preprocessed_content_document_train_indexmap_1729583ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java/gpt2-preprocessed_content_document_train_indexmap_1729583ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.013 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1806678 +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java/gpt2-preprocessed_content_document_valid_indexmap_1400ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java/gpt2-preprocessed_content_document_valid_indexmap_1400ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java/gpt2-preprocessed_content_document_valid_indexmap_1400ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: total number of samples: 930741 +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.011 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: total number of samples: 933066 +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.003090 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: number of documents: 19544285 +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 6380866) total of 6380866 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: document indices in [6380866, 12959283) total of 6578417 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: document indices in [12959283, 19544285) total of 6585002 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/javascript/gpt2-preprocessed_content_document_train_indexmap_1287340ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/javascript/gpt2-preprocessed_content_document_train_indexmap_1287340ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/javascript/gpt2-preprocessed_content_document_train_indexmap_1287340ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.011 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1510953 +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/javascript/gpt2-preprocessed_content_document_valid_indexmap_1042ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/javascript/gpt2-preprocessed_content_document_valid_indexmap_1042ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/javascript/gpt2-preprocessed_content_document_valid_indexmap_1042ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.011 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: total number of samples: 776848 +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/javascript/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/javascript/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/javascript/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.011 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: total number of samples: 776230 +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.002614 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: number of documents: 21029287 +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 6865694) total of 6865694 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: document indices in [6865694, 13943948) total of 7078254 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: document indices in [13943948, 21029287) total of 7085339 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/markdown/gpt2-preprocessed_content_document_train_indexmap_1490657ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/markdown/gpt2-preprocessed_content_document_train_indexmap_1490657ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/markdown/gpt2-preprocessed_content_document_train_indexmap_1490657ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.011 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: total number of samples: 2022322 +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/markdown/gpt2-preprocessed_content_document_valid_indexmap_1207ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/markdown/gpt2-preprocessed_content_document_valid_indexmap_1207ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/markdown/gpt2-preprocessed_content_document_valid_indexmap_1207ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.011 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1046848 +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/markdown/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/markdown/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/markdown/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.011 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1047246 +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.002488 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: number of documents: 15683017 +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 5120231) total of 5120231 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: document indices in [5120231, 10398982) total of 5278751 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: document indices in [10398982, 15683017) total of 5284035 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/php/gpt2-preprocessed_content_document_train_indexmap_1211345ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/php/gpt2-preprocessed_content_document_train_indexmap_1211345ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/php/gpt2-preprocessed_content_document_train_indexmap_1211345ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1341900 +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/php/gpt2-preprocessed_content_document_valid_indexmap_981ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/php/gpt2-preprocessed_content_document_valid_indexmap_981ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/php/gpt2-preprocessed_content_document_valid_indexmap_981ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: total number of samples: 690231 +[ip-26-0-147-141:0]:06/15/2023 12:21:30 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/php/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/php/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/php/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.011 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of samples: 692824 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.002583 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: number of documents: 12866649 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 4200736) total of 4200736 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: document indices in [4200736, 8531525) total of 4330789 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: document indices in [8531525, 12866649) total of 4335124 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/python/gpt2-preprocessed_content_document_train_indexmap_1201597ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/python/gpt2-preprocessed_content_document_train_indexmap_1201597ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/python/gpt2-preprocessed_content_document_train_indexmap_1201597ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1407216 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/python/gpt2-preprocessed_content_document_valid_indexmap_973ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/python/gpt2-preprocessed_content_document_valid_indexmap_973ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/python/gpt2-preprocessed_content_document_valid_indexmap_973ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of samples: 725128 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/python/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/python/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/python/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.011 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of samples: 723779 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.002805 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: number of documents: 10547331 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 3443519) total of 3443519 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: document indices in [3443519, 6993648) total of 3550129 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: document indices in [6993648, 10547331) total of 3553683 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/typescript/gpt2-preprocessed_content_document_train_indexmap_527589ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/typescript/gpt2-preprocessed_content_document_train_indexmap_527589ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/typescript/gpt2-preprocessed_content_document_train_indexmap_527589ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of samples: 592637 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/typescript/gpt2-preprocessed_content_document_valid_indexmap_428ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/typescript/gpt2-preprocessed_content_document_valid_indexmap_428ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/typescript/gpt2-preprocessed_content_document_valid_indexmap_428ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of samples: 303302 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/typescript/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/typescript/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/typescript/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.011 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of samples: 304761 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.004504 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: number of documents: 75 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 25) total of 25 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: document indices in [25, 50) total of 25 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: document indices in [50, 75) total of 25 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/verilog/gpt2-preprocessed_content_document_train_indexmap_20ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/verilog/gpt2-preprocessed_content_document_train_indexmap_20ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/verilog/gpt2-preprocessed_content_document_train_indexmap_20ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of samples: 24 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 4 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/verilog/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/verilog/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/verilog/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of samples: 8 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/verilog/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/verilog/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/verilog/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of samples: 6 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.002087 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: number of documents: 161239 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 52642) total of 52642 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: document indices in [52642, 106913) total of 54271 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: document indices in [106913, 161239) total of 54326 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/visual-basic/gpt2-preprocessed_content_document_train_indexmap_28250ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/visual-basic/gpt2-preprocessed_content_document_train_indexmap_28250ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/visual-basic/gpt2-preprocessed_content_document_train_indexmap_28250ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of samples: 29718 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/visual-basic/gpt2-preprocessed_content_document_valid_indexmap_23ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/visual-basic/gpt2-preprocessed_content_document_valid_indexmap_23ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/visual-basic/gpt2-preprocessed_content_document_valid_indexmap_23ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of samples: 15235 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/visual-basic/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/visual-basic/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/visual-basic/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of samples: 15594 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.000877 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: number of documents: 58208 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 19004) total of 19004 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: document indices in [19004, 38596) total of 19592 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: document indices in [38596, 58208) total of 19612 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/vhdl/gpt2-preprocessed_content_document_train_indexmap_18701ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/vhdl/gpt2-preprocessed_content_document_train_indexmap_18701ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/vhdl/gpt2-preprocessed_content_document_train_indexmap_18701ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of samples: 29022 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/vhdl/gpt2-preprocessed_content_document_valid_indexmap_16ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/vhdl/gpt2-preprocessed_content_document_valid_indexmap_16ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/vhdl/gpt2-preprocessed_content_document_valid_indexmap_16ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of samples: 15492 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/vhdl/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/vhdl/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/vhdl/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of samples: 16244 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.002129 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: number of documents: 4661 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 1522) total of 1522 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: document indices in [1522, 3091) total of 1569 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: document indices in [3091, 4661) total of 1570 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/thrift/gpt2-preprocessed_content_document_train_indexmap_199ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/thrift/gpt2-preprocessed_content_document_train_indexmap_199ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/thrift/gpt2-preprocessed_content_document_train_indexmap_199ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of samples: 300 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/thrift/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/thrift/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/thrift/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of samples: 149 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/thrift/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/thrift/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/thrift/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of samples: 138 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.002943 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: number of documents: 93 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 31) total of 31 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: document indices in [31, 62) total of 31 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: document indices in [62, 93) total of 31 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/matlab/gpt2-preprocessed_content_document_train_indexmap_4ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/matlab/gpt2-preprocessed_content_document_train_indexmap_4ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/matlab/gpt2-preprocessed_content_document_train_indexmap_4ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of samples: 6 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/matlab/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/matlab/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/matlab/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of samples: 4 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/matlab/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/matlab/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/matlab/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of samples: 4 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.002484 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: number of documents: 7451 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 2433) total of 2433 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: document indices in [2433, 4941) total of 2508 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: document indices in [4941, 7451) total of 2510 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yacc/gpt2-preprocessed_content_document_train_indexmap_2189ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yacc/gpt2-preprocessed_content_document_train_indexmap_2189ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yacc/gpt2-preprocessed_content_document_train_indexmap_2189ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of samples: 2982 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yacc/gpt2-preprocessed_content_document_valid_indexmap_2ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yacc/gpt2-preprocessed_content_document_valid_indexmap_2ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yacc/gpt2-preprocessed_content_document_valid_indexmap_2ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.011 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1463 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yacc/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yacc/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yacc/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1483 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.002804 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: number of documents: 15850 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 5175) total of 5175 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: document indices in [5175, 10510) total of 5335 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: document indices in [10510, 15850) total of 5340 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/zig/gpt2-preprocessed_content_document_train_indexmap_3581ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/zig/gpt2-preprocessed_content_document_train_indexmap_3581ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/zig/gpt2-preprocessed_content_document_train_indexmap_3581ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of samples: 4266 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/zig/gpt2-preprocessed_content_document_valid_indexmap_3ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/zig/gpt2-preprocessed_content_document_valid_indexmap_3ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/zig/gpt2-preprocessed_content_document_valid_indexmap_3ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of samples: 2605 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/zig/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/zig/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/zig/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of samples: 2298 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.000642 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: number of documents: 42103 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 13746) total of 13746 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: document indices in [13746, 27917) total of 14171 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: document indices in [27917, 42103) total of 14186 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/xslt/gpt2-preprocessed_content_document_train_indexmap_995ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/xslt/gpt2-preprocessed_content_document_train_indexmap_995ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/xslt/gpt2-preprocessed_content_document_train_indexmap_995ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of samples: 5493 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/xslt/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/xslt/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/xslt/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of samples: 5972 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/xslt/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/xslt/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/xslt/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of samples: 5680 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.003035 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: number of documents: 4751547 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 1551296) total of 1551296 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: document indices in [1551296, 3150621) total of 1599325 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: document indices in [3150621, 4751547) total of 1600926 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/json/gpt2-preprocessed_content_document_train_indexmap_19894ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/json/gpt2-preprocessed_content_document_train_indexmap_19894ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/json/gpt2-preprocessed_content_document_train_indexmap_19894ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of samples: 83479 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/json/gpt2-preprocessed_content_document_valid_indexmap_17ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/json/gpt2-preprocessed_content_document_valid_indexmap_17ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/json/gpt2-preprocessed_content_document_valid_indexmap_17ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of samples: 86240 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/json/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/json/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/json/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of samples: 86171 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.002627 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: number of documents: 3995948 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 1304607) total of 1304607 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: document indices in [1304607, 2649604) total of 1344997 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: document indices in [2649604, 3995948) total of 1346344 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yaml/gpt2-preprocessed_content_document_train_indexmap_19894ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yaml/gpt2-preprocessed_content_document_train_indexmap_19894ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yaml/gpt2-preprocessed_content_document_train_indexmap_19894ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of samples: 47956 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yaml/gpt2-preprocessed_content_document_valid_indexmap_17ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yaml/gpt2-preprocessed_content_document_valid_indexmap_17ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yaml/gpt2-preprocessed_content_document_valid_indexmap_17ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of samples: 49437 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yaml/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yaml/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yaml/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of samples: 49416 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.003440 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: number of documents: 30982955 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 10115392) total of 10115392 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: document indices in [10115392, 20543954) total of 10428562 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: document indices in [20543954, 30982955) total of 10439001 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_issues/gpt2-preprocessed_content_document_train_indexmap_1082233ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_issues/gpt2-preprocessed_content_document_train_indexmap_1082233ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_issues/gpt2-preprocessed_content_document_train_indexmap_1082233ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1436583 +[ip-26-0-147-141:0]:06/15/2023 12:21:31 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_issues/gpt2-preprocessed_content_document_valid_indexmap_876ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_issues/gpt2-preprocessed_content_document_valid_indexmap_876ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_issues/gpt2-preprocessed_content_document_valid_indexmap_876ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: total number of samples: 742304 +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_issues/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_issues/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_issues/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: total number of samples: 740971 +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.002880 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: number of documents: 7634718 +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 2492602) total of 2492602 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: document indices in [2492602, 5062374) total of 2569772 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: document indices in [5062374, 7634718) total of 2572344 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_commits/gpt2-preprocessed_content_document_train_indexmap_636608ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_commits/gpt2-preprocessed_content_document_train_indexmap_636608ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_commits/gpt2-preprocessed_content_document_train_indexmap_636608ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: total number of samples: 647821 +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_commits/gpt2-preprocessed_content_document_valid_indexmap_516ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_commits/gpt2-preprocessed_content_document_valid_indexmap_516ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_commits/gpt2-preprocessed_content_document_valid_indexmap_516ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: total number of samples: 669748 +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_commits/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_commits/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_commits/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: total number of samples: 665861 +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.002223 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: number of documents: 914510 +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 298572) total of 298572 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: document indices in [298572, 606387) total of 307815 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: document indices in [606387, 914510) total of 308123 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_scripts/gpt2-preprocessed_content_document_train_indexmap_141646ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_scripts/gpt2-preprocessed_content_document_train_indexmap_141646ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_scripts/gpt2-preprocessed_content_document_train_indexmap_141646ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: total number of samples: 200647 +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_scripts/gpt2-preprocessed_content_document_valid_indexmap_115ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_scripts/gpt2-preprocessed_content_document_valid_indexmap_115ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_scripts/gpt2-preprocessed_content_document_valid_indexmap_115ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: total number of samples: 101464 +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_scripts/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_scripts/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_scripts/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: total number of samples: 99559 +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: reading sizes... +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: reading pointers... +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: reading document index... +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.002588 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: number of documents: 668743 +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: > dataset split: +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: train: +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 218333) total of 218333 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: validation: +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: document indices in [218333, 443425) total of 225092 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: test: +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: document indices in [443425, 668743) total of 225318 documents +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_structured/gpt2-preprocessed_content_document_train_indexmap_119364ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:3]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' +[ip-26-0-147-141:2]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' +[ip-26-0-147-141:2]:make: Nothing to be done for 'default'. +[ip-26-0-147-141:2]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' +[ip-26-0-147-141:3]:make: Nothing to be done for 'default'. +[ip-26-0-147-141:3]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' +[ip-26-0-147-141:1]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' +[ip-26-0-147-141:1]:make: Nothing to be done for 'default'. +[ip-26-0-147-141:1]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_structured/gpt2-preprocessed_content_document_train_indexmap_119364ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_structured/gpt2-preprocessed_content_document_train_indexmap_119364ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: total number of samples: 160224 +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_structured/gpt2-preprocessed_content_document_valid_indexmap_97ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_structured/gpt2-preprocessed_content_document_valid_indexmap_97ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_structured/gpt2-preprocessed_content_document_valid_indexmap_97ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: total number of samples: 81239 +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_structured/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_structured/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_structured/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.005 seconds +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: total number of samples: 81006 +[ip-26-0-147-141:0]:06/15/2023 12:21:32 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 +[ip-26-0-147-141:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' +[ip-26-0-147-141:0]:make: Nothing to be done for 'default'. +[ip-26-0-147-141:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' +[ip-26-0-147-141:0]:> building indices for blendable datasets ... +[ip-26-0-147-141:3]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' +[ip-26-0-147-141:3]:make: Nothing to be done for 'default'. +[ip-26-0-147-141:3]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' +[ip-26-0-147-141:2]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' +[ip-26-0-147-141:1]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' +[ip-26-0-147-141:0]: > sample ratios: +[ip-26-0-147-141:0]: dataset 0, input: 0.00391159, achieved: 0.00391158 +[ip-26-0-147-141:0]: dataset 1, input: 1.30386e-05, achieved: 1.30425e-05 +[ip-26-0-147-141:0]: dataset 2, input: 0.0702651, achieved: 0.0702651 +[ip-26-0-147-141:0]: dataset 3, input: 0.00232087, achieved: 0.00232085 +[ip-26-0-147-141:0]: dataset 4, input: 0.00110828, achieved: 0.00110829 +[ip-26-0-147-141:0]: dataset 5, input: 0.00740594, achieved: 0.00740593 +[ip-26-0-147-141:0]: dataset 6, input: 1.30386e-05, achieved: 1.30425e-05 +[ip-26-0-147-141:0]: dataset 7, input: 0.00170806, achieved: 0.00170805 +[ip-26-0-147-141:0]: dataset 8, input: 0.00127778, achieved: 0.00127778 +[ip-26-0-147-141:0]: dataset 9, input: 0.000104309, achieved: 0.00010434 +[ip-26-0-147-141:0]: dataset 10, input: 3.91159e-05, achieved: 3.91276e-05 +[ip-26-0-147-141:0]: dataset 11, input: 0.000117348, achieved: 0.000117317 +[ip-26-0-147-141:0]: dataset 12, input: 0.00146033, achieved: 0.00146031 +[ip-26-0-147-141:0]: dataset 13, input: 0.0310058, achieved: 0.0310058 +[ip-26-0-147-141:0]: dataset 14, input: 0.000912704, achieved: 0.000912715 +[ip-26-0-147-141:0]: dataset 15, input: 0.000795356, achieved: 0.000795332 +[ip-26-0-147-141:0]: dataset 16, input: 0.000339004, achieved: 0.000338975 +[ip-26-0-147-141:0]: dataset 17, input: 0.00219049, achieved: 0.00219049 +[ip-26-0-147-141:0]: dataset 18, input: 0.00290761, achieved: 0.00290763 +[ip-26-0-147-141:0]: dataset 19, input: 0.000391159, achieved: 0.000391145 +[ip-26-0-147-141:0]: dataset 20, input: 0.000404197, achieved: 0.000404187 +[ip-26-0-147-141:0]: dataset 21, input: 0.000586738, achieved: 0.000586717 +[ip-26-0-147-141:0]: dataset 22, input: 0.000156463, achieved: 0.000156445 +[ip-26-0-147-141:0]: dataset 23, input: 0.0088793, achieved: 0.00887928 +[ip-26-0-147-141:0]: dataset 24, input: 0.0118782, achieved: 0.0118782 +[ip-26-0-147-141:0]: dataset 25, input: 7.82317e-05, achieved: 7.82552e-05 +[ip-26-0-147-141:0]: dataset 26, input: 0.0582305, achieved: 0.0582305 +[ip-26-0-147-141:0]: dataset 27, input: 0.00075624, achieved: 0.00075627 +[ip-26-0-147-141:0]: dataset 28, input: 0.00290761, achieved: 0.00290763 +[ip-26-0-147-141:0]: dataset 29, input: 1.30386e-05, achieved: 1.30425e-05 +[ip-26-0-147-141:0]: dataset 30, input: 0.00162983, achieved: 0.00162986 +[ip-26-0-147-141:0]: dataset 31, input: 0.00134298, achieved: 0.00134299 +[ip-26-0-147-141:0]: dataset 32, input: 0.00170806, achieved: 0.00170805 +[ip-26-0-147-141:0]: dataset 33, input: 0.00374208, achieved: 0.00374209 +[ip-26-0-147-141:0]: dataset 34, input: 1.30386e-05, achieved: 1.30425e-05 +[ip-26-0-147-141:0]: dataset 35, input: 6.51931e-05, achieved: 6.52127e-05 +[ip-26-0-147-141:0]: dataset 36, input: 0.00432882, achieved: 0.00432881 +[ip-26-0-147-141:0]: dataset 37, input: 3.91159e-05, achieved: 3.91276e-05 +[ip-26-0-147-141:0]: dataset 38, input: 0.000247734, achieved: 0.000247743 +[ip-26-0-147-141:0]: dataset 39, input: 0.000508506, achieved: 0.000508528 +[ip-26-0-147-141:0]: dataset 40, input: 0.00678008, achieved: 0.00678009 +[ip-26-0-147-141:0]: dataset 41, input: 2.60772e-05, achieved: 2.60851e-05 +[ip-26-0-147-141:0]: dataset 42, input: 0.00203403, achieved: 0.00203405 +[ip-26-0-147-141:0]: dataset 43, input: 1.30386e-05, achieved: 1.30425e-05 +[ip-26-0-147-141:0]: dataset 44, input: 9.12704e-05, achieved: 9.12977e-05 +[ip-26-0-147-141:0]: dataset 45, input: 0.000534584, achieved: 0.000534613 +[ip-26-0-147-141:0]: dataset 46, input: 0.00477214, achieved: 0.00477213 +[ip-26-0-147-141:0]: dataset 47, input: 0.000730163, achieved: 0.000730185 +[ip-26-0-147-141:0]: dataset 48, input: 3.91159e-05, achieved: 3.91276e-05 +[ip-26-0-147-141:0]: dataset 49, input: 1.30386e-06, achieved: 1.31081e-06 +[ip-26-0-147-141:0]: dataset 50, input: 0.000299888, achieved: 0.000299913 +[ip-26-0-147-141:0]: dataset 51, input: 2.60772e-05, achieved: 2.60851e-05 +[ip-26-0-147-141:0]: dataset 52, input: 1.30386e-05, achieved: 1.30425e-05 +[ip-26-0-147-141:0]: dataset 53, input: 0.00611511, achieved: 0.00611511 +[ip-26-0-147-141:0]: dataset 54, input: 0.000456352, achieved: 0.000456358 +[ip-26-0-147-141:0]: dataset 55, input: 0.000430275, achieved: 0.000430273 +[ip-26-0-147-141:0]: dataset 56, input: 1.30386e-05, achieved: 1.30425e-05 +[ip-26-0-147-141:0]: dataset 57, input: 0.00402893, achieved: 0.0040289 +[ip-26-0-147-141:0]: dataset 58, input: 0.000599777, achieved: 0.00059976 +[ip-26-0-147-141:0]: dataset 59, input: 0.000260772, achieved: 0.000260785 +[ip-26-0-147-141:0]: dataset 60, input: 6.51931e-05, achieved: 6.52127e-05 +[ip-26-0-147-141:0]: dataset 61, input: 5.21545e-05, achieved: 5.21701e-05 +[ip-26-0-147-141:0]: dataset 62, input: 0.0144598, achieved: 0.0144598 +[ip-26-0-147-141:0]: dataset 63, input: 0.000521545, achieved: 0.00052157 +[ip-26-0-147-141:0]: dataset 64, input: 0.000391159, achieved: 0.000391145 +[ip-26-0-147-141:0]: dataset 65, input: 0.000547622, achieved: 0.00054759 +[ip-26-0-147-141:0]: dataset 66, input: 0.0637849, achieved: 0.0637849 +[ip-26-0-147-141:0]: dataset 67, input: 0.000834472, achieved: 0.00083446 +[ip-26-0-147-141:0]: dataset 68, input: 0.00182541, achieved: 0.00182543 +[ip-26-0-147-141:0]: dataset 69, input: 0.000925742, achieved: 0.000925758 +[ip-26-0-147-141:0]: dataset 70, input: 0.00118651, achieved: 0.00118654 +[ip-26-0-147-141:0]: dataset 71, input: 0.0382814, achieved: 0.0382814 +[ip-26-0-147-141:0]: dataset 72, input: 0.113358, achieved: 0.113358 +[ip-26-0-147-141:0]: dataset 73, input: 0.0843729, achieved: 0.0843729 +[ip-26-0-147-141:0]: dataset 74, input: 0.0976984, achieved: 0.0976983 +[ip-26-0-147-141:0]: dataset 75, input: 0.0793922, achieved: 0.0793921 +[ip-26-0-147-141:0]: dataset 76, input: 0.0787533, achieved: 0.0787532 +[ip-26-0-147-141:0]: dataset 77, input: 0.0345784, achieved: 0.0345784 +[ip-26-0-147-141:0]: dataset 78, input: 1.30386e-06, achieved: 1.31081e-06 +[ip-26-0-147-141:0]: dataset 79, input: 0.00185148, achieved: 0.00185145 +[ip-26-0-147-141:0]: dataset 80, input: 0.00122563, achieved: 0.0012256 +[ip-26-0-147-141:0]: dataset 81, input: 1.30386e-05, achieved: 1.30425e-05 +[ip-26-0-147-141:0]: dataset 82, input: 2.60772e-07, achieved: 2.62161e-07 +[ip-26-0-147-141:0]: dataset 83, input: 0.000143425, achieved: 0.000143402 +[ip-26-0-147-141:0]: dataset 84, input: 0.000234695, achieved: 0.0002347 +[ip-26-0-147-141:0]: dataset 85, input: 6.51931e-05, achieved: 6.52127e-05 +[ip-26-0-147-141:0]: dataset 86, input: 0.00130386, achieved: 0.00130386 +[ip-26-0-147-141:0]: dataset 87, input: 0.00130386, achieved: 0.00130386 +[ip-26-0-147-141:0]: dataset 88, input: 0.0709301, achieved: 0.0709301 +[ip-26-0-147-141:0]: dataset 89, input: 0.0417236, achieved: 0.0417236 +[ip-26-0-147-141:0]: dataset 90, input: 0.0092835, achieved: 0.00928347 +[ip-26-0-147-141:0]: dataset 91, input: 0.00782317, achieved: 0.00782316 +[ip-26-0-147-141:0]:06/15/2023 12:21:34 [INFO|DP=0|PP=0|TP=0]: > elapsed time for building blendable dataset indices: 1.69 (sec) +[ip-26-0-147-141:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' +[ip-26-0-147-141:0]:make: Nothing to be done for 'default'. +[ip-26-0-147-141:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' +[ip-26-0-147-141:3]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' +[ip-26-0-147-141:3]:make: Nothing to be done for 'default'. +[ip-26-0-147-141:3]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' +[ip-26-0-147-141:2]:make: Nothing to be done for 'default'. +[ip-26-0-147-141:2]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' +[ip-26-0-147-141:2]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' +[ip-26-0-147-141:2]:make: Nothing to be done for 'default'. +[ip-26-0-147-141:2]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' +[ip-26-0-147-141:1]:make: Nothing to be done for 'default'. +[ip-26-0-147-141:1]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' +[ip-26-0-147-141:1]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' +[ip-26-0-147-141:1]:make: Nothing to be done for 'default'. +[ip-26-0-147-141:1]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' +[ip-26-0-147-141:0]:> building indices for blendable datasets ... +[ip-26-0-147-141:0]: > sample ratios: +[ip-26-0-147-141:0]: dataset 0, input: 0.00391159, achieved: 0.00395225 +[ip-26-0-147-141:0]: dataset 1, input: 1.30386e-05, achieved: 0 +[ip-26-0-147-141:0]: dataset 2, input: 0.0702651, achieved: 0.0702533 +[ip-26-0-147-141:0]: dataset 3, input: 0.00232087, achieved: 0.00233909 +[ip-26-0-147-141:0]: dataset 4, input: 0.00110828, achieved: 0.00112921 +[ip-26-0-147-141:0]: dataset 5, input: 0.00740594, achieved: 0.00742055 +[ip-26-0-147-141:0]: dataset 6, input: 1.30386e-05, achieved: 0 +[ip-26-0-147-141:0]: dataset 7, input: 0.00170806, achieved: 0.00169382 +[ip-26-0-147-141:0]: dataset 8, input: 0.00127778, achieved: 0.00129053 +[ip-26-0-147-141:0]: dataset 9, input: 0.000104309, achieved: 8.06582e-05 +[ip-26-0-147-141:0]: dataset 10, input: 3.91159e-05, achieved: 8.06582e-05 +[ip-26-0-147-141:0]: dataset 11, input: 0.000117348, achieved: 0.000161316 +[ip-26-0-147-141:0]: dataset 12, input: 0.00146033, achieved: 0.00145185 +[ip-26-0-147-141:0]: dataset 13, input: 0.0310058, achieved: 0.0309727 +[ip-26-0-147-141:0]: dataset 14, input: 0.000912704, achieved: 0.00088724 +[ip-26-0-147-141:0]: dataset 15, input: 0.000795356, achieved: 0.000806582 +[ip-26-0-147-141:0]: dataset 16, input: 0.000339004, achieved: 0.000322633 +[ip-26-0-147-141:0]: dataset 17, input: 0.00219049, achieved: 0.00217777 +[ip-26-0-147-141:0]: dataset 18, input: 0.00290761, achieved: 0.00290369 +[ip-26-0-147-141:0]: dataset 19, input: 0.000391159, achieved: 0.000403291 +[ip-26-0-147-141:0]: dataset 20, input: 0.000404197, achieved: 0.000403291 +[ip-26-0-147-141:0]: dataset 21, input: 0.000586738, achieved: 0.000564607 +[ip-26-0-147-141:0]: dataset 22, input: 0.000156463, achieved: 0.000161316 +[ip-26-0-147-141:0]: dataset 23, input: 0.0088793, achieved: 0.0088724 +[ip-26-0-147-141:0]: dataset 24, input: 0.0118782, achieved: 0.0118568 +[ip-26-0-147-141:0]: dataset 25, input: 7.82317e-05, achieved: 8.06582e-05 +[ip-26-0-147-141:0]: dataset 26, input: 0.0582305, achieved: 0.0582352 +[ip-26-0-147-141:0]: dataset 27, input: 0.00075624, achieved: 0.000725924 +[ip-26-0-147-141:0]: dataset 28, input: 0.00290761, achieved: 0.00290369 +[ip-26-0-147-141:0]: dataset 29, input: 1.30386e-05, achieved: 0 +[ip-26-0-147-141:0]: dataset 30, input: 0.00162983, achieved: 0.00161316 +[ip-26-0-147-141:0]: dataset 31, input: 0.00134298, achieved: 0.00137119 +[ip-26-0-147-141:0]: dataset 32, input: 0.00170806, achieved: 0.00169382 +[ip-26-0-147-141:0]: dataset 33, input: 0.00374208, achieved: 0.00371028 +[ip-26-0-147-141:0]: dataset 34, input: 1.30386e-05, achieved: 0 +[ip-26-0-147-141:0]: dataset 35, input: 6.51931e-05, achieved: 8.06582e-05 +[ip-26-0-147-141:0]: dataset 36, input: 0.00432882, achieved: 0.00435554 +[ip-26-0-147-141:0]: dataset 37, input: 3.91159e-05, achieved: 8.06582e-05 +[ip-26-0-147-141:0]: dataset 38, input: 0.000247734, achieved: 0.000241975 +[ip-26-0-147-141:0]: dataset 39, input: 0.000508506, achieved: 0.000483949 +[ip-26-0-147-141:0]: dataset 40, input: 0.00678008, achieved: 0.00677529 +[ip-26-0-147-141:0]: dataset 41, input: 2.60772e-05, achieved: 0 +[ip-26-0-147-141:0]: dataset 42, input: 0.00203403, achieved: 0.00201645 +[ip-26-0-147-141:0]: dataset 43, input: 1.30386e-05, achieved: 0 +[ip-26-0-147-141:0]: dataset 44, input: 9.12704e-05, achieved: 8.06582e-05 +[ip-26-0-147-141:0]: dataset 45, input: 0.000534584, achieved: 0.000564607 +[ip-26-0-147-141:0]: dataset 46, input: 0.00477214, achieved: 0.00475883 +[ip-26-0-147-141:0]: dataset 47, input: 0.000730163, achieved: 0.000725924 +[ip-26-0-147-141:0]: dataset 48, input: 3.91159e-05, achieved: 8.06582e-05 +[ip-26-0-147-141:0]: dataset 49, input: 1.30386e-06, achieved: 0 +[ip-26-0-147-141:0]: dataset 50, input: 0.000299888, achieved: 0.000322633 +[ip-26-0-147-141:0]: dataset 51, input: 2.60772e-05, achieved: 0 +[ip-26-0-147-141:0]: dataset 52, input: 1.30386e-05, achieved: 0 +[ip-26-0-147-141:0]: dataset 53, input: 0.00611511, achieved: 0.00613002 +[ip-26-0-147-141:0]: dataset 54, input: 0.000456352, achieved: 0.000483949 +[ip-26-0-147-141:0]: dataset 55, input: 0.000430275, achieved: 0.000403291 +[ip-26-0-147-141:0]: dataset 56, input: 1.30386e-05, achieved: 0 +[ip-26-0-147-141:0]: dataset 57, input: 0.00402893, achieved: 0.00403291 +[ip-26-0-147-141:0]: dataset 58, input: 0.000599777, achieved: 0.000645265 +[ip-26-0-147-141:0]: dataset 59, input: 0.000260772, achieved: 0.000241975 +[ip-26-0-147-141:0]: dataset 60, input: 6.51931e-05, achieved: 8.06582e-05 +[ip-26-0-147-141:0]: dataset 61, input: 5.21545e-05, achieved: 8.06582e-05 +[ip-26-0-147-141:0]: dataset 62, input: 0.0144598, achieved: 0.0144378 +[ip-26-0-147-141:0]: dataset 63, input: 0.000521545, achieved: 0.000564607 +[ip-26-0-147-141:0]: dataset 64, input: 0.000391159, achieved: 0.000403291 +[ip-26-0-147-141:0]: dataset 65, input: 0.000547622, achieved: 0.000564607 +[ip-26-0-147-141:0]: dataset 66, input: 0.0637849, achieved: 0.0638006 +[ip-26-0-147-141:0]: dataset 67, input: 0.000834472, achieved: 0.000806582 +[ip-26-0-147-141:0]: dataset 68, input: 0.00182541, achieved: 0.00185514 +[ip-26-0-147-141:0]: dataset 69, input: 0.000925742, achieved: 0.000967898 +[ip-26-0-147-141:0]: dataset 70, input: 0.00118651, achieved: 0.00120987 +[ip-26-0-147-141:0]: dataset 71, input: 0.0382814, achieved: 0.0383126 +[ip-26-0-147-141:0]: dataset 72, input: 0.113358, achieved: 0.113325 +[ip-26-0-147-141:0]: dataset 73, input: 0.0843729, achieved: 0.0843684 +[ip-26-0-147-141:0]: dataset 74, input: 0.0976984, achieved: 0.097677 +[ip-26-0-147-141:0]: dataset 75, input: 0.0793922, achieved: 0.0793676 +[ip-26-0-147-141:0]: dataset 76, input: 0.0787533, achieved: 0.0787224 +[ip-26-0-147-141:0]: dataset 77, input: 0.0345784, achieved: 0.0346024 +[ip-26-0-147-141:0]: dataset 78, input: 1.30386e-06, achieved: 0 +[ip-26-0-147-141:0]: dataset 79, input: 0.00185148, achieved: 0.00185514 +[ip-26-0-147-141:0]: dataset 80, input: 0.00122563, achieved: 0.00120987 +[ip-26-0-147-141:0]: dataset 81, input: 1.30386e-05, achieved: 0 +[ip-26-0-147-141:0]: dataset 82, input: 2.60772e-07, achieved: 0 +[ip-26-0-147-141:0]: dataset 83, input: 0.000143425, achieved: 0.000161316 +[ip-26-0-147-141:0]: dataset 84, input: 0.000234695, achieved: 0.000241975 +[ip-26-0-147-141:0]: dataset 85, input: 6.51931e-05, achieved: 8.06582e-05 +[ip-26-0-147-141:0]: dataset 86, input: 0.00130386, achieved: 0.00129053 +[ip-26-0-147-141:0]: dataset 87, input: 0.00130386, achieved: 0.00129053 +[ip-26-0-147-141:0]: dataset 88, input: 0.0709301, achieved: 0.0708985 +[ip-26-0-147-141:0]: dataset 89, input: 0.0417236, achieved: 0.0417003 +[ip-26-0-147-141:0]: dataset 90, input: 0.0092835, achieved: 0.00927569 +[ip-26-0-147-141:0]: dataset 91, input: 0.00782317, achieved: 0.00782384 +[ip-26-0-147-141:0]:06/15/2023 12:21:34 [INFO|DP=0|PP=0|TP=0]: > elapsed time for building blendable dataset indices: 0.10 (sec) +[ip-26-0-147-141:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' +[ip-26-0-147-141:0]:make: Nothing to be done for 'default'. +[ip-26-0-147-141:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' +[ip-26-0-147-141:0]:> building indices for blendable datasets ... +[ip-26-0-147-141:0]: > sample ratios: +[ip-26-0-147-141:0]: dataset 0, input: 0.00391159, achieved: -nan +[ip-26-0-147-141:0]: dataset 1, input: 1.30386e-05, achieved: -nan +[ip-26-0-147-141:0]: dataset 2, input: 0.0702651, achieved: -nan +[ip-26-0-147-141:0]: dataset 3, input: 0.00232087, achieved: -nan +[ip-26-0-147-141:0]: dataset 4, input: 0.00110828, achieved: -nan +[ip-26-0-147-141:0]: dataset 5, input: 0.00740594, achieved: -nan +[ip-26-0-147-141:0]: dataset 6, input: 1.30386e-05, achieved: -nan +[ip-26-0-147-141:0]: dataset 7, input: 0.00170806, achieved: -nan +[ip-26-0-147-141:0]: dataset 8, input: 0.00127778, achieved: -nan +[ip-26-0-147-141:0]: dataset 9, input: 0.000104309, achieved: -nan +[ip-26-0-147-141:0]: dataset 10, input: 3.91159e-05, achieved: -nan +[ip-26-0-147-141:0]: dataset 11, input: 0.000117348, achieved: -nan +[ip-26-0-147-141:0]: dataset 12, input: 0.00146033, achieved: -nan +[ip-26-0-147-141:0]: dataset 13, input: 0.0310058, achieved: -nan +[ip-26-0-147-141:0]: dataset 14, input: 0.000912704, achieved: -nan +[ip-26-0-147-141:0]: dataset 15, input: 0.000795356, achieved: -nan +[ip-26-0-147-141:0]: dataset 16, input: 0.000339004, achieved: -nan +[ip-26-0-147-141:0]: dataset 17, input: 0.00219049, achieved: -nan +[ip-26-0-147-141:0]: dataset 18, input: 0.00290761, achieved: -nan +[ip-26-0-147-141:0]: dataset 19, input: 0.000391159, achieved: -nan +[ip-26-0-147-141:0]: dataset 20, input: 0.000404197, achieved: -nan +[ip-26-0-147-141:0]: dataset 21, input: 0.000586738, achieved: -nan +[ip-26-0-147-141:0]: dataset 22, input: 0.000156463, achieved: -nan +[ip-26-0-147-141:0]: dataset 23, input: 0.0088793, achieved: -nan +[ip-26-0-147-141:0]: dataset 24, input: 0.0118782, achieved: -nan +[ip-26-0-147-141:0]: dataset 25, input: 7.82317e-05, achieved: -nan +[ip-26-0-147-141:0]: dataset 26, input: 0.0582305, achieved: -nan +[ip-26-0-147-141:0]: dataset 27, input: 0.00075624, achieved: -nan +[ip-26-0-147-141:0]: dataset 28, input: 0.00290761, achieved: -nan +[ip-26-0-147-141:0]: dataset 29, input: 1.30386e-05, achieved: -nan +[ip-26-0-147-141:0]: dataset 30, input: 0.00162983, achieved: -nan +[ip-26-0-147-141:0]: dataset 31, input: 0.00134298, achieved: -nan +[ip-26-0-147-141:0]: dataset 32, input: 0.00170806, achieved: -nan +[ip-26-0-147-141:0]: dataset 33, input: 0.00374208, achieved: -nan +[ip-26-0-147-141:0]: dataset 34, input: 1.30386e-05, achieved: -nan +[ip-26-0-147-141:0]: dataset 35, input: 6.51931e-05, achieved: -nan +[ip-26-0-147-141:0]: dataset 36, input: 0.00432882, achieved: -nan +[ip-26-0-147-141:0]: dataset 37, input: 3.91159e-05, achieved: -nan +[ip-26-0-147-141:0]: dataset 38, input: 0.000247734, achieved: -nan +[ip-26-0-147-141:0]: dataset 39, input: 0.000508506, achieved: -nan +[ip-26-0-147-141:0]: dataset 40, input: 0.00678008, achieved: -nan +[ip-26-0-147-141:0]: dataset 41, input: 2.60772e-05, achieved: -nan +[ip-26-0-147-141:0]: dataset 42, input: 0.00203403, achieved: -nan +[ip-26-0-147-141:0]: dataset 43, input: 1.30386e-05, achieved: -nan +[ip-26-0-147-141:0]: dataset 44, input: 9.12704e-05, achieved: -nan +[ip-26-0-147-141:0]: dataset 45, input: 0.000534584, achieved: -nan +[ip-26-0-147-141:0]: dataset 46, input: 0.00477214, achieved: -nan +[ip-26-0-147-141:0]: dataset 47, input: 0.000730163, achieved: -nan +[ip-26-0-147-141:0]: dataset 48, input: 3.91159e-05, achieved: -nan +[ip-26-0-147-141:0]: dataset 49, input: 1.30386e-06, achieved: -nan +[ip-26-0-147-141:0]: dataset 50, input: 0.000299888, achieved: -nan +[ip-26-0-147-141:0]: dataset 51, input: 2.60772e-05, achieved: -nan +[ip-26-0-147-141:0]: dataset 52, input: 1.30386e-05, achieved: -nan +[ip-26-0-147-141:0]: dataset 53, input: 0.00611511, achieved: -nan +[ip-26-0-147-141:0]: dataset 54, input: 0.000456352, achieved: -nan +[ip-26-0-147-141:0]: dataset 55, input: 0.000430275, achieved: -nan +[ip-26-0-147-141:0]: dataset 56, input: 1.30386e-05, achieved: -nan +[ip-26-0-147-141:0]: dataset 57, input: 0.00402893, achieved: -nan +[ip-26-0-147-141:0]: dataset 58, input: 0.000599777, achieved: -nan +[ip-26-0-147-141:0]: dataset 59, input: 0.000260772, achieved: -nan +[ip-26-0-147-141:0]: dataset 60, input: 6.51931e-05, achieved: -nan +[ip-26-0-147-141:0]: dataset 61, input: 5.21545e-05, achieved: -nan +[ip-26-0-147-141:0]: dataset 62, input: 0.0144598, achieved: -nan +[ip-26-0-147-141:0]: dataset 63, input: 0.000521545, achieved: -nan +[ip-26-0-147-141:0]: dataset 64, input: 0.000391159, achieved: -nan +[ip-26-0-147-141:0]: dataset 65, input: 0.000547622, achieved: -nan +[ip-26-0-147-141:0]: dataset 66, input: 0.0637849, achieved: -nan +[ip-26-0-147-141:0]: dataset 67, input: 0.000834472, achieved: -nan +[ip-26-0-147-141:0]: dataset 68, input: 0.00182541, achieved: -nan +[ip-26-0-147-141:0]: dataset 69, input: 0.000925742, achieved: -nan +[ip-26-0-147-141:0]: dataset 70, input: 0.00118651, achieved: -nan +[ip-26-0-147-141:0]: dataset 71, input: 0.0382814, achieved: -nan +[ip-26-0-147-141:0]: dataset 72, input: 0.113358, achieved: -nan +[ip-26-0-147-141:0]: dataset 73, input: 0.0843729, achieved: -nan +[ip-26-0-147-141:0]: dataset 74, input: 0.0976984, achieved: -nan +[ip-26-0-147-141:0]: dataset 75, input: 0.0793922, achieved: -nan +[ip-26-0-147-141:0]: dataset 76, input: 0.0787533, achieved: -nan +[ip-26-0-147-141:0]: dataset 77, input: 0.0345784, achieved: -nan +[ip-26-0-147-141:0]: dataset 78, input: 1.30386e-06, achieved: -nan +[ip-26-0-147-141:0]: dataset 79, input: 0.00185148, achieved: -nan +[ip-26-0-147-141:0]: dataset 80, input: 0.00122563, achieved: -nan +[ip-26-0-147-141:0]: dataset 81, input: 1.30386e-05, achieved: -nan +[ip-26-0-147-141:0]: dataset 82, input: 2.60772e-07, achieved: -nan +[ip-26-0-147-141:0]: dataset 83, input: 0.000143425, achieved: -nan +[ip-26-0-147-141:0]: dataset 84, input: 0.000234695, achieved: -nan +[ip-26-0-147-141:0]: dataset 85, input: 6.51931e-05, achieved: -nan +[ip-26-0-147-141:0]: dataset 86, input: 0.00130386, achieved: -nan +[ip-26-0-147-141:0]: dataset 87, input: 0.00130386, achieved: -nan +[ip-26-0-147-141:0]: dataset 88, input: 0.0709301, achieved: -nan +[ip-26-0-147-141:0]: dataset 89, input: 0.0417236, achieved: -nan +[ip-26-0-147-141:0]: dataset 90, input: 0.0092835, achieved: -nan +[ip-26-0-147-141:0]: dataset 91, input: 0.00782317, achieved: -nan +[ip-26-0-147-141:0]:06/15/2023 12:21:34 [INFO|DP=0|PP=0|TP=0]: > elapsed time for building blendable dataset indices: 0.05 (sec) +[ip-26-0-147-141:0]:06/15/2023 12:21:34 [INFO|DP=0|PP=0|TP=0]: Building dataloader with consumed samples: 0 +[ip-26-0-147-141:0]:06/15/2023 12:21:34 [INFO|DP=0|PP=0|TP=0]: Instantiating MegatronPretrainingSampler with total_samples: 15257771 and consumed_samples: 0 +[ip-26-0-147-141:0]:06/15/2023 12:21:34 [INFO|DP=0|PP=0|TP=0]: [Before the start of training] datetime: 2023-06-15 12:21:34.214478 +[ip-26-0-148-151:0]:Traceback (most recent call last): +[ip-26-0-148-151:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-151:0]: main() +[ip-26-0-148-151:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-245:2]:Traceback (most recent call last): +[ip-26-0-148-245:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-245:2]: main() +[ip-26-0-148-245:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-245:2]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-245:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-245:2]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-148-245:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-245:2]: result = model(**micro_batch) +[ip-26-0-148-245:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-245:2]: return forward_call(*input, **kwargs) +[ip-26-0-148-245:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-245:2]: sharded_logits = self.model( +[ip-26-0-148-245:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-245:2]: return forward_call(*input, **kwargs) +[ip-26-0-148-245:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-245:2]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-148-245:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-245:2]: return forward_call(*input, **kwargs) +[ip-26-0-148-245:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-148-245:2]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-245:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-245:2]: return forward_call(*input, **kwargs) +[ip-26-0-148-245:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-245:2]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-245:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-245:2]: return forward_call(*input, **kwargs) +[ip-26-0-148-245:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-245:2]: query_states, kv_states = self.qkv( +[ip-26-0-148-245:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-245:2]: return forward_call(*input, **kwargs) +[ip-26-0-148-245:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-245:2]: return column_linear( +[ip-26-0-148-245:2]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-233:0]:Traceback (most recent call last): +[ip-26-0-147-233:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-233:0]: main() +[ip-26-0-147-233:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-233:0]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-233:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-233:0]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-233:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-233:0]: result = model(**micro_batch) +[ip-26-0-147-233:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-233:0]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-233:0]: sharded_logits = self.model( +[ip-26-0-147-233:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-233:0]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-233:0]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-233:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-233:0]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-233:0]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-233:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-233:0]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-233:0]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-233:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-233:0]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-233:0]: query_states, kv_states = self.qkv( +[ip-26-0-147-233:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-233:0]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-233:0]: return column_linear( +[ip-26-0-147-233:0]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-233:1]:Traceback (most recent call last): +[ip-26-0-147-233:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-233:1]: main() +[ip-26-0-147-233:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-233:1]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-233:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-233:1]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-233:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-233:1]: result = model(**micro_batch) +[ip-26-0-147-233:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-233:1]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-233:1]: sharded_logits = self.model( +[ip-26-0-147-233:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-233:1]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-233:1]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-233:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-233:1]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-233:1]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-233:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-233:1]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-233:1]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-233:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-233:1]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-233:1]: query_states, kv_states = self.qkv( +[ip-26-0-147-233:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-233:1]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-233:1]: return column_linear( +[ip-26-0-147-233:1]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-148-193:0]:Traceback (most recent call last): +[ip-26-0-148-193:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-193:0]: main() +[ip-26-0-148-193:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-193:0]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-193:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-193:0]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-148-193:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-193:0]: result = model(**micro_batch) +[ip-26-0-148-193:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-141:3]:Traceback (most recent call last): +[ip-26-0-147-141:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-141:3]: main() +[ip-26-0-147-141:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-141:3]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-141:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-141:3]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-141:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-141:3]: result = model(**micro_batch) +[ip-26-0-147-141:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-141:3]: return forward_call(*input, **kwargs) +[ip-26-0-147-141:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-141:3]: sharded_logits = self.model( +[ip-26-0-147-141:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-141:3]: return forward_call(*input, **kwargs) +[ip-26-0-147-141:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-141:3]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-141:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-141:3]: return forward_call(*input, **kwargs) +[ip-26-0-147-141:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-141:3]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-141:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-141:3]: return forward_call(*input, **kwargs) +[ip-26-0-147-141:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-141:3]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-141:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-141:3]: return forward_call(*input, **kwargs) +[ip-26-0-147-141:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-141:3]: query_states, kv_states = self.qkv( +[ip-26-0-147-141:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-141:3]: return forward_call(*input, **kwargs) +[ip-26-0-147-141:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-141:3]: return column_linear( +[ip-26-0-147-141:3]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-149-1:5]:Traceback (most recent call last): +[ip-26-0-149-1:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-149-1:5]: main() +[ip-26-0-149-1:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-149-1:5]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-149-1:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-149-1:5]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-149-1:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-149-1:5]: result = model(**micro_batch) +[ip-26-0-149-1:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-149-1:5]: return forward_call(*input, **kwargs) +[ip-26-0-149-1:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-149-1:5]: sharded_logits = self.model( +[ip-26-0-149-1:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-149-1:5]: return forward_call(*input, **kwargs) +[ip-26-0-149-1:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-149-1:5]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-149-1:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-149-1:5]: return forward_call(*input, **kwargs) +[ip-26-0-149-1:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-149-1:5]: output = self.pp_block(**new_kwargs) +[ip-26-0-149-1:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-149-1:5]: return forward_call(*input, **kwargs) +[ip-26-0-149-1:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-149-1:5]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-149-1:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-149-1:5]: return forward_call(*input, **kwargs) +[ip-26-0-149-1:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-149-1:5]: query_states, kv_states = self.qkv( +[ip-26-0-149-1:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-149-1:5]: return forward_call(*input, **kwargs) +[ip-26-0-149-1:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-149-1:5]: re[ip-26-0-148-93:2]:Traceback (most recent call last): +[ip-26-0-148-93:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +turn column_linear( +[ip-26-0-149-1:5]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-148-93:2]: main() +[ip-26-0-148-93:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-93:2]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-93:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-93:2]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-148-93:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-93:2]: result = model(**micro_batch) +[ip-26-0-148-93:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-93:2]: return forward_call(*input, **kwargs) +[ip-26-0-148-93:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-93:2]: sharded_logits = self.model( +[ip-26-0-148-93:2]: File "/fsx/lou[ip-26-0-148-151:4]:Traceback (most recent call last): +[ip-26-0-148-151:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-151:4]: main() +[ip-26-0-148-151:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-151:4]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-151:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-151:4]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-148-151:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-151:4]: result = model(**micro_batch) +[ip-26-0-148-151:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-151:4]: return forward_call(*input, **kwargs) +[ip-26-0-148-151:4]: File "/fsx/l[ip-26-0-149-1:0]:Traceback (most recent call last): +[ip-26-0-149-1:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +bna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-93:2]: return forward_call(*input, **kwargs) +[ip-26-0-148-93:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-149-1:2]:Traceback (most recent call last): +[ip-26-0-148-93:2]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-148-93:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-93:2]: return forward_call(*input, **kwargs) +[ip-26-0-148-93:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-148-93:2]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-93:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-93:2]: return forward_call(*input, **kwargs) +[ip-26-0-148-93:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-93:2]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-93:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line[ip-26-0-149-1:0]: main() +[ip-26-0-149-1:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main + 1194, in _call_impl +[ip-26-0-148-93:2]: return forward_call(*input, **kwargs) +[ip-26-0-148-93:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-93:2]: query_states, kv_states = self.qkv( +[ip-26-0-148-93:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +oubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-151:4]: sharded_logits = self.model( +[ip-26-0-148-151:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-151:4]: return forward_call(*input, **kwargs) +[ip-26-0-149-1:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-93:3]:Traceback (most recent call last): +[ip-26-0-148-151:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-151:4]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-148-151:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-151:4]: return forward_call(*input, **kwargs) +[ip-26-0-148-151:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-148-151:4]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-151:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-151:4]: return forward_call(*input, **kwargs) +[ip-26-0-148-151:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-151:4]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask)[ip-26-0-149-1:2]: main() +[ip-26-0-148-93:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in + +[ip-26-0-148-151:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-151:4]: return forward_call(*input, **kwargs) +[ip-26-0-148-151:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-151:4]: query_states, kv_states = self.qkv( +[ip-26-0-148-151:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-151:4]: return forward_call(*input, **kwargs) +[ip-26-0-148-151:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-151:4]: return column_linear( +[ip-26-0-148-151:4]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-149-1:0]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-93:3]: main() +[ip-26-0-148-93:2]: return forward_call(*input, **kwargs) +[ip-26-0-149-1:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-93:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-93:2]: return column_linear( +[ip-26-0-149-1:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-149-1:2]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-93:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-93:3]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-149-1:0]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-149-1:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-93:2]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-149-1:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-149-1:2]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-148-93:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-93:3]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-148-93:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-93:3]: result = model(**micro_batch) +[ip-26-0-149-1:0]: result = model(**micro_batch) +[ip-26-0-149-1:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-93:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-149-1:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-149-1:2]: result = model(**micro_batch) +[ip-26-0-148-93:3]: return forward_call(*input, **kwargs) +[ip-26-0-148-93:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-93:3]: sharded_logits = self.model( +[ip-26-0-149-1:0]: return forward_call(*input, **kwargs) +[ip-26-0-148-93:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-93:3]: return forward_call(*input, **kwargs) +[ip-26-0-148-93:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-93:3]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-148-93:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-149-1:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-247:3]:Traceback (most recent call last): +[ip-26-0-148-93:3]: return forward_call(*input, **kwargs) +[ip-26-0-148-93:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-148-93:3]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-93:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-93:3]: return forward_call(*input, **kwargs) +[ip-26-0-148-93:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-93:3]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-93:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-93:3]: return forward_call(*input, **kwargs) +[ip-26-0-148-93:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-149-1:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-93:3]: query_states, kv_states = self.qkv( +[ip-26-0-148-93:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-93:3]: return forward_call(*input, **kwargs) +[ip-26-0-148-93:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-149-1:2]: return forward_call(*input, **kwargs) +[ip-26-0-148-93:3]: return column_linear( +[ip-26-0-148-93:3]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-149-1:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-149-1:2]: sharded_logits = self.model( +[ip-26-0-149-1:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-247:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-247:3]: main() +[ip-26-0-147-247:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-247:3]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-247:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-247:3]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-247:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-149-1:0]: sharded_logits = self.model( +[ip-26-0-147-247:3]: result = model(**micro_batch) +[ip-26-0-147-247:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-247:3]: return forward_call(*input, **kwargs) +[ip-26-0-147-247:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-247:3]: sharded_logits = self.model( +[ip-26-0-149-1:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-149-1:2]: return forward_call(*input, **kwargs) +[ip-26-0-147-247:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-149-1:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-247:5]:Traceback (most recent call last): +[ip-26-0-147-247:3]: return forward_call(*input, **kwargs) +[ip-26-0-147-247:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-247:5]: main() +[ip-26-0-147-247:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-149-1:0]: return forward_call(*input, **kwargs) +[ip-26-0-147-247:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-247:5]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-149-1:1]:Traceback (most recent call last): +[ip-26-0-147-247:3]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-149-1:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-149-1:0]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-247:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-247:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-149-1:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-149-1:1]: main() +[ip-26-0-147-247:5]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-247:3]: return forward_call(*input, **kwargs) +[ip-26-0-147-247:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-247:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-149-1:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-247:5]: result = model(**micro_batch) +[ip-26-0-147-247:3]: output = self.pp_block(**new_kwargs) +[ip-26-0-149-1:2]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-247:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-247:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-149-1:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-247:5]: return forward_call(*input, **kwargs) +[ip-26-0-147-247:3]: return forward_call(*input, **kwargs) +[ip-26-0-149-1:2]: return forward_call(*input, **kwargs) +[ip-26-0-149-1:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-247:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-247:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-247:5]: sharded_logits = self.model( +[ip-26-0-149-1:3]:Traceback (most recent call last): +[ip-26-0-147-247:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-247:5]: return forward_call(*input, **kwargs) +[ip-26-0-147-247:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-149-1:2]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-247:5]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-247:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-247:5]: return forward_call(*input, **kwargs) +[ip-26-0-149-1:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-149-1:1]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-247:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-247:5]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-247:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-149-1:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-149-1:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-247:3]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-247:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-149-1:3]: main() +[ip-26-0-147-247:5]: return forward_call(*input, **kwargs) +[ip-26-0-149-1:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-247:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-149-1:3]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-247:5]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-247:3]: return forward_call(*input, **kwargs) +[ip-26-0-147-247:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-151:0]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-151:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-151:0]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-148-151:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-151:0]: result = model(**micro_batch) +[ip-26-0-148-151:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-151:0]: return forward_call(*input, **kwargs) +[ip-26-0-148-151:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-149-1:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-247:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-149-1:3]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-247:5]: return forward_call(*input, **kwargs) +[ip-26-0-148-151:0]: sharded_logits = self.model( +[ip-26-0-148-151:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-151:0]: return forward_call(*input, **kwargs) +[ip-26-0-148-151:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-151:0]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-148-151:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-151:0]: return forward_call(*input, **kwargs) +[ip-26-0-148-151:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-149-1:1]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-149-1:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-247:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-247:5]: query_states, kv_states = self.qkv( +[ip-26-0-148-151:0]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-151:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-151:0]: return forward_call(*input, **kwargs) +[ip-26-0-148-151:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-151:0]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-151:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-151:0]: return forward_call(*input, **kwargs) +[ip-26-0-148-151:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-151:0]: query_states, kv_states = self.qkv( +[ip-26-0-148-151:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_imp[ip-26-0-149-1:1]: result = model(**micro_batch) +[ip-26-0-147-247:3]: query_states, kv_states = self.qkv( +l +[ip-26-0-148-151:0]: return forward_call(*input, **kwargs) +[ip-26-0-148-151:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-149-1:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-247:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-151:0]: return column_linear( +[ip-26-0-148-151:0]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-149-1:1]: return forward_call(*input, **kwargs) +[ip-26-0-147-247:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-247:5]: return forward_call(*input, **kwargs) +[ip-26-0-149-1:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-149-1:1]: sharded_logits = self.model( +[ip-26-0-147-247:3]: return forward_call(*input, **kwargs) +[ip-26-0-147-247:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-247:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-149-1:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-247:5]: return column_linear( +[ip-26-0-147-247:3]: return column_linear( +[ip-26-0-149-1:1]: return forward_call(*input, **kwargs) +[ip-26-0-149-1:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-247:3]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-149-1:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-247:5]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-149-1:3]: result = model(**micro_batch) +[ip-26-0-147-247:1]:Traceback (most recent call last): +[ip-26-0-147-247:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-247:1]: main() +[ip-26-0-147-247:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-247:1]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-247:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-151:7]:Traceback (most recent call last): +[ip-26-0-148-151:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-151:7]: main() +[ip-26-0-148-151:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-151:7]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-151:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-151:7]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-148-151:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-151:7]: result = model(**micro_batch) +[ip-26-0-149-1:2]: return forward_call(*input, **kwargs) +[ip-26-0-147-247:1]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-247:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-247:1]: result = model(**micro_batch) +[ip-26-0-149-1:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-247:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-247:1]: return forward_call(*input, **kwargs) +[ip-26-0-147-247:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-247:1]: sharded_logits = self.model( +[ip-26-0-147-247:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-247:1]: return forward_call(*input, **kwargs) +[ip-26-0-147-247:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-247:1]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-247:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-247:1]: return forward_call(*input, **kwargs) +[ip-26-0-148-151:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-151:7]: return forward_call(*input, **kwargs) +[ip-26-0-149-1:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-247:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-247:1]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-247:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-247:1]: return forward_call(*input, **kwargs) +[ip-26-0-147-247:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-247:1]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-151:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-151:7]: sharded_logits = self.model( +[ip-26-0-148-151:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-151:7]: return forward_call(*input, **kwargs) +[ip-26-0-149-1:3]: return forward_call(*input, **kwargs) +[ip-26-0-147-247:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-247:1]: return forward_call(*input, **kwargs) +[ip-26-0-147-247:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-247:1]: query_states, kv_states = self.qkv( +[ip-26-0-147-247:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-247:1]: return forward_call(*input, **kwargs) +[ip-26-0-147-247:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-151:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-149-1:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-247:1]: return column_linear( +[ip-26-0-147-247:1]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-148-151:7]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-148-151:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-151:7]: return forward_call(*input, **kwargs) +[ip-26-0-149-1:2]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-247:6]:Traceback (most recent call last): +[ip-26-0-148-151:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-148-151:7]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-151:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-151:7]: return forward_call(*input, **kwargs) +[ip-26-0-148-151:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-151:7]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-149-1:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-151:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-151:7]: return forward_call(*input, **kwargs) +[ip-26-0-148-151:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-151:7]: query_states, kv_states = self.qkv( +[ip-26-0-149-1:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-151:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-151:7]: return forward_call(*input, **kwargs) +[ip-26-0-148-151:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-151:7]: return column_linear( +[ip-26-0-149-1:3]: sharded_logits = self.model( +[ip-26-0-149-1:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-247:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-151:7]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-149-1:3]: return forward_call(*input, **kwargs) +[ip-26-0-147-247:6]: main() +[ip-26-0-149-1:0]: return forward_call(*input, **kwargs) +[ip-26-0-147-247:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-149-1:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-247:6]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-247:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-149-1:1]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-247:7]:Traceback (most recent call last): +[ip-26-0-147-247:6]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-149-1:2]: return forward_call(*input, **kwargs) +[ip-26-0-147-247:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-149-1:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-247:7]: main() +[ip-26-0-149-1:0]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-247:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-149-1:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-247:7]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-149-1:1]: return forward_call(*input, **kwargs) +[ip-26-0-149-1:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-247:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-149-1:1]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-247:7]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-247:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-149-1:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-247:7]: result = model(**micro_batch) +[ip-26-0-149-1:4]:Traceback (most recent call last): +[ip-26-0-147-247:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-151:2]:Traceback (most recent call last): +[ip-26-0-148-151:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-149-1:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-247:7]: return forward_call(*input, **kwargs) +[ip-26-0-149-1:2]: query_states, kv_states = self.qkv( +[ip-26-0-147-247:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-149-1:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-149-1:3]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-247:7]: sharded_logits = self.model( +[ip-26-0-149-1:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-247:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-151:2]: main() +[ip-26-0-148-151:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-151:2]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-149-1:3]: return forward_call(*input, **kwargs) +[ip-26-0-147-247:6]: result = model(**micro_batch) +[ip-26-0-147-247:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-151:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-149-1:6]:Traceback (most recent call last): +[ip-26-0-147-247:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-247:7]: return forward_call(*input, **kwargs) +[ip-26-0-148-151:2]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-148-151:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-151:2]: result = model(**micro_batch) +[ip-26-0-148-151:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-149-1:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-149-1:2]: return forward_call(*input, **kwargs) +[ip-26-0-147-247:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-151:2]: return forward_call(*input, **kwargs) +[ip-26-0-148-151:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-151:2]: sharded_logits = self.model( +[ip-26-0-148-151:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-151:2]: return forward_call(*input, **kwargs) +[ip-26-0-148-151:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-149-1:1]: return forward_call(*input, **kwargs) +[ip-26-0-147-247:6]: return forward_call(*input, **kwargs) +[ip-26-0-147-247:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-247:6]: sharded_logits = self.model( +[ip-26-0-149-1:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-247:7]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-149-1:1]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-149-1:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-247:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-149-1:1]: return forward_call(*input, **kwargs) +[ip-26-0-147-247:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-247:6]: return forward_call(*input, **kwargs) +[ip-26-0-149-1:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-247:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-149-1:1]: query_states, kv_states = self.qkv( +[ip-26-0-147-247:7]: return forward_call(*input, **kwargs) +[ip-26-0-149-1:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-247:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-149-1:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-149-1:6]: main() +[ip-26-0-147-247:7]: output = self.pp_block(**new_kwargs) +[ip-26-0-149-1:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-247:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-149-1:6]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-149-1:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-149-1:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-247:7]: return forward_call(*input, **kwargs) +[ip-26-0-149-1:2]: return column_linear( +[ip-26-0-149-1:2]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-247:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-149-1:0]: return forward_call(*input, **kwargs) +[ip-26-0-147-247:6]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-247:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-247:6]: return forward_call(*input, **kwargs) +[ip-26-0-149-1:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-247:7]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-149-1:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-247:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-149-1:6]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-149-1:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-247:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-247:6]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-247:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-149-1:0]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-247:7]: return forward_call(*input, **kwargs) +[ip-26-0-149-1:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-247:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-247:7]: query_states, kv_states = self.qkv( +[ip-26-0-149-1:0]: return forward_call(*input, **kwargs) +[ip-26-0-149-1:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-247:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-149-1:3]: output = self.pp_block(**new_kwargs) +[ip-26-0-149-1:4]: main() +[ip-26-0-147-247:7]: return forward_call(*input, **kwargs) +[ip-26-0-147-247:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-247:7]: return column_linear( +[ip-26-0-149-1:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-247:7]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-149-1:6]: result = model(**micro_batch) +[ip-26-0-149-1:1]: return forward_call(*input, **kwargs) +[ip-26-0-149-1:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-149-1:1]: return column_linear( +[ip-26-0-149-1:1]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-149-1:3]: return forward_call(*input, **kwargs) +[ip-26-0-149-1:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-149-1:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-247:2]:Traceback (most recent call last): +[ip-26-0-147-247:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-149-1:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-247:2]: main() +[ip-26-0-147-247:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-247:2]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-247:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-149-1:4]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-247:2]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-247:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-247:2]: result = model(**micro_batch) +[ip-26-0-147-247:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-149-1:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-247:2]: return forward_call(*input, **kwargs) +[ip-26-0-147-247:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-247:2]: sharded_logits = self.model( +[ip-26-0-147-247:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-149-1:6]: return forward_call(*input, **kwargs) +[ip-26-0-149-1:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-247:2]: return forward_call(*input, **kwargs) +[ip-26-0-147-247:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-247:2]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-149-1:6]: sharded_logits = self.model( +[ip-26-0-149-1:4]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-247:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-247:2]: return forward_call(*input, **kwargs) +[ip-26-0-147-247:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-247:2]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-247:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-149-1:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-149-1:4]: result = model(**micro_batch) +[ip-26-0-147-247:2]: return forward_call(*input, **kwargs) +[ip-26-0-147-247:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-247:2]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-247:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-149-1:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-247:2]: return forward_call(*input, **kwargs) +[ip-26-0-147-247:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-247:2]: query_states, kv_states = self.qkv( +[ip-26-0-147-247:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-247:2]: return forward_call(*input, **kwargs) +[ip-26-0-147-247:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-149-1:7]:Traceback (most recent call last): +[ip-26-0-149-1:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-247:2]: return column_linear( +[ip-26-0-147-247:2]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-149-1:7]: main() +[ip-26-0-149-1:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-149-1:7]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-149-1:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-149-1:4]: return forward_call(*input, **kwargs) +[ip-26-0-149-1:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-149-1:4]: sharded_logits = self.model( +[ip-26-0-149-1:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-149-1:3]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-149-1:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-149-1:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-149-1:7]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-187:7]:Traceback (most recent call last): +[ip-26-0-147-187:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-187:7]: main() +[ip-26-0-147-187:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-187:7]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-187:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-187:7]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-187:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-187:7]: result = model(**micro_batch) +[ip-26-0-147-187:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-187:7]: return forward_call(*input, **kwargs) +[ip-26-0-147-187:7]: File "/fsx/l[ip-26-0-149-1:0]: query_states, kv_states = self.qkv( +[ip-26-0-149-1:4]: return forward_call(*input, **kwargs) +[ip-26-0-149-1:6]: return forward_call(*input, **kwargs) +[ip-26-0-149-1:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-149-1:6]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-247:4]:Traceback (most recent call last): +[ip-26-0-147-247:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-93:5]:Traceback (most recent call last): +[ip-26-0-148-93:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-93:5]: main() +[ip-26-0-149-1:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-149-1:4]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-149-1:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-149-1:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-247:4]: main() +[ip-26-0-147-247:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-247:4]: outputs = pipeline_engine.train_batch_iter( +oubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-187:7]: sharded_logits = self.model( +[ip-26-0-149-1:0]: return forward_call(*input, **kwargs) +[ip-26-0-147-247:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-187:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-187:7]: return forward_call(*input, **kwargs) +[ip-26-0-147-187:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-187:7]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-187:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-187:7]: return forward_call(*input, **kwargs) +[ip-26-0-147-187:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-187:7]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-187:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-187:7]: return forward_call(*input, **kwargs) +[ip-26-0[ip-26-0-149-1:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-149-1:6]: return forward_call(*input, **kwargs) +[ip-26-0-147-247:4]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-247:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-247:4]: result = model(**micro_batch) +[ip-26-0-147-247:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +-147-187:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-187:7]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-187:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-187:7]: return forward_call(*input, **kwargs) +[ip-26-0-147-187:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-187:7]: query_states, kv_states = self.qkv( +[ip-26-0-147-187:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-187:7]: return forward_call(*input, **kwargs) +[ip-26-0-147-187:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-187:7]: return column_linear( +[ip-26-0-147-187:7]:TypeError: split() missing requ[ip-26-0-149-1:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-149-1:7]: result = model(**micro_batch) +[ip-26-0-149-1:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-149-1:4]: return forward_call(*input, **kwargs) +[ip-26-0-147-247:0]:Traceback (most recent call last): +[ip-26-0-148-93:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-93:5]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-93:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-93:5]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +ired argument 'split_size_or_sections' (pos 2) +[ip-26-0-149-1:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-149-1:3]: return forward_call(*input, **kwargs) +[ip-26-0-147-247:4]: return forward_call(*input, **kwargs) +[ip-26-0-148-93:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-93:5]: result = model(**micro_batch) +[ip-26-0-148-93:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-93:5]: return forward_call(*input, **kwargs) +[ip-26-0-148-93:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-93:5]: sharded_logits = self.model( +[ip-26-0-148-93:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-149-1:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-149-1:3]: query_states, kv_states = self.qkv( +[ip-26-0-147-247:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-93:5]: return forward_call(*input, **kwargs) +[ip-26-0-148-93:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-93:5]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-148-93:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-93:5]: return forward_call(*input, **kwargs) +[ip-26-0-148-93:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-148-93:5]: output = self.pp_block(**new_kwargs) +[ip-26-0-149-1:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-247:0]: main() +[ip-26-0-147-247:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-93:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-93:5]: return forward_call(*input, **kwargs) +[ip-26-0-148-93:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-149-1:4]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-247:0]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-247:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-93:5]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-93:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-93:5]: return forward_call(*input, **kwargs) +[ip-26-0-149-1:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-149-1:0]: return column_linear( +[ip-26-0-147-247:0]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-247:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-93:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-93:5]: query_states, kv_states = self.qkv( +[ip-26-0-148-93:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-93:5]: return forward_call(*input, **kwargs) +[ip-26-0-149-1:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-247:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-247:4]: sharded_logits = self.model( +[ip-26-0-147-247:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-93:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-93:5]: return column_linear( +[ip-26-0-148-93:5]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-149-1:6]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-247:4]: return forward_call(*input, **kwargs) +[ip-26-0-149-1:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-149-1:6]: return forward_call(*input, **kwargs) +[ip-26-0-149-1:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-247:0]: result = model(**micro_batch) +[ip-26-0-147-247:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-247:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-247:0]: return forward_call(*input, **kwargs) +[ip-26-0-147-247:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-149-1:4]: return forward_call(*input, **kwargs) +[ip-26-0-147-247:4]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-247:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-149-1:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-149-1:4]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-247:4]: return forward_call(*input, **kwargs) +[ip-26-0-149-1:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-149-1:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-247:0]: sharded_logits = self.model( +[ip-26-0-147-247:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-247:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-247:4]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-245:1]:Traceback (most recent call last): +[ip-26-0-148-245:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-245:1]: main() +[ip-26-0-148-245:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-245:1]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-245:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-245:1]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-148-245:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-245:1]: result = model(**micro_batch) +[ip-26-0-148-245:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-149-1:6]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-149-1:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-247:0]: return forward_call(*input, **kwargs) +[ip-26-0-149-1:6]: return forward_call(*input, **kwargs) +[ip-26-0-147-247:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-149-1:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-247:0]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-149-1:6]: query_states, kv_states = self.qkv( +[ip-26-0-149-1:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-247:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-149-1:6]: return forward_call(*input, **kwargs) +[ip-26-0-147-247:0]: return forward_call(*input, **kwargs) +[ip-26-0-148-245:1]: return forward_call(*input, **kwargs) +[ip-26-0-148-245:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-245:1]: sharded_logits = self.model( +[ip-26-0-148-245:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-245:1]: return forward_call(*input, **kwargs) +[ip-26-0-148-245:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-149-1:0]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-247:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-247:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-148-245:1]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-149-1:4]: return forward_call(*input, **kwargs) +[ip-26-0-147-247:0]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-245:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-245:1]: return forward_call(*input, **kwargs) +[ip-26-0-149-1:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-247:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-245:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-149-1:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-247:4]: return forward_call(*input, **kwargs) +[ip-26-0-147-247:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-245:1]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-245:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-245:1]: return forward_call(*input, **kwargs) +[ip-26-0-149-1:3]: return forward_call(*input, **kwargs) +[ip-26-0-147-247:4]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-247:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-245:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-245:1]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-245:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-245:1]: return forward_call(*input, **kwargs) +[ip-26-0-148-245:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-245:1]: query_states, kv_states = self.qkv( +[ip-26-0-149-1:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-149-1:3]: return column_linear( +[ip-26-0-147-247:0]: return forward_call(*input, **kwargs) +[ip-26-0-148-245:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-245:1]: return forward_call(*input, **kwargs) +[ip-26-0-148-245:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-245:1]: return column_linear( +[ip-26-0-149-1:3]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-247:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-245:1]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-148-151:3]:Traceback (most recent call last): +[ip-26-0-148-151:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-151:3]: main() +[ip-26-0-149-1:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-247:0]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-247:4]: return forward_call(*input, **kwargs) +[ip-26-0-148-245:4]:Traceback (most recent call last): +[ip-26-0-149-1:6]: return column_linear( +[ip-26-0-147-247:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-245:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-149-1:7]: return forward_call(*input, **kwargs) +[ip-26-0-147-247:0]: return forward_call(*input, **kwargs) +[ip-26-0-149-1:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-247:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-149-1:6]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-247:0]: query_states, kv_states = self.qkv( +[ip-26-0-147-247:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-151:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-151:3]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-151:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-151:3]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-148-151:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-149-1:4]: query_states, kv_states = self.qkv( +[ip-26-0-149-1:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-247:0]: return forward_call(*input, **kwargs) +[ip-26-0-147-247:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-247:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-245:4]: main() +[ip-26-0-148-245:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-245:4]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-245:0]:Traceback (most recent call last): +[ip-26-0-148-245:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-245:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-245:0]: main() +[ip-26-0-148-245:3]:Traceback (most recent call last): +[ip-26-0-148-245:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-245:3]: main() +[ip-26-0-148-245:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-245:6]:Traceback (most recent call last): +[ip-26-0-148-151:3]: result = model(**micro_batch) +[ip-26-0-148-151:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-151:3]: return forward_call(*input, **kwargs) +[ip-26-0-148-151:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-151:3]: sharded_logits = self.model( +[ip-26-0-148-151:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-151:3]: return forward_call(*input, **kwargs) +[ip-26-0-148-151:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-55:5]:Traceback (most recent call last): +[ip-26-0-148-55:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-55:5]: main() +[ip-26-0-148-55:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-55:5]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-55:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-55:5]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-148-55:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-55:5]: result = model(**micro_batch) +[ip-26-0-148-55:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-55:5]: return forward_call(*input, **kwargs) +[ip-26-0-148-55:5]: File "/fsx/loubna/code/f[ip-26-0-149-1:7]: sharded_logits = self.model( +[ip-26-0-147-247:0]: return column_linear( +[ip-26-0-147-247:4]: query_states, kv_states = self.qkv( +[ip-26-0-148-151:3]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-148-151:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-151:3]: return forward_call(*input, **kwargs) +[ip-26-0-149-1:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-247:0]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-148-151:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-148-151:3]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-151:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-149-1:7]: return forward_call(*input, **kwargs) +[ip-26-0-147-247:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-247:4]: return forward_call(*input, **kwargs) +[ip-26-0-147-247:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-151:3]: return forward_call(*input, **kwargs) +[ip-26-0-148-151:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-151:3]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-151:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-151:3]: return forward_call(*input, **kwargs) +[ip-26-0-149-1:4]: return forward_call(*input, **kwargs) +[ip-26-0-149-1:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-247:4]: return column_linear( +[ip-26-0-147-247:4]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-148-245:4]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-148-245:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-245:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-151:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-151:3]: query_states, kv_states = self.qkv( +[ip-26-0-148-151:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-151:3]: return forward_call(*input, **kwargs) +[ip-26-0-149-1:4]: return column_linear( +[ip-26-0-148-245:0]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-245:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-245:0]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-148-151:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-151:3]: return column_linear( +[ip-26-0-148-151:3]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-149-1:4]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-148-245:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-149-1:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-245:6]: main() +[ip-26-0-149-1:7]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-148-245:3]: outputs = pipeline_engine.train_batch_iter( +ork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-55:5]: sharded_logits = self.model( +[ip-26-0-148-55:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-55:5]: return forward_call(*input, **kwargs) +[ip-26-0-148-55:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-55:5]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-148-55:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-55:5]: return forward_call(*input, **kwargs) +[ip-26-0-148-55:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-148-55:5]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-55:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/[ip-26-0-149-1:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-245:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +modules/module.py", line 1194, in _call_impl +[ip-26-0-148-55:5]: return forward_call(*input, **kwargs) +[ip-26-0-148-55:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-55:5]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-55:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-55:5]: return forward_call(*input, **kwargs) +[ip-26-0-148-55:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-55:5]: query_states, kv_states = self.qkv( +[ip-26-0-148-55:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-55:5]: return forward_call(*input, **kwargs) +[ip-26-0-148-55:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in [ip-26-0-149-1:7]: return forward_call(*input, **kwargs) +[ip-26-0-149-1:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-149-1:7]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-245:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +forward +[ip-26-0-148-55:5]: return column_linear( +[ip-26-0-148-55:5]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-149-1:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-149-1:7]: return forward_call(*input, **kwargs) +[ip-26-0-149-1:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-149-1:7]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-149-1:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-245:6]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-149-1:7]: return forward_call(*input, **kwargs) +[ip-26-0-149-1:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-149-1:7]: query_states, kv_states = self.qkv( +[ip-26-0-149-1:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-149-1:7]: return forward_call(*input, **kwargs) +[ip-26-0-149-1:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-149-1:7]: return column_linear( +[ip-26-0-149-1:7]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-148-245:0]: result = model(**micro_batch) +[ip-26-0-148-170:0]:Traceback (most recent call last): +[ip-26-0-148-170:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-170:0]: main() +[ip-26-0-148-170:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-245:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-245:0]: return forward_call(*input, **kwargs) +[ip-26-0-148-245:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-245:0]: sharded_logits = self.model( +[ip-26-0-148-245:4]: result = model(**micro_batch) +[ip-26-0-148-245:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-245:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-233:3]:Traceback (most recent call last): +[ip-26-0-147-233:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-233:3]: main() +[ip-26-0-148-245:3]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-148-245:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-170:0]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-170:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-170:0]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-148-170:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-170:0]: result = model(**micro_batch) +[ip-26-0-148-170:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-170:0]: return forward_call(*input, **kwargs) +[ip-26-0-148-170:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-170:0]: sharded_logits = self.model( +[ip-26-0-148-170:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-[ip-26-0-148-245:3]: result = model(**micro_batch) +148-170:0]: return forward_call(*input, **kwargs) +[ip-26-0-148-170:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-170:0]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-148-170:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-170:0]: return forward_call(*input, **kwargs) +[ip-26-0-148-170:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-148-170:0]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-170:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-170:0]: return forward_call(*input, **kwargs) +[ip-26-0-148-170:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-170:0]: output = self.attn(hidd[ip-26-0-148-245:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +en_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-170:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-170:0]: return forward_call(*input, **kwargs) +[ip-26-0-148-245:4]: return forward_call(*input, **kwargs) +[ip-26-0-148-170:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-170:0]: query_states, kv_states = self.qkv( +[ip-26-0-148-170:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-170:0]: return forward_call(*input, **kwargs) +[ip-26-0-148-170:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-170:0]: return column_linear( +[ip-26-0-148-170:0]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-148-245:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-233:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-233:3]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-233:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-233:3]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-233:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-233:3]: result = model(**micro_batch) +[ip-26-0-147-233:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-233:3]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-233:3]: sharded_logits = self.model( +[ip-26-0-147-233:3]: File "/fsx/loubna/miniconda3/envs[ip-26-0-148-245:4]: sharded_logits = self.model( +/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-233:3]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-233:3]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-233:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-233:3]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-233:3]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-233:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-233:3]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:3]: File "/fsx/loubna/code/fork/brrr/examples[ip-26-0-148-245:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-187:2]:Traceback (most recent call last): +[ip-26-0-147-187:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-187:2]: main() +[ip-26-0-147-187:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-233:3]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-233:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-233:3]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-233:3]: query_states, kv_states = self.qkv( +[ip-26-0-148-245:3]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-233:3]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-245:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-233:3]: return column_linear( +[ip-26-0-148-245:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-233:3]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-148-245:4]: return forward_call(*input, **kwargs) +[ip-26-0-147-187:2]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-187:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-187:2]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-187:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-187:2]: result = model(**micro_batch) +[ip-26-0-148-245:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-187:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-233:4]:Traceback (most recent call last): +[ip-26-0-148-245:4]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-148-245:3]: sharded_logits = self.model( +[ip-26-0-148-245:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-245:3]: return forward_call(*input, **kwargs) +[ip-26-0-148-245:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-245:3]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-148-245:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-245:6]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-187:3]:Traceback (most recent call last): +[ip-26-0-147-187:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-151:5]:Traceback (most recent call last): +[ip-26-0-148-151:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-151:5]: main() +[ip-26-0-147-233:6]:Traceback (most recent call last): +[ip-26-0-148-245:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-245:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-245:3]: return forward_call(*input, **kwargs) +[ip-26-0-147-187:3]: main() +[ip-26-0-147-187:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-187:3]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-245:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-187:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-187:3]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-187:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-187:3]: result = model(**micro_batch) +[ip-26-0-147-187:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-187:3]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-170:4]:Traceback (most recent call last): +[ip-26-0-148-170:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-245:5]:Traceback (most recent call last): +[ip-26-0-147-187:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-187:3]: sharded_logits = self.model( +[ip-26-0-147-187:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-187:3]: return forward_call(*input, **kwargs) +[ip-26-0-147-187:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-187:3]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-187:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-151:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-151:5]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-151:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-151:5]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-148-151:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-151:5]: result = model(**micro_batch) +[ip-26-0-148-151:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-151:5]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:2]:Traceback (most recent call last): +[ip-26-0-148-245:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-187:3]: return forward_call(*input, **kwargs) +[ip-26-0-147-187:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-187:3]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-187:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-151:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-151:5]: sharded_logits = self.model( +[ip-26-0-148-151:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-151:5]: return forward_call(*input, **kwargs) +[ip-26-0-148-151:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-151:5]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-148-151:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-151:5]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-233:6]: main() +[ip-26-0-148-245:5]: main() +[ip-26-0-148-93:6]:Traceback (most recent call last): +[ip-26-0-148-93:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-93:6]: main() +[ip-26-0-148-93:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-187:3]: return forward_call(*input, **kwargs) +[ip-26-0-147-187:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-187:3]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-187:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-151:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-148-151:5]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-151:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-151:5]: return forward_call(*input, **kwargs) +[ip-26-0-148-151:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-233:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-170:4]: main() +[ip-26-0-148-245:3]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-245:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-245:4]: return forward_call(*input, **kwargs) +[ip-26-0-148-245:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-187:3]: return forward_call(*input, **kwargs) +[ip-26-0-147-187:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-187:3]: query_states, kv_states = self.qkv( +[ip-26-0-147-187:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-187:3]: return forward_call(*input, **kwargs) +[ip-26-0-147-187:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-151:5]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-151:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-151:5]: return forward_call(*input, **kwargs) +[ip-26-0-148-151:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-151:5]: query_states, kv_states = self.qkv( +[ip-26-0-148-151:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-233:6]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-170:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-245:4]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-93:6]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-93:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-187:3]: return column_linear( +[ip-26-0-147-187:3]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-148-151:5]: return forward_call(*input, **kwargs) +[ip-26-0-148-151:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-151:5]: return column_linear( +[ip-26-0-147-233:4]: main() +[ip-26-0-148-170:4]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-170:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-170:4]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-148-170:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-170:4]: result = model(**micro_batch) +[ip-26-0-148-245:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-93:6]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-148-93:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-93:6]: result = model(**micro_batch) +[ip-26-0-148-93:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-93:6]: return forward_call(*input, **kwargs) +[ip-26-0-148-93:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-151:5]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-233:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-170:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-170:4]: return forward_call(*input, **kwargs) +[ip-26-0-148-170:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-245:6]: result = model(**micro_batch) +[ip-26-0-148-93:7]:Traceback (most recent call last): +[ip-26-0-148-170:4]: sharded_logits = self.model( +[ip-26-0-148-170:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-170:4]: return forward_call(*input, **kwargs) +[ip-26-0-148-170:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-245:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-245:6]: return forward_call(*input, **kwargs) +[ip-26-0-148-93:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-170:4]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-148-170:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-170:4]: return forward_call(*input, **kwargs) +[ip-26-0-148-245:4]: return forward_call(*input, **kwargs) +[ip-26-0-148-93:6]: sharded_logits = self.model( +[ip-26-0-148-115:6]:Traceback (most recent call last): +[ip-26-0-148-115:2]:Traceback (most recent call last): +[ip-26-0-148-170:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-148-245:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-93:7]: main() +[ip-26-0-148-93:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-170:7]:Traceback (most recent call last): +[ip-26-0-148-245:5]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-245:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-93:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-93:6]: return forward_call(*input, **kwargs) +[ip-26-0-148-170:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-245:6]: sharded_logits = self.model( +[ip-26-0-148-93:7]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-233:6]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-204:1]:Traceback (most recent call last): +[ip-26-0-147-204:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-204:1]: main() +[ip-26-0-148-170:7]: main() +[ip-26-0-148-170:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-245:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-93:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-93:7]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-233:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-170:4]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-170:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-245:6]: return forward_call(*input, **kwargs) +[ip-26-0-148-93:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-93:6]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-233:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-170:4]: return forward_call(*input, **kwargs) +[ip-26-0-148-245:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-93:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-233:2]: main() +[ip-26-0-147-204:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-204:1]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-204:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-204:1]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-204:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-204:1]: result = model(**micro_batch) +[ip-26-0-147-204:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-204:1]: return forward_call(*input, **kwargs) +[ip-26-0-148-170:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-170:4]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-245:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-93:7]: result = model(**micro_batch) +[ip-26-0-148-93:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-233:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-204:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-170:7]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-245:3]: return forward_call(*input, **kwargs) +[ip-26-0-148-93:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-93:6]: return forward_call(*input, **kwargs) +[ip-26-0-148-115:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-115:6]: main() +[ip-26-0-148-115:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-233:6]: result = model(**micro_batch) +[ip-26-0-147-204:1]: sharded_logits = self.model( +[ip-26-0-148-170:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-245:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-93:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-148-93:7]: return forward_call(*input, **kwargs) +[ip-26-0-148-115:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-115:2]: main() +[ip-26-0-148-115:6]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-233:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-204:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-170:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-170:4]: return forward_call(*input, **kwargs) +[ip-26-0-148-245:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-245:4]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-93:6]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-115:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-115:6]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-233:6]: return forward_call(*input, **kwargs) +[ip-26-0-147-204:1]: return forward_call(*input, **kwargs) +[ip-26-0-147-204:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-170:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-170:7]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-148-245:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-93:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-93:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-115:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-233:2]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-233:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-204:1]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-148-170:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-170:4]: query_states, kv_states = self.qkv( +[ip-26-0-148-245:4]: return forward_call(*input, **kwargs) +[ip-26-0-148-93:7]: sharded_logits = self.model( +[ip-26-0-148-93:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-115:2]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-233:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-204:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-170:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-245:6]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-148-245:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-93:6]: return forward_call(*input, **kwargs) +[ip-26-0-148-115:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-151:1]:Traceback (most recent call last): +[ip-26-0-148-151:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-151:1]: main() +[ip-26-0-148-151:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-233:2]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-204:1]: return forward_call(*input, **kwargs) +[ip-26-0-148-170:7]: result = model(**micro_batch) +[ip-26-0-148-245:6]: return forward_call(*input, **kwargs) +[ip-26-0-148-93:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-115:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-233:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-204:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-204:1]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-170:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-170:4]: return forward_call(*input, **kwargs) +[ip-26-0-148-170:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-245:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-148-93:7]: return forward_call(*input, **kwargs) +[ip-26-0-148-93:6]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-115:2]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-148-115:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-115:2]: result = model(**micro_batch) +[ip-26-0-147-233:2]: result = model(**micro_batch) +[ip-26-0-147-204:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-204:1]: return forward_call(*input, **kwargs) +[ip-26-0-148-170:7]: return forward_call(*input, **kwargs) +[ip-26-0-148-170:4]: return column_linear( +[ip-26-0-148-245:5]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-148-93:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-93:7]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-187:0]:Traceback (most recent call last): +[ip-26-0-147-187:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-187:0]: main() +[ip-26-0-147-187:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-187:0]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-115:6]: result = model(**micro_batch) +[ip-26-0-148-151:1]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-151:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-151:1]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-148-151:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-151:1]: result = model(**micro_batch) +[ip-26-0-148-151:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-151:1]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-204:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-170:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-245:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-93:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-93:6]: return forward_call(*input, **kwargs) +[ip-26-0-147-187:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-187:0]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-187:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-187:0]: result = model(**micro_batch) +[ip-26-0-147-187:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-115:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-151:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-151:1]: sharded_logits = self.model( +[ip-26-0-148-151:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-151:1]: return forward_call(*input, **kwargs) +[ip-26-0-148-151:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-151:1]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-148-151:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-151:1]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:6]: sharded_logits = self.model( +[ip-26-0-147-204:6]:Traceback (most recent call last): +[ip-26-0-148-170:7]: sharded_logits = self.model( +[ip-26-0-148-170:4]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-148-170:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-245:3]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-245:5]: result = model(**micro_batch) +[ip-26-0-148-93:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-187:0]: return forward_call(*input, **kwargs) +[ip-26-0-147-187:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-187:0]: sharded_logits = self.model( +[ip-26-0-147-187:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-187:0]: return forward_call(*input, **kwargs) +[ip-26-0-147-187:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-187:0]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-187:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-187:0]: return forward_call(*input, **kwargs) +[ip-26-0-147-187:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-187:0]: output = self.pp_block[ip-26-0-148-115:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-151:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-148-151:1]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-151:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-151:1]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-204:1]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-204:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-170:7]: return forward_call(*input, **kwargs) +[ip-26-0-148-170:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-245:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-93:7]: return forward_call(*input, **kwargs) +(**new_kwargs) +[ip-26-0-148-115:6]: return forward_call(*input, **kwargs) +[ip-26-0-148-151:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-151:1]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-151:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-151:1]: return forward_call(*input, **kwargs) +[ip-26-0-148-151:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-151:1]: query_states, kv_states = self.qkv( +[ip-26-0-148-151:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-233:2]: return forward_call(*input, **kwargs) +[ip-26-0-147-204:1]: return forward_call(*input, **kwargs) +[ip-26-0-148-170:7]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-148-170:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-245:4]: query_states, kv_states = self.qkv( +[ip-26-0-148-93:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-187:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-187:0]: return forward_call(*input, **kwargs) +[ip-26-0-147-187:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-187:0]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-187:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-187:0]: return forward_call(*input, **kwargs) +[ip-26-0-148-115:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-151:1]: return forward_call(*input, **kwargs) +[ip-26-0-148-151:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-151:1]: return column_linear( +[ip-26-0-148-151:1]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-233:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-204:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-170:7]: return forward_call(*input, **kwargs) +[ip-26-0-148-170:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-148-245:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-245:4]: return forward_call(*input, **kwargs) +[ip-26-0-148-93:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-148-93:7]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-187:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-187:0]: query_states, kv_states = self.qkv( +[ip-26-0-147-187:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-187:0]: return forward_call(*input, **kwargs) +[ip-26-0-147-187:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-187:0]: return column_linear( +[ip-26-0-148-115:6]: sharded_logits = self.model( +[ip-26-0-147-233:6]: return forward_call(*input, **kwargs) +[ip-26-0-147-204:1]: query_states, kv_states = self.qkv( +[ip-26-0-148-170:7]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-170:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-245:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-187:0]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-148-115:5]:Traceback (most recent call last): +[ip-26-0-148-115:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-233:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-204:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-170:1]:Traceback (most recent call last): +[ip-26-0-148-245:4]: return column_linear( +[ip-26-0-148-115:6]: return forward_call(*input, **kwargs) +[ip-26-0-148-151:6]:Traceback (most recent call last): +[ip-26-0-148-151:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-151:6]: main() +[ip-26-0-147-233:4]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-204:6]: main() +[ip-26-0-148-170:7]: return forward_call(*input, **kwargs) +[ip-26-0-148-170:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-245:4]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-148-93:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-93:7]: return forward_call(*input, **kwargs) +[ip-26-0-148-115:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-151:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-151:6]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-151:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-151:6]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-233:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-204:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-170:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-170:1]: main() +[ip-26-0-148-245:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-93:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-93:7]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-93:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-93:7]: return forward_call(*input, **kwargs) +[ip-26-0-148-93:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-115:5]: main() +[ip-26-0-148-151:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-151:6]: result = model(**micro_batch) +[ip-26-0-148-151:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-151:6]: return forward_call(*input, **kwargs) +[ip-26-0-148-151:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-151:6]: sharded_logits = self.model( +[ip-26-0-147-233:4]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-204:6]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-170:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-245:5]: return forward_call(*input, **kwargs) +[ip-26-0-148-93:7]: query_states, kv_states = self.qkv( +[ip-26-0-148-115:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-115:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-115:6]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-148-151:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-151:6]: return forward_call(*input, **kwargs) +[ip-26-0-148-151:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-151:6]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-148-151:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-233:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-204:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-170:7]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-170:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-245:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-245:0]: return forward_call(*input, **kwargs) +[ip-26-0-148-93:4]:Traceback (most recent call last): +[ip-26-0-148-115:5]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-151:6]: return forward_call(*input, **kwargs) +[ip-26-0-148-151:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-148-151:6]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-151:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-151:6]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-204:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-204:1]: return forward_call(*input, **kwargs) +[ip-26-0-148-170:1]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-245:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-93:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-93:7]: return forward_call(*input, **kwargs) +[ip-26-0-148-93:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-93:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-115:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-151:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-151:6]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-151:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-151:6]: return forward_call(*input, **kwargs) +[ip-26-0-148-151:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-151:6]: query_states, kv_states = self.qkv( +[ip-26-0-148-151:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-151:6]: return forward_call(*input, **kwargs) +[ip-26-0-148-151:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-151:6]: return column_linear( +[ip-26-0-147-233:2]: sharded_logits = self.model( +[ip-26-0-147-204:6]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-204:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-170:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-170:7]: return forward_call(*input, **kwargs) +[ip-26-0-148-245:5]: sharded_logits = self.model( +[ip-26-0-148-245:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-93:7]: return column_linear( +[ip-26-0-148-93:7]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-148-115:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-115:6]: return forward_call(*input, **kwargs) +[ip-26-0-148-151:6]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-233:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-204:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-204:1]: return column_linear( +[ip-26-0-148-170:1]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-148-170:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-245:5]: return forward_call(*input, **kwargs) +[ip-26-0-148-115:2]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:2]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-204:6]: result = model(**micro_batch) +[ip-26-0-147-204:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-170:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-170:7]: query_states, kv_states = self.qkv( +[ip-26-0-148-245:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-115:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-233:2]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-204:1]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-148-170:1]: result = model(**micro_batch) +[ip-26-0-148-245:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-93:1]:Traceback (most recent call last): +[ip-26-0-148-93:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-115:2]: sharded_logits = self.model( +[ip-26-0-147-233:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-204:6]: return forward_call(*input, **kwargs) +[ip-26-0-147-204:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-204:6]: sharded_logits = self.model( +[ip-26-0-148-170:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-245:0]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-148-245:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-115:5]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-233:2]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-204:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-204:6]: return forward_call(*input, **kwargs) +[ip-26-0-147-204:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-170:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-170:7]: return forward_call(*input, **kwargs) +[ip-26-0-148-245:0]: return forward_call(*input, **kwargs) +[ip-26-0-148-245:6]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-187:1]:Traceback (most recent call last): +[ip-26-0-147-187:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-187:1]: main() +[ip-26-0-148-115:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-115:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-233:2]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-204:6]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-148-170:1]: return forward_call(*input, **kwargs) +[ip-26-0-148-170:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-245:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-115:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-233:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-204:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-204:6]: return forward_call(*input, **kwargs) +[ip-26-0-147-204:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-204:6]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-170:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-170:7]: return column_linear( +[ip-26-0-148-170:7]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-148-245:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-148-115:2]: return forward_call(*input, **kwargs) +[ip-26-0-148-115:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-233:4]: result = model(**micro_batch) +[ip-26-0-147-204:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-204:6]: return forward_call(*input, **kwargs) +[ip-26-0-147-204:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-170:1]: sharded_logits = self.model( +[ip-26-0-148-170:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-170:1]: return forward_call(*input, **kwargs) +[ip-26-0-148-245:0]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-187:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-187:1]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-187:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-115:5]: result = model(**micro_batch) +[ip-26-0-147-233:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-204:6]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-204:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-204:6]: return forward_call(*input, **kwargs) +[ip-26-0-147-204:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-170:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-170:1]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-148-245:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-115:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-233:2]: return forward_call(*input, **kwargs) +[ip-26-0-147-204:6]: query_states, kv_states = self.qkv( +[ip-26-0-147-204:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-204:6]: return forward_call(*input, **kwargs) +[ip-26-0-147-204:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-204:6]: return column_linear( +[ip-26-0-147-204:6]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-148-170:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-170:1]: return forward_call(*input, **kwargs) +[ip-26-0-148-170:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-148-170:1]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-245:3]: return forward_call(*input, **kwargs) +[ip-26-0-148-115:5]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:6]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-233:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-170:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-170:1]: return forward_call(*input, **kwargs) +[ip-26-0-148-170:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-170:1]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-245:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-115:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-115:5]: sharded_logits = self.model( +[ip-26-0-148-115:6]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-115:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-233:4]: return forward_call(*input, **kwargs) +[ip-26-0-148-170:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-170:1]: return forward_call(*input, **kwargs) +[ip-26-0-148-170:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-245:3]: query_states, kv_states = self.qkv( +[ip-26-0-148-245:5]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-148-115:6]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-204:0]:Traceback (most recent call last): +[ip-26-0-147-204:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-245:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-115:2]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-148-115:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-233:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-204:0]: main() +[ip-26-0-147-204:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-245:5]: return forward_call(*input, **kwargs) +[ip-26-0-148-115:2]: return forward_call(*input, **kwargs) +[ip-26-0-147-204:0]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-204:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-204:0]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-204:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-204:0]: result = model(**micro_batch) +[ip-26-0-148-245:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-148-245:5]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-115:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-204:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-204:0]: return forward_call(*input, **kwargs) +[ip-26-0-147-204:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-170:1]: query_states, kv_states = self.qkv( +[ip-26-0-148-170:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-170:1]: return forward_call(*input, **kwargs) +[ip-26-0-148-170:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-245:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-115:5]: return forward_call(*input, **kwargs) +[ip-26-0-147-204:0]: sharded_logits = self.model( +[ip-26-0-147-204:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-204:0]: return forward_call(*input, **kwargs) +[ip-26-0-147-204:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-170:1]: return column_linear( +[ip-26-0-148-170:1]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-148-245:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-245:0]: return forward_call(*input, **kwargs) +[ip-26-0-148-115:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-204:0]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-204:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-204:0]: return forward_call(*input, **kwargs) +[ip-26-0-148-170:3]:Traceback (most recent call last): +[ip-26-0-148-170:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-245:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-245:3]: return forward_call(*input, **kwargs) +[ip-26-0-148-115:5]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-204:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-204:0]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-204:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-204:0]: return forward_call(*input, **kwargs) +[ip-26-0-147-204:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-204:0]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-170:3]: main() +[ip-26-0-148-170:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-170:3]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-245:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-115:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-204:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-204:0]: return forward_call(*input, **kwargs) +[ip-26-0-147-204:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-204:0]: query_states, kv_states = self.qkv( +[ip-26-0-148-170:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-170:3]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-148-170:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-170:3]: result = model(**micro_batch) +[ip-26-0-148-170:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-245:0]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-245:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-115:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-204:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-204:0]: return forward_call(*input, **kwargs) +[ip-26-0-147-204:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-170:3]: return forward_call(*input, **kwargs) +[ip-26-0-148-170:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-170:3]: sharded_logits = self.model( +[ip-26-0-148-170:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-245:0]: return forward_call(*input, **kwargs) +[ip-26-0-147-187:4]:Traceback (most recent call last): +[ip-26-0-147-187:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-187:4]: main() +[ip-26-0-148-115:6]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-204:0]: return column_linear( +[ip-26-0-147-204:0]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-148-170:3]: return forward_call(*input, **kwargs) +[ip-26-0-148-170:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-170:3]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-148-245:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-245:3]: return column_linear( +[ip-26-0-148-115:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-204:7]:Traceback (most recent call last): +[ip-26-0-147-204:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-170:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-170:3]: return forward_call(*input, **kwargs) +[ip-26-0-148-170:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-148-170:3]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-245:3]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-148-115:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-170:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-170:3]: return forward_call(*input, **kwargs) +[ip-26-0-148-170:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-170:3]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-170:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-170:3]: return forward_call(*input, **kwargs) +[ip-26-0-148-245:0]: query_states, kv_states = self.qkv( +[ip-26-0-148-245:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-115:2]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-170:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-170:3]: query_states, kv_states = self.qkv( +[ip-26-0-148-170:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-170:3]: return forward_call(*input, **kwargs) +[ip-26-0-148-245:0]: return forward_call(*input, **kwargs) +[ip-26-0-147-187:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-187:4]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-187:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-115:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-115:2]: return forward_call(*input, **kwargs) +[ip-26-0-148-170:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-170:3]: return column_linear( +[ip-26-0-148-170:3]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-148-245:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-187:4]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-187:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-187:4]: result = model(**micro_batch) +[ip-26-0-147-187:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-187:4]: return forward_call(*input, **kwargs) +[ip-26-0-147-187:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-187:4]: sharded_logits = self.model( +[ip-26-0-147-187:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-115:5]: return forward_call(*input, **kwargs) +[ip-26-0-148-115:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-148-170:2]:Traceback (most recent call last): +[ip-26-0-148-170:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-245:0]: return column_linear( +[ip-26-0-147-187:4]: return forward_call(*input, **kwargs) +[ip-26-0-147-187:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-187:4]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-187:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-187:4]: return forward_call(*input, **kwargs) +[ip-26-0-147-187:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-187:4]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-115:5]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-170:2]: main() +[ip-26-0-148-170:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-245:0]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-187:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-187:4]: return forward_call(*input, **kwargs) +[ip-26-0-147-187:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-187:4]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-115:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-115:2]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-170:2]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-170:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-170:2]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-148-245:6]: return forward_call(*input, **kwargs) +[ip-26-0-148-245:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-187:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-187:4]: return forward_call(*input, **kwargs) +[ip-26-0-147-187:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-187:4]: query_states, kv_states = self.qkv( +[ip-26-0-148-115:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-170:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-170:2]: result = model(**micro_batch) +[ip-26-0-148-170:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-245:6]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-187:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-187:4]: return forward_call(*input, **kwargs) +[ip-26-0-147-187:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-115:2]: return forward_call(*input, **kwargs) +[ip-26-0-148-170:2]: return forward_call(*input, **kwargs) +[ip-26-0-148-170:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-170:2]: sharded_logits = self.model( +[ip-26-0-148-170:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-170:2]: return forward_call(*input, **kwargs) +[ip-26-0-148-170:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-245:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-245:6]: return forward_call(*input, **kwargs) +[ip-26-0-147-187:4]: return column_linear( +[ip-26-0-147-187:4]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-148-115:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-115:5]: return forward_call(*input, **kwargs) +[ip-26-0-148-170:2]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-148-170:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-170:2]: return forward_call(*input, **kwargs) +[ip-26-0-148-245:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-245:6]: query_states, kv_states = self.qkv( +[ip-26-0-148-115:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-170:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-148-170:2]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-170:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-170:2]: return forward_call(*input, **kwargs) +[ip-26-0-148-245:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-115:5]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-170:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-170:2]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-170:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-170:2]: return forward_call(*input, **kwargs) +[ip-26-0-148-170:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-245:6]: return forward_call(*input, **kwargs) +[ip-26-0-148-115:6]: return forward_call(*input, **kwargs) +[ip-26-0-148-170:2]: query_states, kv_states = self.qkv( +[ip-26-0-148-170:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-170:2]: return forward_call(*input, **kwargs) +[ip-26-0-148-170:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-245:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-245:5]: return forward_call(*input, **kwargs) +[ip-26-0-148-115:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-170:2]: return column_linear( +[ip-26-0-148-170:2]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-148-245:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-115:6]: query_states, kv_states = self.qkv( +[ip-26-0-148-245:5]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-245:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-115:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-245:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-115:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-245:6]: return column_linear( +[ip-26-0-148-115:5]: return forward_call(*input, **kwargs) +[ip-26-0-148-245:6]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-148-245:5]: return forward_call(*input, **kwargs) +[ip-26-0-148-115:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-55:1]:Traceback (most recent call last): +[ip-26-0-148-55:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-55:1]: main() +[ip-26-0-148-55:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-55:1]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-55:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-55:1]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-148-55:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-55:1]: result = model(**micro_batch) +[ip-26-0-148-55:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-55:1]: return forward_call(*input, **kwargs) +[ip-26-0-148-55:1]: File "/fsx/loubna/code/f[ip-26-0-148-245:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-115:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-245:5]: query_states, kv_states = self.qkv( +[ip-26-0-148-245:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-245:5]: return forward_call(*input, **kwargs) +[ip-26-0-148-115:2]: query_states, kv_states = self.qkv( +[ip-26-0-148-115:6]: return forward_call(*input, **kwargs) +[ip-26-0-148-245:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-245:5]: return column_linear( +[ip-26-0-148-245:5]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-148-115:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-115:5]: query_states, kv_states = self.qkv( +[ip-26-0-148-115:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-115:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-115:2]: return forward_call(*input, **kwargs) +[ip-26-0-148-115:5]: return forward_call(*input, **kwargs) +[ip-26-0-148-115:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-115:5]: return column_linear( +[ip-26-0-148-245:7]:Traceback (most recent call last): +[ip-26-0-148-245:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-245:7]: main() +[ip-26-0-148-245:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-245:7]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-245:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-245:7]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-148-245:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-245:7]: result = model(**micro_batch) +[ip-26-0-148-245:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-245:7]: return forward_call(*input, **kwargs) +[ip-26-0-148-115:5]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-148-245:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-245:7]: sharded_logits = self.model( +[ip-26-0-148-245:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-245:7]: return forward_call(*input, **kwargs) +[ip-26-0-148-245:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-245:7]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-148-245:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-245:7]: return forward_call(*input, **kwargs) +[ip-26-0-148-245:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-148-245:7]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-245:7]: File "/fsx/loubna/minicon[ip-26-0-148-115:6]: return column_linear( +da3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-245:7]: return forward_call(*input, **kwargs) +[ip-26-0-148-245:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-245:7]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-245:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-245:7]: return forward_call(*input, **kwargs) +[ip-26-0-148-245:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-245:7]: query_states, kv_states = self.qkv( +[ip-26-0-148-245:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-245:7]: return forward_call(*input, **kwargs) +[ip-26-0-148-245:7]: File "/fsx/loubna/code[ip-26-0-148-115:6]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +ork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-55:1]: sharded_logits = self.model( +[ip-26-0-148-55:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-55:1]: return forward_call(*input, **kwargs) +[ip-26-0-148-55:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-55:1]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-148-55:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-55:1]: return forward_call(*input, **kwargs) +[ip-26-0-148-55:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-148-55:1]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-55:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn//fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-245:7]: return column_linear( +[ip-26-0-148-245:7]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-245:3]:Traceback (most recent call last): +[ip-26-0-147-245:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-245:3]: main() +[ip-26-0-147-245:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-245:3]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-245:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-245:3]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-245:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-245:3]: result = model(**micro_batch) +[ip-26-0-147-245:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-245:3]: return forward_call(*input, **kwargs) +[ip-26-0-147-245:3]: File "/fsx/l[ip-26-0-148-115:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +modules/module.py", line 1194, in _call_impl +[ip-26-0-148-55:1]: return forward_call(*input, **kwargs) +[ip-26-0-148-55:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-55:1]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-55:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-55:1]: return forward_call(*input, **kwargs) +[ip-26-0-148-55:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-55:1]: query_states, kv_states = self.qkv( +[ip-26-0-148-55:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-55:1]: return forward_call(*input, **kwargs) +[ip-26-0-148-55:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in oubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-245:3]: sharded_logits = self.model( +[ip-26-0-147-245:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-245:3]: return forward_call(*input, **kwargs) +[ip-26-0-147-245:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-245:3]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-245:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-245:3]: return forward_call(*input, **kwargs) +[ip-26-0-147-245:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-245:3]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-245:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/s[ip-26-0-148-115:2]: return column_linear( +forward +[ip-26-0-148-55:1]: return column_linear( +[ip-26-0-148-55:1]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +ite-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-245:3]: return forward_call(*input, **kwargs) +[ip-26-0-147-245:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-245:3]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-245:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-245:3]: return forward_call(*input, **kwargs) +[ip-26-0-147-245:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-245:3]: query_states, kv_states = self.qkv( +[ip-26-0-147-245:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-245:3]: return forward_call(*input, **kwargs) +[ip-26-0-147-245:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/model[ip-26-0-148-115:2]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +ing_gpt2_fast.py", line 281, in forward +[ip-26-0-147-245:3]: return column_linear( +[ip-26-0-147-245:3]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-148-115:7]:Traceback (most recent call last): +[ip-26-0-148-115:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-115:7]: main() +[ip-26-0-148-115:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-115:7]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-115:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-115:7]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-148-115:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-115:7]: result = model(**micro_batch) +[ip-26-0-148-115:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-115:7]: return forward_call(*input, **kwargs) +[ip-26-0-148-115:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-115:7]: sharded_logits = self.model( +[ip-26-0-148-115:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-115:7]: return forward_call(*input, **kwargs) +[ip-26-0-148-115:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-115:7]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-148-115:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-115:7]: return forward_call(*input, **kwargs) +[ip-26-0-148-115:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-148-115:7]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-115:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-115:7]: return forward_call(*input, **kwargs) +[ip-26-0-148-115:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-115:7]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-115:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-115:7]: return forward_call(*input, **kwargs) +[ip-26-0-148-115:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-115:7]: query_states, kv_states = self.qkv( +[ip-26-0-148-115:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-115:7]: return forward_call(*input, **kwargs) +[ip-26-0-148-115:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-115:7]: return column_linear( +[ip-26-0-148-115:7]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-148-115:3]:Traceback (most recent call last): +[ip-26-0-148-115:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-115:3]: main() +[ip-26-0-148-115:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-115:3]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-115:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-115:3]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-148-115:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-115:3]: result = model(**micro_batch) +[ip-26-0-148-115:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-115:3]: return forward_call(*input, **kwargs) +[ip-26-0-148-115:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-115:3]: sharded_logits = self.model( +[ip-26-0-148-115:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-115:3]: return forward_call(*input, **kwargs) +[ip-26-0-148-115:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-115:3]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-148-115:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-115:3]: return forward_call(*input, **kwargs) +[ip-26-0-148-115:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-148-115:3]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-115:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-115:3]: return forward_call(*input, **kwargs) +[ip-26-0-148-115:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-115:3]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-115:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-115:3]: return forward_call(*input, **kwargs) +[ip-26-0-148-115:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-115:3]: query_states, kv_states = self.qkv( +[ip-26-0-148-115:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-115:3]: return forward_call(*input, **kwargs) +[ip-26-0-148-115:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-115:3]: return column_linear( +[ip-26-0-148-115:3]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-148-115:4]:Traceback (most recent call last): +[ip-26-0-148-115:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-115:4]: main() +[ip-26-0-148-115:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-115:4]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-115:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-115:4]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-148-115:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-115:4]: result = model(**micro_batch) +[ip-26-0-148-115:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-115:4]: return forward_call(*input, **kwargs) +[ip-26-0-148-115:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-115:4]: sharded_logits = self.model( +[ip-26-0-148-115:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-115:4]: return forward_call(*input, **kwargs) +[ip-26-0-148-115:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-115:4]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-148-115:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-115:4]: return forward_call(*input, **kwargs) +[ip-26-0-148-115:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-148-115:4]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-115:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-115:4]: return forward_call(*input, **kwargs) +[ip-26-0-148-115:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-115:4]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-115:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-115:4]: return forward_call(*input, **kwargs) +[ip-26-0-148-115:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-115:4]: query_states, kv_states = self.qkv( +[ip-26-0-148-115:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-115:4]: return forward_call(*input, **kwargs) +[ip-26-0-148-115:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-115:4]: return column_linear( +[ip-26-0-148-115:4]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-148-115:0]:Traceback (most recent call last): +[ip-26-0-148-115:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-115:0]: main() +[ip-26-0-148-115:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-115:0]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-115:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-115:0]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-148-115:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-115:0]: result = model(**micro_batch) +[ip-26-0-148-115:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-115:0]: return forward_call(*input, **kwargs) +[ip-26-0-148-115:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-115:0]: sharded_logits = self.model( +[ip-26-0-148-115:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-115:0]: return forward_call(*input, **kwargs) +[ip-26-0-148-115:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-115:0]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-148-115:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-115:0]: return forward_call(*input, **kwargs) +[ip-26-0-148-115:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-148-115:0]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-115:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-115:0]: return forward_call(*input, **kwargs) +[ip-26-0-148-115:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-115:0]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-115:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-115:0]: return forward_call(*input, **kwargs) +[ip-26-0-148-115:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-115:0]: query_states, kv_states = self.qkv( +[ip-26-0-148-115:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-115:0]: return forward_call(*input, **kwargs) +[ip-26-0-148-115:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-115:0]: return column_linear( +[ip-26-0-148-115:0]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-189:4]:Traceback (most recent call last): +[ip-26-0-147-189:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-189:4]: main() +[ip-26-0-147-189:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-189:4]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-189:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-189:4]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-189:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-189:4]: result = model(**micro_batch) +[ip-26-0-147-189:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-189:4]: return forward_call(*input, **kwargs) +[ip-26-0-147-189:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-189:4]: sharded_logits = self.model( +[ip-26-0-147-189:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-189:4]: return forward_call(*input, **kwargs) +[ip-26-0-147-189:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-55:7]:Traceback (most recent call last): +[ip-26-0-148-55:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-55:7]: main() +[ip-26-0-148-55:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-55:7]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-55:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-55:7]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-148-55:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-55:7]: result = model(**micro_batch) +[ip-26-0-148-55:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-55:7]: return forward_call(*input, **kwargs) +[ip-26-0-148-55:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-55:7]: sharded_logits = self.model( +[ip-26-0-148-55:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-55:7]: return forward_call(*input, **kwargs) +[ip-26-0-148-55:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-55:7]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-148-55:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-55:7]: return forward_call(*input, **kwargs) +[ip-26-0-148-55:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-148-55:7]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-55:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-55:7]: return forward_call(*input, **kwargs) +[ip-26-0-148-55:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-55:7]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-55:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-55:7]: return forward_call(*input, **kwargs) +[ip-26-0-148-55:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-55:7]: query_states, kv_states = self.qkv( +[ip-26-0-148-55:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-55:7]: return forward_call(*input, **kwargs) +[ip-26-0-148-55:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-55:7]: return column_linear( +[ip-26-0-148-55:7]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-189:2]:Traceback (most recent call last): +[ip-26-0-147-189:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-189:2]: main() +[ip-26-0-147-189:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-189:2]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-189:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-189:2]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-189:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-189:2]: result = model(**micro_batch) +[ip-26-0-147-189:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-189:2]: return forward_call(*input, **kwargs) +[ip-26-0-147-189:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-189:2]: sharded_logits = self.model( +[ip-26-0-147-189:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-189:2]: return forward_call(*input, **kwargs) +[ip-26-0-147-189:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-189:2]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-189:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-189:2]: return forward_call(*input, **kwargs) +[ip-26-0-147-189:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-189:2]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-189:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-189:2]: return forward_call(*input, **kwargs) +[ip-26-0-147-189:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-189:2]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-189:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-189:2]: return forward_call(*input, **kwargs) +[ip-26-0-147-189:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-189:2]: query_states, kv_states = self.qkv( +[ip-26-0-147-189:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-189:2]: return forward_call(*input, **kwargs) +[ip-26-0-147-189:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-189:2]: return column_linear( +[ip-26-0-147-189:2]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-189:5]:Traceback (most recent call last): +[ip-26-0-147-189:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-189:5]: main() +[ip-26-0-147-189:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-189:5]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-189:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-189:5]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-189:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-189:5]: result = model(**micro_batch) +[ip-26-0-147-189:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-189:5]: return forward_call(*input, **kwargs) +[ip-26-0-147-189:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-189:5]: sharded_logits = self.model( +[ip-26-0-147-189:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-189:5]: return forward_call(*input, **kwargs) +[ip-26-0-147-189:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-189:5]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-189:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-189:5]: return forward_call(*input, **kwargs) +[ip-26-0-147-189:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-189:5]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-189:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-189:5]: return forward_call(*input, **kwargs) +[ip-26-0-147-189:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-189:5]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-189:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-189:5]: return forward_call(*input, **kwargs) +[ip-26-0-147-189:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-189:5]: query_states, kv_states = self.qkv( +[ip-26-0-147-189:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-189:5]: return forward_call(*input, **kwargs) +[ip-26-0-147-189:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-189:5]: return column_linear( +[ip-26-0-147-189:5]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-189:3]:Traceback (most recent call last): +[ip-26-0-147-189:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-189:3]: main() +[ip-26-0-147-189:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-189:3]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-189:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-189:3]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-189:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-189:3]: result = model(**micro_batch) +[ip-26-0-147-189:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-189:3]: return forward_call(*input, **kwargs) +[ip-26-0-147-189:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-189:3]: sharded_logits = self.model( +[ip-26-0-147-189:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-189:3]: return forward_call(*input, **kwargs) +[ip-26-0-147-189:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-189:3]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-189:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-189:3]: return forward_call(*input, **kwargs) +[ip-26-0-147-189:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-189:3]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-189:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-189:3]: return forward_call(*input, **kwargs) +[ip-26-0-147-189:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-189:3]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-189:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-189:3]: return forward_call(*input, **kwargs) +[ip-26-0-147-189:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-189:3]: query_states, kv_states = self.qkv( +[ip-26-0-147-189:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-189:3]: return forward_call(*input, **kwargs) +[ip-26-0-147-189:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-189:3]: return column_linear( +[ip-26-0-147-189:3]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-189:7]:Traceback (most recent call last): +[ip-26-0-147-189:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-189:7]: main() +[ip-26-0-147-189:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-189:7]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-189:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-189:7]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-189:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-189:7]: result = model(**micro_batch) +[ip-26-0-147-189:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-189:7]: return forward_call(*input, **kwargs) +[ip-26-0-147-189:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-189:7]: sharded_logits = self.model( +[ip-26-0-147-189:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-189:7]: return forward_call(*input, **kwargs) +[ip-26-0-147-189:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-189:7]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-189:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-189:7]: return forward_call(*input, **kwargs) +[ip-26-0-147-189:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-189:7]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-189:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-189:7]: return forward_call(*input, **kwargs) +[ip-26-0-147-189:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-189:7]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-189:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-189:7]: return forward_call(*input, **kwargs) +[ip-26-0-147-189:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-189:7]: query_states, kv_states = self.qkv( +[ip-26-0-147-189:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-189:7]: return forward_call(*input, **kwargs) +[ip-26-0-147-189:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-189:7]: return column_linear( +[ip-26-0-147-189:7]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-148-193:5]:Traceback (most recent call last): +[ip-26-0-148-193:1]:Traceback (most recent call last): +[ip-26-0-148-193:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-193:5]: main() +[ip-26-0-148-193:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-193:1]: main() +[ip-26-0-148-193:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-193:5]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-193:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-193:1]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-193:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-193:5]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-148-193:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-193:1]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-148-193:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-193:5]: result = model(**micro_batch) +[ip-26-0-148-193:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-193:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-193:1]: result = model(**micro_batch) +[ip-26-0-148-193:5]: return forward_call(*input, **kwargs) +[ip-26-0-148-193:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-193:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-193:1]: return forward_call(*input, **kwargs) +[ip-26-0-148-193:5]: sharded_logits = self.model( +[ip-26-0-148-193:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-193:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-193:1]: sharded_logits = self.model( +[ip-26-0-148-193:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-193:1]: return forward_call(*input, **kwargs) +[ip-26-0-148-193:5]: return forward_call(*input, **kwargs) +[ip-26-0-148-193:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-193:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-193:1]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-148-193:5]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-148-193:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-193:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-193:1]: return forward_call(*input, **kwargs) +[ip-26-0-148-193:5]: return forward_call(*input, **kwargs) +[ip-26-0-148-193:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-148-193:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-148-193:1]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-193:5]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-193:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-193:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-193:1]: return forward_call(*input, **kwargs) +[ip-26-0-148-193:5]: return forward_call(*input, **kwargs) +[ip-26-0-148-193:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-193:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-193:1]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-193:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-193:1]: return forward_call(*input, **kwargs) +[ip-26-0-148-193:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-193:1]: query_states, kv_states = self.qkv( +[ip-26-0-148-193:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-193:1]: return forward_call(*input, **kwargs) +[ip-26-0-148-193:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-193:1]: return column_linear( +[ip-26-0-148-193:1]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-148-193:5]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-193:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-193:5]: return forward_call(*input, **kwargs) +[ip-26-0-148-193:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-193:5]: query_states, kv_states = self.qkv( +[ip-26-0-148-193:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-193:5]: return forward_call(*input, **kwargs) +[ip-26-0-148-193:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-193:5]: return column_linear( +[ip-26-0-148-193:5]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-148-193:4]:Traceback (most recent call last): +[ip-26-0-148-193:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-193:4]: main() +[ip-26-0-148-193:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-193:4]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-193:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-193:4]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-148-193:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-193:4]: result = model(**micro_batch) +[ip-26-0-148-193:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-193:4]: return forward_call(*input, **kwargs) +[ip-26-0-148-193:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-193:4]: sharded_logits = self.model( +[ip-26-0-148-193:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-193:4]: return forward_call(*input, **kwargs) +[ip-26-0-148-193:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-193:4]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-148-193:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-193:4]: return forward_call(*input, **kwargs) +[ip-26-0-148-193:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-148-193:4]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-193:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-193:4]: return forward_call(*input, **kwargs) +[ip-26-0-148-193:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-193:4]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-193:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-193:4]: return forward_call(*input, **kwargs) +[ip-26-0-148-193:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-193:4]: query_states, kv_states = self.qkv( +[ip-26-0-148-193:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-193:4]: return forward_call(*input, **kwargs) +[ip-26-0-148-193:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-193:4]: return column_linear( +[ip-26-0-148-193:4]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-148-193:6]:Traceback (most recent call last): +[ip-26-0-148-193:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-193:6]: main() +[ip-26-0-148-193:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-193:6]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-193:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-193:6]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-148-193:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-193:6]: result = model(**micro_batch) +[ip-26-0-148-193:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-193:6]: return forward_call(*input, **kwargs) +[ip-26-0-148-193:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-193:6]: sharded_logits = self.model( +[ip-26-0-148-193:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-193:6]: return forward_call(*input, **kwargs) +[ip-26-0-148-193:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-193:6]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-148-193:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-193:6]: return forward_call(*input, **kwargs) +[ip-26-0-148-193:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-148-193:6]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-193:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-193:6]: return forward_call(*input, **kwargs) +[ip-26-0-148-193:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-193:6]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-193:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-193:6]: return forward_call(*input, **kwargs) +[ip-26-0-148-193:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-193:6]: query_states, kv_states = self.qkv( +[ip-26-0-148-193:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-193:6]: return forward_call(*input, **kwargs) +[ip-26-0-148-193:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-193:6]: return column_linear( +[ip-26-0-148-193:6]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-148-193:2]:Traceback (most recent call last): +[ip-26-0-148-193:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-193:2]: main() +[ip-26-0-148-193:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-193:2]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-193:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-193:2]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-148-193:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-193:2]: result = model(**micro_batch) +[ip-26-0-148-193:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-193:2]: return forward_call(*input, **kwargs) +[ip-26-0-148-193:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-193:2]: sharded_logits = self.model( +[ip-26-0-148-193:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-193:2]: return forward_call(*input, **kwargs) +[ip-26-0-148-193:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-193:2]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-148-193:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-193:2]: return forward_call(*input, **kwargs) +[ip-26-0-148-193:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-148-193:2]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-193:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-193:2]: return forward_call(*input, **kwargs) +[ip-26-0-148-193:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-193:2]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-193:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-193:2]: return forward_call(*input, **kwargs) +[ip-26-0-148-193:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-193:2]: query_states, kv_states = self.qkv( +[ip-26-0-148-193:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-193:2]: return forward_call(*input, **kwargs) +[ip-26-0-148-193:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-193:2]: return column_linear( +[ip-26-0-148-193:2]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-148-193:0]: return forward_call(*input, **kwargs) +[ip-26-0-148-193:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-193:0]: sharded_logits = self.model( +[ip-26-0-148-193:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-193:0]: return forward_call(*input, **kwargs) +[ip-26-0-148-193:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-193:0]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-148-193:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-193:0]: return forward_call(*input, **kwargs) +[ip-26-0-148-193:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-148-193:0]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-193:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-193:0]: return forward_call(*input, **kwargs) +[ip-26-0-148-193:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-193:0]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-193:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-193:0]: return forward_call(*input, **kwargs) +[ip-26-0-148-193:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-193:0]: query_states, kv_states = self.qkv( +[ip-26-0-148-193:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-193:0]: return forward_call(*input, **kwargs) +[ip-26-0-148-193:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-193:0]: return column_linear( +[ip-26-0-148-193:0]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-193:0]:Traceback (most recent call last): +[ip-26-0-147-193:5]:Traceback (most recent call last): +[ip-26-0-147-193:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-193:5]: main() +[ip-26-0-147-193:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-193:5]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-193:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-193:5]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-193:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-193:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-193:5]: result = model(**micro_batch) +[ip-26-0-147-193:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-193:3]:Traceback (most recent call last): +[ip-26-0-147-193:2]:Traceback (most recent call last): +[ip-26-0-147-193:7]:Traceback (most recent call last): +[ip-26-0-147-193:0]: main() +[ip-26-0-147-193:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-193:0]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-193:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-193:0]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-193:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-193:0]: result = model(**micro_batch) +[ip-26-0-147-193:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-193:0]: return forward_call(*input, **kwargs) +[ip-26-0-147-193:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-115:1]:Traceback (most recent call last): +[ip-26-0-148-115:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-115:1]: main() +[ip-26-0-148-115:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-115:1]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-115:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-115:1]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-148-115:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-115:1]: result = model(**micro_batch) +[ip-26-0-148-115:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-115:1]: return forward_call(*input, **kwargs) +[ip-26-0-148-115:1]: File "/fsx/l[ip-26-0-147-193:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-193:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-193:7]: main() +[ip-26-0-147-193:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-193:7]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-193:3]: main() +[ip-26-0-147-193:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-193:0]: sharded_logits = self.model( +[ip-26-0-147-193:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-193:2]: main() +[ip-26-0-147-193:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +oubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-115:1]: sharded_logits = self.model( +[ip-26-0-148-115:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-115:1]: return forward_call(*input, **kwargs) +[ip-26-0-148-115:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-115:1]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-148-115:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-115:1]: return forward_call(*input, **kwargs) +[ip-26-0-148-115:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-148-115:1]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-115:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/s[ip-26-0-147-193:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +ite-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-115:1]: return forward_call(*input, **kwargs) +[ip-26-0-148-115:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-115:1]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-115:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-115:1]: return forward_call(*input, **kwargs) +[ip-26-0-148-115:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-115:1]: query_states, kv_states = self.qkv( +[ip-26-0-148-115:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-115:1]: return forward_call(*input, **kwargs) +[ip-26-0-148-115:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/model[ip-26-0-147-193:0]: return forward_call(*input, **kwargs) +[ip-26-0-147-193:5]: return forward_call(*input, **kwargs) +ing_gpt2_fast.py", line 281, in forward +[ip-26-0-148-115:1]: return column_linear( +[ip-26-0-148-115:1]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-193:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-193:7]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-193:3]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-193:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-193:3]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-193:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-193:2]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-193:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-193:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-193:0]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-193:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-193:2]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-193:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-193:2]: result = model(**micro_batch) +[ip-26-0-147-193:4]:Traceback (most recent call last): +[ip-26-0-147-193:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-193:0]: return forward_call(*input, **kwargs) +[ip-26-0-147-193:3]: result = model(**micro_batch) +[ip-26-0-147-193:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-193:6]:Traceback (most recent call last): +[ip-26-0-147-193:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-193:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-193:7]: result = model(**micro_batch) +[ip-26-0-147-193:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-193:7]: return forward_call(*input, **kwargs) +[ip-26-0-147-193:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-193:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-193:2]: return forward_call(*input, **kwargs) +[ip-26-0-147-193:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-193:2]: sharded_logits = self.model( +[ip-26-0-147-193:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-193:2]: return forward_call(*input, **kwargs) +[ip-26-0-147-193:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-193:5]: sharded_logits = self.model( +[ip-26-0-147-193:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-193:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-193:4]: main() +[ip-26-0-147-193:3]: return forward_call(*input, **kwargs) +[ip-26-0-147-193:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-193:3]: sharded_logits = self.model( +[ip-26-0-147-193:6]: main() +[ip-26-0-147-193:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-193:6]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-193:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-193:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-193:5]: return forward_call(*input, **kwargs) +[ip-26-0-147-193:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-193:3]: return forward_call(*input, **kwargs) +[ip-26-0-147-193:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-193:3]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-193:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-193:4]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-193:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-193:2]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-193:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-193:2]: return forward_call(*input, **kwargs) +[ip-26-0-147-193:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-193:2]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-193:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-193:2]: return forward_call(*input, **kwargs) +[ip-26-0-147-193:6]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-193:3]: return forward_call(*input, **kwargs) +[ip-26-0-147-193:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-193:3]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-193:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-193:2]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-193:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-193:5]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-193:4]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-193:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-193:4]: result = model(**micro_batch) +[ip-26-0-147-193:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-193:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-193:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-193:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-193:7]: sharded_logits = self.model( +[ip-26-0-147-193:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-193:7]: return forward_call(*input, **kwargs) +[ip-26-0-147-193:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-193:5]: return forward_call(*input, **kwargs) +[ip-26-0-147-193:6]: result = model(**micro_batch) +[ip-26-0-147-193:2]: return forward_call(*input, **kwargs) +[ip-26-0-147-193:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-193:2]: query_states, kv_states = self.qkv( +[ip-26-0-147-193:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-193:2]: return forward_call(*input, **kwargs) +[ip-26-0-147-193:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-193:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-193:6]: return forward_call(*input, **kwargs) +[ip-26-0-147-193:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-193:6]: sharded_logits = self.model( +[ip-26-0-147-193:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-193:6]: return forward_call(*input, **kwargs) +[ip-26-0-147-193:2]: return column_linear( +[ip-26-0-147-193:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-193:0]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-193:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-193:0]: return forward_call(*input, **kwargs) +[ip-26-0-147-193:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-193:0]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-193:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-193:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-193:3]: return forward_call(*input, **kwargs) +[ip-26-0-147-193:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-193:3]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-193:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-193:5]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-193:2]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-193:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-193:3]: return forward_call(*input, **kwargs) +[ip-26-0-147-193:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-193:3]: query_states, kv_states = self.qkv( +[ip-26-0-147-193:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-193:3]: return forward_call(*input, **kwargs) +[ip-26-0-147-193:4]: return forward_call(*input, **kwargs) +[ip-26-0-147-193:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-193:4]: sharded_logits = self.model( +[ip-26-0-147-193:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-193:3]: return column_linear( +[ip-26-0-147-193:3]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-193:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-193:4]: return forward_call(*input, **kwargs) +[ip-26-0-147-193:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-193:4]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-193:0]: return forward_call(*input, **kwargs) +[ip-26-0-147-193:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-193:0]: query_states, kv_states = self.qkv( +[ip-26-0-147-193:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-193:4]: return forward_call(*input, **kwargs) +[ip-26-0-147-193:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-193:7]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-193:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-193:7]: return forward_call(*input, **kwargs) +[ip-26-0-147-193:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-193:7]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-193:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-193:7]: return forward_call(*input, **kwargs) +[ip-26-0-147-193:6]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-193:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-193:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-193:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-193:4]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-193:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-193:4]: return forward_call(*input, **kwargs) +[ip-26-0-147-193:0]: return forward_call(*input, **kwargs) +[ip-26-0-147-193:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-193:0]: return column_linear( +[ip-26-0-147-193:0]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-193:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-193:4]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-193:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-193:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-193:7]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-193:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-193:5]: return forward_call(*input, **kwargs) +[ip-26-0-147-193:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-193:4]: return forward_call(*input, **kwargs) +[ip-26-0-147-193:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-193:4]: query_states, kv_states = self.qkv( +[ip-26-0-147-193:6]: return forward_call(*input, **kwargs) +[ip-26-0-147-193:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-193:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-193:4]: return forward_call(*input, **kwargs) +[ip-26-0-147-193:5]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-193:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-193:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-193:7]: return forward_call(*input, **kwargs) +[ip-26-0-147-193:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-193:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-193:4]: return column_linear( +[ip-26-0-147-193:4]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-193:7]: query_states, kv_states = self.qkv( +[ip-26-0-147-193:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-193:5]: return forward_call(*input, **kwargs) +[ip-26-0-147-193:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-193:5]: query_states, kv_states = self.qkv( +[ip-26-0-147-193:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-193:5]: return forward_call(*input, **kwargs) +[ip-26-0-147-193:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-193:5]: return column_linear( +[ip-26-0-147-193:7]: return forward_call(*input, **kwargs) +[ip-26-0-147-193:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-193:5]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-193:7]: return column_linear( +[ip-26-0-147-193:7]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-193:1]:Traceback (most recent call last): +[ip-26-0-147-193:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-193:1]: main() +[ip-26-0-147-193:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-193:1]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-193:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-193:1]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-193:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-193:1]: result = model(**micro_batch) +[ip-26-0-147-193:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-193:1]: return forward_call(*input, **kwargs) +[ip-26-0-147-193:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-193:1]: sharded_logits = self.model( +[ip-26-0-147-193:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-193:1]: return forward_call(*input, **kwargs) +[ip-26-0-147-193:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-193:1]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-193:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-193:1]: return forward_call(*input, **kwargs) +[ip-26-0-147-193:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-193:1]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-193:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-193:1]: return forward_call(*input, **kwargs) +[ip-26-0-147-193:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-193:1]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-193:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-193:1]: return forward_call(*input, **kwargs) +[ip-26-0-147-193:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-193:1]: query_states, kv_states = self.qkv( +[ip-26-0-147-193:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-193:1]: return forward_call(*input, **kwargs) +[ip-26-0-147-193:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-193:1]: return column_linear( +[ip-26-0-147-193:1]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-141:2]:Traceback (most recent call last): +[ip-26-0-147-141:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-141:2]: main() +[ip-26-0-147-141:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-141:2]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-141:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-141:2]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-141:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-141:2]: result = model(**micro_batch) +[ip-26-0-147-141:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-141:2]: return forward_call(*input, **kwargs) +[ip-26-0-147-141:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-141:2]: sharded_logits = self.model( +[ip-26-0-147-141:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-141:2]: return forward_call(*input, **kwargs) +[ip-26-0-147-141:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-141:2]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-141:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-141:2]: return forward_call(*input, **kwargs) +[ip-26-0-147-141:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-141:2]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-141:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-141:2]: return forward_call(*input, **kwargs) +[ip-26-0-147-141:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-141:2]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-141:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-141:2]: return forward_call(*input, **kwargs) +[ip-26-0-147-141:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-141:2]: query_states, kv_states = self.qkv( +[ip-26-0-147-141:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-141:2]: return forward_call(*input, **kwargs) +[ip-26-0-147-141:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-141:2]: return column_linear( +[ip-26-0-147-141:2]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-141:7]:Traceback (most recent call last): +[ip-26-0-147-141:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-141:7]: main() +[ip-26-0-147-141:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-141:7]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-141:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-141:7]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-141:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-141:7]: result = model(**micro_batch) +[ip-26-0-147-141:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-141:7]: return forward_call(*input, **kwargs) +[ip-26-0-147-141:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-141:7]: sharded_logits = self.model( +[ip-26-0-147-141:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-141:7]: return forward_call(*input, **kwargs) +[ip-26-0-147-141:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-141:7]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-141:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-141:7]: return forward_call(*input, **kwargs) +[ip-26-0-147-141:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-141:7]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-141:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-141:7]: return forward_call(*input, **kwargs) +[ip-26-0-147-141:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-141:7]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-141:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-141:7]: return forward_call(*input, **kwargs) +[ip-26-0-147-141:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-141:7]: query_states, kv_states = self.qkv( +[ip-26-0-147-141:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-141:7]: return forward_call(*input, **kwargs) +[ip-26-0-147-141:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-141:7]: return column_linear( +[ip-26-0-147-141:7]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-141:4]:Traceback (most recent call last): +[ip-26-0-147-141:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-141:4]: main() +[ip-26-0-147-141:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-141:4]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-141:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-141:4]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-141:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-141:4]: result = model(**micro_batch) +[ip-26-0-147-141:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-141:4]: return forward_call(*input, **kwargs) +[ip-26-0-147-141:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-141:4]: sharded_logits = self.model( +[ip-26-0-147-141:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-141:4]: return forward_call(*input, **kwargs) +[ip-26-0-147-141:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-141:4]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-141:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-141:4]: return forward_call(*input, **kwargs) +[ip-26-0-147-141:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-141:4]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-141:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-141:4]: return forward_call(*input, **kwargs) +[ip-26-0-147-141:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-141:4]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-141:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-141:4]: return forward_call(*input, **kwargs) +[ip-26-0-147-141:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-141:4]: query_states, kv_states = self.qkv( +[ip-26-0-147-141:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-141:4]: return forward_call(*input, **kwargs) +[ip-26-0-147-141:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-141:4]: return column_linear( +[ip-26-0-147-141:4]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-141:1]:Traceback (most recent call last): +[ip-26-0-147-141:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-141:1]: main() +[ip-26-0-147-141:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-141:1]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-141:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-141:1]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-141:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-141:1]: result = model(**micro_batch) +[ip-26-0-147-141:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-141:1]: return forward_call(*input, **kwargs) +[ip-26-0-147-141:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-141:1]: sharded_logits = self.model( +[ip-26-0-147-141:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-141:1]: return forward_call(*input, **kwargs) +[ip-26-0-147-141:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-141:1]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-141:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-141:1]: return forward_call(*input, **kwargs) +[ip-26-0-147-141:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-141:1]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-141:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-141:1]: return forward_call(*input, **kwargs) +[ip-26-0-147-141:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-141:1]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-141:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-141:1]: return forward_call(*input, **kwargs) +[ip-26-0-147-141:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-141:1]: query_states, kv_states = self.qkv( +[ip-26-0-147-141:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-141:1]: return forward_call(*input, **kwargs) +[ip-26-0-147-141:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-141:1]: return column_linear( +[ip-26-0-147-141:1]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-141:0]:Traceback (most recent call last): +[ip-26-0-147-141:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-141:0]: main() +[ip-26-0-147-141:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-141:0]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-141:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-141:0]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-141:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-141:0]: result = model(**micro_batch) +[ip-26-0-147-141:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-141:0]: return forward_call(*input, **kwargs) +[ip-26-0-147-141:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-141:0]: sharded_logits = self.model( +[ip-26-0-147-141:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-141:0]: return forward_call(*input, **kwargs) +[ip-26-0-147-141:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-141:0]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-141:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-141:0]: return forward_call(*input, **kwargs) +[ip-26-0-147-141:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-141:0]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-141:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-141:0]: return forward_call(*input, **kwargs) +[ip-26-0-147-141:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-141:0]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-141:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-141:0]: return forward_call(*input, **kwargs) +[ip-26-0-147-141:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-141:0]: query_states, kv_states = self.qkv( +[ip-26-0-147-141:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-141:0]: return forward_call(*input, **kwargs) +[ip-26-0-147-141:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-141:0]: return column_linear( +[ip-26-0-147-141:0]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-141:5]:Traceback (most recent call last): +[ip-26-0-147-141:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-141:5]: main() +[ip-26-0-147-141:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-141:5]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-141:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-141:5]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-141:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-141:5]: result = model(**micro_batch) +[ip-26-0-147-141:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-141:5]: return forward_call(*input, **kwargs) +[ip-26-0-147-141:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-141:5]: sharded_logits = self.model( +[ip-26-0-147-141:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-141:5]: return forward_call(*input, **kwargs) +[ip-26-0-147-141:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-141:5]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-141:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-141:5]: return forward_call(*input, **kwargs) +[ip-26-0-147-141:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-141:5]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-141:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-141:5]: return forward_call(*input, **kwargs) +[ip-26-0-147-141:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-141:5]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-141:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-141:5]: return forward_call(*input, **kwargs) +[ip-26-0-147-141:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-141:5]: query_states, kv_states = self.qkv( +[ip-26-0-147-141:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-141:5]: return forward_call(*input, **kwargs) +[ip-26-0-147-141:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-141:5]: return column_linear( +[ip-26-0-147-141:5]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-233:2]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-233:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-233:2]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:6]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-233:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-233:2]: query_states, kv_states = self.qkv( +[ip-26-0-147-233:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-233:2]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-233:6]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-233:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-233:2]: return column_linear( +[ip-26-0-147-233:2]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-233:6]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-233:6]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-233:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-233:6]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-233:6]: query_states, kv_states = self.qkv( +[ip-26-0-147-233:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-233:6]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-233:6]: return column_linear( +[ip-26-0-147-233:6]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-233:7]:Traceback (most recent call last): +[ip-26-0-147-233:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-233:7]: main() +[ip-26-0-147-233:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-233:7]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-233:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-233:7]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-233:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-233:7]: result = model(**micro_batch) +[ip-26-0-147-233:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-233:7]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-233:7]: sharded_logits = self.model( +[ip-26-0-147-233:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-233:7]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-233:7]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-233:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-233:7]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-233:7]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-233:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-233:7]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-233:7]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-233:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-233:7]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-233:7]: query_states, kv_states = self.qkv( +[ip-26-0-147-233:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-233:7]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-233:7]: return column_linear( +[ip-26-0-147-233:7]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-247:6]: return forward_call(*input, **kwargs) +[ip-26-0-147-247:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-247:6]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-247:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-247:6]: return forward_call(*input, **kwargs) +[ip-26-0-147-247:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-247:6]: query_states, kv_states = self.qkv( +[ip-26-0-147-247:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-247:6]: return forward_call(*input, **kwargs) +[ip-26-0-147-247:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-247:6]: ret[ip-26-0-148-151:2]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-148-151:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-151:2]: return forward_call(*input, **kwargs) +[ip-26-0-148-151:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-148-151:2]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-151:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-151:2]: return forward_call(*input, **kwargs) +[ip-26-0-148-151:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-151:2]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-151:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/moduleurn column_linear( +[ip-26-0-147-247:6]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +.py", line 1194, in _call_impl +[ip-26-0-148-151:2]: return forward_call(*input, **kwargs) +[ip-26-0-148-151:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-151:2]: query_states, kv_states = self.qkv( +[ip-26-0-148-151:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-151:2]: return forward_call(*input, **kwargs) +[ip-26-0-148-151:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-151:2]: return column_linear( +[ip-26-0-148-151:2]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-187:6]:Traceback (most recent call last): +[ip-26-0-147-187:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-187:6]: main() +[ip-26-0-147-187:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-187:6]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-187:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-187:6]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-187:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-187:6]: result = model(**micro_batch) +[ip-26-0-147-187:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-187:6]: return forward_call(*input, **kwargs) +[ip-26-0-147-187:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-187:6]: sharded_logits = self.model( +[ip-26-0-147-187:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-187:6]: return forward_call(*input, **kwargs) +[ip-26-0-147-187:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-187:6]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-187:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-187:6]: return forward_call(*input, **kwargs) +[ip-26-0-147-187:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-187:6]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-187:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-187:6]: return forward_call(*input, **kwargs) +[ip-26-0-147-187:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-187:6]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-187:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-187:6]: return forward_call(*input, **kwargs) +[ip-26-0-147-187:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-187:6]: query_states, kv_states = self.qkv( +[ip-26-0-147-187:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-187:6]: return forward_call(*input, **kwargs) +[ip-26-0-147-187:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-187:6]: return column_linear( +[ip-26-0-147-187:6]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-148-170:6]:Traceback (most recent call last): +[ip-26-0-147-187:2]: return forward_call(*input, **kwargs) +[ip-26-0-147-187:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-187:2]: sharded_logits = self.model( +[ip-26-0-147-187:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-187:2]: return forward_call(*input, **kwargs) +[ip-26-0-147-187:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-187:2]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-187:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-187:2]: return forward_call(*input, **kwargs) +[ip-26-0-147-187:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-187:2]: output = self.pp_block[ip-26-0-148-170:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-170:6]: main() +[ip-26-0-148-170:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-170:6]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-170:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-170:6]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +(**new_kwargs) +[ip-26-0-147-187:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-187:2]: return forward_call(*input, **kwargs) +[ip-26-0-147-187:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-187:2]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-187:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-187:2]: return forward_call(*input, **kwargs) +[ip-26-0-147-187:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-187:2]: query_states, kv_states = self.qkv( +[ip-26-0-147-187:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-187:2]: return forward_call([ip-26-0-148-170:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +*input, **kwargs) +[ip-26-0-147-187:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-170:6]: result = model(**micro_batch) +[ip-26-0-148-170:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-187:2]: return column_linear( +[ip-26-0-147-187:2]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-148-170:6]: return forward_call(*input, **kwargs) +[ip-26-0-148-170:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-170:6]: sharded_logits = self.model( +[ip-26-0-148-170:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-170:6]: return forward_call(*input, **kwargs) +[ip-26-0-148-170:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-170:6]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-148-170:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-170:6]: return forward_call(*input, **kwargs) +[ip-26-0-148-170:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-148-170:6]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-170:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-170:6]: return forward_call(*input, **kwargs) +[ip-26-0-148-170:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-170:6]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-170:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-170:6]: return forward_call(*input, **kwargs) +[ip-26-0-148-170:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-170:6]: query_states, kv_states = self.qkv( +[ip-26-0-148-170:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-170:6]: return forward_call(*input, **kwargs) +[ip-26-0-148-170:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-170:6]: return column_linear( +[ip-26-0-148-170:6]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-148-170:5]:Traceback (most recent call last): +[ip-26-0-148-170:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-170:5]: main() +[ip-26-0-148-170:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-170:5]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-170:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-170:5]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-148-170:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-170:5]: result = model(**micro_batch) +[ip-26-0-148-170:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-170:5]: return forward_call(*input, **kwargs) +[ip-26-0-148-170:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-170:5]: sharded_logits = self.model( +[ip-26-0-148-93:6]: query_states, kv_states = self.qkv( +[ip-26-0-148-93:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-93:6]: return forward_call(*input, **kwargs) +[ip-26-0-148-93:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-93:6]: return column_linear( +[ip-26-0-148-93:6]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-148-170:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-170:5]: return forward_call(*input, **kwargs) +[ip-26-0-148-170:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-170:5]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-148-170:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-170:5]: return forward_call(*input, **kwargs) +[ip-26-0-148-170:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-148-170:5]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-170:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-170:5]: return forward_call(*input, **kwargs) +[ip-26-0-148-170:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-170:5]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-170:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-170:5]: return forward_call(*input, **kwargs) +[ip-26-0-148-170:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-170:5]: query_states, kv_states = self.qkv( +[ip-26-0-148-170:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-170:5]: return forward_call(*input, **kwargs) +[ip-26-0-148-93:4]: main() +[ip-26-0-148-170:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-170:5]: return column_linear( +[ip-26-0-148-170:5]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-148-93:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-93:4]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-93:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-93:4]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-148-93:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-93:4]: result = model(**micro_batch) +[ip-26-0-148-93:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-93:4]: return forward_call(*input, **kwargs) +[ip-26-0-147-187:5]:Traceback (most recent call last): +[ip-26-0-147-187:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-187:5]: main() +[ip-26-0-147-187:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-187:5]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-187:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-187:5]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-187:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-187:5]: result = model(**micro_batch) +[ip-26-0-147-187:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-187:5]: return forward_call(*input, **kwargs) +[ip-26-0-148-93:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-93:4]: sharded_logits = self.model( +[ip-26-0-147-233:4]: sharded_logits = self.model( +[ip-26-0-147-233:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-233:4]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-233:4]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-233:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-233:4]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-233:4]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-233:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-233:[ip-26-0-148-93:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-204:2]:Traceback (most recent call last): +[ip-26-0-147-187:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-187:5]: sharded_logits = self.model( +[ip-26-0-147-187:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-187:5]: return forward_call(*input, **kwargs) +[ip-26-0-147-187:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-187:5]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-187:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-187:5]: return forward_call(*input, **kwargs) +[ip-26-0-147-187:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-187:5]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-187:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-187:5]: return forward_call(*input, **kwargs) +[ip-26-0-147-187:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-187:5]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-187:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-187:5]: return forward_call(*input, **kwargs) +[ip-26-0-147-187:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-187:5]: query_states, kv_states = self.qkv( +[ip-26-0-147-187:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_imp[ip-26-0-148-93:4]: return forward_call(*input, **kwargs) +[ip-26-0-148-93:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-93:4]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-148-93:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-93:4]: return forward_call(*input, **kwargs) +[ip-26-0-148-93:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +l +[ip-26-0-147-187:5]: return forward_call(*input, **kwargs) +[ip-26-0-147-187:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-187:5]: return column_linear( +[ip-26-0-147-187:5]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +4]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-233:4]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-233:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-233:4]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-233:4]: query_states, kv_states = self.qkv( +[ip-26-0-147-233:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-233:4]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-93:4]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-233:4]: return column_linear( +[ip-26-0-147-233:4]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-148-93:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-93:4]: return forward_call(*input, **kwargs) +[ip-26-0-148-93:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-93:4]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-93:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-93:4]: return forward_call(*input, **kwargs) +[ip-26-0-148-93:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-93:4]: query_states, kv_states = self.qkv( +[ip-26-0-148-93:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-93:4]: return forward_call(*input, **kwargs) +[ip-26-0-148-93:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-93:4]: return column_linear( +[ip-26-0-148-93:4]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-204:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-204:2]: main() +[ip-26-0-147-204:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-204:2]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-93:0]:Traceback (most recent call last): +[ip-26-0-148-93:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-93:0]: main() +[ip-26-0-148-93:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-93:0]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-204:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-204:2]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-204:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-204:2]: result = model(**micro_batch) +[ip-26-0-148-93:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-93:0]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-148-93:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-93:0]: result = model(**micro_batch) +[ip-26-0-148-93:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-204:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-93:0]: return forward_call(*input, **kwargs) +[ip-26-0-148-93:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-93:0]: sharded_logits = self.model( +[ip-26-0-148-93:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-93:0]: return forward_call(*input, **kwargs) +[ip-26-0-148-93:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-93:0]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-204:2]: return forward_call(*input, **kwargs) +[ip-26-0-148-93:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-93:0]: return forward_call(*input, **kwargs) +[ip-26-0-148-93:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-148-93:0]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-93:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-93:0]: return forward_call(*input, **kwargs) +[ip-26-0-148-93:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-93:0]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-204:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-93:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-93:0]: return forward_call(*input, **kwargs) +[ip-26-0-148-93:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-93:0]: query_states, kv_states = self.qkv( +[ip-26-0-148-93:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-93:0]: return forward_call(*input, **kwargs) +[ip-26-0-147-204:2]: sharded_logits = self.model( +[ip-26-0-148-93:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-93:0]: return column_linear( +[ip-26-0-148-93:0]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-204:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-204:2]: return forward_call(*input, **kwargs) +[ip-26-0-147-204:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-204:4]:Traceback (most recent call last): +[ip-26-0-148-93:1]: main() +[ip-26-0-148-93:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-204:2]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-204:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-204:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-93:1]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-93:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-93:1]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-148-93:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-204:4]: main() +[ip-26-0-147-204:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-93:1]: result = model(**micro_batch) +[ip-26-0-148-93:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-93:1]: return forward_call(*input, **kwargs) +[ip-26-0-147-204:4]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-93:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-93:1]: sharded_logits = self.model( +[ip-26-0-148-93:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-93:1]: return forward_call(*input, **kwargs) +[ip-26-0-148-93:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-93:1]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-187:1]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-187:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-187:1]: result = model(**micro_batch) +[ip-26-0-147-187:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-187:1]: return forward_call(*input, **kwargs) +[ip-26-0-147-187:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-187:1]: sharded_logits = self.model( +[ip-26-0-147-187:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-187:1]: return forward_call(*input, **kwargs) +[ip-26-0-147-187:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-187:1]: hid[ip-26-0-147-204:2]: return forward_call(*input, **kwargs) +[ip-26-0-148-93:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-93:1]: return forward_call(*input, **kwargs) +[ip-26-0-147-204:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-148-93:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-148-93:1]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-204:2]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-93:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-93:1]: return forward_call(*input, **kwargs) +[ip-26-0-148-93:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-93:1]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-93:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-204:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-204:4]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-148-93:1]: return forward_call(*input, **kwargs) +[ip-26-0-148-93:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-93:1]: query_states, kv_states = self.qkv( +[ip-26-0-148-93:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-93:1]: return forward_call(*input, **kwargs) +[ip-26-0-148-93:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-93:1]: return column_linear( +[ip-26-0-148-93:1]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +den_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-187:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-187:1]: return forward_call(*input, **kwargs) +[ip-26-0-147-187:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-187:1]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-204:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-187:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-187:1]: return forward_call(*input, **kwargs) +[ip-26-0-147-187:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-187:1]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-187:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-187:1]: return forward_call(*input, **kwargs) +[ip-26-0-147-187:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-187:1]: query_states, kv_states = self.qkv( +[ip-26-0-147-187:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-187:1]: return forward_call(*input, **kwarg[ip-26-0-147-204:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +s) +[ip-26-0-147-187:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-233:5]:Traceback (most recent call last): +[ip-26-0-147-233:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-233:5]: main() +[ip-26-0-147-204:4]: result = model(**micro_batch) +[ip-26-0-147-187:1]: return column_linear( +[ip-26-0-147-187:1]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-204:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-204:5]:Traceback (most recent call last): +[ip-26-0-147-204:4]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-204:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-204:2]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:5]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-233:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-233:5]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-204:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-233:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-233:5]: result = model(**micro_batch) +[ip-26-0-147-204:2]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-233:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-233:5]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-204:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-233:5]: sharded_logits = self.model( +[ip-26-0-147-233:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-233:5]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-233:5]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-204:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-204:5]: main() +[ip-26-0-147-233:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-233:5]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-233:5]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-233:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-233:5]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-233:5]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-233:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-233:5]: return forward_call(*input, **kw[ip-26-0-147-204:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +args) +[ip-26-0-147-233:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-233:5]: query_states, kv_states = self.qkv( +[ip-26-0-147-233:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-233:5]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-233:5]: return column_linear( +[ip-26-0-147-204:2]: return forward_call(*input, **kwargs) +[ip-26-0-147-233:5]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-204:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-245:5]:Traceback (most recent call last): +[ip-26-0-147-245:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-245:5]: main() +[ip-26-0-147-245:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-245:5]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-245:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-245:5]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-245:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-245:5]: result = model(**micro_batch) +[ip-26-0-147-245:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-245:5]: return forward_call(*input, **kwargs) +[ip-26-0-147-245:5]: File "/fsx/l[ip-26-0-147-204:2]: query_states, kv_states = self.qkv( +[ip-26-0-147-204:5]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-204:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +oubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-245:5]: sharded_logits = self.model( +[ip-26-0-147-245:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-245:5]: return forward_call(*input, **kwargs) +[ip-26-0-147-245:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-245:5]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-245:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-245:5]: return forward_call(*input, **kwargs) +[ip-26-0-147-245:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-245:5]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-245:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/s[ip-26-0-147-204:5]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +ite-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-245:5]: return forward_call(*input, **kwargs) +[ip-26-0-147-245:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-245:5]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-245:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-245:5]: return forward_call(*input, **kwargs) +[ip-26-0-147-245:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-245:5]: query_states, kv_states = self.qkv( +[ip-26-0-147-245:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-245:5]: return forward_call(*input, **kwargs) +[ip-26-0-147-245:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/model[ip-26-0-147-204:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +ing_gpt2_fast.py", line 281, in forward +[ip-26-0-147-245:5]: return column_linear( +[ip-26-0-147-245:5]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-204:5]: result = model(**micro_batch) +[ip-26-0-147-204:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-204:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-204:5]: return forward_call(*input, **kwargs) +[ip-26-0-147-204:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-204:2]: return forward_call(*input, **kwargs) +[ip-26-0-147-204:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-204:2]: return column_linear( +[ip-26-0-147-204:5]: sharded_logits = self.model( +[ip-26-0-147-204:4]: sharded_logits = self.model( +[ip-26-0-147-204:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-204:5]: return forward_call(*input, **kwargs) +[ip-26-0-147-204:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-204:2]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-204:4]: return forward_call(*input, **kwargs) +[ip-26-0-147-204:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-204:4]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-204:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-204:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-204:4]: return forward_call(*input, **kwargs) +[ip-26-0-147-204:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-148-55:3]:Traceback (most recent call last): +[ip-26-0-148-55:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-55:3]: main() +[ip-26-0-148-55:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-55:3]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-55:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-55:3]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-148-55:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-55:3]: result = model(**micro_batch) +[ip-26-0-148-55:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-55:3]: return forward_call(*input, **kwargs) +[ip-26-0-148-55:3]: File "/fsx/loubna/code/f[ip-26-0-147-204:4]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-204:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-204:5]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-204:4]: return forward_call(*input, **kwargs) +[ip-26-0-147-204:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-204:5]: return forward_call(*input, **kwargs) +[ip-26-0-147-204:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-204:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-204:5]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-204:4]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-245:4]:Traceback (most recent call last): +[ip-26-0-147-245:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-245:4]: main() +[ip-26-0-147-204:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-204:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-245:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-245:4]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-245:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-245:4]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-245:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-245:4]: result = model(**micro_batch) +[ip-26-0-147-245:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-245:4]: return forward_call(*input, **kwargs) +ork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-55:3]: sharded_logits = self.model( +[ip-26-0-148-55:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-55:3]: return forward_call(*input, **kwargs) +[ip-26-0-148-55:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-55:3]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-148-55:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-55:3]: return forward_call(*input, **kwargs) +[ip-26-0-148-55:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-148-55:3]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-55:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/[ip-26-0-147-204:3]:Traceback (most recent call last): +[ip-26-0-147-245:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-245:4]: sharded_logits = self.model( +[ip-26-0-147-245:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +modules/module.py", line 1194, in _call_impl +[ip-26-0-148-55:3]: return forward_call(*input, **kwargs) +[ip-26-0-148-55:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-55:3]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-55:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-55:3]: return forward_call(*input, **kwargs) +[ip-26-0-148-55:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-55:3]: query_states, kv_states = self.qkv( +[ip-26-0-148-55:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-55:3]: return forward_call(*input, **kwargs) +[ip-26-0-148-55:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in [ip-26-0-147-204:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-245:4]: return forward_call(*input, **kwargs) +[ip-26-0-147-245:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-245:4]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-245:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +forward +[ip-26-0-148-55:3]: return column_linear( +[ip-26-0-148-55:3]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-204:3]: main() +[ip-26-0-147-245:4]: return forward_call(*input, **kwargs) +[ip-26-0-147-245:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-245:4]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-204:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-204:5]: return forward_call(*input, **kwargs) +[ip-26-0-147-245:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-245:4]: return forward_call(*input, **kwargs) +[ip-26-0-147-245:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-245:4]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-245:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-245:4]: return forward_call(*input, **kwargs) +[ip-26-0-147-204:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-245:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-245:4]: query_states, kv_states = self.qkv( +[ip-26-0-147-245:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-245:4]: return forward_call(*input, **kwargs) +[ip-26-0-147-204:5]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-245:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-204:7]: main() +[ip-26-0-147-245:4]: return column_linear( +[ip-26-0-147-204:4]: return forward_call(*input, **kwargs) +[ip-26-0-147-204:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-245:4]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-204:4]: query_states, kv_states = self.qkv( +[ip-26-0-147-245:7]:Traceback (most recent call last): +[ip-26-0-147-245:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-204:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-245:7]: main() +[ip-26-0-147-245:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-245:7]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-204:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-204:5]: return forward_call(*input, **kwargs) +[ip-26-0-147-245:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-245:7]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-245:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-245:7]: result = model(**micro_batch) +[ip-26-0-147-245:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-204:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-204:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-245:7]: return forward_call(*input, **kwargs) +[ip-26-0-147-245:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-245:7]: sharded_logits = self.model( +[ip-26-0-147-204:7]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-204:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-245:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-245:7]: return forward_call(*input, **kwargs) +[ip-26-0-147-245:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-245:7]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-204:7]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-204:5]: query_states, kv_states = self.qkv( +[ip-26-0-147-245:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-245:7]: return forward_call(*input, **kwargs) +[ip-26-0-147-245:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-245:7]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-245:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-245:7]: return forward_call(*input, **kwargs) +[ip-26-0-147-204:3]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-245:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-245:7]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-245:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-245:7]: return forward_call(*input, **kwargs) +[ip-26-0-147-204:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-204:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-245:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-245:7]: query_states, kv_states = self.qkv( +[ip-26-0-147-245:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-245:7]: return forward_call(*input, **kwargs) +[ip-26-0-147-204:5]: return forward_call(*input, **kwargs) +[ip-26-0-147-245:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-245:7]: return column_linear( +[ip-26-0-147-245:7]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-204:3]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-204:4]: return forward_call(*input, **kwargs) +[ip-26-0-147-204:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-204:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-204:3]: result = model(**micro_batch) +[ip-26-0-147-204:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-204:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-204:5]: return column_linear( +[ip-26-0-147-204:5]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-204:4]: return column_linear( +[ip-26-0-147-204:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-204:7]: result = model(**micro_batch) +[ip-26-0-147-204:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-204:7]: return forward_call(*input, **kwargs) +[ip-26-0-147-245:6]:Traceback (most recent call last): +[ip-26-0-147-204:4]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-245:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-245:6]: main() +[ip-26-0-147-245:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-245:6]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-245:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-245:6]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-245:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-204:3]: return forward_call(*input, **kwargs) +[ip-26-0-147-245:6]: result = model(**micro_batch) +[ip-26-0-147-245:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-245:6]: return forward_call(*input, **kwargs) +[ip-26-0-147-245:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-245:6]: sharded_logits = self.model( +[ip-26-0-147-245:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-245:6]: return forward_call(*input, **kwargs) +[ip-26-0-147-245:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-204:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-245:6]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-245:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-245:6]: return forward_call(*input, **kwargs) +[ip-26-0-147-245:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-245:6]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-245:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-245:6]: return forward_call(*input, **kwargs) +[ip-26-0-147-245:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-204:3]: sharded_logits = self.model( +[ip-26-0-147-204:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-245:6]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-245:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-245:6]: return forward_call(*input, **kwargs) +[ip-26-0-147-245:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-204:7]: sharded_logits = self.model( +[ip-26-0-147-245:6]: query_states, kv_states = self.qkv( +[ip-26-0-147-245:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-245:6]: return forward_call(*input, **kwargs) +[ip-26-0-147-245:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-245:6]: return column_linear( +[ip-26-0-147-204:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-245:6]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-204:7]: return forward_call(*input, **kwargs) +[ip-26-0-147-204:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-204:7]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-204:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-204:7]: return forward_call(*input, **kwargs) +[ip-26-0-147-204:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-204:3]: return forward_call(*input, **kwargs) +[ip-26-0-147-204:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-204:7]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-204:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-204:3]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-204:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-204:7]: return forward_call(*input, **kwargs) +[ip-26-0-147-204:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-204:3]: return forward_call(*input, **kwargs) +[ip-26-0-148-55:2]:Traceback (most recent call last): +[ip-26-0-148-55:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-55:2]: main() +[ip-26-0-148-55:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-55:2]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-204:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-204:7]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-204:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-204:3]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-204:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-204:7]: return forward_call(*input, **kwargs) +[ip-26-0-148-55:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-55:2]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-148-55:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-204:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-204:3]: return forward_call(*input, **kwargs) +[ip-26-0-147-204:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-55:2]: result = model(**micro_batch) +[ip-26-0-148-55:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-55:2]: return forward_call(*input, **kwargs) +[ip-26-0-148-55:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-55:2]: sharded_logits = self.model( +[ip-26-0-148-55:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-55:2]: return forward_call(*input, **kwargs) +[ip-26-0-148-55:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-55:2]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-148-55:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-204:7]: query_states, kv_states = self.qkv( +[ip-26-0-147-204:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-204:3]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-55:2]: return forward_call(*input, **kwargs) +[ip-26-0-148-55:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-148-55:2]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-55:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-55:2]: return forward_call(*input, **kwargs) +[ip-26-0-148-55:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-55:2]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-55:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-55:2]: return forward_call(*input, **kwargs) +[ip-26-0-148-55:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-55:2]: query_states, kv_states = self.qkv( +[ip-26-0-148-55:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-55:2]: return forward_call(*input, **kwargs) +[ip-26-0-148-55:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-55:2]: return column_linear( +[ip-26-0-148-55:2]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-204:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-204:7]: return forward_call(*input, **kwargs) +[ip-26-0-147-204:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-204:7]: return column_linear( +[ip-26-0-147-204:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-204:3]: return forward_call(*input, **kwargs) +[ip-26-0-147-204:7]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-189:0]:Traceback (most recent call last): +[ip-26-0-147-189:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-189:0]: main() +[ip-26-0-147-189:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-189:0]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-189:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-189:0]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-189:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-189:0]: result = model(**micro_batch) +[ip-26-0-147-189:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-189:0]: return forward_call(*input, **kwargs) +[ip-26-0-147-189:0]: File "/fsx/l[ip-26-0-148-55:6]:Traceback (most recent call last): +[ip-26-0-148-55:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-204:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-204:3]: query_states, kv_states = self.qkv( +[ip-26-0-148-55:6]: main() +[ip-26-0-147-204:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-204:3]: return forward_call(*input, **kwargs) +[ip-26-0-147-204:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-204:3]: return column_linear( +[ip-26-0-147-204:3]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-148-55:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-55:6]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-55:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-55:6]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-148-55:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-55:6]: result = model(**micro_batch) +[ip-26-0-148-55:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-55:6]: return forward_call(*input, **kwargs) +[ip-26-0-148-55:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-55:6]: sharded_logits = self.model( +oubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-189:0]: sharded_logits = self.model( +[ip-26-0-147-189:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-189:0]: return forward_call(*input, **kwargs) +[ip-26-0-147-189:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-189:0]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-189:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-189:0]: return forward_call(*input, **kwargs) +[ip-26-0-147-189:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-148-55:0]:Traceback (most recent call last): +[ip-26-0-147-189:4]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-189:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-189:4]: return forward_call(*input, **kwargs) +[ip-26-0-148-55:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-189:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-189:4]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-189:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-55:6]: return forward_call(*input, **kwargs) +[ip-26-0-147-189:4]: return forward_call(*input, **kwargs) +[ip-26-0-147-189:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-189:4]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-55:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-189:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-189:4]: return forward_call(*input, **kwargs) +[ip-26-0-147-189:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-189:4]: query_states, kv_states = self.qkv( +[ip-26-0-148-55:0]: main() +[ip-26-0-147-189:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-189:4]: return forward_call(*input, **kwargs) +[ip-26-0-147-189:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-189:4]: return column_linear( +[ip-26-0-147-189:4]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-148-55:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-55:0]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-55:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-55:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-55:6]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-148-55:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-55:6]: return forward_call(*input, **kwargs) +[ip-26-0-148-55:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-148-55:0]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-148-55:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-55:0]: result = model(**micro_batch) +[ip-26-0-148-55:6]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-55:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-189:1]:Traceback (most recent call last): +[ip-26-0-148-55:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-55:0]: return forward_call(*input, **kwargs) +[ip-26-0-148-55:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-55:6]: return forward_call(*input, **kwargs) +[ip-26-0-148-55:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-55:0]: sharded_logits = self.model( +[ip-26-0-148-55:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-55:0]: return forward_call(*input, **kwargs) +[ip-26-0-147-189:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-189:1]: main() +[ip-26-0-148-55:6]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-189:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-189:1]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-189:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-55:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-55:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-55:0]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-148-55:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-55:6]: return forward_call(*input, **kwargs) +[ip-26-0-148-55:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-55:6]: query_states, kv_states = self.qkv( +[ip-26-0-148-55:0]: return forward_call(*input, **kwargs) +[ip-26-0-148-55:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-148-55:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-189:6]:Traceback (most recent call last): +[ip-26-0-148-55:6]: return forward_call(*input, **kwargs) +[ip-26-0-148-55:0]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-55:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-55:6]: return column_linear( +[ip-26-0-147-189:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-189:6]: main() +[ip-26-0-147-189:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-55:6]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-189:6]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-189:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-55:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-55:0]: return forward_call(*input, **kwargs) +[ip-26-0-147-189:6]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-189:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-55:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-189:6]: result = model(**micro_batch) +[ip-26-0-148-55:0]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-55:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-55:0]: return forward_call(*input, **kwargs) +[ip-26-0-147-189:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-55:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-55:0]: query_states, kv_states = self.qkv( +[ip-26-0-148-55:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-189:6]: return forward_call(*input, **kwargs) +[ip-26-0-147-189:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-189:6]: sharded_logits = self.model( +[ip-26-0-148-55:0]: return forward_call(*input, **kwargs) +[ip-26-0-148-55:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-55:0]: return column_linear( +[ip-26-0-148-55:0]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-189:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-189:6]: return forward_call(*input, **kwargs) +[ip-26-0-147-189:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-189:6]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-189:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-189:6]: return forward_call(*input, **kwargs) +[ip-26-0-147-189:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-189:6]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-189:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-189:6]: return forward_call(*input, **kwargs) +[ip-26-0-147-189:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-189:6]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-189:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-189:6]: return forward_call(*input, **kwargs) +[ip-26-0-147-189:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-189:6]: query_states, kv_states = self.qkv( +[ip-26-0-147-189:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-189:6]: return forward_call(*input, **kwargs) +[ip-26-0-147-189:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-189:6]: return column_linear( +[ip-26-0-147-189:6]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-245:0]:Traceback (most recent call last): +[ip-26-0-147-245:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-245:0]: main() +[ip-26-0-147-245:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-245:0]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-245:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-245:0]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-245:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-245:0]: result = model(**micro_batch) +[ip-26-0-147-245:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-245:0]: return forward_call(*input, **kwargs) +[ip-26-0-147-245:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-245:0]: sharded_logits = self.model( +[ip-26-0-147-245:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-245:0]: return forward_call(*input, **kwargs) +[ip-26-0-147-245:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-245:0]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-245:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-245:0]: return forward_call(*input, **kwargs) +[ip-26-0-147-245:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-245:0]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-245:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-245:0]: return forward_call(*input, **kwargs) +[ip-26-0-147-245:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-245:0]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-245:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-245:0]: return forward_call(*input, **kwargs) +[ip-26-0-147-245:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-245:0]: query_states, kv_states = self.qkv( +[ip-26-0-147-245:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-245:0]: return forward_call(*input, **kwargs) +[ip-26-0-147-245:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-245:0]: return column_linear( +[ip-26-0-147-245:0]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-245:2]:Traceback (most recent call last): +[ip-26-0-147-245:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-245:2]: main() +[ip-26-0-147-245:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-245:2]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-245:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-245:2]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-245:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-245:2]: result = model(**micro_batch) +[ip-26-0-147-245:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-245:2]: return forward_call(*input, **kwargs) +[ip-26-0-147-245:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-245:2]: sharded_logits = self.model( +[ip-26-0-147-245:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-245:2]: return forward_call(*input, **kwargs) +[ip-26-0-147-245:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-245:2]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-245:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-245:2]: return forward_call(*input, **kwargs) +[ip-26-0-147-245:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-245:2]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-245:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-245:2]: return forward_call(*input, **kwargs) +[ip-26-0-147-245:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-245:2]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-245:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-245:2]: return forward_call(*input, **kwargs) +[ip-26-0-147-245:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-245:2]: query_states, kv_states = self.qkv( +[ip-26-0-147-245:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-245:2]: return forward_call(*input, **kwargs) +[ip-26-0-147-245:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-245:2]: return column_linear( +[ip-26-0-147-245:2]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-193:6]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-193:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-193:6]: return forward_call(*input, **kwargs) +[ip-26-0-147-193:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-193:6]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-193:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-193:6]: return forward_call(*input, **kwargs) +[ip-26-0-147-193:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-193:6]: query_states, kv_states = self.qkv( +[ip-26-0-147-193:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-193:6]: return forward_call(*input, **kwargs) +[ip-26-0-147-193:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-193:6]: return column_linear( +[ip-26-0-147-193:6]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-141:6]:Traceback (most recent call last): +[ip-26-0-147-141:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-141:6]: main() +[ip-26-0-147-141:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-141:6]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-141:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-141:6]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-141:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-141:6]: result = model(**micro_batch) +[ip-26-0-147-141:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-141:6]: return forward_call(*input, **kwargs) +[ip-26-0-147-141:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-141:6]: sharded_logits = self.model( +[ip-26-0-147-141:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-141:6]: return forward_call(*input, **kwargs) +[ip-26-0-147-141:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-141:6]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-141:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-141:6]: return forward_call(*input, **kwargs) +[ip-26-0-147-141:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-141:6]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-141:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-141:6]: return forward_call(*input, **kwargs) +[ip-26-0-147-141:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-141:6]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-141:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-141:6]: return forward_call(*input, **kwargs) +[ip-26-0-147-141:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-141:6]: query_states, kv_states = self.qkv( +[ip-26-0-147-141:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-141:6]: return forward_call(*input, **kwargs) +[ip-26-0-147-141:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-141:6]: return column_linear( +[ip-26-0-147-141:6]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-148-55:4]:Traceback (most recent call last): +[ip-26-0-148-55:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-55:4]: main() +[ip-26-0-148-55:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-55:4]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-55:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-55:4]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-148-55:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-55:4]: result = model(**micro_batch) +[ip-26-0-148-55:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-55:4]: return forward_call(*input, **kwargs) +[ip-26-0-148-55:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-55:4]: sharded_logits = self.model( +[ip-26-0-148-55:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-55:4]: return forward_call(*input, **kwargs) +[ip-26-0-148-55:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-55:4]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-148-55:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-55:4]: return forward_call(*input, **kwargs) +[ip-26-0-148-55:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-148-55:4]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-55:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-55:4]: return forward_call(*input, **kwargs) +[ip-26-0-148-55:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-55:4]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-55:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-55:4]: return forward_call(*input, **kwargs) +[ip-26-0-148-55:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-55:4]: query_states, kv_states = self.qkv( +[ip-26-0-148-55:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-55:4]: return forward_call(*input, **kwargs) +[ip-26-0-148-55:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-55:4]: return column_linear( +[ip-26-0-148-55:4]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-189:0]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-189:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-189:0]: return forward_call(*input, **kwargs) +[ip-26-0-147-189:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-189:0]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-189:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-189:0]: return forward_call(*input, **kwargs) +[ip-26-0-147-189:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-189:0]: query_states, kv_states = self.qkv( +[ip-26-0-147-189:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-189:0]: return forward_call(*input, **kwargs) +[ip-26-0-147-189:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-189:0]: return column_linear( +[ip-26-0-147-189:0]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-189:1]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-189:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-189:1]: result = model(**micro_batch) +[ip-26-0-147-189:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-189:1]: return forward_call(*input, **kwargs) +[ip-26-0-147-189:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-189:1]: sharded_logits = self.model( +[ip-26-0-147-189:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-189:1]: return forward_call(*input, **kwargs) +[ip-26-0-147-189:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-189:1]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-189:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-189:1]: return forward_call(*input, **kwargs) +[ip-26-0-147-189:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-189:1]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-189:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-189:1]: return forward_call(*input, **kwargs) +[ip-26-0-147-189:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-189:1]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-189:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-189:1]: return forward_call(*input, **kwargs) +[ip-26-0-147-189:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-189:1]: query_states, kv_states = self.qkv( +[ip-26-0-147-189:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-189:1]: return forward_call(*input, **kwargs) +[ip-26-0-147-189:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-189:1]: return column_linear( +[ip-26-0-147-189:1]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-148-193:3]:Traceback (most recent call last): +[ip-26-0-148-193:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-193:3]: main() +[ip-26-0-148-193:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-193:3]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-193:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-193:3]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-148-193:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-193:3]: result = model(**micro_batch) +[ip-26-0-148-193:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-193:3]: return forward_call(*input, **kwargs) +[ip-26-0-148-193:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-193:3]: sharded_logits = self.model( +[ip-26-0-148-193:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-193:3]: return forward_call(*input, **kwargs) +[ip-26-0-148-193:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-193:3]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-148-193:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-193:3]: return forward_call(*input, **kwargs) +[ip-26-0-148-193:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-148-193:3]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-193:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-193:3]: return forward_call(*input, **kwargs) +[ip-26-0-148-193:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-193:3]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-193:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-193:3]: return forward_call(*input, **kwargs) +[ip-26-0-148-193:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-193:3]: query_states, kv_states = self.qkv( +[ip-26-0-148-193:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-193:3]: return forward_call(*input, **kwargs) +[ip-26-0-148-193:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-193:3]: return column_linear( +[ip-26-0-148-193:3]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-147-245:1]:Traceback (most recent call last): +[ip-26-0-147-245:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-147-245:1]: main() +[ip-26-0-147-245:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-147-245:1]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-147-245:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-147-245:1]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-147-245:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-147-245:1]: result = model(**micro_batch) +[ip-26-0-147-245:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-245:1]: return forward_call(*input, **kwargs) +[ip-26-0-147-245:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-147-245:1]: sharded_logits = self.model( +[ip-26-0-147-245:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-245:1]: return forward_call(*input, **kwargs) +[ip-26-0-147-245:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-147-245:1]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-147-245:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-245:1]: return forward_call(*input, **kwargs) +[ip-26-0-147-245:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-147-245:1]: output = self.pp_block(**new_kwargs) +[ip-26-0-147-245:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-245:1]: return forward_call(*input, **kwargs) +[ip-26-0-147-245:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-147-245:1]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-147-245:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-245:1]: return forward_call(*input, **kwargs) +[ip-26-0-147-245:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-147-245:1]: query_states, kv_states = self.qkv( +[ip-26-0-147-245:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-147-245:1]: return forward_call(*input, **kwargs) +[ip-26-0-147-245:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-147-245:1]: return column_linear( +[ip-26-0-147-245:1]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-148-193:7]:Traceback (most recent call last): +[ip-26-0-148-193:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in +[ip-26-0-148-193:7]: main() +[ip-26-0-148-193:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main +[ip-26-0-148-193:7]: outputs = pipeline_engine.train_batch_iter( +[ip-26-0-148-193:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter +[ip-26-0-148-193:7]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) +[ip-26-0-148-193:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward +[ip-26-0-148-193:7]: result = model(**micro_batch) +[ip-26-0-148-193:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-193:7]: return forward_call(*input, **kwargs) +[ip-26-0-148-193:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward +[ip-26-0-148-193:7]: sharded_logits = self.model( +[ip-26-0-148-193:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-193:7]: return forward_call(*input, **kwargs) +[ip-26-0-148-193:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward +[ip-26-0-148-193:7]: hidden_encoder_states = encoder_block(**hidden_encoder_states) +[ip-26-0-148-193:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-193:7]: return forward_call(*input, **kwargs) +[ip-26-0-148-193:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward +[ip-26-0-148-193:7]: output = self.pp_block(**new_kwargs) +[ip-26-0-148-193:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-193:7]: return forward_call(*input, **kwargs) +[ip-26-0-148-193:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward +[ip-26-0-148-193:7]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) +[ip-26-0-148-193:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-193:7]: return forward_call(*input, **kwargs) +[ip-26-0-148-193:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward +[ip-26-0-148-193:7]: query_states, kv_states = self.qkv( +[ip-26-0-148-193:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl +[ip-26-0-148-193:7]: return forward_call(*input, **kwargs) +[ip-26-0-148-193:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward +[ip-26-0-148-193:7]: return column_linear( +[ip-26-0-148-193:7]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) +[ip-26-0-148-245:3]:libfabric:345565:1686831696::efa:ep_ctrl:efa_rdm_peer_clear():123 Closing EP with unacked CONNREQs in flight +[ip-26-0-148-115:7]:libfabric:2598809:1686831696::efa:ep_ctrl:efa_rdm_peer_clear():123 Closing EP with unacked CONNREQs in flight +[ip-26-0-148-115:3]:libfabric:2598805:1686831696::efa:ep_ctrl:efa_rdm_peer_clear():123 Closing EP with unacked CONNREQs in flight +[ip-26-0-148-115:5]:libfabric:2598807:1686831696::efa:ep_ctrl:efa_rdm_peer_clear():123 Closing EP with unacked CONNREQs in flight +[ip-26-0-148-245:7]:libfabric:345569:1686831696::efa:ep_ctrl:efa_rdm_peer_clear():123 Closing EP with unacked CONNREQs in flight +[ip-26-0-147-204:3]:libfabric:3473903:1686831696::efa:ep_ctrl:efa_rdm_peer_clear():123 Closing EP with unacked CONNREQs in flight +[ip-26-0-147-204:7]:libfabric:3473907:1686831696::efa:ep_ctrl:efa_rdm_peer_clear():123 Closing EP with unacked CONNREQs in flight +[ip-26-0-148-245:5]:libfabric:345567:1686831696::efa:ep_ctrl:efa_rdm_peer_clear():123 Closing EP with unacked CONNREQs in flight +[ip-26-0-147-204:5]:libfabric:3473905:1686831696::efa:ep_ctrl:efa_rdm_peer_clear():123 Closing EP with unacked CONNREQs in flight +ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 346331) of binary: /fsx/loubna/miniconda3/envs/megatron/bin/python +ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 2873803) of binary: /fsx/loubna/miniconda3/envs/megatron/bin/python +ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 148753) of binary: /fsx/loubna/miniconda3/envs/megatron/bin/python +ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 345562) of binary: /fsx/loubna/miniconda3/envs/megatron/bin/python +ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 2492612) of binary: /fsx/loubna/miniconda3/envs/megatron/bin/python +ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 3972943) of binary: /fsx/loubna/miniconda3/envs/megatron/bin/python +ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 3234666) of binary: /fsx/loubna/miniconda3/envs/megatron/bin/python +ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 2598802) of binary: /fsx/loubna/miniconda3/envs/megatron/bin/python +ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 1361) of binary: /fsx/loubna/miniconda3/envs/megatron/bin/python +ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 3473900) of binary: /fsx/loubna/miniconda3/envs/megatron/bin/python +ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 4103480) of binary: /fsx/loubna/miniconda3/envs/megatron/bin/python +ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 840174) of binary: /fsx/loubna/miniconda3/envs/megatron/bin/python +ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 3653324) of binary: /fsx/loubna/miniconda3/envs/megatron/bin/python +ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 3255534) of binary: /fsx/loubna/miniconda3/envs/megatron/bin/python +ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 3607182) of binary: /fsx/loubna/miniconda3/envs/megatron/bin/python +ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 3679420) of binary: /fsx/loubna/miniconda3/envs/megatron/bin/python +Traceback (most recent call last): + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 196, in _run_module_as_main +Traceback (most recent call last): + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 196, in _run_module_as_main + return _run_code(code, main_globals, None, + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 86, in _run_code +Traceback (most recent call last): + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 196, in _run_module_as_main + return _run_code(code, main_globals, None, + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 86, in _run_code +Traceback (most recent call last): + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 196, in _run_module_as_main + exec(code, run_globals) +Traceback (most recent call last): + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 196, in _run_module_as_main + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 766, in + exec(code, run_globals) + return _run_code(code, main_globals, None, + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 766, in + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 86, in _run_code +Traceback (most recent call last): + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 196, in _run_module_as_main +Traceback (most recent call last): + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 196, in _run_module_as_main + return _run_code(code, main_globals, None, +Traceback (most recent call last): + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 196, in _run_module_as_main + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 86, in _run_code +Traceback (most recent call last): + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 196, in _run_module_as_main +Traceback (most recent call last): + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 196, in _run_module_as_main + exec(code, run_globals) + main() + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper + return _run_code(code, main_globals, None, + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 766, in + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 86, in _run_code +Traceback (most recent call last): + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 196, in _run_module_as_main + return _run_code(code, main_globals, None, + return _run_code(code, main_globals, None, + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 86, in _run_code + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 86, in _run_code + exec(code, run_globals) + return _run_code(code, main_globals, None, + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 766, in + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 86, in _run_code + return _run_code(code, main_globals, None, + main() + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 86, in _run_code + return f(*args, **kwargs) + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main + exec(code, run_globals) + exec(code, run_globals) + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 766, in + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 766, in + exec(code, run_globals) + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 766, in + exec(code, run_globals) + return _run_code(code, main_globals, None, + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 766, in + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 86, in _run_code + main() +Traceback (most recent call last): + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 196, in _run_module_as_main + return f(*args, **kwargs) + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper +Traceback (most recent call last): + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 196, in _run_module_as_main + main() + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper + return _run_code(code, main_globals, None, + exec(code, run_globals) + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 766, in + main() + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 86, in _run_code + main() + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper + run(args) + exec(code, run_globals) + main() + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run +Traceback (most recent call last): + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 196, in _run_module_as_main + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 766, in + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper + return f(*args, **kwargs) + main() + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper + return f(*args, **kwargs) + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main + return _run_code(code, main_globals, None, + run(args) + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run + return f(*args, **kwargs) + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 86, in _run_code + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main +Traceback (most recent call last): + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 196, in _run_module_as_main + return f(*args, **kwargs) + main() + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper + return f(*args, **kwargs) + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main + return _run_code(code, main_globals, None, + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main + elastic_launch( + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 86, in _run_code + return f(*args, **kwargs) + exec(code, run_globals) + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main + main() + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 766, in + exec(code, run_globals) + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper + run(args) + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 766, in + run(args) + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run + return f(*args, **kwargs) + return _run_code(code, main_globals, None, + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 86, in _run_code + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main + run(args) + run(args) + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run + return launch_agent(self._config, self._entrypoint, list(args)) + exec(code, run_globals) + run(args) + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent + run(args) + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 766, in + return _run_code(code, main_globals, None, + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 86, in _run_code + elastic_launch( + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ + exec(code, run_globals) + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 766, in + return f(*args, **kwargs) + elastic_launch( + main() + main() + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ + elastic_launch( + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper + elastic_launch( + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ + elastic_launch( + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +main.py FAILED +------------------------------------------------------------ +Failures: +[1]: + time : 2023-06-15_12:21:38 + host : ip-26-0-148-170.us-west-2.compute.internal + rank : 81 (local_rank: 1) + exitcode : 1 (pid: 1362) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[2]: + time : 2023-06-15_12:21:38 + host : ip-26-0-148-170.us-west-2.compute.internal + rank : 82 (local_rank: 2) + exitcode : 1 (pid: 1363) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[3]: + time : 2023-06-15_12:21:38 + host : ip-26-0-148-170.us-west-2.compute.internal + rank : 83 (local_rank: 3) + exitcode : 1 (pid: 1364) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[4]: + time : 2023-06-15_12:21:38 + host : ip-26-0-148-170.us-west-2.compute.internal + rank : 84 (local_rank: 4) + exitcode : 1 (pid: 1365) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[5]: + time : 2023-06-15_12:21:38 + host : ip-26-0-148-170.us-west-2.compute.internal + rank : 85 (local_rank: 5) + exitcode : 1 (pid: 1366) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[6]: + time : 2023-06-15_12:21:38 + host : ip-26-0-148-170.us-west-2.compute.internal + rank : 86 (local_rank: 6) + exitcode : 1 (pid: 1367) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[7]: + time : 2023-06-15_12:21:38 + host : ip-26-0-148-170.us-west-2.compute.internal + rank : 87 (local_rank: 7) + exitcode : 1 (pid: 1368) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +---------------------- elastic_launch( + return f(*args, **kwargs) + return launch_agent(self._config, self._entrypoint, list(args)) +-------------------------------------- +Root Cause (first observed failure): +[0]: + time : 2023-06-15_12:21:38 + host : ip-26-0-148-170.us-west-2.compute.internal + rank : 80 (local_rank: 0) + exitcode : 1 (pid: 1361) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ + return launch_agent(self._config, self._entrypoint, list(args)) + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent + return launch_agent(self._config, self._entrypoint, list(args)) + main() + return launch_agent(self._config, self._entrypoint, list(args)) + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent + elastic_launch( + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent + run(args) + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ + exec(code, run_globals) + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 766, in + return launch_agent(self._config, self._entrypoint, list(args)) + main() + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent + raise ChildFailedError( + raise ChildFailedError( + return launch_agent(self._config, self._entrypoint, list(args)) + return f(*args, **kwargs) + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +main.py FAILED +------------------------------------------------------------ +Failures: +[1]: + time : 2023-06-15_12:21:38 + host : ip-26-0-148-245.us-west-2.compute.internal + rank : 97 (local_rank: 1) + exitcode : 1 (pid: 345563) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[2]: + time : 2023-06-15_12:21:38 + host : ip-26-0-148-245.us-west-2.compute.internal + rank : 98 (local_rank: 2) + exitcode : 1 (pid: 345564) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[3]: + time : 2023-06-15_12:21:38 + host : ip-26-0-148-245.us-west-2.compute.internal + rank : 99 (local_rank: 3) + exitcode : 1 (pid: 345565) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[4]: + time : 2023-06-15_12:21:38 + host : ip-26-0-148-2 return launch_agent(self._config, self._entrypoint, list(args)) +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +main.py FAILED +------------------------------------------------------------ +Failures: +[1]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-233.us-west-2.compute.internal + rank : 41 (local_rank: 1) + exitcode : 1 (pid: 3234667) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[2]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-233.us-west-2.compute.internal + rank : 42 (local_rank: 2) + exitcode : 1 (pid: 3234668) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[3]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-233.us-west-2.compute.internal + rank : 43 (local_rank: 3) + exitcode : 1 (pid: 3234669) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.h45.us-west-2.compute.internal + rank : 100 (local_rank: 4) + exitcode : 1 (pid: 345566) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[5]: + time : 2023-06-15_12:21:38 + host : ip-26-0-148-245.us-west-2.compute.internal + rank : 101 (local_rank: 5) + exitcode : 1 (pid: 345567) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[6]: + time : 2023-06-15_12:21:38 + host : ip-26-0-148-245.us-west-2.compute.internal + rank : 102 (local_rank: 6) + exitcode : 1 (pid: 345568) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[7]: + time : 2023-06-15_12:21:38 + host : ip-26-0-148-245.us-west-2.compute.internal + rank : 103 (local_rank: 7) + exitcode : 1 (pid: 345569) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +------ File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent +tml +[4]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-233.us-west-2.compute.internal + rank : 44 (local_rank: 4) + exitcode : 1 (pid: 3234670) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[5]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-233.us-west-2.compute.internal + rank : 45 (local_rank: 5) + exitcode : 1 (pid: 3234671) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[6]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-233.us-west-2.compute.internal + rank : 46 (local_rank: 6) + exitcode : 1 (pid: 3234672) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[7]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-233.us-west-2.compute.internal + rank : 47 (local_rank: 7) + exitcode : 1 (pid: 3234673) + error_file: + traceback : To enable tra------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2023-06-15_12:21:38 + host : ip-26-0-148-245.us-west-2.compute.internal + rank : 96 (local_rank: 0) + exitcode : 1 (pid: 345562) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ +ceback see: https://pytorch.org/docs/stable/elastic/errors.html +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-233.us-west-2.compute.internal + rank : 40 (local_rank: 0) + exitcode : 1 (pid: 3234666) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main +torch.distributed.elastic.multiprocessing.errors.ChildFailedError raise ChildFailedError( +: +============================================================ +main.py FAILED +------------------------------------------------------------ +Failures: +[1]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-193.us-west-2.compute.internal + rank : 25 (local_rank: 1) + exitcode : 1 (pid: 3607183) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[2]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-193.us-west-2.compute.internal + rank : 26 (local_rank: 2) + exitcode : 1 (pid: 3607184) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[3]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-193.us-west-2.compute.internal + rank : 27 (local_rank: 3) + exitcode : 1 (pid: 3607185) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[4]: + time : 2023-06-15_12:21:38 + host : ip-26-0- return f(*args, **kwargs) + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent +147-193.us-west-2.compute.internal + rank : 28 (local_rank: 4) + exitcode : 1 (pid: 3607186) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[5]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-193.us-west-2.compute.internal + rank : 29 (local_rank: 5) + exitcode : 1 (pid: 3607187) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[6]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-193.us-west-2.compute.internal + rank : 30 (local_rank: 6) + exitcode : 1 (pid: 3607188) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[7]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-193.us-west-2.compute.internal + rank : 31 (local_rank: 7) + exitcode : 1 (pid: 3607189) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +- File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main +torch.distributed.elastic.multiprocessing.errors run(args) +----------------------------------------------------------- +Root Cause (first observed failure): +[0]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-193.us-west-2.compute.internal + rank : 24 (local_rank: 0) + exitcode : 1 (pid: 3607182) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ +.ChildFailedError: +============================================================ +main.py FAILED +------------------------------------------------------------ +Failures: +[1]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-187.us-west-2.compute.internal + rank : 9 (local_rank: 1) + exitcode : 1 (pid: 2492613) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[2]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-187.us-west-2.compute.internal + rank : 10 (local_rank: 2) + exitcode : 1 (pid: 2492614) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[3]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-187.us-west-2.compute.internal + rank : 11 (local_rank: 3) + exitcode : 1 (pid: 2492615) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[4]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-187.us-west-2.compute.internal + rank : 12 (local_rank: 4) + exitcode : 1 (pid: 2492616) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[5]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-187.us-west-2.compute.internal + rank : 13 (local_rank: 5) + exitcode : 1 (pid: 2492617) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[6]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-187.us-west-2.compute.internal + rank : 14 (local_rank: 6) + exitcode : 1 (pid: 2492618) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[7]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-187.us-west-2.compute.internal + rank : 15 (local_rank: 7) + exitcode : 1 (pid: 2492619) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elast raise ChildFailedError( + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run +Traceback (most recent call last): + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 196, in _run_module_as_main + run(args) + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run + elastic_launch( + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ +ic/errors.html +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-187.us-west-2.compute.internal + rank : 8 (local_rank: 0) + exitcode : 1 (pid: 2492612) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ + return f(*args, **kwargs) + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main + raise ChildFailedError( + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +main.py FAILED +------------------------------------------------------------ +Failures: +[1]: + time : 2023-06-15_12:21:38 + host : ip-26-0-148-115.us-west-2.compute.internal + rank : 65 (local_rank: 1) + exitcode : 1 (pid: 2598803) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[2]: + time : 2023-06-15_12:21:38 + host : ip-26-0-148-115.us-west-2.compute.internal + rank : 66 (local_rank: 2) + exitcode : 1 (pid: 2598804) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[3]: + time : 2023-06-15_12:21:38 + host : ip-26-0-148-115.us-west-2.compute.internal + rank : 67 (local_rank: 3) + exitcode : 1 (pid: 2598805) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.htorch.distributed.elastic.multiprocessing.errors.ChildFailedErrortml +[4]: + time : 2023-06-15_12:21:38 + host : ip-26-0-148-115.us-west-2.compute.internal + rank : 68 (local_rank: 4) + exitcode : 1 (pid: 2598806) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[5]: + time : 2023-06-15_12:21:38 + host : ip-26-0-148-115.us-west-2.compute.internal + rank : 69 (local_rank: 5) + exitcode : 1 (pid: 2598807) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[6]: + time : 2023-06-15_12:21:38 + host : ip-26-0-148-115.us-west-2.compute.internal + rank : 70 (local_rank: 6) + exitcode : 1 (pid: 2598808) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[7]: + time : 2023-06-15_12:21:38 + host : ip-26-0-148-115.us-west-2.compute.internal + rank : 71 (local_rank: 7) + exitcode : 1 (pid: 2598809) + error_file: + traceback : To enable tra: +============================================================ +main.py FAILED +------------------------------------------------------------ +Failures: +[1]: + time : 2023-06-15_12:21:38 + host : ip-26-0-148-55.us-west-2.compute.internal + rank : 105 (local_rank: 1) + exitcode : 1 (pid: 4103481) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[2]: + time : 2023-06-15_12:21:38 + host : ip-26-0-148-55.us-west-2.compute.internal + rank : 106 (local_rank: 2) + exitcode : 1 (pid: 4103482) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[3]: + time : 2023-06-15_12:21:38 + host : ip-26-0-148-55.us-west-2.compute.internal + rank : 107 (local_rank: 3) + exitcode : 1 (pid: 4103483) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[4]: + time : 2023-06-15_12:21:38 + host : ip-26-0- run(args) + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: ceback see: https://pytorch.org/docs/stable/elastic/errors.html +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2023-06-15_12:21:38 + host : ip-26-0-148-115.us-west-2.compute.internal + rank : 64 (local_rank: 0) + exitcode : 1 (pid: 2598802) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ +148-55.us-west-2.compute.internal + rank : 108 (local_rank: 4) + exitcode : 1 (pid: 4103484) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[5]: + time : 2023-06-15_12:21:38 + host : ip-26-0-148-55.us-west-2.compute.internal + rank : 109 (local_rank: 5) + exitcode : 1 (pid: 4103485) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[6]: + time : 2023-06-15_12:21:38 + host : ip-26-0-148-55.us-west-2.compute.internal + rank : 110 (local_rank: 6) + exitcode : 1 (pid: 4103486) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[7]: + time : 2023-06-15_12:21:38 + host : ip-26-0-148-55.us-west-2.compute.internal + rank : 111 (local_rank: 7) + exitcode : 1 (pid: 4103487) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +- +============================================================ +main.py FAILED +------------------------------------------------------------ +Failures: +[1]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-245.us-west-2.compute.internal + rank : 49 (local_rank: 1) + exitcode : 1 (pid: 3653325) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[2]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-245.us-west-2.compute.internal + rank : 50 (local_rank: 2) + exitcode : 1 (pid: 3653326) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[3]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-245.us-west-2.compute.internal + rank : 51 (local_rank: 3) + exitcode : 1 (pid: 3653327) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[4]: + time : 2023-06-15_12:21:38 + host : ip-26-0-14----------------------------------------------------------- +Root Cause (first observed failure): +[0]: + time : 2023-06-15_12:21:38 + host : ip-26-0-148-55.us-west-2.compute.internal + rank : 104 (local_rank: 0) + exitcode : 1 (pid: 4103480) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ +7-245.us-west-2.compute.internal + rank : 52 (local_rank: 4) + exitcode : 1 (pid: 3653328) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[5]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-245.us-west-2.compute.internal + rank : 53 (local_rank: 5) + exitcode : 1 (pid: 3653329) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[6]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-245.us-west-2.compute.internal + rank : 54 (local_rank: 6) + exitcode : 1 (pid: 3653330) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[7]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-245.us-west-2.compute.internal + rank : 55 (local_rank: 7) + exitcode : 1 (pid: 3653331) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-245.us-west-2.compute.internal + rank : 48 (local_rank: 0) + exitcode : 1 (pid: 3653324) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ + run(args) + return launch_agent(self._config, self._entrypoint, list(args)) + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent + elastic_launch( + elastic_launch( + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ + raise ChildFailedError( + return _run_code(code, main_globals, None, + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 86, in _run_code + elastic_launch( + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ + run(args) + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +main.py FAILED +------------------------------------------------------------ +Failures: +[1]: + time : 2023-06-15_12:21:38 + host : ip-26-0-148-93.us-west-2.compute.internal + rank : 113 (local_rank: 1) + exitcode : 1 (pid: 3972944) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[2]: + time : 2023-06-15_12:21:38 + host : ip-26-0-148-93.us-west-2.compute.internal + rank : 114 (local_rank: 2) + exitcode : 1 (pid: 3972945) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[3]: + time : 2023-06-15_12:21:38 + host : ip-26-0-148-93.us-west-2.compute.internal + rank : 115 (local_rank: 3) + exitcode : 1 (pid: 3972946) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[4]: + time : 2023-06-15_12:21:38 + host : ip-26-0- return launch_agent(self._config, self._entrypoint, list(args)) + main() + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper +148-93.us-west-2.compute.internal + rank : 116 (local_rank: 4) + exitcode : 1 (pid: 3972947) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[5]: + time : 2023-06-15_12:21:38 + host : ip-26-0-148-93.us-west-2.compute.internal + rank : 117 (local_rank: 5) + exitcode : 1 (pid: 3972948) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[6]: + time : 2023-06-15_12:21:38 + host : ip-26-0-148-93.us-west-2.compute.internal + rank : 118 (local_rank: 6) + exitcode : 1 (pid: 3972949) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[7]: + time : 2023-06-15_12:21:38 + host : ip-26-0-148-93.us-west-2.compute.internal + rank : 119 (local_rank: 7) + exitcode : 1 (pid: 3972950) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2023-06-15_12:21:38 + host : ip-26-0-148-93.us-west-2.compute.internal + rank : 112 (local_rank: 0) + exitcode : 1 (pid: 3972943) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent + return launch_agent(self._config, self._entrypoint, list(args)) + return launch_agent(self._config, self._entrypoint, list(args)) + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent + raise ChildFailedError( + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +main.py FAILED +------------------------------------------------------------ +Failures: +[1]: + time : 2023-06-15_12:21:38 + host : ip-26-0-148-151.us-west-2.compute.internal + rank : 73 (local_rank: 1) + exitcode : 1 (pid: 2873804) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[2]: + time : 2023-06-15_12:21:38 + host : ip-26-0-148-151.us-west-2.compute.internal + rank : 74 (local_rank: 2) + exitcode : 1 (pid: 2873805) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[3]: + time : 2023-06-15_12:21:38 + host : ip-26-0-148-151.us-west-2.compute.internal + rank : 75 (local_rank: 3) + exitcode : 1 (pid: 2873806) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[4]: + time : 2023-06-15_12:21:38 + host : ip-26-0-14 raise ChildFailedError( +8-151.us-west-2.compute.internal + rank : 76 (local_rank: 4) + exitcode : 1 (pid: 2873807) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[5]: + time : 2023-06-15_12:21:38 + host : ip-26-0-148-151.us-west-2.compute.internal + rank : 77 (local_rank: 5) + exitcode : 1 (pid: 2873808) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[6]: + time : 2023-06-15_12:21:38 + host : ip-26-0-148-151.us-west-2.compute.internal + rank : 78 (local_rank: 6) + exitcode : 1 (pid: 2873809) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[7]: + time : 2023-06-15_12:21:38 + host : ip-26-0-148-151.us-west-2.compute.internal + rank : 79 (local_rank: 7) + exitcode : 1 (pid: 2873810) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +--- elastic_launch( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError--------------------------------------------------------- +Root Cause (first observed failure): +[0]: + time : 2023-06-15_12:21:38 + host : ip-26-0-148-151.us-west-2.compute.internal + rank : 72 (local_rank: 0) + exitcode : 1 (pid: 2873803) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ + exec(code, run_globals) +: +============================================================ +main.py FAILED +------------------------------------------------------------ +Failures: +[1]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-141.us-west-2.compute.internal + rank : 1 (local_rank: 1) + exitcode : 1 (pid: 3679421) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[2]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-141.us-west-2.compute.internal + rank : 2 (local_rank: 2) + exitcode : 1 (pid: 3679422) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[3]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-141.us-west-2.compute.internal + rank : 3 (local_rank: 3) + exitcode : 1 (pid: 3679423) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[4]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147 return f(*args, **kwargs) + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main +-141.us-west-2.compute.internal + rank : 4 (local_rank: 4) + exitcode : 1 (pid: 3679424) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[5]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-141.us-west-2.compute.internal + rank : 5 (local_rank: 5) + exitcode : 1 (pid: 3679425) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[6]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-141.us-west-2.compute.internal + rank : 6 (local_rank: 6) + exitcode : 1 (pid: 3679426) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[7]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-141.us-west-2.compute.internal + rank : 7 (local_rank: 7) + exitcode : 1 (pid: 3679427) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +-------- raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ +---------------------------------------------------- +Root Cause (first observed failure): +[0]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-141.us-west-2.compute.internal + rank : 0 (local_rank: 0) + exitcode : 1 (pid: 3679420) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 766, in + elastic_launch( + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ +.ChildFailedError: +============================================================ +main.py FAILED +------------------------------------------------------------ +Failures: +[1]: + time : 2023-06-15_12:21:38 + host : ip-26-0-149-1.us-west-2.compute.internal + rank : 121 (local_rank: 1) + exitcode : 1 (pid: 346332) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[2]: + time : 2023-06-15_12:21:38 + host : ip-26-0-149-1.us-west-2.compute.internal + rank : 122 (local_rank: 2) + exitcode : 1 (pid: 346333) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[3]: + time : 2023-06-15_12:21:38 + host : ip-26-0-149-1.us-west-2.compute.internal + rank : 123 (local_rank: 3) + exitcode : 1 (pid: 346334) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[4]: + time : 2023-06-15_12:21:38 + host : ip-26-0-149-1.us-west-2.compute.internal + rank : 124 (local_rank: 4) + exitcode : 1 (pid: 346335) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[5]: + time : 2023-06-15_12:21:38 + host : ip-26-0-149-1.us-west-2.compute.internal + rank : 125 (local_rank: 5) + exitcode : 1 (pid: 346336) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[6]: + time : 2023-06-15_12:21:38 + host : ip-26-0-149-1.us-west-2.compute.internal + rank : 126 (local_rank: 6) + exitcode : 1 (pid: 346337) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[7]: + time : 2023-06-15_12:21:38 + host : ip-26-0-149-1.us-west-2.compute.internal + rank : 127 (local_rank: 7) + exitcode : 1 (pid: 346338) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2023-06-15_12:21:38 + host : ip-26-0-149-1.us-west-2.compute.internal + rank : 120 (local_rank: 0) + exitcode : 1 (pid: 346331) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ + return launch_agent(self._config, self._entrypoint, list(args)) + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent + main() + return launch_agent(self._config, self._entrypoint, list(args)) + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors run(args) + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run + raise ChildFailedError( +.ChildFailedError: +============================================================ +main.py FAILED +------------------------------------------------------------ +Failures: +[1]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-189.us-west-2.compute.internal + rank : 17 (local_rank: 1) + exitcode : 1 (pid: 840175) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[2]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-189.us-west-2.compute.internal + rank : 18 (local_rank: 2) + exitcode : 1 (pid: 840176) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[3]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-189.us-west-2.compute.internal + rank : 19 (local_rank: 3) + exitcode : 1 (pid: 840177) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[4]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-1 return f(*args, **kwargs) +89.us-west-2.compute.internal + rank : 20 (local_rank: 4) + exitcode : 1 (pid: 840178) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[5]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-189.us-west-2.compute.internal + rank : 21 (local_rank: 5) + exitcode : 1 (pid: 840179) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[6]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-189.us-west-2.compute.internal + rank : 22 (local_rank: 6) + exitcode : 1 (pid: 840180) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[7]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-189.us-west-2.compute.internal + rank : 23 (local_rank: 7) + exitcode : 1 (pid: 840181) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-189.us-west-2.compute.internal + rank : 16 (local_rank: 0) + exitcode : 1 (pid: 840174) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +main.py FAILED +------------------------------------------------------------ +Failures: +[1]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-247.us-west-2.compute.internal + rank : 57 (local_rank: 1) + exitcode : 1 (pid: 148754) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[2]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-247.us-west-2.compute.internal + rank : 58 (local_rank: 2) + exitcode : 1 (pid: 148755) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[3]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-247.us-west-2.compute.internal + rank : 59 (local_rank: 3) + exitcode : 1 (pid: 148756) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[4]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-247.us-west-2.compute.internal + rank : 60 (local_rank: 4) + exitcode : 1 (pid: 148757) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[5]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-247.us-west-2.compute.internal + rank : 61 (local_rank: 5) + exitcode : 1 (pid: 148758) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[6]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-247.us-west-2.compute.internal + rank : 62 (local_rank: 6) + exitcode : 1 (pid: 148759) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[7]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-247.us-west-2.compute.internal + rank : 63 (local_rank: 7) + exitcode : 1 (pid: 148760) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-247.us-west-2.compute.internal + rank : 56 (local_rank: 0) + exitcode : 1 (pid: 148753) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ + elastic_launch( + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ + run(args) + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run + return launch_agent(self._config, self._entrypoint, list(args)) + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent + elastic_launch( + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +main.py FAILED +------------------------------------------------------------ +Failures: +[1]: + time : 2023-06-15_12:21:38 + host : ip-26-0-148-193.us-west-2.compute.internal + rank : 89 (local_rank: 1) + exitcode : 1 (pid: 3255535) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[2]: + time : 2023-06-15_12:21:38 + host : ip-26-0-148-193.us-west-2.compute.internal + rank : 90 (local_rank: 2) + exitcode : 1 (pid: 3255536) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[3]: + time : 2023-06-15_12:21:38 + host : ip-26-0-148-193.us-west-2.compute.internal + rank : 91 (local_rank: 3) + exitcode : 1 (pid: 3255537) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[4]: + time : 2023-06-15_12:21:38 + host : ip-26-0-148-193.us-west-2.compute.internal + rank : 92 (local_rank: 4) + exitcode : 1 (pid: 3255538) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[5]: + time : 2023-06-15_12:21:38 + host : ip-26-0-148-193.us-west-2.compute.internal + rank : 93 (local_rank: 5) + exitcode : 1 (pid: 3255539) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[6]: + time : 2023-06-15_12:21:38 + host : ip-26-0-148-193.us-west-2.compute.internal + rank : 94 (local_rank: 6) + exitcode : 1 (pid: 3255540) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[7]: + time : 2023-06-15_12:21:38 + host : ip-26-0-148-193.us-west-2.compute.internal + rank : 95 (local_rank: 7) + exitcode : 1 (pid: 3255541) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2023-06-15_12:21:38 + host : ip-26-0-148-193.us-west-2.compute.internal + rank : 88 (local_rank: 0) + exitcode : 1 (pid: 3255534) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ + return launch_agent(self._config, self._entrypoint, list(args)) + File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +main.py FAILED +------------------------------------------------------------ +Failures: +[1]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-204.us-west-2.compute.internal + rank : 33 (local_rank: 1) + exitcode : 1 (pid: 3473901) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[2]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-204.us-west-2.compute.internal + rank : 34 (local_rank: 2) + exitcode : 1 (pid: 3473902) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[3]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-204.us-west-2.compute.internal + rank : 35 (local_rank: 3) + exitcode : 1 (pid: 3473903) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[4]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-204.us-west-2.compute.internal + rank : 36 (local_rank: 4) + exitcode : 1 (pid: 3473904) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[5]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-204.us-west-2.compute.internal + rank : 37 (local_rank: 5) + exitcode : 1 (pid: 3473905) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[6]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-204.us-west-2.compute.internal + rank : 38 (local_rank: 6) + exitcode : 1 (pid: 3473906) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[7]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-204.us-west-2.compute.internal + rank : 39 (local_rank: 7) + exitcode : 1 (pid: 3473907) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2023-06-15_12:21:38 + host : ip-26-0-147-204.us-west-2.compute.internal + rank : 32 (local_rank: 0) + exitcode : 1 (pid: 3473900) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ +srun: error: ip-26-0-149-1: task 15: Exited with exit code 1 +srun: launch/slurm: _step_signal: Terminating StepId=158049.0 +srun: error: ip-26-0-147-189: task 2: Exited with exit code 1 +srun: error: ip-26-0-147-204: task 4: Exited with exit code 1 +srun: error: ip-26-0-147-247: task 7: Exited with exit code 1 +srun: error: ip-26-0-148-193: task 13: Exited with exit code 1 +slurmstepd: error: *** STEP 158049.0 ON ip-26-0-147-141 CANCELLED AT 2023-06-15T12:21:38 *** +srun: error: ip-26-0-147-187: task 1: Terminated +srun: error: ip-26-0-147-193: task 3: Terminated +srun: error: ip-26-0-148-115: task 10: Terminated +srun: error: ip-26-0-147-141: task 0: Terminated +srun: error: ip-26-0-148-93: task 9: Terminated +srun: error: ip-26-0-148-151: task 11: Terminated +srun: error: ip-26-0-148-170: task 12: Terminated +srun: error: ip-26-0-147-245: task 6: Terminated +srun: error: ip-26-0-147-233: task 5: Terminated +srun: error: ip-26-0-148-55: task 8: Terminated +srun: error: ip-26-0-148-245: task 14: Terminated +srun: Force Terminated StepId=158049.0