Upload folder using huggingface_hub

.gitignore

@@ -0,0 +1,174 @@
+tmp
+dataset_visuals
+old
+.DS_Store
+zz
+eval_tables
+models
+slurm
+wandb
+runs
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# vscode
+.vscode/

.pre-commit-config.yaml

@@ -0,0 +1,32 @@
+repos:
+  - repo: https://github.com/asottile/pyupgrade
+    rev: v2.31.1
+    hooks:
+      - id: pyupgrade
+        args: 
+          - --py37-plus
+  - repo: https://github.com/PyCQA/isort
+    rev: 5.12.0
+    hooks:
+      - id: isort
+        args:
+          - --profile=black
+          - --skip-glob=wandb/**/*
+          - --thirdparty=wandb
+  - repo: https://github.com/myint/autoflake
+    rev: v1.4
+    hooks:
+      - id: autoflake
+        args:
+          - -r
+          - --exclude=wandb
+          - --in-place
+          - --remove-unused-variables
+          - --remove-all-unused-imports
+  - repo: https://github.com/python/black
+    rev: 22.3.0
+    hooks:
+      - id: black
+        args:
+          - --line-length=127
+          - --exclude=wandb

.ruff_cache/.gitignore

@@ -0,0 +1 @@
+*

.ruff_cache/CACHEDIR.TAG

@@ -0,0 +1 @@
+Signature: 8a477f597d28d172789f06886806bc55

.venv/.gitignore

@@ -0,0 +1,2 @@
+# created by virtualenv automatically
+*

.venv/bin/accelerate

@@ -0,0 +1,8 @@
+#!/fsx/costa/summarize_from_feedback_details/.venv/bin/python
+# -*- coding: utf-8 -*-
+import re
+import sys
+from accelerate.commands.accelerate_cli import main
+if __name__ == "__main__":
+    sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
+    sys.exit(main())

.venv/bin/accelerate-config

@@ -0,0 +1,8 @@
+#!/fsx/costa/summarize_from_feedback_details/.venv/bin/python
+# -*- coding: utf-8 -*-
+import re
+import sys
+from accelerate.commands.config import main
+if __name__ == "__main__":
+    sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
+    sys.exit(main())

.venv/bin/accelerate-estimate-memory

@@ -0,0 +1,8 @@
+#!/fsx/costa/summarize_from_feedback_details/.venv/bin/python
+# -*- coding: utf-8 -*-
+import re
+import sys
+from accelerate.commands.estimate import main
+if __name__ == "__main__":
+    sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
+    sys.exit(main())

.venv/bin/accelerate-launch

@@ -0,0 +1,8 @@
+#!/fsx/costa/summarize_from_feedback_details/.venv/bin/python
+# -*- coding: utf-8 -*-
+import re
+import sys
+from accelerate.commands.launch import main
+if __name__ == "__main__":
+    sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
+    sys.exit(main())

.venv/bin/activate

@@ -0,0 +1,87 @@
+# This file must be used with "source bin/activate" *from bash*
+# you cannot run it directly
+
+
+if [ "${BASH_SOURCE-}" = "$0" ]; then
+    echo "You must source this script: \$ source $0" >&2
+    exit 33
+fi
+
+deactivate () {
+    unset -f pydoc >/dev/null 2>&1 || true
+
+    # reset old environment variables
+    # ! [ -z ${VAR+_} ] returns true if VAR is declared at all
+    if ! [ -z "${_OLD_VIRTUAL_PATH:+_}" ] ; then
+        PATH="$_OLD_VIRTUAL_PATH"
+        export PATH
+        unset _OLD_VIRTUAL_PATH
+    fi
+    if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
+        PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
+        export PYTHONHOME
+        unset _OLD_VIRTUAL_PYTHONHOME
+    fi
+
+    # The hash command must be called to get it to forget past
+    # commands. Without forgetting past commands the $PATH changes
+    # we made may not be respected
+    hash -r 2>/dev/null
+
+    if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
+        PS1="$_OLD_VIRTUAL_PS1"
+        export PS1
+        unset _OLD_VIRTUAL_PS1
+    fi
+
+    unset VIRTUAL_ENV
+    unset VIRTUAL_ENV_PROMPT
+    if [ ! "${1-}" = "nondestructive" ] ; then
+    # Self destruct!
+        unset -f deactivate
+    fi
+}
+
+# unset irrelevant variables
+deactivate nondestructive
+
+VIRTUAL_ENV='/fsx/costa/summarize_from_feedback_details/.venv'
+if ([ "$OSTYPE" = "cygwin" ] || [ "$OSTYPE" = "msys" ]) && $(command -v cygpath &> /dev/null) ; then
+    VIRTUAL_ENV=$(cygpath -u "$VIRTUAL_ENV")
+fi
+export VIRTUAL_ENV
+
+_OLD_VIRTUAL_PATH="$PATH"
+PATH="$VIRTUAL_ENV/bin:$PATH"
+export PATH
+
+if [ "xlm-human-preference-details-py3.10" != x ] ; then
+    VIRTUAL_ENV_PROMPT="lm-human-preference-details-py3.10"
+else
+    VIRTUAL_ENV_PROMPT=$(basename "$VIRTUAL_ENV")
+fi
+export VIRTUAL_ENV_PROMPT
+
+# unset PYTHONHOME if set
+if ! [ -z "${PYTHONHOME+_}" ] ; then
+    _OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
+    unset PYTHONHOME
+fi
+
+if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
+    _OLD_VIRTUAL_PS1="${PS1-}"
+    PS1="(${VIRTUAL_ENV_PROMPT}) ${PS1-}"
+    export PS1
+fi
+
+# Make sure to unalias pydoc if it's already there
+alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
+
+pydoc () {
+    python -m pydoc "$@"
+}
+
+# The hash command must be called to get it to forget past
+# commands. Without forgetting past commands the $PATH changes
+# we made may not be respected
+hash -r 2>/dev/null

.venv/bin/activate.csh

@@ -0,0 +1,55 @@
+# This file must be used with "source bin/activate.csh" *from csh*.
+# You cannot run it directly.
+# Created by Davide Di Blasi <davidedb@gmail.com>.
+
+set newline='\
+'
+
+alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PATH:q" && unset _OLD_VIRTUAL_PATH; rehash; test $?_OLD_VIRTUAL_PROMPT != 0 && set prompt="$_OLD_VIRTUAL_PROMPT:q" && unset _OLD_VIRTUAL_PROMPT; unsetenv VIRTUAL_ENV; unsetenv VIRTUAL_ENV_PROMPT; test "\!:*" != "nondestructive" && unalias deactivate && unalias pydoc'
+
+# Unset irrelevant variables.
+deactivate nondestructive
+
+setenv VIRTUAL_ENV '/fsx/costa/summarize_from_feedback_details/.venv'
+
+set _OLD_VIRTUAL_PATH="$PATH:q"
+setenv PATH "$VIRTUAL_ENV:q/bin:$PATH:q"
+
+
+
+if ('lm-human-preference-details-py3.10' != "") then
+    setenv VIRTUAL_ENV_PROMPT 'lm-human-preference-details-py3.10'
+else
+    setenv VIRTUAL_ENV_PROMPT "$VIRTUAL_ENV:t:q"
+endif
+
+if ( $?VIRTUAL_ENV_DISABLE_PROMPT ) then
+    if ( $VIRTUAL_ENV_DISABLE_PROMPT == "" ) then
+        set do_prompt = "1"
+    else
+        set do_prompt = "0"
+    endif
+else
+    set do_prompt = "1"
+endif
+
+if ( $do_prompt == "1" ) then
+    # Could be in a non-interactive environment,
+    # in which case, $prompt is undefined and we wouldn't
+    # care about the prompt anyway.
+    if ( $?prompt ) then
+        set _OLD_VIRTUAL_PROMPT="$prompt:q"
+        if ( "$prompt:q" =~ *"$newline:q"* ) then
+            :
+        else
+            set prompt = '('"$VIRTUAL_ENV_PROMPT:q"') '"$prompt:q"
+        endif
+    endif
+endif
+
+unset env_name
+unset do_prompt
+
+alias pydoc python -m pydoc
+
+rehash

.venv/bin/activate.fish

@@ -0,0 +1,103 @@
+# This file must be used using `source bin/activate.fish` *within a running fish ( http://fishshell.com ) session*.
+# Do not run it directly.
+
+function _bashify_path -d "Converts a fish path to something bash can recognize"
+    set fishy_path $argv
+    set bashy_path $fishy_path[1]
+    for path_part in $fishy_path[2..-1]
+        set bashy_path "$bashy_path:$path_part"
+    end
+    echo $bashy_path
+end
+
+function _fishify_path -d "Converts a bash path to something fish can recognize"
+    echo $argv | tr ':' '\n'
+end
+
+function deactivate -d 'Exit virtualenv mode and return to the normal environment.'
+    # reset old environment variables
+    if test -n "$_OLD_VIRTUAL_PATH"
+        # https://github.com/fish-shell/fish-shell/issues/436 altered PATH handling
+        if test (echo $FISH_VERSION | head -c 1) -lt 3
+            set -gx PATH (_fishify_path "$_OLD_VIRTUAL_PATH")
+        else
+            set -gx PATH $_OLD_VIRTUAL_PATH
+        end
+        set -e _OLD_VIRTUAL_PATH
+    end
+
+    if test -n "$_OLD_VIRTUAL_PYTHONHOME"
+        set -gx PYTHONHOME "$_OLD_VIRTUAL_PYTHONHOME"
+        set -e _OLD_VIRTUAL_PYTHONHOME
+    end
+
+    if test -n "$_OLD_FISH_PROMPT_OVERRIDE"
+       and functions -q _old_fish_prompt
+        # Set an empty local `$fish_function_path` to allow the removal of `fish_prompt` using `functions -e`.
+        set -l fish_function_path
+
+        # Erase virtualenv's `fish_prompt` and restore the original.
+        functions -e fish_prompt
+        functions -c _old_fish_prompt fish_prompt
+        functions -e _old_fish_prompt
+        set -e _OLD_FISH_PROMPT_OVERRIDE
+    end
+
+    set -e VIRTUAL_ENV
+    set -e VIRTUAL_ENV_PROMPT
+
+    if test "$argv[1]" != 'nondestructive'
+        # Self-destruct!
+        functions -e pydoc
+        functions -e deactivate
+        functions -e _bashify_path
+        functions -e _fishify_path
+    end
+end
+
+# Unset irrelevant variables.
+deactivate nondestructive
+
+set -gx VIRTUAL_ENV '/fsx/costa/summarize_from_feedback_details/.venv'
+
+# https://github.com/fish-shell/fish-shell/issues/436 altered PATH handling
+if test (echo $FISH_VERSION | head -c 1) -lt 3
+   set -gx _OLD_VIRTUAL_PATH (_bashify_path $PATH)
+else
+    set -gx _OLD_VIRTUAL_PATH $PATH
+end
+set -gx PATH "$VIRTUAL_ENV"'/bin' $PATH
+
+# Prompt override provided?
+# If not, just use the environment name.
+if test -n 'lm-human-preference-details-py3.10'
+    set -gx VIRTUAL_ENV_PROMPT 'lm-human-preference-details-py3.10'
+else
+    set -gx VIRTUAL_ENV_PROMPT (basename "$VIRTUAL_ENV")
+end
+
+# Unset `$PYTHONHOME` if set.
+if set -q PYTHONHOME
+    set -gx _OLD_VIRTUAL_PYTHONHOME $PYTHONHOME
+    set -e PYTHONHOME
+end
+
+function pydoc
+    python -m pydoc $argv
+end
+
+if test -z "$VIRTUAL_ENV_DISABLE_PROMPT"
+    # Copy the current `fish_prompt` function as `_old_fish_prompt`.
+    functions -c fish_prompt _old_fish_prompt
+
+    function fish_prompt
+        # Run the user's prompt first; it might depend on (pipe)status.
+        set -l prompt (_old_fish_prompt)
+
+        printf '(%s) ' $VIRTUAL_ENV_PROMPT
+
+        string join -- \n $prompt # handle multi-line prompts
+    end
+
+    set -gx _OLD_FISH_PROMPT_OVERRIDE "$VIRTUAL_ENV"
+end

.venv/bin/activate.nu

@@ -0,0 +1,96 @@
+# virtualenv activation module
+# Activate with `overlay use activate.nu`
+# Deactivate with `deactivate`, as usual
+#
+# To customize the overlay name, you can call `overlay use activate.nu as foo`,
+# but then simply `deactivate` won't work because it is just an alias to hide
+# the "activate" overlay. You'd need to call `overlay hide foo` manually.
+
+export-env {
+    def is-string [x] {
+        ($x | describe) == 'string'
+    }
+
+    def has-env [...names] {
+        $names | each {|n|
+            $n in $env
+        } | all {|i| $i == true}
+    }
+
+    # Emulates a `test -z`, but btter as it handles e.g 'false'
+    def is-env-true [name: string] {
+      if (has-env $name) {
+        # Try to parse 'true', '0', '1', and fail if not convertible
+        let parsed = (do -i { $env | get $name | into bool })
+        if ($parsed | describe) == 'bool' {
+          $parsed
+        } else {
+          not ($env | get -i $name | is-empty)
+        }
+      } else {
+        false
+      }
+    }
+
+    let virtual_env = '/fsx/costa/summarize_from_feedback_details/.venv'
+    let bin = 'bin'
+
+    let is_windows = ($nu.os-info.family) == 'windows'
+    let path_name = (if (has-env 'Path') {
+            'Path'
+        } else {
+            'PATH'
+        }
+    )
+
+    let venv_path = ([$virtual_env $bin] | path join)
+    let new_path = ($env | get $path_name | prepend $venv_path)
+
+    # If there is no default prompt, then use the env name instead
+    let virtual_env_prompt = (if ('lm-human-preference-details-py3.10' | is-empty) {
+        ($virtual_env | path basename)
+    } else {
+        'lm-human-preference-details-py3.10'
+    })
+
+    let new_env = {
+        $path_name         : $new_path
+        VIRTUAL_ENV        : $virtual_env
+        VIRTUAL_ENV_PROMPT : $virtual_env_prompt
+    }
+
+    let new_env = (if (is-env-true 'VIRTUAL_ENV_DISABLE_PROMPT') {
+      $new_env
+    } else {
+      # Creating the new prompt for the session
+      let virtual_prefix = $'(char lparen)($virtual_env_prompt)(char rparen) '
+
+      # Back up the old prompt builder
+      let old_prompt_command = (if (has-env 'PROMPT_COMMAND') {
+              $env.PROMPT_COMMAND
+          } else {
+              ''
+        })
+
+      let new_prompt = (if (has-env 'PROMPT_COMMAND') {
+          if 'closure' in ($old_prompt_command | describe) {
+              {|| $'($virtual_prefix)(do $old_prompt_command)' }
+          } else {
+              {|| $'($virtual_prefix)($old_prompt_command)' }
+          }
+      } else {
+          {|| $'($virtual_prefix)' }
+      })
+
+      $new_env | merge {
+        PROMPT_COMMAND      : $new_prompt
+        VIRTUAL_PREFIX      : $virtual_prefix
+      }
+    })
+
+    # Environment variables that will be loaded as the virtual env
+    load-env $new_env
+}
+
+export alias pydoc = python -m pydoc
+export alias deactivate = overlay hide activate

.venv/bin/activate.ps1

@@ -0,0 +1,61 @@
+$script:THIS_PATH = $myinvocation.mycommand.path
+$script:BASE_DIR = Split-Path (Resolve-Path "$THIS_PATH/..") -Parent
+
+function global:deactivate([switch] $NonDestructive) {
+    if (Test-Path variable:_OLD_VIRTUAL_PATH) {
+        $env:PATH = $variable:_OLD_VIRTUAL_PATH
+        Remove-Variable "_OLD_VIRTUAL_PATH" -Scope global
+    }
+
+    if (Test-Path function:_old_virtual_prompt) {
+        $function:prompt = $function:_old_virtual_prompt
+        Remove-Item function:\_old_virtual_prompt
+    }
+
+    if ($env:VIRTUAL_ENV) {
+        Remove-Item env:VIRTUAL_ENV -ErrorAction SilentlyContinue
+    }
+
+    if ($env:VIRTUAL_ENV_PROMPT) {
+        Remove-Item env:VIRTUAL_ENV_PROMPT -ErrorAction SilentlyContinue
+    }
+
+    if (!$NonDestructive) {
+        # Self destruct!
+        Remove-Item function:deactivate
+        Remove-Item function:pydoc
+    }
+}
+
+function global:pydoc {
+    python -m pydoc $args
+}
+
+# unset irrelevant variables
+deactivate -nondestructive
+
+$VIRTUAL_ENV = $BASE_DIR
+$env:VIRTUAL_ENV = $VIRTUAL_ENV
+
+if ("lm-human-preference-details-py3.10" -ne "") {
+    $env:VIRTUAL_ENV_PROMPT = "lm-human-preference-details-py3.10"
+}
+else {
+    $env:VIRTUAL_ENV_PROMPT = $( Split-Path $env:VIRTUAL_ENV -Leaf )
+}
+
+New-Variable -Scope global -Name _OLD_VIRTUAL_PATH -Value $env:PATH
+
+$env:PATH = "$env:VIRTUAL_ENV/bin:" + $env:PATH
+if (!$env:VIRTUAL_ENV_DISABLE_PROMPT) {
+    function global:_old_virtual_prompt {
+        ""
+    }
+    $function:_old_virtual_prompt = $function:prompt
+
+    function global:prompt {
+        # Add the custom prefix to the existing prompt
+        $previous_prompt_value = & $function:_old_virtual_prompt
+        ("(" + $env:VIRTUAL_ENV_PROMPT + ") " + $previous_prompt_value)
+    }
+}

.venv/bin/activate_this.py

@@ -0,0 +1,36 @@
+"""
+Activate virtualenv for current interpreter:
+
+Use exec(open(this_file).read(), {'__file__': this_file}).
+
+This can be used when you must use an existing Python interpreter, not the virtualenv bin/python.
+"""  # noqa: D415
+from __future__ import annotations
+
+import os
+import site
+import sys
+
+try:
+    abs_file = os.path.abspath(__file__)
+except NameError as exc:
+    msg = "You must use exec(open(this_file).read(), {'__file__': this_file})"
+    raise AssertionError(msg) from exc
+
+bin_dir = os.path.dirname(abs_file)
+base = bin_dir[: -len("bin") - 1]  # strip away the bin part from the __file__, plus the path separator
+
+# prepend bin to PATH (this file is inside the bin directory)
+os.environ["PATH"] = os.pathsep.join([bin_dir, *os.environ.get("PATH", "").split(os.pathsep)])
+os.environ["VIRTUAL_ENV"] = base  # virtual env is right above bin directory
+os.environ["VIRTUAL_ENV_PROMPT"] = "lm-human-preference-details-py3.10" or os.path.basename(base)  # noqa: SIM222
+
+# add the virtual environments libraries to the host python import mechanism
+prev_length = len(sys.path)
+for lib in "../lib/python3.10/site-packages".split(os.pathsep):
+    path = os.path.realpath(os.path.join(bin_dir, lib))
+    site.addsitedir(path.decode("utf-8") if "" else path)
+sys.path[:] = sys.path[prev_length:] + sys.path[0:prev_length]
+
+sys.real_prefix = sys.prefix
+sys.prefix = base

.venv/bin/convert-caffe2-to-onnx

@@ -0,0 +1,8 @@
+#!/fsx/costa/summarize_from_feedback_details/.venv/bin/python
+# -*- coding: utf-8 -*-
+import re
+import sys
+from caffe2.python.onnx.bin.conversion import caffe2_to_onnx
+if __name__ == "__main__":
+    sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
+    sys.exit(caffe2_to_onnx())

.venv/bin/convert-onnx-to-caffe2

@@ -0,0 +1,8 @@
+#!/fsx/costa/summarize_from_feedback_details/.venv/bin/python
+# -*- coding: utf-8 -*-
+import re
+import sys
+from caffe2.python.onnx.bin.conversion import onnx_to_caffe2
+if __name__ == "__main__":
+    sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
+    sys.exit(onnx_to_caffe2())

.venv/bin/cpuinfo

@@ -0,0 +1,8 @@
+#!/fsx/costa/summarize_from_feedback_details/.venv/bin/python
+# -*- coding: utf-8 -*-
+import re
+import sys
+from cpuinfo import main
+if __name__ == "__main__":
+    sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
+    sys.exit(main())

.venv/bin/datasets-cli

@@ -0,0 +1,8 @@
+#!/fsx/costa/summarize_from_feedback_details/.venv/bin/python
+# -*- coding: utf-8 -*-
+import re
+import sys
+from datasets.commands.datasets_cli import main
+if __name__ == "__main__":
+    sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
+    sys.exit(main())

.venv/bin/deepspeed

@@ -0,0 +1,6 @@
+#!/fsx/costa/summarize_from_feedback_details/.venv/bin/python
+
+from deepspeed.launcher.runner import main
+
+if __name__ == '__main__':
+    main()

.venv/bin/deepspeed.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d96fb3c18c6f12810533006f8c145f568bf68d5b760c41cf556e01163aeb8282
+size 145

.venv/bin/distro

@@ -0,0 +1,8 @@
+#!/fsx/costa/summarize_from_feedback_details/.venv/bin/python
+# -*- coding: utf-8 -*-
+import re
+import sys
+from distro.distro import main
+if __name__ == "__main__":
+    sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
+    sys.exit(main())

.venv/bin/ds

@@ -0,0 +1,6 @@
+#!/fsx/costa/summarize_from_feedback_details/.venv/bin/python
+
+from deepspeed.launcher.runner import main
+
+if __name__ == '__main__':
+    main()

.venv/bin/ds_bench

@@ -0,0 +1,17 @@
+#!/fsx/costa/summarize_from_feedback_details/.venv/bin/python
+
+from benchmarks.communication.run_all import main
+from benchmarks.communication.constants import *
+from benchmarks.communication.utils import *
+import os
+import sys
+
+# Run the same file with deepspeed launcher. This is required since setuptools will auto-detect python files and insert a python shebang for both 'scripts' and 'entry_points', and this benchmarks require the DS launcher
+required_env = ["RANK", "WORLD_SIZE", "MASTER_ADDR", "MASTER_PORT", "LOCAL_RANK"]
+if not all(map(lambda v: v in os.environ, required_env)):
+    import subprocess
+    subprocess.run("deepspeed $(which ds_bench) " + " ".join(sys.argv[1:]), shell=True)
+else:
+    args = benchmark_parser().parse_args()
+    rank = args.local_rank
+    main(args, rank)

.venv/bin/ds_elastic

@@ -0,0 +1,40 @@
+#!/fsx/costa/summarize_from_feedback_details/.venv/bin/python
+
+import argparse
+import json
+
+import deepspeed
+from deepspeed.elasticity import compute_elastic_config
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-c', '--config', type=str, help="DeepSpeed config json")
+    parser.add_argument('-w', '--world-size', type=int, default=0, help="Intended/current world size")
+    args = parser.parse_args()
+    ds_config = json.load(open(args.config, 'r'))
+
+    ds_version = deepspeed.__version__
+
+    elastic_config = ds_config['elasticity']
+    print('------------------------------------------')
+    print("Elasticity config:")
+    print('------------------------------------------')
+    print(json.dumps(elastic_config, indent=4, sort_keys=True))
+
+    if args.world_size > 0:
+        final_batch_size, valid_gpus, micro_batch_size = compute_elastic_config(ds_config=ds_config,
+                                                                                target_deepspeed_version=ds_version,
+                                                                                world_size=args.world_size)
+        print('------------------------------------------')
+        print(f"Calculated results for world size {args.world_size}:")
+        print('------------------------------------------')
+        print(f'final_batch_size .... {final_batch_size}')
+        print(f'valid_gpus .......... {valid_gpus}')
+        print(f'micro_batch_size .... {micro_batch_size}')
+    else:
+        final_batch_size, valid_gpus = compute_elastic_config(ds_config=ds_config, target_deepspeed_version=ds_version)
+        print('------------------------------------------')
+        print("Calculated results:")
+        print('------------------------------------------')
+        print(f'final_batch_size .... {final_batch_size}')
+        print(f'valid_gpus .......... {valid_gpus}')

.venv/bin/ds_report

@@ -0,0 +1,6 @@
+#!/fsx/costa/summarize_from_feedback_details/.venv/bin/python
+
+from deepspeed.env_report import cli_main
+
+if __name__ == '__main__':
+    cli_main()

.venv/bin/ds_ssh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+# Copyright 2020 The Microsoft DeepSpeed Team
+
+command -v pdsh
+if [ $? != 0 ]; then
+    echo "Cannot find pdsh, please install via 'apt-get install -y pdsh'"
+    exit 1
+fi
+
+hostfile=/job/hostfile
+
+while getopts "h?f:" opt; do
+  case "$opt" in
+    h|\?)
+      echo "-f <hostfile>: specify a hostfile, defaults to /job/hostfile"
+      exit 0
+      ;;
+    f)
+      hostfile=$OPTARG
+      shift $((OPTIND-1))
+      ;;
+  esac
+done
+
+echo "hostfile=$hostfile"
+
+if [ -f $hostfile ]; then
+    hosts=`cat $hostfile | awk '{print $1}' | paste -sd "," -`
+    export PDSH_RCMD_TYPE=ssh
+    pdsh -w ${hosts} $@
+else
+    echo "Missing hostfile at ${hostfile}, unable to proceed"
+fi

.venv/bin/dsr

@@ -0,0 +1,6 @@
+#!/fsx/costa/summarize_from_feedback_details/.venv/bin/python
+
+from deepspeed.env_report import cli_main
+
+if __name__ == '__main__':
+    cli_main()

.venv/bin/evaluate-cli

@@ -0,0 +1,8 @@
+#!/fsx/costa/summarize_from_feedback_details/.venv/bin/python
+# -*- coding: utf-8 -*-
+import re
+import sys
+from evaluate.commands.evaluate_cli import main
+if __name__ == "__main__":
+    sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
+    sys.exit(main())

.venv/bin/f2py

@@ -0,0 +1,8 @@
+#!/fsx/costa/summarize_from_feedback_details/.venv/bin/python
+# -*- coding: utf-8 -*-
+import re
+import sys
+from numpy.f2py.f2py2e import main
+if __name__ == "__main__":
+    sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
+    sys.exit(main())

.venv/bin/fonttools

@@ -0,0 +1,8 @@
+#!/fsx/costa/summarize_from_feedback_details/.venv/bin/python
+# -*- coding: utf-8 -*-
+import re
+import sys
+from fontTools.__main__ import main
+if __name__ == "__main__":
+    sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
+    sys.exit(main())

.venv/bin/ftfy

@@ -0,0 +1,8 @@
+#!/fsx/costa/summarize_from_feedback_details/.venv/bin/python
+# -*- coding: utf-8 -*-
+import re
+import sys
+from ftfy.cli import main
+if __name__ == "__main__":
+    sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
+    sys.exit(main())

.venv/bin/get_objgraph

@@ -0,0 +1,54 @@
+#!/fsx/costa/summarize_from_feedback_details/.venv/bin/python
+#
+# Author: Mike McKerns (mmckerns @caltech and @uqfoundation)
+# Copyright (c) 2008-2016 California Institute of Technology.
+# Copyright (c) 2016-2022 The Uncertainty Quantification Foundation.
+# License: 3-clause BSD.  The full license text is available at:
+#  - https://github.com/uqfoundation/dill/blob/master/LICENSE
+"""
+display the reference paths for objects in ``dill.types`` or a .pkl file
+
+Notes:
+    the generated image is useful in showing the pointer references in
+    objects that are or can be pickled.  Any object in ``dill.objects``
+    listed in ``dill.load_types(picklable=True, unpicklable=True)`` works.
+
+Examples::
+
+    $ get_objgraph FrameType
+    Image generated as FrameType.png
+"""
+
+import dill as pickle
+#pickle.debug.trace(True)
+#import pickle
+
+# get all objects for testing
+from dill import load_types
+load_types(pickleable=True,unpickleable=True)
+from dill import objects
+
+if __name__ == "__main__":
+    import sys
+    if len(sys.argv) != 2:
+        print ("Please provide exactly one file or type name (e.g. 'IntType')")
+        msg = "\n"
+        for objtype in list(objects.keys())[:40]:
+            msg += objtype + ', '
+        print (msg + "...")
+    else:
+        objtype = str(sys.argv[-1])
+        try:
+            obj = objects[objtype]
+        except KeyError:
+            obj = pickle.load(open(objtype,'rb'))
+            import os
+            objtype = os.path.splitext(objtype)[0]
+        try:
+            import objgraph
+            objgraph.show_refs(obj, filename=objtype+'.png')
+        except ImportError:
+            print ("Please install 'objgraph' to view object graphs")
+
+
+# EOF

.venv/bin/google-oauthlib-tool

@@ -0,0 +1,8 @@
+#!/fsx/costa/summarize_from_feedback_details/.venv/bin/python
+# -*- coding: utf-8 -*-
+import re
+import sys
+from google_auth_oauthlib.tool.__main__ import main
+if __name__ == "__main__":
+    sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
+    sys.exit(main())

.venv/bin/hjson

@@ -0,0 +1,8 @@
+#!/fsx/costa/summarize_from_feedback_details/.venv/bin/python
+# -*- coding: utf-8 -*-
+import re
+import sys
+from hjson.tool import main
+if __name__ == "__main__":
+    sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
+    sys.exit(main())

.venv/bin/httpx

@@ -0,0 +1,8 @@
+#!/fsx/costa/summarize_from_feedback_details/.venv/bin/python
+# -*- coding: utf-8 -*-
+import re
+import sys
+from httpx import main
+if __name__ == "__main__":
+    sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
+    sys.exit(main())

.venv/bin/huggingface-cli

@@ -0,0 +1,8 @@
+#!/fsx/costa/summarize_from_feedback_details/.venv/bin/python
+# -*- coding: utf-8 -*-
+import re
+import sys
+from huggingface_hub.commands.huggingface_cli import main
+if __name__ == "__main__":
+    sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
+    sys.exit(main())