Add files using upload-large-folder tool

bin/Activate.ps1

@@ -0,0 +1,247 @@
+<#
+.Synopsis
+Activate a Python virtual environment for the current PowerShell session.
+
+.Description
+Pushes the python executable for a virtual environment to the front of the
+$Env:PATH environment variable and sets the prompt to signify that you are
+in a Python virtual environment. Makes use of the command line switches as
+well as the `pyvenv.cfg` file values present in the virtual environment.
+
+.Parameter VenvDir
+Path to the directory that contains the virtual environment to activate. The
+default value for this is the parent of the directory that the Activate.ps1
+script is located within.
+
+.Parameter Prompt
+The prompt prefix to display when this virtual environment is activated. By
+default, this prompt is the name of the virtual environment folder (VenvDir)
+surrounded by parentheses and followed by a single space (ie. '(.venv) ').
+
+.Example
+Activate.ps1
+Activates the Python virtual environment that contains the Activate.ps1 script.
+
+.Example
+Activate.ps1 -Verbose
+Activates the Python virtual environment that contains the Activate.ps1 script,
+and shows extra information about the activation as it executes.
+
+.Example
+Activate.ps1 -VenvDir C:\Users\MyUser\Common\.venv
+Activates the Python virtual environment located in the specified location.
+
+.Example
+Activate.ps1 -Prompt "MyPython"
+Activates the Python virtual environment that contains the Activate.ps1 script,
+and prefixes the current prompt with the specified string (surrounded in
+parentheses) while the virtual environment is active.
+
+.Notes
+On Windows, it may be required to enable this Activate.ps1 script by setting the
+execution policy for the user. You can do this by issuing the following PowerShell
+command:
+
+PS C:\> Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser
+
+For more information on Execution Policies: 
+https://go.microsoft.com/fwlink/?LinkID=135170
+
+#>
+Param(
+    [Parameter(Mandatory = $false)]
+    [String]
+    $VenvDir,
+    [Parameter(Mandatory = $false)]
+    [String]
+    $Prompt
+)
+
+<# Function declarations --------------------------------------------------- #>
+
+<#
+.Synopsis
+Remove all shell session elements added by the Activate script, including the
+addition of the virtual environment's Python executable from the beginning of
+the PATH variable.
+
+.Parameter NonDestructive
+If present, do not remove this function from the global namespace for the
+session.
+
+#>
+function global:deactivate ([switch]$NonDestructive) {
+    # Revert to original values
+
+    # The prior prompt:
+    if (Test-Path -Path Function:_OLD_VIRTUAL_PROMPT) {
+        Copy-Item -Path Function:_OLD_VIRTUAL_PROMPT -Destination Function:prompt
+        Remove-Item -Path Function:_OLD_VIRTUAL_PROMPT
+    }
+
+    # The prior PYTHONHOME:
+    if (Test-Path -Path Env:_OLD_VIRTUAL_PYTHONHOME) {
+        Copy-Item -Path Env:_OLD_VIRTUAL_PYTHONHOME -Destination Env:PYTHONHOME
+        Remove-Item -Path Env:_OLD_VIRTUAL_PYTHONHOME
+    }
+
+    # The prior PATH:
+    if (Test-Path -Path Env:_OLD_VIRTUAL_PATH) {
+        Copy-Item -Path Env:_OLD_VIRTUAL_PATH -Destination Env:PATH
+        Remove-Item -Path Env:_OLD_VIRTUAL_PATH
+    }
+
+    # Just remove the VIRTUAL_ENV altogether:
+    if (Test-Path -Path Env:VIRTUAL_ENV) {
+        Remove-Item -Path env:VIRTUAL_ENV
+    }
+
+    # Just remove VIRTUAL_ENV_PROMPT altogether.
+    if (Test-Path -Path Env:VIRTUAL_ENV_PROMPT) {
+        Remove-Item -Path env:VIRTUAL_ENV_PROMPT
+    }
+
+    # Just remove the _PYTHON_VENV_PROMPT_PREFIX altogether:
+    if (Get-Variable -Name "_PYTHON_VENV_PROMPT_PREFIX" -ErrorAction SilentlyContinue) {
+        Remove-Variable -Name _PYTHON_VENV_PROMPT_PREFIX -Scope Global -Force
+    }
+
+    # Leave deactivate function in the global namespace if requested:
+    if (-not $NonDestructive) {
+        Remove-Item -Path function:deactivate
+    }
+}
+
+<#
+.Description
+Get-PyVenvConfig parses the values from the pyvenv.cfg file located in the
+given folder, and returns them in a map.
+
+For each line in the pyvenv.cfg file, if that line can be parsed into exactly
+two strings separated by `=` (with any amount of whitespace surrounding the =)
+then it is considered a `key = value` line. The left hand string is the key,
+the right hand is the value.
+
+If the value starts with a `'` or a `"` then the first and last character is
+stripped from the value before being captured.
+
+.Parameter ConfigDir
+Path to the directory that contains the `pyvenv.cfg` file.
+#>
+function Get-PyVenvConfig(
+    [String]
+    $ConfigDir
+) {
+    Write-Verbose "Given ConfigDir=$ConfigDir, obtain values in pyvenv.cfg"
+
+    # Ensure the file exists, and issue a warning if it doesn't (but still allow the function to continue).
+    $pyvenvConfigPath = Join-Path -Resolve -Path $ConfigDir -ChildPath 'pyvenv.cfg' -ErrorAction Continue
+
+    # An empty map will be returned if no config file is found.
+    $pyvenvConfig = @{ }
+
+    if ($pyvenvConfigPath) {
+
+        Write-Verbose "File exists, parse `key = value` lines"
+        $pyvenvConfigContent = Get-Content -Path $pyvenvConfigPath
+
+        $pyvenvConfigContent | ForEach-Object {
+            $keyval = $PSItem -split "\s*=\s*", 2
+            if ($keyval[0] -and $keyval[1]) {
+                $val = $keyval[1]
+
+                # Remove extraneous quotations around a string value.
+                if ("'""".Contains($val.Substring(0, 1))) {
+                    $val = $val.Substring(1, $val.Length - 2)
+                }
+
+                $pyvenvConfig[$keyval[0]] = $val
+                Write-Verbose "Adding Key: '$($keyval[0])'='$val'"
+            }
+        }
+    }
+    return $pyvenvConfig
+}
+
+
+<# Begin Activate script --------------------------------------------------- #>
+
+# Determine the containing directory of this script
+$VenvExecPath = Split-Path -Parent $MyInvocation.MyCommand.Definition
+$VenvExecDir = Get-Item -Path $VenvExecPath
+
+Write-Verbose "Activation script is located in path: '$VenvExecPath'"
+Write-Verbose "VenvExecDir Fullname: '$($VenvExecDir.FullName)"
+Write-Verbose "VenvExecDir Name: '$($VenvExecDir.Name)"
+
+# Set values required in priority: CmdLine, ConfigFile, Default
+# First, get the location of the virtual environment, it might not be
+# VenvExecDir if specified on the command line.
+if ($VenvDir) {
+    Write-Verbose "VenvDir given as parameter, using '$VenvDir' to determine values"
+}
+else {
+    Write-Verbose "VenvDir not given as a parameter, using parent directory name as VenvDir."
+    $VenvDir = $VenvExecDir.Parent.FullName.TrimEnd("\\/")
+    Write-Verbose "VenvDir=$VenvDir"
+}
+
+# Next, read the `pyvenv.cfg` file to determine any required value such
+# as `prompt`.
+$pyvenvCfg = Get-PyVenvConfig -ConfigDir $VenvDir
+
+# Next, set the prompt from the command line, or the config file, or
+# just use the name of the virtual environment folder.
+if ($Prompt) {
+    Write-Verbose "Prompt specified as argument, using '$Prompt'"
+}
+else {
+    Write-Verbose "Prompt not specified as argument to script, checking pyvenv.cfg value"
+    if ($pyvenvCfg -and $pyvenvCfg['prompt']) {
+        Write-Verbose "  Setting based on value in pyvenv.cfg='$($pyvenvCfg['prompt'])'"
+        $Prompt = $pyvenvCfg['prompt'];
+    }
+    else {
+        Write-Verbose "  Setting prompt based on parent's directory's name. (Is the directory name passed to venv module when creating the virtual environment)"
+        Write-Verbose "  Got leaf-name of $VenvDir='$(Split-Path -Path $venvDir -Leaf)'"
+        $Prompt = Split-Path -Path $venvDir -Leaf
+    }
+}
+
+Write-Verbose "Prompt = '$Prompt'"
+Write-Verbose "VenvDir='$VenvDir'"
+
+# Deactivate any currently active virtual environment, but leave the
+# deactivate function in place.
+deactivate -nondestructive
+
+# Now set the environment variable VIRTUAL_ENV, used by many tools to determine
+# that there is an activated venv.
+$env:VIRTUAL_ENV = $VenvDir
+
+if (-not $Env:VIRTUAL_ENV_DISABLE_PROMPT) {
+
+    Write-Verbose "Setting prompt to '$Prompt'"
+
+    # Set the prompt to include the env name
+    # Make sure _OLD_VIRTUAL_PROMPT is global
+    function global:_OLD_VIRTUAL_PROMPT { "" }
+    Copy-Item -Path function:prompt -Destination function:_OLD_VIRTUAL_PROMPT
+    New-Variable -Name _PYTHON_VENV_PROMPT_PREFIX -Description "Python virtual environment prompt prefix" -Scope Global -Option ReadOnly -Visibility Public -Value $Prompt
+
+    function global:prompt {
+        Write-Host -NoNewline -ForegroundColor Green "($_PYTHON_VENV_PROMPT_PREFIX) "
+        _OLD_VIRTUAL_PROMPT
+    }
+    $env:VIRTUAL_ENV_PROMPT = $Prompt
+}
+
+# Clear PYTHONHOME
+if (Test-Path -Path Env:PYTHONHOME) {
+    Copy-Item -Path Env:PYTHONHOME -Destination Env:_OLD_VIRTUAL_PYTHONHOME
+    Remove-Item -Path Env:PYTHONHOME
+}
+
+# Add the venv to the PATH
+Copy-Item -Path Env:PATH -Destination Env:_OLD_VIRTUAL_PATH
+$Env:PATH = "$VenvExecDir$([System.IO.Path]::PathSeparator)$Env:PATH"

bin/accelerate

@@ -0,0 +1,8 @@
+#!/storage/openpsi/users/zzy/.threadweaver/bin/python3
+# -*- coding: utf-8 -*-
+import re
+import sys
+from accelerate.commands.accelerate_cli import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

bin/accelerate-config

@@ -0,0 +1,8 @@
+#!/storage/openpsi/users/zzy/.threadweaver/bin/python3
+# -*- coding: utf-8 -*-
+import re
+import sys
+from accelerate.commands.config import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

bin/accelerate-estimate-memory

@@ -0,0 +1,8 @@
+#!/storage/openpsi/users/zzy/.threadweaver/bin/python3
+# -*- coding: utf-8 -*-
+import re
+import sys
+from accelerate.commands.estimate import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

bin/accelerate-launch

@@ -0,0 +1,8 @@
+#!/storage/openpsi/users/zzy/.threadweaver/bin/python3
+# -*- coding: utf-8 -*-
+import re
+import sys
+from accelerate.commands.launch import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

bin/accelerate-merge-weights

@@ -0,0 +1,8 @@
+#!/storage/openpsi/users/zzy/.threadweaver/bin/python3
+# -*- coding: utf-8 -*-
+import re
+import sys
+from accelerate.commands.merge import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

bin/activate.csh

@@ -0,0 +1,27 @@
+# This file must be used with "source bin/activate.csh" *from csh*.
+# You cannot run it directly.
+
+# Created by Davide Di Blasi <davidedb@gmail.com>.
+# Ported to Python 3.3 venv by Andrew Svetlov <andrew.svetlov@gmail.com>
+
+alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PATH" && unset _OLD_VIRTUAL_PATH; rehash; test $?_OLD_VIRTUAL_PROMPT != 0 && set prompt="$_OLD_VIRTUAL_PROMPT" && unset _OLD_VIRTUAL_PROMPT; unsetenv VIRTUAL_ENV; unsetenv VIRTUAL_ENV_PROMPT; test "\!:*" != "nondestructive" && unalias deactivate'
+
+# Unset irrelevant variables.
+deactivate nondestructive
+
+setenv VIRTUAL_ENV /storage/openpsi/users/zzy/.threadweaver
+
+set _OLD_VIRTUAL_PATH="$PATH"
+setenv PATH "$VIRTUAL_ENV/"bin":$PATH"
+
+
+set _OLD_VIRTUAL_PROMPT="$prompt"
+
+if (! "$?VIRTUAL_ENV_DISABLE_PROMPT") then
+    set prompt = '(.threadweaver) '"$prompt"
+    setenv VIRTUAL_ENV_PROMPT '(.threadweaver) '
+endif
+
+alias pydoc python -m pydoc
+
+rehash

bin/activate.fish

@@ -0,0 +1,69 @@
+# This file must be used with "source <venv>/bin/activate.fish" *from fish*
+# (https://fishshell.com/). You cannot run it directly.
+
+function deactivate  -d "Exit virtual environment and return to normal shell environment"
+    # reset old environment variables
+    if test -n "$_OLD_VIRTUAL_PATH"
+        set -gx PATH $_OLD_VIRTUAL_PATH
+        set -e _OLD_VIRTUAL_PATH
+    end
+    if test -n "$_OLD_VIRTUAL_PYTHONHOME"
+        set -gx PYTHONHOME $_OLD_VIRTUAL_PYTHONHOME
+        set -e _OLD_VIRTUAL_PYTHONHOME
+    end
+
+    if test -n "$_OLD_FISH_PROMPT_OVERRIDE"
+        set -e _OLD_FISH_PROMPT_OVERRIDE
+        # prevents error when using nested fish instances (Issue #93858)
+        if functions -q _old_fish_prompt
+            functions -e fish_prompt
+            functions -c _old_fish_prompt fish_prompt
+            functions -e _old_fish_prompt
+        end
+    end
+
+    set -e VIRTUAL_ENV
+    set -e VIRTUAL_ENV_PROMPT
+    if test "$argv[1]" != "nondestructive"
+        # Self-destruct!
+        functions -e deactivate
+    end
+end
+
+# Unset irrelevant variables.
+deactivate nondestructive
+
+set -gx VIRTUAL_ENV /storage/openpsi/users/zzy/.threadweaver
+
+set -gx _OLD_VIRTUAL_PATH $PATH
+set -gx PATH "$VIRTUAL_ENV/"bin $PATH
+
+# Unset PYTHONHOME if set.
+if set -q PYTHONHOME
+    set -gx _OLD_VIRTUAL_PYTHONHOME $PYTHONHOME
+    set -e PYTHONHOME
+end
+
+if test -z "$VIRTUAL_ENV_DISABLE_PROMPT"
+    # fish uses a function instead of an env var to generate the prompt.
+
+    # Save the current fish_prompt function as the function _old_fish_prompt.
+    functions -c fish_prompt _old_fish_prompt
+
+    # With the original prompt function renamed, we can override with our own.
+    function fish_prompt
+        # Save the return status of the last command.
+        set -l old_status $status
+
+        # Output the venv prompt; color taken from the blue of the Python logo.
+        printf "%s%s%s" (set_color 4B8BBE) '(.threadweaver) ' (set_color normal)
+
+        # Restore the return status of the previous command.
+        echo "exit $old_status" | .
+        # Output the original/"old" prompt.
+        _old_fish_prompt
+    end
+
+    set -gx _OLD_FISH_PROMPT_OVERRIDE "$VIRTUAL_ENV"
+    set -gx VIRTUAL_ENV_PROMPT '(.threadweaver) '
+end

bin/cpuinfo

@@ -0,0 +1,8 @@
+#!/storage/openpsi/users/zzy/.threadweaver/bin/python3
+# -*- coding: utf-8 -*-
+import re
+import sys
+from cpuinfo import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

bin/datasets-cli

@@ -0,0 +1,8 @@
+#!/storage/openpsi/users/zzy/.threadweaver/bin/python3
+# -*- coding: utf-8 -*-
+import re
+import sys
+from datasets.commands.datasets_cli import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

bin/distro

@@ -0,0 +1,8 @@
+#!/storage/openpsi/users/zzy/.threadweaver/bin/python3
+# -*- coding: utf-8 -*-
+import re
+import sys
+from distro.distro import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

bin/ds

@@ -0,0 +1,6 @@
+#!/storage/openpsi/users/zzy/.threadweaver/bin/python3
+
+from deepspeed.launcher.runner import main
+
+if __name__ == '__main__':
+    main()

bin/ds_bench

@@ -0,0 +1,20 @@
+#!/storage/openpsi/users/zzy/.threadweaver/bin/python3
+
+from benchmarks.communication.run_all import main
+from benchmarks.communication.constants import *
+from benchmarks.communication.utils import *
+import os
+import sys
+
+# Run the same file with deepspeed launcher. This is required since setuptools will auto-detect python files and insert a python shebang for both 'scripts' and 'entry_points', and this benchmarks require the DS launcher
+required_env = ["RANK", "WORLD_SIZE", "MASTER_ADDR", "MASTER_PORT", "LOCAL_RANK"]
+if not all(map(lambda v: v in os.environ, required_env)):
+    import subprocess
+    r = subprocess.check_output(["which", "ds_bench"])
+    ds_bench_bin = r.decode('utf-8').strip()
+    safe_cmd = ["deepspeed", ds_bench_bin] + sys.argv[1:]
+    subprocess.run(safe_cmd)
+else:
+    args = benchmark_parser().parse_args()
+    rank = args.local_rank
+    main(args, rank)

bin/ds_elastic

@@ -0,0 +1,40 @@
+#!/storage/openpsi/users/zzy/.threadweaver/bin/python3
+
+import argparse
+import json
+
+import deepspeed
+from deepspeed.elasticity import compute_elastic_config
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-c', '--config', type=str, help="DeepSpeed config json")
+    parser.add_argument('-w', '--world-size', type=int, default=0, help="Intended/current world size")
+    args = parser.parse_args()
+    ds_config = json.load(open(args.config, 'r'))
+
+    ds_version = deepspeed.__version__
+
+    elastic_config = ds_config['elasticity']
+    print('------------------------------------------')
+    print("Elasticity config:")
+    print('------------------------------------------')
+    print(json.dumps(elastic_config, indent=4, sort_keys=True))
+
+    if args.world_size > 0:
+        final_batch_size, valid_gpus, micro_batch_size = compute_elastic_config(ds_config=ds_config,
+                                                                                target_deepspeed_version=ds_version,
+                                                                                world_size=args.world_size)
+        print('------------------------------------------')
+        print(f"Calculated results for world size {args.world_size}:")
+        print('------------------------------------------')
+        print(f'final_batch_size .... {final_batch_size}')
+        print(f'valid_gpus .......... {valid_gpus}')
+        print(f'micro_batch_size .... {micro_batch_size}')
+    else:
+        final_batch_size, valid_gpus = compute_elastic_config(ds_config=ds_config, target_deepspeed_version=ds_version)
+        print('------------------------------------------')
+        print("Calculated results:")
+        print('------------------------------------------')
+        print(f'final_batch_size .... {final_batch_size}')
+        print(f'valid_gpus .......... {valid_gpus}')

bin/ds_io

@@ -0,0 +1,6 @@
+#!/storage/openpsi/users/zzy/.threadweaver/bin/python3
+
+from deepspeed.nvme import ds_io_main
+
+if __name__ == '__main__':
+    ds_io_main()

bin/ds_nvme_tune

@@ -0,0 +1,9 @@
+#!/storage/openpsi/users/zzy/.threadweaver/bin/python3
+
+from deepspeed.nvme import sweep_main, generate_main, parse_sweep_arguments
+
+if __name__ == '__main__':
+    args = parse_sweep_arguments()
+    print(f"Running DeepNVMe performance tuning on {args.nvme_dir}")
+    sweep_main(args)
+    generate_main(args.log_dir)

bin/ds_ssh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+# Copyright 2020 The Microsoft DeepSpeed Team
+
+command -v pdsh
+if [ $? != 0 ]; then
+    echo "Cannot find pdsh, please install via 'apt-get install -y pdsh'"
+    exit 1
+fi
+
+hostfile=/job/hostfile
+
+while getopts "h?f:" opt; do
+  case "$opt" in
+    h|\?)
+      echo "-f <hostfile>: specify a hostfile, defaults to /job/hostfile"
+      exit 0
+      ;;
+    f)
+      hostfile=$OPTARG
+      shift $((OPTIND-1))
+      ;;
+  esac
+done
+
+echo "hostfile=$hostfile"
+
+if [ -f $hostfile ]; then
+    hosts=`cat $hostfile | awk '{print $1}' | paste -sd "," -`
+    export PDSH_RCMD_TYPE=ssh
+    pdsh -w ${hosts} $@
+else
+    echo "Missing hostfile at ${hostfile}, unable to proceed"
+fi

bin/dsr

@@ -0,0 +1,6 @@
+#!/storage/openpsi/users/zzy/.threadweaver/bin/python3
+
+from deepspeed.env_report import cli_main
+
+if __name__ == '__main__':
+    cli_main()

bin/f2py

@@ -0,0 +1,8 @@
+#!/storage/openpsi/users/zzy/.threadweaver/bin/python3
+# -*- coding: utf-8 -*-
+import re
+import sys
+from numpy.f2py.f2py2e import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

bin/flask

@@ -0,0 +1,8 @@
+#!/storage/openpsi/users/zzy/.threadweaver/bin/python3
+# -*- coding: utf-8 -*-
+import re
+import sys
+from flask.cli import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

bin/fonttools

@@ -0,0 +1,8 @@
+#!/storage/openpsi/users/zzy/.threadweaver/bin/python3
+# -*- coding: utf-8 -*-
+import re
+import sys
+from fontTools.__main__ import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

bin/get_gprof

@@ -0,0 +1,75 @@
+#!/storage/openpsi/users/zzy/.threadweaver/bin/python3
+#
+# Author: Mike McKerns (mmckerns @caltech and @uqfoundation)
+# Copyright (c) 2008-2016 California Institute of Technology.
+# Copyright (c) 2016-2024 The Uncertainty Quantification Foundation.
+# License: 3-clause BSD.  The full license text is available at:
+#  - https://github.com/uqfoundation/dill/blob/master/LICENSE
+'''
+build profile graph for the given instance
+
+running:
+  $ get_gprof <args> <instance>
+
+executes:
+  gprof2dot -f pstats <args> <type>.prof | dot -Tpng -o <type>.call.png
+
+where:
+  <args> are arguments for gprof2dot, such as "-n 5 -e 5"
+  <instance> is code to create the instance to profile
+  <type> is the class of the instance (i.e. type(instance))
+
+For example:
+  $ get_gprof -n 5 -e 1 "import numpy; numpy.array([1,2])"
+
+will create 'ndarray.call.png' with the profile graph for numpy.array([1,2]),
+where '-n 5' eliminates nodes below 5% threshold, similarly '-e 1' eliminates
+edges below 1% threshold
+'''
+
+if __name__ == "__main__":
+    import sys
+    if len(sys.argv) < 2:
+        print ("Please provide an object instance (e.g. 'import math; math.pi')")
+        sys.exit()
+    # grab args for gprof2dot
+    args = sys.argv[1:-1]
+    args = ' '.join(args)
+    # last arg builds the object
+    obj = sys.argv[-1]
+    obj = obj.split(';')
+    # multi-line prep for generating an instance
+    for line in obj[:-1]:
+        exec(line)
+    # one-line generation of an instance
+    try:
+        obj = eval(obj[-1])
+    except Exception:
+        print ("Error processing object instance")
+        sys.exit()
+
+    # get object 'name'
+    objtype = type(obj)
+    name = getattr(objtype, '__name__', getattr(objtype, '__class__', objtype))
+
+    # profile dumping an object
+    import dill
+    import os
+    import cProfile
+    #name = os.path.splitext(os.path.basename(__file__))[0]
+    cProfile.run("dill.dumps(obj)", filename="%s.prof" % name)
+    msg = "gprof2dot -f pstats %s %s.prof | dot -Tpng -o %s.call.png" % (args, name, name)
+    try:
+        res = os.system(msg)
+    except Exception:
+        print ("Please verify install of 'gprof2dot' to view profile graphs")
+    if res:
+        print ("Please verify install of 'gprof2dot' to view profile graphs")
+
+    # get stats
+    f_prof = "%s.prof" % name
+    import pstats
+    stats = pstats.Stats(f_prof, stream=sys.stdout)
+    stats.strip_dirs().sort_stats('cumtime')
+    stats.print_stats(20) #XXX: save to file instead of print top 20?
+    os.remove(f_prof)

bin/get_objgraph

@@ -0,0 +1,54 @@
+#!/storage/openpsi/users/zzy/.threadweaver/bin/python3
+#
+# Author: Mike McKerns (mmckerns @caltech and @uqfoundation)
+# Copyright (c) 2008-2016 California Institute of Technology.
+# Copyright (c) 2016-2024 The Uncertainty Quantification Foundation.
+# License: 3-clause BSD.  The full license text is available at:
+#  - https://github.com/uqfoundation/dill/blob/master/LICENSE
+"""
+display the reference paths for objects in ``dill.types`` or a .pkl file
+
+Notes:
+    the generated image is useful in showing the pointer references in
+    objects that are or can be pickled.  Any object in ``dill.objects``
+    listed in ``dill.load_types(picklable=True, unpicklable=True)`` works.
+
+Examples::
+
+    $ get_objgraph ArrayType
+    Image generated as ArrayType.png
+"""
+
+import dill as pickle
+#pickle.debug.trace(True)
+#import pickle
+
+# get all objects for testing
+from dill import load_types
+load_types(pickleable=True,unpickleable=True)
+from dill import objects
+
+if __name__ == "__main__":
+    import sys
+    if len(sys.argv) != 2:
+        print ("Please provide exactly one file or type name (e.g. 'IntType')")
+        msg = "\n"
+        for objtype in list(objects.keys())[:40]:
+            msg += objtype + ', '
+        print (msg + "...")
+    else:
+        objtype = str(sys.argv[-1])
+        try:
+            obj = objects[objtype]
+        except KeyError:
+            obj = pickle.load(open(objtype,'rb'))
+            import os
+            objtype = os.path.splitext(objtype)[0]
+        try:
+            import objgraph
+            objgraph.show_refs(obj, filename=objtype+'.png')
+        except ImportError:
+            print ("Please install 'objgraph' to view object graphs")
+
+
+# EOF

bin/hjson

@@ -0,0 +1,8 @@
+#!/storage/openpsi/users/zzy/.threadweaver/bin/python3
+# -*- coding: utf-8 -*-
+import re
+import sys
+from hjson.tool import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

bin/httpx

@@ -0,0 +1,8 @@
+#!/storage/openpsi/users/zzy/.threadweaver/bin/python3
+# -*- coding: utf-8 -*-
+import re
+import sys
+from httpx import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

bin/ipython

@@ -0,0 +1,8 @@
+#!/storage/openpsi/users/zzy/.threadweaver/bin/python3
+# -*- coding: utf-8 -*-
+import re
+import sys
+from IPython import start_ipython
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(start_ipython())

bin/json-playground

@@ -0,0 +1,8 @@
+#!/storage/openpsi/users/zzy/.threadweaver/bin/python3
+# -*- coding: utf-8 -*-
+import re
+import sys
+from partial_json_parser.playground import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

bin/jsonschema

@@ -0,0 +1,8 @@
+#!/storage/openpsi/users/zzy/.threadweaver/bin/python3
+# -*- coding: utf-8 -*-
+import re
+import sys
+from jsonschema.cli import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

bin/latex2text

@@ -0,0 +1,8 @@
+#!/storage/openpsi/users/zzy/.threadweaver/bin/python3
+# -*- coding: utf-8 -*-
+import re
+import sys
+from pylatexenc.latex2text.__main__ import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

bin/latexencode

@@ -0,0 +1,8 @@
+#!/storage/openpsi/users/zzy/.threadweaver/bin/python3
+# -*- coding: utf-8 -*-
+import re
+import sys
+from pylatexenc.latexencode.__main__ import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

bin/latexwalker

@@ -0,0 +1,8 @@
+#!/storage/openpsi/users/zzy/.threadweaver/bin/python3
+# -*- coding: utf-8 -*-
+import re
+import sys
+from pylatexenc.latexwalker.__main__ import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

bin/normalizer

@@ -0,0 +1,8 @@
+#!/storage/openpsi/users/zzy/.threadweaver/bin/python3
+# -*- coding: utf-8 -*-
+import re
+import sys
+from charset_normalizer.cli import cli_detect
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(cli_detect())

bin/openai

@@ -0,0 +1,8 @@
+#!/storage/openpsi/users/zzy/.threadweaver/bin/python3
+# -*- coding: utf-8 -*-
+import re
+import sys
+from openai.cli import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

bin/pip

@@ -0,0 +1,8 @@
+#!/storage/openpsi/users/zzy/.threadweaver/bin/python3
+# -*- coding: utf-8 -*-
+import re
+import sys
+from pip._internal.cli.main import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

bin/proton-viewer

@@ -0,0 +1,8 @@
+#!/storage/openpsi/users/zzy/.threadweaver/bin/python3
+# -*- coding: utf-8 -*-
+import re
+import sys
+from triton.profiler.viewer import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

bin/pyftsubset

@@ -0,0 +1,8 @@
+#!/storage/openpsi/users/zzy/.threadweaver/bin/python3
+# -*- coding: utf-8 -*-
+import re
+import sys
+from fontTools.subset import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

bin/pygmentize

@@ -0,0 +1,8 @@
+#!/storage/openpsi/users/zzy/.threadweaver/bin/python3
+# -*- coding: utf-8 -*-
+import re
+import sys
+from pygments.cmdline import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

bin/pygrun

@@ -0,0 +1,164 @@
+#!/storage/openpsi/users/zzy/.threadweaver/bin/python3
+__author__ = 'jszheng'
+import optparse
+import sys
+import os
+import importlib
+from antlr4 import *
+
+
+# this is a python version of TestRig
+def beautify_lisp_string(in_string):
+    indent_size = 3
+    add_indent = ' '*indent_size
+    out_string = in_string[0]  # no indent for 1st (
+    indent = ''
+    for i in range(1, len(in_string)):
+        if in_string[i] == '(' and in_string[i+1] != ' ':
+            indent += add_indent
+            out_string += "\n" + indent + '('
+        elif in_string[i] == ')':
+            out_string += ')'
+            if len(indent) > 0:
+                indent = indent.replace(add_indent, '', 1)
+        else:
+            out_string += in_string[i]
+    return out_string
+
+
+if __name__ == '__main__':
+
+    #############################################################
+    # parse options
+    # not support -gui -encoding -ps
+    #############################################################
+    usage = "Usage: %prog [options] Grammar_Name Start_Rule"
+    parser = optparse.OptionParser(usage=usage)
+    # parser.add_option('-t', '--tree',
+    #                   dest="out_file",
+    #                   default="default.out",
+    #                   help='set output file name',
+    #                   )
+    parser.add_option('-t', '--tree',
+                      default=False,
+                      action='store_true',
+                      help='Print AST tree'
+                      )
+    parser.add_option('-k', '--tokens',
+                      dest="token",
+                      default=False,
+                      action='store_true',
+                      help='Show Tokens'
+                      )
+    parser.add_option('-s', '--sll',
+                      dest="sll",
+                      default=False,
+                      action='store_true',
+                      help='Show SLL'
+                      )
+    parser.add_option('-d', '--diagnostics',
+                      dest="diagnostics",
+                      default=False,
+                      action='store_true',
+                      help='Enable diagnostics error listener'
+                      )
+    parser.add_option('-a', '--trace',
+                      dest="trace",
+                      default=False,
+                      action='store_true',
+                      help='Enable Trace'
+                      )
+
+    options, remainder = parser.parse_args()
+    if len(remainder) < 2:
+        print('ERROR: You have to provide at least 2 arguments!')
+        parser.print_help()
+        exit(1)
+    else:
+        grammar = remainder.pop(0)
+        start_rule = remainder.pop(0)
+        file_list = remainder
+
+    #############################################################
+    # check and load antlr generated files
+    #############################################################
+    # dynamic load the module and class
+    lexerName = grammar + 'Lexer'
+    parserName = grammar + 'Parser'
+    # check if the generate file exist
+    lexer_file = lexerName + '.py'
+    parser_file = parserName + '.py'
+    if not os.path.exists(lexer_file):
+        print("[ERROR] Can't find lexer file {}!".format(lexer_file))
+        print(os.path.realpath('.'))
+        exit(1)
+    if not os.path.exists(parser_file):
+        print("[ERROR] Can't find parser file {}!".format(lexer_file))
+        print(os.path.realpath('.'))
+        exit(1)
+
+    # current directory is where the generated file loaded
+    # the script might be in different place.
+    sys.path.append('.')
+    # print(sys.path)
+
+    # print("Load Lexer {}".format(lexerName))
+    module_lexer = __import__(lexerName, globals(), locals(), lexerName)
+    class_lexer = getattr(module_lexer, lexerName)
+    # print(class_lexer)
+
+    # print("Load Parser {}".format(parserName))
+    module_parser = __import__(parserName, globals(), locals(), parserName)
+    class_parser = getattr(module_parser, parserName)
+    # print(class_parser)
+
+    #############################################################
+    # main process steps.
+    #############################################################
+    def process(input_stream, class_lexer, class_parser):
+        lexer = class_lexer(input_stream)
+        token_stream = CommonTokenStream(lexer)
+        token_stream.fill()
+        if options.token:  # need to show token
+            for tok in token_stream.tokens:
+                print(tok)
+        if start_rule == 'tokens':
+            return
+
+        parser = class_parser(token_stream)
+
+        if options.diagnostics:
+            parser.addErrorListener(DiagnosticErrorListener())
+            parser._interp.predictionMode = PredictionMode.LL_EXACT_AMBIG_DETECTION
+        if options.tree:
+            parser.buildParseTrees = True
+        if options.sll:
+            parser._interp.predictionMode = PredictionMode.SLL
+        #parser.setTokenStream(token_stream)
+        parser.setTrace(options.trace)
+        if hasattr(parser, start_rule):
+            func_start_rule = getattr(parser, start_rule)
+            parser_ret = func_start_rule()
+            if options.tree:
+                lisp_tree_str = parser_ret.toStringTree(recog=parser)
+                print(beautify_lisp_string(lisp_tree_str))
+        else:
+            print("[ERROR] Can't find start rule '{}' in parser '{}'".format(start_rule, parserName))
+
+    #############################################################
+    # use stdin if not provide file as input stream
+    #############################################################
+    if len(file_list) == 0:
+        input_stream = InputStream(sys.stdin.read())
+        process(input_stream, class_lexer, class_parser)
+        exit(0)
+
+    #############################################################
+    # iterate all input file
+    #############################################################
+    for file_name in file_list:
+        if os.path.exists(file_name) and os.path.isfile(file_name):
+            input_stream = FileStream(file_name)
+            process(input_stream, class_lexer, class_parser)
+        else:
+            print("[ERROR] file {} not exist".format(os.path.normpath(file_name)))

bin/tensorboard

@@ -0,0 +1,8 @@
+#!/storage/openpsi/users/zzy/.threadweaver/bin/python3
+# -*- coding: utf-8 -*-
+import re
+import sys
+from tensorboard.main import run_main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(run_main())

bin/torchfrtrace

@@ -0,0 +1,8 @@
+#!/storage/openpsi/users/zzy/.threadweaver/bin/python3
+# -*- coding: utf-8 -*-
+import re
+import sys
+from tools.flight_recorder.fr_trace import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

bin/torchrun

@@ -0,0 +1,8 @@
+#!/storage/openpsi/users/zzy/.threadweaver/bin/python3
+# -*- coding: utf-8 -*-
+import re
+import sys
+from torch.distributed.run import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

bin/tqdm

@@ -0,0 +1,8 @@
+#!/storage/openpsi/users/zzy/.threadweaver/bin/python3
+# -*- coding: utf-8 -*-
+import re
+import sys
+from tqdm.cli import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

bin/transformers-cli

@@ -0,0 +1,8 @@
+#!/storage/openpsi/users/zzy/.threadweaver/bin/python3
+# -*- coding: utf-8 -*-
+import re
+import sys
+from transformers.commands.transformers_cli import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

bin/trl

@@ -0,0 +1,8 @@
+#!/storage/openpsi/users/zzy/.threadweaver/bin/python3
+# -*- coding: utf-8 -*-
+import re
+import sys
+from trl.cli import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

bin/ttx

@@ -0,0 +1,8 @@
+#!/storage/openpsi/users/zzy/.threadweaver/bin/python3
+# -*- coding: utf-8 -*-
+import re
+import sys
+from fontTools.ttx import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

bin/tune

@@ -0,0 +1,8 @@
+#!/storage/openpsi/users/zzy/.threadweaver/bin/python3
+# -*- coding: utf-8 -*-
+import re
+import sys
+from ray.tune.cli.scripts import cli
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(cli())

bin/undill

@@ -0,0 +1,22 @@
+#!/storage/openpsi/users/zzy/.threadweaver/bin/python3
+#
+# Author: Mike McKerns (mmckerns @caltech and @uqfoundation)
+# Copyright (c) 2008-2016 California Institute of Technology.
+# Copyright (c) 2016-2024 The Uncertainty Quantification Foundation.
+# License: 3-clause BSD.  The full license text is available at:
+#  - https://github.com/uqfoundation/dill/blob/master/LICENSE
+"""
+unpickle the contents of a pickled object file
+
+Examples::
+
+    $ undill hello.pkl
+    ['hello', 'world']
+"""
+
+if __name__ == '__main__':
+    import sys
+    import dill
+    for file in sys.argv[1:]:
+        print (dill.load(open(file,'rb')))
+

bin/uvicorn

@@ -0,0 +1,8 @@
+#!/storage/openpsi/users/zzy/.threadweaver/bin/python3
+# -*- coding: utf-8 -*-
+import re
+import sys
+from uvicorn.main import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

pyvenv.cfg

@@ -0,0 +1,5 @@
+home = /usr/bin
+include-system-site-packages = false
+version = 3.12.12
+executable = /usr/bin/python3.12
+command = /usr/bin/python3 -m venv /storage/openpsi/users/zzy/.threadweaver