diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..ae183ded6e061f73014b3317ce3ffda87f413d95 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,12 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +.venv/bin/python filter=lfs diff=lfs merge=lfs -text +.venv/bin/python3 filter=lfs diff=lfs merge=lfs -text +.venv/bin/python3.14 filter=lfs diff=lfs merge=lfs -text +.venv/bin/𝜋thon filter=lfs diff=lfs merge=lfs -text +.venv/lib/python3.14/site-packages/__pycache__/typing_extensions.cpython-314.pyc filter=lfs diff=lfs merge=lfs -text +.venv/lib/python3.14/site-packages/anyio/_backends/__pycache__/_asyncio.cpython-314.pyc filter=lfs diff=lfs merge=lfs -text +.venv/lib/python3.14/site-packages/anyio/_backends/__pycache__/_trio.cpython-314.pyc filter=lfs diff=lfs merge=lfs -text +.venv/lib/python3.14/site-packages/click/__pycache__/core.cpython-314.pyc filter=lfs diff=lfs merge=lfs -text +.venv/lib/python3.14/site-packages/filelock/__pycache__/_api.cpython-314.pyc filter=lfs diff=lfs merge=lfs -text diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..0594ff99d0f69626b9a2f2d73bf6d7cfbff3cab6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +conversations/ + +.venv \ No newline at end of file diff --git a/.venv/.gitignore b/.venv/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..f514b74c5f2fbd0a3b33874cf5c6cf4719ee9bbf --- /dev/null +++ b/.venv/.gitignore @@ -0,0 +1,2 @@ +# Created by venv; see https://docs.python.org/3/library/venv.html +* diff --git a/.venv/bin/Activate.ps1 b/.venv/bin/Activate.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..16ba5290faef35e77277fddb1afd0c48573bc54d --- /dev/null +++ b/.venv/bin/Activate.ps1 @@ -0,0 +1,248 @@ +<# +.Synopsis +Activate a Python virtual environment for the current PowerShell session. + +.Description +Pushes the python executable for a virtual environment to the front of the +$Env:PATH environment variable and sets the prompt to signify that you are +in a Python virtual environment. Makes use of the command line switches as +well as the `pyvenv.cfg` file values present in the virtual environment. + +.Parameter VenvDir +Path to the directory that contains the virtual environment to activate. The +default value for this is the parent of the directory that the Activate.ps1 +script is located within. + +.Parameter Prompt +The prompt prefix to display when this virtual environment is activated. By +default, this prompt is the name of the virtual environment folder (VenvDir) +surrounded by parentheses and followed by a single space (ie. '(.venv) '). + +.Example +Activate.ps1 +Activates the Python virtual environment that contains the Activate.ps1 script. + +.Example +Activate.ps1 -Verbose +Activates the Python virtual environment that contains the Activate.ps1 script, +and shows extra information about the activation as it executes. + +.Example +Activate.ps1 -VenvDir C:\Users\MyUser\Common\.venv +Activates the Python virtual environment located in the specified location. + +.Example +Activate.ps1 -Prompt "MyPython" +Activates the Python virtual environment that contains the Activate.ps1 script, +and prefixes the current prompt with the specified string (surrounded in +parentheses) while the virtual environment is active. + +.Notes +On Windows, it may be required to enable this Activate.ps1 script by setting the +execution policy for the user. You can do this by issuing the following PowerShell +command: + +PS C:\> Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser + +For more information on Execution Policies: +https://go.microsoft.com/fwlink/?LinkID=135170 + +#> +Param( + [Parameter(Mandatory = $false)] + [String] + $VenvDir, + [Parameter(Mandatory = $false)] + [String] + $Prompt +) + +<# Function declarations --------------------------------------------------- #> + +<# +.Synopsis +Remove all shell session elements added by the Activate script, including the +addition of the virtual environment's Python executable from the beginning of +the PATH variable. + +.Parameter NonDestructive +If present, do not remove this function from the global namespace for the +session. + +#> +function global:deactivate ([switch]$NonDestructive) { + # Revert to original values + + # The prior prompt: + if (Test-Path -Path Function:_OLD_VIRTUAL_PROMPT) { + Copy-Item -Path Function:_OLD_VIRTUAL_PROMPT -Destination Function:prompt + Remove-Item -Path Function:_OLD_VIRTUAL_PROMPT + } + + # The prior PYTHONHOME: + if (Test-Path -Path Env:_OLD_VIRTUAL_PYTHONHOME) { + Copy-Item -Path Env:_OLD_VIRTUAL_PYTHONHOME -Destination Env:PYTHONHOME + Remove-Item -Path Env:_OLD_VIRTUAL_PYTHONHOME + } + + # The prior PATH: + if (Test-Path -Path Env:_OLD_VIRTUAL_PATH) { + Copy-Item -Path Env:_OLD_VIRTUAL_PATH -Destination Env:PATH + Remove-Item -Path Env:_OLD_VIRTUAL_PATH + } + + # Just remove the VIRTUAL_ENV altogether: + if (Test-Path -Path Env:VIRTUAL_ENV) { + Remove-Item -Path env:VIRTUAL_ENV + } + + # Just remove VIRTUAL_ENV_PROMPT altogether. + if (Test-Path -Path Env:VIRTUAL_ENV_PROMPT) { + Remove-Item -Path env:VIRTUAL_ENV_PROMPT + } + + # Just remove the _PYTHON_VENV_PROMPT_PREFIX altogether: + if (Get-Variable -Name "_PYTHON_VENV_PROMPT_PREFIX" -ErrorAction SilentlyContinue) { + Remove-Variable -Name _PYTHON_VENV_PROMPT_PREFIX -Scope Global -Force + } + + # Leave deactivate function in the global namespace if requested: + if (-not $NonDestructive) { + Remove-Item -Path function:deactivate + } +} + +<# +.Description +Get-PyVenvConfig parses the values from the pyvenv.cfg file located in the +given folder, and returns them in a map. + +For each line in the pyvenv.cfg file, if that line can be parsed into exactly +two strings separated by `=` (with any amount of whitespace surrounding the =) +then it is considered a `key = value` line. The left hand string is the key, +the right hand is the value. + +If the value starts with a `'` or a `"` then the first and last character is +stripped from the value before being captured. + +.Parameter ConfigDir +Path to the directory that contains the `pyvenv.cfg` file. +#> +function Get-PyVenvConfig( + [String] + $ConfigDir +) { + Write-Verbose "Given ConfigDir=$ConfigDir, obtain values in pyvenv.cfg" + + # Ensure the file exists, and issue a warning if it doesn't (but still allow the function to continue). + $pyvenvConfigPath = Join-Path -Resolve -Path $ConfigDir -ChildPath 'pyvenv.cfg' -ErrorAction Continue + + # An empty map will be returned if no config file is found. + $pyvenvConfig = @{ } + + if ($pyvenvConfigPath) { + + Write-Verbose "File exists, parse `key = value` lines" + $pyvenvConfigContent = Get-Content -Path $pyvenvConfigPath + + $pyvenvConfigContent | ForEach-Object { + $keyval = $PSItem -split "\s*=\s*", 2 + if ($keyval[0] -and $keyval[1]) { + $val = $keyval[1] + + # Remove extraneous quotations around a string value. + if ("'""".Contains($val.Substring(0, 1))) { + $val = $val.Substring(1, $val.Length - 2) + } + + $pyvenvConfig[$keyval[0]] = $val + Write-Verbose "Adding Key: '$($keyval[0])'='$val'" + } + } + } + return $pyvenvConfig +} + + +<# Begin Activate script --------------------------------------------------- #> + +# Determine the containing directory of this script +$VenvExecPath = Split-Path -Parent $MyInvocation.MyCommand.Definition +$VenvExecDir = Get-Item -Path $VenvExecPath + +Write-Verbose "Activation script is located in path: '$VenvExecPath'" +Write-Verbose "VenvExecDir Fullname: '$($VenvExecDir.FullName)" +Write-Verbose "VenvExecDir Name: '$($VenvExecDir.Name)" + +# Set values required in priority: CmdLine, ConfigFile, Default +# First, get the location of the virtual environment, it might not be +# VenvExecDir if specified on the command line. +if ($VenvDir) { + Write-Verbose "VenvDir given as parameter, using '$VenvDir' to determine values" +} +else { + Write-Verbose "VenvDir not given as a parameter, using parent directory name as VenvDir." + $VenvDir = $VenvExecDir.Parent.FullName.TrimEnd("\\/") + Write-Verbose "VenvDir=$VenvDir" +} + +# Next, read the `pyvenv.cfg` file to determine any required value such +# as `prompt`. +$pyvenvCfg = Get-PyVenvConfig -ConfigDir $VenvDir + +# Next, set the prompt from the command line, or the config file, or +# just use the name of the virtual environment folder. +if ($Prompt) { + Write-Verbose "Prompt specified as argument, using '$Prompt'" +} +else { + Write-Verbose "Prompt not specified as argument to script, checking pyvenv.cfg value" + if ($pyvenvCfg -and $pyvenvCfg['prompt']) { + Write-Verbose " Setting based on value in pyvenv.cfg='$($pyvenvCfg['prompt'])'" + $Prompt = $pyvenvCfg['prompt']; + } + else { + Write-Verbose " Setting prompt based on parent's directory's name. (Is the directory name passed to venv module when creating the virtual environment)" + Write-Verbose " Got leaf-name of $VenvDir='$(Split-Path -Path $venvDir -Leaf)'" + $Prompt = Split-Path -Path $venvDir -Leaf + } +} + +Write-Verbose "Prompt = '$Prompt'" +Write-Verbose "VenvDir='$VenvDir'" + +# Deactivate any currently active virtual environment, but leave the +# deactivate function in place. +deactivate -nondestructive + +# Now set the environment variable VIRTUAL_ENV, used by many tools to determine +# that there is an activated venv. +$env:VIRTUAL_ENV = $VenvDir + +$env:VIRTUAL_ENV_PROMPT = $Prompt + +if (-not $Env:VIRTUAL_ENV_DISABLE_PROMPT) { + + Write-Verbose "Setting prompt to '$Prompt'" + + # Set the prompt to include the env name + # Make sure _OLD_VIRTUAL_PROMPT is global + function global:_OLD_VIRTUAL_PROMPT { "" } + Copy-Item -Path function:prompt -Destination function:_OLD_VIRTUAL_PROMPT + New-Variable -Name _PYTHON_VENV_PROMPT_PREFIX -Description "Python virtual environment prompt prefix" -Scope Global -Option ReadOnly -Visibility Public -Value $Prompt + + function global:prompt { + Write-Host -NoNewline -ForegroundColor Green "($_PYTHON_VENV_PROMPT_PREFIX) " + _OLD_VIRTUAL_PROMPT + } +} + +# Clear PYTHONHOME +if (Test-Path -Path Env:PYTHONHOME) { + Copy-Item -Path Env:PYTHONHOME -Destination Env:_OLD_VIRTUAL_PYTHONHOME + Remove-Item -Path Env:PYTHONHOME +} + +# Add the venv to the PATH +Copy-Item -Path Env:PATH -Destination Env:_OLD_VIRTUAL_PATH +$Env:PATH = "$VenvExecDir$([System.IO.Path]::PathSeparator)$Env:PATH" diff --git a/.venv/bin/activate b/.venv/bin/activate new file mode 100644 index 0000000000000000000000000000000000000000..2d77659e3477cab33f0b22b55e3cb7698e2a6c0f --- /dev/null +++ b/.venv/bin/activate @@ -0,0 +1,76 @@ +# This file must be used with "source bin/activate" *from bash* +# You cannot run it directly + +deactivate () { + # reset old environment variables + if [ -n "${_OLD_VIRTUAL_PATH:-}" ] ; then + PATH="${_OLD_VIRTUAL_PATH:-}" + export PATH + unset _OLD_VIRTUAL_PATH + fi + if [ -n "${_OLD_VIRTUAL_PYTHONHOME:-}" ] ; then + PYTHONHOME="${_OLD_VIRTUAL_PYTHONHOME:-}" + export PYTHONHOME + unset _OLD_VIRTUAL_PYTHONHOME + fi + + # Call hash to forget past locations. Without forgetting + # past locations the $PATH changes we made may not be respected. + # See "man bash" for more details. hash is usually a builtin of your shell + hash -r 2> /dev/null + + if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then + PS1="${_OLD_VIRTUAL_PS1:-}" + export PS1 + unset _OLD_VIRTUAL_PS1 + fi + + unset VIRTUAL_ENV + unset VIRTUAL_ENV_PROMPT + if [ ! "${1:-}" = "nondestructive" ] ; then + # Self destruct! + unset -f deactivate + fi +} + +# unset irrelevant variables +deactivate nondestructive + +# on Windows, a path can contain colons and backslashes and has to be converted: +case "$(uname)" in + CYGWIN*|MSYS*|MINGW*) + # transform D:\path\to\venv to /d/path/to/venv on MSYS and MINGW + # and to /cygdrive/d/path/to/venv on Cygwin + VIRTUAL_ENV=$(cygpath /run/media/chop/NEWBULK/Workspace/ChopGuard/run_20261018_010954_625209/.venv) + export VIRTUAL_ENV + ;; + *) + # use the path as-is + export VIRTUAL_ENV=/run/media/chop/NEWBULK/Workspace/ChopGuard/run_20261018_010954_625209/.venv + ;; +esac + +_OLD_VIRTUAL_PATH="$PATH" +PATH="$VIRTUAL_ENV/"bin":$PATH" +export PATH + +VIRTUAL_ENV_PROMPT=.venv +export VIRTUAL_ENV_PROMPT + +# unset PYTHONHOME if set +# this will fail if PYTHONHOME is set to the empty string (which is bad anyway) +# could use `if (set -u; : $PYTHONHOME) ;` in bash +if [ -n "${PYTHONHOME:-}" ] ; then + _OLD_VIRTUAL_PYTHONHOME="${PYTHONHOME:-}" + unset PYTHONHOME +fi + +if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then + _OLD_VIRTUAL_PS1="${PS1:-}" + PS1="(".venv") ${PS1:-}" + export PS1 +fi + +# Call hash to forget past commands. Without forgetting +# past commands the $PATH changes we made may not be respected +hash -r 2> /dev/null diff --git a/.venv/bin/activate.csh b/.venv/bin/activate.csh new file mode 100644 index 0000000000000000000000000000000000000000..ef79695111d27e85cd8b587a940ec4b64de25f18 --- /dev/null +++ b/.venv/bin/activate.csh @@ -0,0 +1,27 @@ +# This file must be used with "source bin/activate.csh" *from csh*. +# You cannot run it directly. + +# Created by Davide Di Blasi . +# Ported to Python 3.3 venv by Andrew Svetlov + +alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PATH" && unset _OLD_VIRTUAL_PATH; rehash; test $?_OLD_VIRTUAL_PROMPT != 0 && set prompt="$_OLD_VIRTUAL_PROMPT" && unset _OLD_VIRTUAL_PROMPT; unsetenv VIRTUAL_ENV; unsetenv VIRTUAL_ENV_PROMPT; test "\!:*" != "nondestructive" && unalias deactivate' + +# Unset irrelevant variables. +deactivate nondestructive + +setenv VIRTUAL_ENV /run/media/chop/NEWBULK/Workspace/ChopGuard/run_20261018_010954_625209/.venv + +set _OLD_VIRTUAL_PATH="$PATH" +setenv PATH "$VIRTUAL_ENV/"bin":$PATH" +setenv VIRTUAL_ENV_PROMPT .venv + + +set _OLD_VIRTUAL_PROMPT="$prompt" + +if (! "$?VIRTUAL_ENV_DISABLE_PROMPT") then + set prompt = "(".venv") $prompt:q" +endif + +alias pydoc python -m pydoc + +rehash diff --git a/.venv/bin/activate.fish b/.venv/bin/activate.fish new file mode 100644 index 0000000000000000000000000000000000000000..1007c31f0c1e0cd034525e452132aff0b97905cb --- /dev/null +++ b/.venv/bin/activate.fish @@ -0,0 +1,69 @@ +# This file must be used with "source /bin/activate.fish" *from fish* +# (https://fishshell.com/). You cannot run it directly. + +function deactivate -d "Exit virtual environment and return to normal shell environment" + # reset old environment variables + if test -n "$_OLD_VIRTUAL_PATH" + set -gx PATH $_OLD_VIRTUAL_PATH + set -e _OLD_VIRTUAL_PATH + end + if test -n "$_OLD_VIRTUAL_PYTHONHOME" + set -gx PYTHONHOME $_OLD_VIRTUAL_PYTHONHOME + set -e _OLD_VIRTUAL_PYTHONHOME + end + + if test -n "$_OLD_FISH_PROMPT_OVERRIDE" + set -e _OLD_FISH_PROMPT_OVERRIDE + # prevents error when using nested fish instances (Issue #93858) + if functions -q _old_fish_prompt + functions -e fish_prompt + functions -c _old_fish_prompt fish_prompt + functions -e _old_fish_prompt + end + end + + set -e VIRTUAL_ENV + set -e VIRTUAL_ENV_PROMPT + if test "$argv[1]" != "nondestructive" + # Self-destruct! + functions -e deactivate + end +end + +# Unset irrelevant variables. +deactivate nondestructive + +set -gx VIRTUAL_ENV /run/media/chop/NEWBULK/Workspace/ChopGuard/run_20261018_010954_625209/.venv + +set -gx _OLD_VIRTUAL_PATH $PATH +set -gx PATH "$VIRTUAL_ENV/"bin $PATH +set -gx VIRTUAL_ENV_PROMPT .venv + +# Unset PYTHONHOME if set. +if set -q PYTHONHOME + set -gx _OLD_VIRTUAL_PYTHONHOME $PYTHONHOME + set -e PYTHONHOME +end + +if test -z "$VIRTUAL_ENV_DISABLE_PROMPT" + # fish uses a function instead of an env var to generate the prompt. + + # Save the current fish_prompt function as the function _old_fish_prompt. + functions -c fish_prompt _old_fish_prompt + + # With the original prompt function renamed, we can override with our own. + function fish_prompt + # Save the return status of the last command. + set -l old_status $status + + # Output the venv prompt; color taken from the blue of the Python logo. + printf "%s(%s)%s " (set_color 4B8BBE) .venv (set_color normal) + + # Restore the return status of the previous command. + echo "exit $old_status" | . + # Output the original/"old" prompt. + _old_fish_prompt + end + + set -gx _OLD_FISH_PROMPT_OVERRIDE "$VIRTUAL_ENV" +end diff --git a/.venv/bin/hf b/.venv/bin/hf new file mode 100644 index 0000000000000000000000000000000000000000..1edbcb4d826670260c226b156733c28ad10d9fdd --- /dev/null +++ b/.venv/bin/hf @@ -0,0 +1,8 @@ +#!/run/media/chop/NEWBULK/Workspace/ChopGuard/run_20261018_010954_625209/.venv/bin/python3 +# -*- coding: utf-8 -*- +import re +import sys +from huggingface_hub.cli.hf import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/.venv/bin/httpx2 b/.venv/bin/httpx2 new file mode 100644 index 0000000000000000000000000000000000000000..73d393e8f575e3de3031aab12563a8133f677b66 --- /dev/null +++ b/.venv/bin/httpx2 @@ -0,0 +1,8 @@ +#!/run/media/chop/NEWBULK/Workspace/ChopGuard/run_20261018_010954_625209/.venv/bin/python3 +# -*- coding: utf-8 -*- +import re +import sys +from httpx2._main import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/.venv/bin/idna b/.venv/bin/idna new file mode 100644 index 0000000000000000000000000000000000000000..b85deb1b0800b7f28ff2e4e937922fe2967a78d9 --- /dev/null +++ b/.venv/bin/idna @@ -0,0 +1,8 @@ +#!/run/media/chop/NEWBULK/Workspace/ChopGuard/run_20261018_010954_625209/.venv/bin/python3 +# -*- coding: utf-8 -*- +import re +import sys +from idna.cli import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/.venv/bin/pip b/.venv/bin/pip new file mode 100644 index 0000000000000000000000000000000000000000..f31e81551974f89945c47c8a96155068033451bf --- /dev/null +++ b/.venv/bin/pip @@ -0,0 +1,8 @@ +#!/run/media/chop/NEWBULK/Workspace/ChopGuard/run_20261018_010954_625209/.venv/bin/python3 +# -*- coding: utf-8 -*- +import re +import sys +from pip._internal.cli.main import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/.venv/bin/pip3 b/.venv/bin/pip3 new file mode 100644 index 0000000000000000000000000000000000000000..f31e81551974f89945c47c8a96155068033451bf --- /dev/null +++ b/.venv/bin/pip3 @@ -0,0 +1,8 @@ +#!/run/media/chop/NEWBULK/Workspace/ChopGuard/run_20261018_010954_625209/.venv/bin/python3 +# -*- coding: utf-8 -*- +import re +import sys +from pip._internal.cli.main import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/.venv/bin/pip3.14 b/.venv/bin/pip3.14 new file mode 100644 index 0000000000000000000000000000000000000000..f31e81551974f89945c47c8a96155068033451bf --- /dev/null +++ b/.venv/bin/pip3.14 @@ -0,0 +1,8 @@ +#!/run/media/chop/NEWBULK/Workspace/ChopGuard/run_20261018_010954_625209/.venv/bin/python3 +# -*- coding: utf-8 -*- +import re +import sys +from pip._internal.cli.main import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/.venv/bin/python b/.venv/bin/python new file mode 100644 index 0000000000000000000000000000000000000000..d247aad4e499981d071cebc03767c687c32735f3 --- /dev/null +++ b/.venv/bin/python @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8d8288faefdd300201f43fcf00f6f539a27218eeed3a3dff5ab10b9c4c99700 +size 7481192 diff --git a/.venv/bin/python3 b/.venv/bin/python3 new file mode 100644 index 0000000000000000000000000000000000000000..d247aad4e499981d071cebc03767c687c32735f3 --- /dev/null +++ b/.venv/bin/python3 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8d8288faefdd300201f43fcf00f6f539a27218eeed3a3dff5ab10b9c4c99700 +size 7481192 diff --git a/.venv/bin/python3.14 b/.venv/bin/python3.14 new file mode 100644 index 0000000000000000000000000000000000000000..d247aad4e499981d071cebc03767c687c32735f3 --- /dev/null +++ b/.venv/bin/python3.14 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8d8288faefdd300201f43fcf00f6f539a27218eeed3a3dff5ab10b9c4c99700 +size 7481192 diff --git a/.venv/bin/tiny-agents b/.venv/bin/tiny-agents new file mode 100644 index 0000000000000000000000000000000000000000..6fd15cee1461b73e427fc2467c4b114f5be63469 --- /dev/null +++ b/.venv/bin/tiny-agents @@ -0,0 +1,8 @@ +#!/run/media/chop/NEWBULK/Workspace/ChopGuard/run_20261018_010954_625209/.venv/bin/python3 +# -*- coding: utf-8 -*- +import re +import sys +from huggingface_hub.inference._mcp.cli import app +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(app()) diff --git a/.venv/bin/tqdm b/.venv/bin/tqdm new file mode 100644 index 0000000000000000000000000000000000000000..cfdc45f825d119c42539c59f1bac005149d2b82d --- /dev/null +++ b/.venv/bin/tqdm @@ -0,0 +1,8 @@ +#!/run/media/chop/NEWBULK/Workspace/ChopGuard/run_20261018_010954_625209/.venv/bin/python3 +# -*- coding: utf-8 -*- +import re +import sys +from tqdm.cli import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git "a/.venv/bin/\360\235\234\213thon" "b/.venv/bin/\360\235\234\213thon" new file mode 100644 index 0000000000000000000000000000000000000000..d247aad4e499981d071cebc03767c687c32735f3 --- /dev/null +++ "b/.venv/bin/\360\235\234\213thon" @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8d8288faefdd300201f43fcf00f6f539a27218eeed3a3dff5ab10b9c4c99700 +size 7481192 diff --git a/.venv/lib/python3.14/site-packages/__pycache__/typing_extensions.cpython-314.pyc b/.venv/lib/python3.14/site-packages/__pycache__/typing_extensions.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c8e1a282f032c224416f8f4f4015cfe549be8388 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/__pycache__/typing_extensions.cpython-314.pyc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e337f18c92750414c4fb602467988640887646dca3ac07e1f707169eb340718 +size 179457 diff --git a/.venv/lib/python3.14/site-packages/_yaml/__init__.py b/.venv/lib/python3.14/site-packages/_yaml/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7baa8c4b68127d5cdf0be9a799429e61347c2694 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/_yaml/__init__.py @@ -0,0 +1,33 @@ +# This is a stub package designed to roughly emulate the _yaml +# extension module, which previously existed as a standalone module +# and has been moved into the `yaml` package namespace. +# It does not perfectly mimic its old counterpart, but should get +# close enough for anyone who's relying on it even when they shouldn't. +import yaml + +# in some circumstances, the yaml module we imoprted may be from a different version, so we need +# to tread carefully when poking at it here (it may not have the attributes we expect) +if not getattr(yaml, '__with_libyaml__', False): + from sys import version_info + + exc = ModuleNotFoundError if version_info >= (3, 6) else ImportError + raise exc("No module named '_yaml'") +else: + from yaml._yaml import * + import warnings + warnings.warn( + 'The _yaml extension module is now located at yaml._yaml' + ' and its location is subject to change. To use the' + ' LibYAML-based parser and emitter, import from `yaml`:' + ' `from yaml import CLoader as Loader, CDumper as Dumper`.', + DeprecationWarning + ) + del warnings + # Don't `del yaml` here because yaml is actually an existing + # namespace member of _yaml. + +__name__ = '_yaml' +# If the module is top-level (i.e. not a part of any specific package) +# then the attribute should be set to ''. +# https://docs.python.org/3.8/library/types.html +__package__ = '' diff --git a/.venv/lib/python3.14/site-packages/_yaml/__pycache__/__init__.cpython-314.pyc b/.venv/lib/python3.14/site-packages/_yaml/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2aa63d0b223c99aa9fe17f66712c4361282c7b2c Binary files /dev/null and b/.venv/lib/python3.14/site-packages/_yaml/__pycache__/__init__.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/anyio-4.15.1.dist-info/INSTALLER b/.venv/lib/python3.14/site-packages/anyio-4.15.1.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/anyio-4.15.1.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/.venv/lib/python3.14/site-packages/anyio-4.15.1.dist-info/METADATA b/.venv/lib/python3.14/site-packages/anyio-4.15.1.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..13734ecac7501953843b99eef680987bad705350 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/anyio-4.15.1.dist-info/METADATA @@ -0,0 +1,108 @@ +Metadata-Version: 2.4 +Name: anyio +Version: 4.15.1 +Summary: High-level concurrency and networking framework on top of asyncio or Trio +Author-email: Alex Grönholm +License-Expression: MIT +Project-URL: Documentation, https://anyio.readthedocs.io/en/latest/ +Project-URL: Changelog, https://anyio.readthedocs.io/en/stable/versionhistory.html +Project-URL: Source code, https://github.com/agronholm/anyio +Project-URL: Issue tracker, https://github.com/agronholm/anyio/issues +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: Framework :: AnyIO +Classifier: Typing :: Typed +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 +Classifier: Programming Language :: Python :: 3.14 +Classifier: Programming Language :: Python :: 3.15 +Classifier: Programming Language :: Python :: Free Threading :: 2 - Beta +Requires-Python: >=3.10 +Description-Content-Type: text/x-rst +License-File: LICENSE +Requires-Dist: exceptiongroup>=1.0.2; python_version < "3.11" +Requires-Dist: idna>=2.8 +Requires-Dist: typing_extensions>=4.16.0; python_version < "3.15" +Provides-Extra: trio +Requires-Dist: trio>=0.32.0; extra == "trio" +Dynamic: license-file + +.. image:: https://github.com/agronholm/anyio/actions/workflows/test.yml/badge.svg + :target: https://github.com/agronholm/anyio/actions/workflows/test.yml + :alt: Build Status +.. image:: https://coveralls.io/repos/github/agronholm/anyio/badge.svg?branch=master + :target: https://coveralls.io/github/agronholm/anyio?branch=master + :alt: Code Coverage +.. image:: https://readthedocs.org/projects/anyio/badge/?version=latest + :target: https://anyio.readthedocs.io/en/latest/?badge=latest + :alt: Documentation +.. image:: https://badges.gitter.im/gitterHQ/gitter.svg + :target: https://gitter.im/python-trio/AnyIO + :alt: Gitter chat +.. image:: https://tidelift.com/badges/package/pypi/anyio + :target: https://tidelift.com/subscription/pkg/pypi-anyio + :alt: Tidelift + +AnyIO is an asynchronous networking and concurrency library that works on top of either asyncio_ or +Trio_. It implements Trio-like `structured concurrency`_ (SC) on top of asyncio and works in harmony +with the native SC of Trio itself. + +Applications and libraries written against AnyIO's API will run unmodified on either asyncio_ or +Trio_. AnyIO can also be adopted into a library or application incrementally – bit by bit, no full +refactoring necessary. It will blend in with the native libraries of your chosen backend. + +To find out why you might want to use AnyIO's APIs instead of asyncio's, you can read about it +`here `_. + +Documentation +------------- + +View full documentation at: https://anyio.readthedocs.io/ + +Features +-------- + +AnyIO offers the following functionality: + +* Task groups (nurseries_ in trio terminology) +* High-level networking (TCP, UDP and UNIX sockets) + + * `Happy eyeballs`_ algorithm for TCP connections (more robust than that of asyncio on Python + 3.8) + * async/await style UDP sockets (unlike asyncio where you still have to use Transports and + Protocols) + +* A versatile API for byte streams and object streams +* Inter-task synchronization and communication (locks, conditions, events, semaphores, object + streams and futures) +* Worker threads +* Subprocesses +* Subinterpreter support for code parallelization (on Python 3.13 and later) +* Asynchronous file I/O (using worker threads) +* Signal handling +* Asynchronous versions of the functools_ and itertools_ modules + +AnyIO also comes with its own pytest_ plugin which also supports asynchronous fixtures. +It even works with the popular Hypothesis_ library. + +.. _asyncio: https://docs.python.org/3/library/asyncio.html +.. _Trio: https://github.com/python-trio/trio +.. _structured concurrency: https://en.wikipedia.org/wiki/Structured_concurrency +.. _nurseries: https://trio.readthedocs.io/en/stable/reference-core.html#nurseries-and-spawning +.. _Happy eyeballs: https://en.wikipedia.org/wiki/Happy_Eyeballs +.. _pytest: https://docs.pytest.org/en/latest/ +.. _functools: https://docs.python.org/3/library/functools.html +.. _itertools: https://docs.python.org/3/library/itertools.html +.. _Hypothesis: https://hypothesis.works/ + +Security contact information +---------------------------- + +To report a security vulnerability, please use the `Tidelift security contact`_. +Tidelift will coordinate the fix and disclosure. + +.. _Tidelift security contact: https://tidelift.com/security diff --git a/.venv/lib/python3.14/site-packages/anyio-4.15.1.dist-info/RECORD b/.venv/lib/python3.14/site-packages/anyio-4.15.1.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..1523e782ea1bc9d7f7732ba20d8005222cc53b71 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/anyio-4.15.1.dist-info/RECORD @@ -0,0 +1,100 @@ +anyio-4.15.1.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +anyio-4.15.1.dist-info/METADATA,sha256=BDrstXGvcYnopsUo9djjJUXxss36r3YdUebVtn6jyoE,4733 +anyio-4.15.1.dist-info/RECORD,, +anyio-4.15.1.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91 +anyio-4.15.1.dist-info/entry_points.txt,sha256=_d6Yu6uiaZmNe0CydowirE9Cmg7zUL2g08tQpoS3Qvc,39 +anyio-4.15.1.dist-info/licenses/LICENSE,sha256=U2GsncWPLvX9LpsJxoKXwX8ElQkJu8gCO9uC6s8iwrA,1081 +anyio-4.15.1.dist-info/top_level.txt,sha256=QglSMiWX8_5dpoVAEIHdEYzvqFMdSYWmCj6tYw2ITkQ,6 +anyio/__init__.py,sha256=OfroYRjIUw_JFj0Zwh1JAaajMQg4VqtoAt0XoOHDRwE,7602 +anyio/__pycache__/__init__.cpython-314.pyc,, +anyio/__pycache__/_lazyimport.cpython-314.pyc,, +anyio/__pycache__/from_thread.cpython-314.pyc,, +anyio/__pycache__/functools.cpython-314.pyc,, +anyio/__pycache__/itertools.cpython-314.pyc,, +anyio/__pycache__/lowlevel.cpython-314.pyc,, +anyio/__pycache__/pytest_plugin.cpython-314.pyc,, +anyio/__pycache__/to_interpreter.cpython-314.pyc,, +anyio/__pycache__/to_process.cpython-314.pyc,, +anyio/__pycache__/to_thread.cpython-314.pyc,, +anyio/_backends/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +anyio/_backends/__pycache__/__init__.cpython-314.pyc,, +anyio/_backends/__pycache__/_asyncio.cpython-314.pyc,, +anyio/_backends/__pycache__/_trio.cpython-314.pyc,, +anyio/_backends/_asyncio.py,sha256=Vqy7O1ymmEpxTiPzehSuBCVTziGALNV-BbsIRaKXsg4,107767 +anyio/_backends/_trio.py,sha256=Gs8XzX0IeBvje8IEZtJmpNylKCUAChKv7I_YZi3wlmg,45715 +anyio/_core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +anyio/_core/__pycache__/__init__.cpython-314.pyc,, +anyio/_core/__pycache__/_asyncio_selector_thread.cpython-314.pyc,, +anyio/_core/__pycache__/_concurrency_utils.cpython-314.pyc,, +anyio/_core/__pycache__/_contextmanagers.cpython-314.pyc,, +anyio/_core/__pycache__/_eventloop.cpython-314.pyc,, +anyio/_core/__pycache__/_exceptions.cpython-314.pyc,, +anyio/_core/__pycache__/_fileio.cpython-314.pyc,, +anyio/_core/__pycache__/_futures.cpython-314.pyc,, +anyio/_core/__pycache__/_resources.cpython-314.pyc,, +anyio/_core/__pycache__/_signals.cpython-314.pyc,, +anyio/_core/__pycache__/_sockets.cpython-314.pyc,, +anyio/_core/__pycache__/_streams.cpython-314.pyc,, +anyio/_core/__pycache__/_subprocesses.cpython-314.pyc,, +anyio/_core/__pycache__/_synchronization.cpython-314.pyc,, +anyio/_core/__pycache__/_tasks.cpython-314.pyc,, +anyio/_core/__pycache__/_tempfile.cpython-314.pyc,, +anyio/_core/__pycache__/_testing.cpython-314.pyc,, +anyio/_core/__pycache__/_typedattr.cpython-314.pyc,, +anyio/_core/_asyncio_selector_thread.py,sha256=2PdxFM3cs02Kp6BSppbvmRT7q7asreTW5FgBxEsflBo,5626 +anyio/_core/_concurrency_utils.py,sha256=kvRXMrnDqgQTBkZc-f279-lBnZM4YbSToXqhnAUCfSc,4522 +anyio/_core/_contextmanagers.py,sha256=YInBCabiEeS-UaP_Jdxa1CaFC71ETPW8HZTHIM8Rsc8,7215 +anyio/_core/_eventloop.py,sha256=ByZUeJD9alMfcyTseRo5IzTO0IltEul_Gyq9iqSjqDk,6658 +anyio/_core/_exceptions.py,sha256=B4ZhLkBQT4tI6vShv4-GJui492IJsoIDTw1FQJEkmAI,5618 +anyio/_core/_fileio.py,sha256=q--ft9h5Y_FRLzhAnoYrXf3tOuZ8ZyIN2A9mCfMbZRc,34163 +anyio/_core/_futures.py,sha256=YQq5uhnCnKQMQD62tOLfzEm7dUXY3GQorJdH6ghfgIQ,6177 +anyio/_core/_resources.py,sha256=Ld5duCe9NwgPfI7EIOh3CMIBYQ7rq6ZQEa7CFS0u9Ec,450 +anyio/_core/_signals.py,sha256=mjTBB2hTKNPRlU0IhnijeQedpWOGERDiMjSlJQsFrug,1016 +anyio/_core/_sockets.py,sha256=C8b-3w5H9n0Fo4g7X8WJvk0m6CGh6NVAAFtieBGHK3g,35289 +anyio/_core/_streams.py,sha256=FczFwIgDpnkK0bODWJXMpsUJYdvAD04kaUaGzJU8DK0,1806 +anyio/_core/_subprocesses.py,sha256=TMBuGNILz-47w5t9vayJ1GpCpDT6LxsMohJDFCewIRc,7938 +anyio/_core/_synchronization.py,sha256=ggsd74QPd846faA2HQeT07ONQ5HQ1S78gy8mvB3qCrw,21658 +anyio/_core/_tasks.py,sha256=J2dhM_Jq3Grsfg_enNWpRz2SxfM15uLt-pEfvoQq3uA,15391 +anyio/_core/_tempfile.py,sha256=IgoBCFaPhrFvsF9KHcCcrnCRn-UqMfw0WRdSDWgxAiI,19726 +anyio/_core/_testing.py,sha256=ZRucLh5kqhiFWAJMMbd8yZ6uIvz0rq3WwL72ldosxAU,2340 +anyio/_core/_typedattr.py,sha256=gt9Dzn-IL0QW14aHPDD8PAtq95FYKNNupHyf37xZGPQ,2607 +anyio/_lazyimport.py,sha256=LCjnYanIfYE9G-Dv_EsqNLDuVRppNicWZjCTfS9-sJU,6082 +anyio/abc/__init__.py,sha256=XIVvCI2c6oyaOcqXe94k2rzU5_s7-ZNnlxGTk34H4T0,3050 +anyio/abc/__pycache__/__init__.cpython-314.pyc,, +anyio/abc/__pycache__/_eventloop.cpython-314.pyc,, +anyio/abc/__pycache__/_resources.cpython-314.pyc,, +anyio/abc/__pycache__/_sockets.cpython-314.pyc,, +anyio/abc/__pycache__/_streams.cpython-314.pyc,, +anyio/abc/__pycache__/_subprocesses.cpython-314.pyc,, +anyio/abc/__pycache__/_tasks.cpython-314.pyc,, +anyio/abc/__pycache__/_testing.cpython-314.pyc,, +anyio/abc/_eventloop.py,sha256=0-UwgB351rkocCt8m3fjTmcNx2Fxn52nUoujG5DzMvU,11127 +anyio/abc/_resources.py,sha256=hrb0LXUgHSp3i8G6W8JJchdzKXgg1m0T-L7udXmv2y0,900 +anyio/abc/_sockets.py,sha256=jQWnPX7iDIDyvyvVkbuFhjezw4oI2WggzbW6Vl2-eqs,13162 +anyio/abc/_streams.py,sha256=IMeOPPutFXRbyeAutxV1HJe7FUQZYp2qKrJJpP1DCv0,7620 +anyio/abc/_subprocesses.py,sha256=CDqU1p0HuBS0pXUzBW_hbfhihTJRM67n__7qZmoPU84,2123 +anyio/abc/_tasks.py,sha256=HBPpT72aOy0bwMAxBCKmnXsR328wCYRjYm3iaVw2fQo,7070 +anyio/abc/_testing.py,sha256=A9Y8RAClIPn1A0aNHBN8YoaRy_MKOpbFGEbTHAXgB6U,2145 +anyio/from_thread.py,sha256=iZZbfcNySi4wAcx1_cxAQE4J0YZC88qRMIYqQ6SF9sc,19415 +anyio/functools.py,sha256=T4JS8IXq-x1S0Lbo2owF8l9fza2KypO147QLeyz4cjs,11797 +anyio/itertools.py,sha256=TS7_UCa2Y3cggtaLlWHbjOyf6QhPfQUdPduNW-ddXTE,16255 +anyio/lowlevel.py,sha256=QyWkvfetbCH0mHkqv1yxGwOaiZ7fyxhpZkHWOXPV5JU,6280 +anyio/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +anyio/pytest_plugin.py,sha256=oLXyOstvRZW3VVbqIODhZsPt4jSz2UGh44YeG2nJXx0,14516 +anyio/streams/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +anyio/streams/__pycache__/__init__.cpython-314.pyc,, +anyio/streams/__pycache__/buffered.cpython-314.pyc,, +anyio/streams/__pycache__/file.cpython-314.pyc,, +anyio/streams/__pycache__/memory.cpython-314.pyc,, +anyio/streams/__pycache__/stapled.cpython-314.pyc,, +anyio/streams/__pycache__/text.cpython-314.pyc,, +anyio/streams/__pycache__/tls.cpython-314.pyc,, +anyio/streams/buffered.py,sha256=u7hCD8SNrYHcutG6K5wEiy1F88-pizgjvEFM22Kq2Cw,6746 +anyio/streams/file.py,sha256=6jujI2m-QJITqqKFamrupX_DNsU7y2Fz3omLZxOLuY0,4524 +anyio/streams/memory.py,sha256=rpxNzDMag9PCOynBqN64KyuAfAPB41vFyRP-Npo-lE0,10793 +anyio/streams/stapled.py,sha256=UJ2Rgt0NUmM1VovRNOoZoic960lcbDjb3vhfZg3b_Z0,4845 +anyio/streams/text.py,sha256=BcVAGJw1VRvtIqnv-o0Rb0pwH7p8vwlvl21xHq522ag,5765 +anyio/streams/tls.py,sha256=fqLmxvULFO9rvxGyJ33VPrOIN7E3eiU1q90BkmKigQ8,15818 +anyio/to_interpreter.py,sha256=ATnoT6kei9wDmQZtXTA3SsY4CK7XKr8Ta4_x5i6pbTw,7100 +anyio/to_process.py,sha256=p6n8j0nreJUl1DEkR_9PL6KRoKtNqFxORTn8qZBvn94,9850 +anyio/to_thread.py,sha256=8MPVrrQq8ttGvV21cSd68rMto5bhxJ0-i9CupXag5Gs,2750 diff --git a/.venv/lib/python3.14/site-packages/anyio-4.15.1.dist-info/WHEEL b/.venv/lib/python3.14/site-packages/anyio-4.15.1.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..13013d7b49be5a84c3ee5931b06285b9553cf5d0 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/anyio-4.15.1.dist-info/WHEEL @@ -0,0 +1,5 @@ +Wheel-Version: 1.0 +Generator: setuptools (84.0.0) +Root-Is-Purelib: true +Tag: py3-none-any + diff --git a/.venv/lib/python3.14/site-packages/anyio-4.15.1.dist-info/entry_points.txt b/.venv/lib/python3.14/site-packages/anyio-4.15.1.dist-info/entry_points.txt new file mode 100644 index 0000000000000000000000000000000000000000..44dd9bdc3039122cc98014c1439ca254313fd014 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/anyio-4.15.1.dist-info/entry_points.txt @@ -0,0 +1,2 @@ +[pytest11] +anyio = anyio.pytest_plugin diff --git a/.venv/lib/python3.14/site-packages/anyio-4.15.1.dist-info/licenses/LICENSE b/.venv/lib/python3.14/site-packages/anyio-4.15.1.dist-info/licenses/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..104eebf5a3002fccdaceef3a4cb936173c1c2035 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/anyio-4.15.1.dist-info/licenses/LICENSE @@ -0,0 +1,20 @@ +The MIT License (MIT) + +Copyright (c) 2018 Alex Grönholm + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/.venv/lib/python3.14/site-packages/anyio-4.15.1.dist-info/top_level.txt b/.venv/lib/python3.14/site-packages/anyio-4.15.1.dist-info/top_level.txt new file mode 100644 index 0000000000000000000000000000000000000000..c77c069ecc9b7f8b1f97dbcfec905725db0253a8 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/anyio-4.15.1.dist-info/top_level.txt @@ -0,0 +1 @@ +anyio diff --git a/.venv/lib/python3.14/site-packages/anyio/__init__.py b/.venv/lib/python3.14/site-packages/anyio/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..30a3a16151bef66d432cb8e06ad858c3ada002fd --- /dev/null +++ b/.venv/lib/python3.14/site-packages/anyio/__init__.py @@ -0,0 +1,154 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from ._lazyimport import ( + fix_package_names, + install_lazy_importer, + set_deprecated_aliases, +) + +if TYPE_CHECKING or not install_lazy_importer(): + from ._core._concurrency_utils import amap as amap + from ._core._concurrency_utils import as_completed as as_completed + from ._core._concurrency_utils import gather as gather + from ._core._contextmanagers import ( + AsyncContextManagerMixin as AsyncContextManagerMixin, + ) + from ._core._contextmanagers import ContextManagerMixin as ContextManagerMixin + from ._core._eventloop import current_time as current_time + from ._core._eventloop import get_all_backends as get_all_backends + from ._core._eventloop import get_available_backends as get_available_backends + from ._core._eventloop import get_cancelled_exc_class as get_cancelled_exc_class + from ._core._eventloop import run as run + from ._core._eventloop import sleep as sleep + from ._core._eventloop import sleep_forever as sleep_forever + from ._core._eventloop import sleep_until as sleep_until + from ._core._exceptions import BrokenResourceError as BrokenResourceError + from ._core._exceptions import BrokenWorkerInterpreter as BrokenWorkerInterpreter + from ._core._exceptions import BrokenWorkerProcess as BrokenWorkerProcess + from ._core._exceptions import BusyResourceError as BusyResourceError + from ._core._exceptions import ClosedResourceError as ClosedResourceError + from ._core._exceptions import ConnectionFailed as ConnectionFailed + from ._core._exceptions import DelimiterNotFound as DelimiterNotFound + from ._core._exceptions import EndOfStream as EndOfStream + from ._core._exceptions import FutureAlreadyFinished as FutureAlreadyFinished + from ._core._exceptions import FutureCancelled as FutureCancelled + from ._core._exceptions import FutureFailed as FutureFailed + from ._core._exceptions import FutureNotFinished as FutureNotFinished + from ._core._exceptions import IncompleteRead as IncompleteRead + from ._core._exceptions import NoEventLoopError as NoEventLoopError + from ._core._exceptions import RunFinishedError as RunFinishedError + from ._core._exceptions import TaskCancelled as TaskCancelled + from ._core._exceptions import TaskFailed as TaskFailed + from ._core._exceptions import TaskNotFinished as TaskNotFinished + from ._core._exceptions import ( + TypedAttributeLookupError as TypedAttributeLookupError, + ) + from ._core._exceptions import WouldBlock as WouldBlock + from ._core._fileio import AsyncFile as AsyncFile + from ._core._fileio import Path as Path + from ._core._fileio import open_file as open_file + from ._core._fileio import wrap_file as wrap_file + from ._core._futures import Future as Future + from ._core._resources import aclose_forcefully as aclose_forcefully + from ._core._signals import open_signal_receiver as open_signal_receiver + from ._core._sockets import TCPConnectable as TCPConnectable + from ._core._sockets import UNIXConnectable as UNIXConnectable + from ._core._sockets import as_connectable as as_connectable + from ._core._sockets import connect_tcp as connect_tcp + from ._core._sockets import connect_unix as connect_unix + from ._core._sockets import ( + create_connected_udp_socket as create_connected_udp_socket, + ) + from ._core._sockets import ( + create_connected_unix_datagram_socket as create_connected_unix_datagram_socket, + ) + from ._core._sockets import create_tcp_listener as create_tcp_listener + from ._core._sockets import create_udp_socket as create_udp_socket + from ._core._sockets import ( + create_unix_datagram_socket as create_unix_datagram_socket, + ) + from ._core._sockets import create_unix_listener as create_unix_listener + from ._core._sockets import getaddrinfo as getaddrinfo + from ._core._sockets import getnameinfo as getnameinfo + from ._core._sockets import notify_closing as notify_closing + from ._core._sockets import wait_readable as wait_readable + from ._core._sockets import wait_socket_readable as wait_socket_readable + from ._core._sockets import wait_socket_writable as wait_socket_writable + from ._core._sockets import wait_writable as wait_writable + from ._core._streams import ( + create_memory_object_stream as create_memory_object_stream, + ) + from ._core._subprocesses import open_process as open_process + from ._core._subprocesses import run_process as run_process + from ._core._synchronization import CapacityLimiter as CapacityLimiter + from ._core._synchronization import ( + CapacityLimiterStatistics as CapacityLimiterStatistics, + ) + from ._core._synchronization import Condition as Condition + from ._core._synchronization import ConditionStatistics as ConditionStatistics + from ._core._synchronization import Event as Event + from ._core._synchronization import EventStatistics as EventStatistics + from ._core._synchronization import Lock as Lock + from ._core._synchronization import LockStatistics as LockStatistics + from ._core._synchronization import ResourceGuard as ResourceGuard + from ._core._synchronization import Semaphore as Semaphore + from ._core._synchronization import SemaphoreStatistics as SemaphoreStatistics + from ._core._tasks import TASK_STATUS_IGNORED as TASK_STATUS_IGNORED + from ._core._tasks import CancelScope as CancelScope + from ._core._tasks import TaskHandle as TaskHandle + from ._core._tasks import create_task_group as create_task_group + from ._core._tasks import current_effective_deadline as current_effective_deadline + from ._core._tasks import fail_after as fail_after + from ._core._tasks import fail_at as fail_at + from ._core._tasks import move_on_after as move_on_after + from ._core._tasks import move_on_at as move_on_at + from ._core._tempfile import NamedTemporaryFile as NamedTemporaryFile + from ._core._tempfile import SpooledTemporaryFile as SpooledTemporaryFile + from ._core._tempfile import TemporaryDirectory as TemporaryDirectory + from ._core._tempfile import TemporaryFile as TemporaryFile + from ._core._tempfile import gettempdir as gettempdir + from ._core._tempfile import gettempdirb as gettempdirb + from ._core._tempfile import mkdtemp as mkdtemp + from ._core._tempfile import mkstemp as mkstemp + from ._core._testing import TaskInfo as TaskInfo + from ._core._testing import get_current_task as get_current_task + from ._core._testing import get_running_tasks as get_running_tasks + from ._core._testing import wait_all_tasks_blocked as wait_all_tasks_blocked + from ._core._typedattr import TypedAttributeProvider as TypedAttributeProvider + from ._core._typedattr import TypedAttributeSet as TypedAttributeSet + from ._core._typedattr import typed_attribute as typed_attribute + + # ruff: isort: off + from . import ( + abc as abc, + ) + from . import ( + from_thread as from_thread, + ) + from . import ( + functools as functools, + ) + from . import ( + itertools as itertools, + ) + from . import ( + lowlevel as lowlevel, + ) + from . import ( + to_interpreter as to_interpreter, + ) + from . import ( + to_process as to_process, + ) + from . import ( + to_thread as to_thread, + ) + + fix_package_names() + set_deprecated_aliases( + { + "BrokenWorkerIntepreter": "anyio.BrokenWorkerInterpreter", + } + ) diff --git a/.venv/lib/python3.14/site-packages/anyio/__pycache__/__init__.cpython-314.pyc b/.venv/lib/python3.14/site-packages/anyio/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5697bd803f09a6d0d6b30894048cddb2a2b1741c Binary files /dev/null and b/.venv/lib/python3.14/site-packages/anyio/__pycache__/__init__.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/anyio/__pycache__/_lazyimport.cpython-314.pyc b/.venv/lib/python3.14/site-packages/anyio/__pycache__/_lazyimport.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b1c8264cfd2a741b987ca2e1c5edbac9c053966a Binary files /dev/null and b/.venv/lib/python3.14/site-packages/anyio/__pycache__/_lazyimport.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/anyio/__pycache__/from_thread.cpython-314.pyc b/.venv/lib/python3.14/site-packages/anyio/__pycache__/from_thread.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b49c087d7b0a20b69392ccbf5b3daec2844f5376 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/anyio/__pycache__/from_thread.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/anyio/__pycache__/functools.cpython-314.pyc b/.venv/lib/python3.14/site-packages/anyio/__pycache__/functools.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3edc91c98e96778315af1055a74b5538801efba0 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/anyio/__pycache__/functools.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/anyio/__pycache__/itertools.cpython-314.pyc b/.venv/lib/python3.14/site-packages/anyio/__pycache__/itertools.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9227f18e9d40650f7edb90996c93bda9d911ff8c Binary files /dev/null and b/.venv/lib/python3.14/site-packages/anyio/__pycache__/itertools.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/anyio/__pycache__/lowlevel.cpython-314.pyc b/.venv/lib/python3.14/site-packages/anyio/__pycache__/lowlevel.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6f824ff602a13f2b579ae08dad148c36c51b8930 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/anyio/__pycache__/lowlevel.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/anyio/__pycache__/pytest_plugin.cpython-314.pyc b/.venv/lib/python3.14/site-packages/anyio/__pycache__/pytest_plugin.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bea189ff4e8ba4962edec553d5b11cd218ce8b85 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/anyio/__pycache__/pytest_plugin.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/anyio/__pycache__/to_interpreter.cpython-314.pyc b/.venv/lib/python3.14/site-packages/anyio/__pycache__/to_interpreter.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..31eca09725ec1c33801bbd0dcccbc21d3711ae1b Binary files /dev/null and b/.venv/lib/python3.14/site-packages/anyio/__pycache__/to_interpreter.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/anyio/__pycache__/to_process.cpython-314.pyc b/.venv/lib/python3.14/site-packages/anyio/__pycache__/to_process.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5782e7cdd3faa51d1e78ed91b95a0bc3d0d191d1 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/anyio/__pycache__/to_process.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/anyio/__pycache__/to_thread.cpython-314.pyc b/.venv/lib/python3.14/site-packages/anyio/__pycache__/to_thread.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f5ffc6fc93d8fc5e65f71052a355b0f067c32aee Binary files /dev/null and b/.venv/lib/python3.14/site-packages/anyio/__pycache__/to_thread.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/anyio/_backends/__init__.py b/.venv/lib/python3.14/site-packages/anyio/_backends/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.venv/lib/python3.14/site-packages/anyio/_backends/__pycache__/__init__.cpython-314.pyc b/.venv/lib/python3.14/site-packages/anyio/_backends/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7d529eab22a4e866509a7803574bef7cea52f256 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/anyio/_backends/__pycache__/__init__.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/anyio/_backends/__pycache__/_asyncio.cpython-314.pyc b/.venv/lib/python3.14/site-packages/anyio/_backends/__pycache__/_asyncio.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..efc484ccccd9428183ecb117be6153f8e108ebf0 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/anyio/_backends/__pycache__/_asyncio.cpython-314.pyc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e188e14da7a55a3731f1c9ed351e35fd48616734cb483582843f14b2b19159f +size 182577 diff --git a/.venv/lib/python3.14/site-packages/anyio/_backends/__pycache__/_trio.cpython-314.pyc b/.venv/lib/python3.14/site-packages/anyio/_backends/__pycache__/_trio.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..83f9eafef5d10b87f73ee76ac7f486382e96b714 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/anyio/_backends/__pycache__/_trio.cpython-314.pyc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6789b427c9f38804c593ea6a8947aedc00166998aa7904b8dfca3d629485e06d +size 101861 diff --git a/.venv/lib/python3.14/site-packages/anyio/_backends/_asyncio.py b/.venv/lib/python3.14/site-packages/anyio/_backends/_asyncio.py new file mode 100644 index 0000000000000000000000000000000000000000..7eba257c962819dbf8192b51185a4f5faa4746fd --- /dev/null +++ b/.venv/lib/python3.14/site-packages/anyio/_backends/_asyncio.py @@ -0,0 +1,3201 @@ +from __future__ import annotations + +import array +import asyncio +import concurrent.futures +import math +import os +import socket +import sys +import threading +import weakref +from asyncio import ( + AbstractEventLoop, + CancelledError, + all_tasks, + create_task, + current_task, + get_running_loop, + sleep, +) +from asyncio.base_events import _run_until_complete_cb # type: ignore[attr-defined] +from collections import OrderedDict, deque +from collections.abc import ( + AsyncGenerator, + AsyncIterator, + Awaitable, + Callable, + Collection, + Coroutine, + Iterable, + Sequence, +) +from concurrent.futures import Future +from contextlib import AbstractContextManager, suppress +from contextvars import Context, copy_context +from dataclasses import dataclass, field +from functools import partial, wraps +from inspect import ( + CORO_RUNNING, + CORO_SUSPENDED, + getcoroutinestate, +) +from io import IOBase +from os import PathLike +from queue import Queue +from signal import Signals +from socket import AddressFamily, SocketKind +from threading import Thread +from types import CodeType, TracebackType +from typing import ( + IO, + TYPE_CHECKING, + Any, + Literal, + ParamSpec, + TypeVar, + cast, +) +from weakref import WeakKeyDictionary + +from .. import ( + CapacityLimiterStatistics, + EventStatistics, + LockStatistics, + TaskInfo, + abc, +) +from .._core._eventloop import ( + claim_worker_thread, + set_current_async_library, + threadlocals, +) +from .._core._exceptions import ( + BrokenResourceError, + BusyResourceError, + ClosedResourceError, + EndOfStream, + RunFinishedError, + WouldBlock, +) +from .._core._sockets import convert_ipv6_sockaddr +from .._core._streams import create_memory_object_stream +from .._core._synchronization import ( + CapacityLimiter as BaseCapacityLimiter, +) +from .._core._synchronization import Event as BaseEvent +from .._core._synchronization import Lock as BaseLock +from .._core._synchronization import ( + ResourceGuard, + SemaphoreStatistics, +) +from .._core._synchronization import Semaphore as BaseSemaphore +from .._core._tasks import CancelScope as BaseCancelScope +from .._core._tasks import TaskHandle +from ..abc import ( + AsyncBackend, + IPSockAddrType, + SocketListener, + UDPPacketType, + UNIXDatagramPacketType, +) +from ..abc._tasks import call_for_coroutine, get_callable_name, get_coro_name +from ..lowlevel import RunVar, _run_vars + +if TYPE_CHECKING: + from _typeshed import FileDescriptorLike + + from ..abc._eventloop import StrOrBytesPath + from ..streams.memory import MemoryObjectReceiveStream, MemoryObjectSendStream +else: + FileDescriptorLike = object + +if sys.version_info >= (3, 11): + from asyncio import Runner + from typing import Self, TypeVarTuple, Unpack +else: + import contextvars + import enum + import signal + from asyncio import coroutines, events, exceptions, tasks + + from exceptiongroup import BaseExceptionGroup + from typing_extensions import Self, TypeVarTuple, Unpack + + class _State(enum.Enum): + CREATED = "created" + INITIALIZED = "initialized" + CLOSED = "closed" + + class Runner: + # Copied from CPython 3.11 + def __init__( + self, + *, + debug: bool | None = None, + loop_factory: Callable[[], AbstractEventLoop] | None = None, + ): + self._state = _State.CREATED + self._debug = debug + self._loop_factory = loop_factory + self._loop: AbstractEventLoop | None = None + self._context = None + self._interrupt_count = 0 + self._set_event_loop = False + + def __enter__(self) -> Self: + self._lazy_init() + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> None: + self.close() + + def close(self) -> None: + """Shutdown and close event loop.""" + loop = self._loop + if self._state is not _State.INITIALIZED or loop is None: + return + try: + _cancel_all_tasks(loop) + loop.run_until_complete(loop.shutdown_asyncgens()) + if hasattr(loop, "shutdown_default_executor"): + loop.run_until_complete(loop.shutdown_default_executor()) + else: + loop.run_until_complete(_shutdown_default_executor(loop)) + finally: + if self._set_event_loop: + events.set_event_loop(None) + loop.close() + self._loop = None + self._state = _State.CLOSED + + def get_loop(self) -> AbstractEventLoop: + """Return embedded event loop.""" + self._lazy_init() + return self._loop + + def run(self, coro: Coroutine[T_Retval], *, context=None) -> T_Retval: + """Run a coroutine inside the embedded event loop.""" + if not coroutines.iscoroutine(coro): + raise ValueError(f"a coroutine was expected, got {coro!r}") + + if events._get_running_loop() is not None: + # fail fast with short traceback + raise RuntimeError( + "Runner.run() cannot be called from a running event loop" + ) + + self._lazy_init() + + if context is None: + context = self._context + task = context.run(self._loop.create_task, coro) + + if ( + threading.current_thread() is threading.main_thread() + and signal.getsignal(signal.SIGINT) is signal.default_int_handler + ): + sigint_handler = partial(self._on_sigint, main_task=task) + try: + signal.signal(signal.SIGINT, sigint_handler) + except ValueError: + # `signal.signal` may throw if `threading.main_thread` does + # not support signals (e.g. embedded interpreter with signals + # not registered - see gh-91880) + sigint_handler = None + else: + sigint_handler = None + + self._interrupt_count = 0 + try: + return self._loop.run_until_complete(task) + except exceptions.CancelledError: + if self._interrupt_count > 0: + uncancel = getattr(task, "uncancel", None) + if uncancel is not None and uncancel() == 0: + raise KeyboardInterrupt # noqa: B904 + raise # CancelledError + finally: + if ( + sigint_handler is not None + and signal.getsignal(signal.SIGINT) is sigint_handler + ): + signal.signal(signal.SIGINT, signal.default_int_handler) + + def _lazy_init(self) -> None: + if self._state is _State.CLOSED: + raise RuntimeError("Runner is closed") + if self._state is _State.INITIALIZED: + return + if self._loop_factory is None: + self._loop = events.new_event_loop() + if not self._set_event_loop: + # Call set_event_loop only once to avoid calling + # attach_loop multiple times on child watchers + events.set_event_loop(self._loop) + self._set_event_loop = True + else: + self._loop = self._loop_factory() + if self._debug is not None: + self._loop.set_debug(self._debug) + self._context = contextvars.copy_context() + self._state = _State.INITIALIZED + + def _on_sigint(self, signum, frame, main_task: asyncio.Task) -> None: + self._interrupt_count += 1 + if self._interrupt_count == 1 and not main_task.done(): + main_task.cancel() + # wakeup loop if it is blocked by select() with long timeout + self._loop.call_soon_threadsafe(lambda: None) + return + raise KeyboardInterrupt() + + def _cancel_all_tasks(loop: AbstractEventLoop) -> None: + to_cancel = tasks.all_tasks(loop) + if not to_cancel: + return + + for task in to_cancel: + task.cancel() + + loop.run_until_complete(tasks.gather(*to_cancel, return_exceptions=True)) + + for task in to_cancel: + if task.cancelled(): + continue + if task.exception() is not None: + loop.call_exception_handler( + { + "message": "unhandled exception during asyncio.run() shutdown", + "exception": task.exception(), + "task": task, + } + ) + + async def _shutdown_default_executor(loop: AbstractEventLoop) -> None: + """Schedule the shutdown of the default executor.""" + + def _do_shutdown(future: asyncio.futures.Future) -> None: + try: + loop._default_executor.shutdown(wait=True) # type: ignore[attr-defined] + loop.call_soon_threadsafe(future.set_result, None) + except Exception as ex: + loop.call_soon_threadsafe(future.set_exception, ex) + + loop._executor_shutdown_called = True + if loop._default_executor is None: + return + future = loop.create_future() + thread = threading.Thread(target=_do_shutdown, args=(future,)) + thread.start() + try: + await future + finally: + thread.join() + + +T_Retval = TypeVar("T_Retval") +T_co = TypeVar("T_co", covariant=True) +T_contra = TypeVar("T_contra", contravariant=True) +PosArgsT = TypeVarTuple("PosArgsT") +P = ParamSpec("P") + +_root_task: RunVar[asyncio.Task[Any] | None] = RunVar("_root_task") + + +def find_root_task() -> asyncio.Task: + root_task = _root_task.get(None) + if root_task is not None and not root_task.done(): + return root_task + + # Look for a task that has been started via run_until_complete() + for task in all_tasks(): + if task._callbacks and not task.done(): + for cb, context in task._callbacks: + if ( + cb is _run_until_complete_cb + or getattr(cb, "__module__", None) == "uvloop.loop" + ): + _root_task.set(task) + + def _unset(t: asyncio.Task[Any]) -> None: + if vars := _run_vars.get(t.get_loop()): + vars.pop(_root_task, None) + + # Register a callback to break the task -> loop -> _run_var[loop][_root_task] -> task cycle + # Also run it in its own context to not create another reference. + # We can't use RunVar.reset() here since these are called synchronously + # and thus lowlevel.current_token() (which RunVar.reset() depends on) fails. + task.add_done_callback(_unset, context=context) + return task + + # Look up the topmost task in the AnyIO task tree, if possible + task = cast(asyncio.Task, current_task()) + state = _task_states.get(task) + if state: + cancel_scope = state.cancel_scope + while cancel_scope and cancel_scope._parent_scope is not None: + cancel_scope = cancel_scope._parent_scope + + if cancel_scope is not None: + return cast(asyncio.Task, cancel_scope._host_task) + + return task + + +def _task_started(task: asyncio.Task) -> bool: + """Return ``True`` if the task has been started and has not finished.""" + # The task coro should never be None here, as we never add finished tasks to the + # task list + coro = task.get_coro() + assert coro is not None + return getcoroutinestate(coro) in (CORO_RUNNING, CORO_SUSPENDED) + + +# +# Timeouts and cancellation +# + + +def is_anyio_cancellation(exc: CancelledError) -> bool: + # Sometimes third party frameworks catch a CancelledError and raise a new one, so as + # a workaround we have to look at the previous ones in __context__ too for a + # matching cancel message + while True: + if ( + exc.args + and isinstance(exc.args[0], str) + and exc.args[0].startswith("Cancelled via cancel scope ") + ): + return True + + if isinstance(exc.__context__, CancelledError): + exc = exc.__context__ + continue + + return False + + +class CancelScope(BaseCancelScope): + __slots__ = ( + "_active", + "_cancel_called", + "_cancel_handle", + "_cancel_reason", + "_cancelled_caught", + "_child_scopes", + "_deadline", + "_host_task", + "_parent_scope", + "_pending_uncancellations", + "_shield", + "_tasks", + "_timeout_handle", + ) + + def __new__(cls, *, deadline: float = math.inf, shield: bool = False) -> Self: + return object.__new__(cls) + + def __init__(self, deadline: float = math.inf, shield: bool = False): + self._deadline = deadline + self._shield = shield + self._parent_scope: CancelScope | None = None + self._child_scopes: set[CancelScope] = set() + self._cancel_called = False + self._cancel_reason: str | None = None + self._cancelled_caught = False + self._active = False + self._timeout_handle: asyncio.TimerHandle | None = None + self._cancel_handle: asyncio.Handle | None = None + self._tasks: set[asyncio.Task] = set() + self._host_task: asyncio.Task | None = None + if sys.version_info >= (3, 11): + self._pending_uncancellations: int | None = 0 + else: + self._pending_uncancellations = None + + def __enter__(self) -> Self: + if self._active: + raise RuntimeError( + "Each CancelScope may only be used for a single 'with' block" + ) + + self._host_task = host_task = cast(asyncio.Task, current_task()) + self._tasks.add(host_task) + try: + task_state = _task_states[host_task] + except KeyError: + task_state = TaskState(None, self) + _task_states[host_task] = task_state + else: + self._parent_scope = task_state.cancel_scope + task_state.cancel_scope = self + if self._parent_scope is not None: + # If using an eager task factory, the parent scope may not even contain + # the host task + self._parent_scope._child_scopes.add(self) + self._parent_scope._tasks.discard(host_task) + + self._timeout() + self._active = True + + # Start cancelling the host task if the scope was cancelled before entering + if self._cancel_called: + self._deliver_cancellation(self) + + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> bool: + del exc_tb + + if not self._active: + raise RuntimeError("This cancel scope is not active") + if current_task() is not self._host_task: + raise RuntimeError( + "Attempted to exit cancel scope in a different task than it was " + "entered in" + ) + + assert self._host_task is not None + host_task_state = _task_states.get(self._host_task) + if host_task_state is None or host_task_state.cancel_scope is not self: + raise RuntimeError( + "Attempted to exit a cancel scope that isn't the current tasks's " + "current cancel scope" + ) + + try: + self._active = False + if self._timeout_handle: + self._timeout_handle.cancel() + self._timeout_handle = None + + self._tasks.remove(self._host_task) + if self._parent_scope is not None: + self._parent_scope._child_scopes.remove(self) + self._parent_scope._tasks.add(self._host_task) + + host_task_state.cancel_scope = self._parent_scope + + # Restart the cancellation effort in the closest visible, cancelled parent + # scope if necessary + self._restart_cancellation_in_parent() + + # We only swallow the exception iff it was an AnyIO CancelledError, either + # directly as exc_val or inside an exception group and there are no cancelled + # parent cancel scopes visible to us here + if self._cancel_called and not self._parent_cancellation_is_visible_to_us: + # For each level-cancel() call made on the host task, call uncancel() + while self._pending_uncancellations: + self._host_task.uncancel() + self._pending_uncancellations -= 1 + + # Update cancelled_caught and check for exceptions we must not swallow + if isinstance(exc_val, BaseExceptionGroup): + cancelleds_caught, remaining = exc_val.split( + lambda exc: ( + isinstance(exc, CancelledError) + and is_anyio_cancellation(exc) + ) + ) + + if cancelleds_caught is None: + return False + + self._cancelled_caught = True + + if remaining is None: + return True + + context = remaining.__context__ + try: + # Preserve __cause__ and __suppress_context__ by avoiding `raise + # ... from ...` + raise remaining + finally: + # Preserve __context__ + remaining.__context__ = context + del context + else: + if isinstance(exc_val, CancelledError) and is_anyio_cancellation( + exc_val + ): + self._cancelled_caught = True + return True + else: + return False + else: + if self._pending_uncancellations: + assert self._parent_scope is not None + assert self._parent_scope._pending_uncancellations is not None + self._parent_scope._pending_uncancellations += ( + self._pending_uncancellations + ) + self._pending_uncancellations = 0 + + return False + finally: + self._host_task = None + del exc_val + + @property + def _effectively_cancelled(self) -> bool: + cancel_scope: CancelScope | None = self + while cancel_scope is not None: + if cancel_scope._cancel_called: + return True + + if cancel_scope.shield: + return False + + cancel_scope = cancel_scope._parent_scope + + return False + + @property + def _parent_cancellation_is_visible_to_us(self) -> bool: + return ( + self._parent_scope is not None + and not self.shield + and self._parent_scope._effectively_cancelled + ) + + def _timeout(self) -> None: + if self._deadline != math.inf: + loop = get_running_loop() + if loop.time() >= self._deadline: + self.cancel("deadline exceeded") + else: + self._timeout_handle = loop.call_at(self._deadline, self._timeout) + + def _deliver_cancellation(self, origin: CancelScope) -> bool: + """ + Deliver cancellation to directly contained tasks and nested cancel scopes. + + Schedule another run at the end if we still have tasks eligible for + cancellation. + + :param origin: the cancel scope that originated the cancellation + :return: ``True`` if the delivery needs to be retried on the next cycle + + """ + should_retry = False + current = current_task() + for task in self._tasks: + # Always skip tasks that are already done (see issue #1111) + if task.done(): + continue + + should_retry = True + if task._must_cancel: # type: ignore[attr-defined] + continue + + # The task is eligible for cancellation if it has started + if task is not current and (task is self._host_task or _task_started(task)): + waiter = task._fut_waiter # type: ignore[attr-defined] + if not isinstance(waiter, asyncio.Future) or not waiter.done(): + task.cancel(origin._cancel_reason) + if ( + task is origin._host_task + and origin._pending_uncancellations is not None + ): + origin._pending_uncancellations += 1 + + # Deliver cancellation to child scopes that aren't shielded or running their own + # cancellation callbacks + for scope in self._child_scopes: + if not scope._shield and not scope.cancel_called: + should_retry = scope._deliver_cancellation(origin) or should_retry + + # Schedule another callback if there are still tasks left + if origin is self: + if should_retry: + self._cancel_handle = get_running_loop().call_soon( + self._deliver_cancellation, origin + ) + else: + self._cancel_handle = None + + return should_retry + + def _restart_cancellation_in_parent(self) -> None: + """ + Restart the cancellation effort in the closest directly cancelled parent scope. + + """ + scope = self._parent_scope + while scope is not None: + if scope._cancel_called: + if scope._cancel_handle is None: + scope._deliver_cancellation(scope) + + break + + # No point in looking beyond any shielded scope + if scope._shield: + break + + scope = scope._parent_scope + + def _reparent(self, new_parent: CancelScope) -> None: + """ + Move this active scope from its current parent to ``new_parent``. + + Used by :meth:`TaskGroup.start` to move a task that has just called + ``task_status.started()`` into the target task group's cancel scope. + """ + if self._parent_scope is new_parent: + return + + if self._parent_scope is not None: + self._parent_scope._child_scopes.discard(self) + + self._parent_scope = new_parent + new_parent._child_scopes.add(self) + + # If the new parent (or an ancestor) is already cancelled, (re)start the + # delivery loop to ensure we're cancelled at next checkpoint like Trio. + self._restart_cancellation_in_parent() + + def cancel(self, reason: str | None = None) -> None: + if not self._cancel_called: + if self._timeout_handle: + self._timeout_handle.cancel() + self._timeout_handle = None + + self._cancel_called = True + self._cancel_reason = f"Cancelled via cancel scope {id(self):x}" + if task := current_task(): + self._cancel_reason += f" by {task}" + + if reason: + self._cancel_reason += f"; reason: {reason}" + + if self._host_task is not None: + self._deliver_cancellation(self) + + @property + def deadline(self) -> float: + return self._deadline + + @deadline.setter + def deadline(self, value: float) -> None: + self._deadline = float(value) + if self._timeout_handle is not None: + self._timeout_handle.cancel() + self._timeout_handle = None + + if self._active and not self._cancel_called: + self._timeout() + + @property + def cancel_called(self) -> bool: + return self._cancel_called + + @property + def cancelled_caught(self) -> bool: + return self._cancelled_caught + + @property + def shield(self) -> bool: + return self._shield + + @shield.setter + def shield(self, value: bool) -> None: + if self._shield != value: + self._shield = value + if not value: + self._restart_cancellation_in_parent() + + +# +# Task states +# + + +class TaskState: + """ + Encapsulates auxiliary task information that cannot be added to the Task instance + itself because there are no guarantees about its implementation. + """ + + __slots__ = "__weakref__", "cancel_scope", "parent_id" + + def __init__(self, parent_id: int | None, cancel_scope: CancelScope | None): + self.parent_id = parent_id + self.cancel_scope = cancel_scope + + +_task_states: WeakKeyDictionary[asyncio.Task, TaskState] = WeakKeyDictionary() + + +# +# Task groups +# + + +class _AsyncioTaskStatus(abc.TaskStatus[T_contra]): + def __init__( + self, + future: asyncio.Future, + parent_id: int, + target_scope: CancelScope, + spawn_scope: CancelScope, + ): + self._future = future + self._parent_id = parent_id + # The eventual parent scope for this spawn_scope + # (after task_status.started() has been called) + self._target_scope = target_scope + # The task's own cancel scope, also held by its TaskHandle + self._spawn_scope = spawn_scope + + def started(self, value: T_contra | None = None) -> None: + task = cast(asyncio.Task, current_task()) + _task_states[task].parent_id = self._parent_id + if self._future.done(): + if not self._future.cancelled(): + raise RuntimeError("called 'started' twice on the same task status") + else: + # Caller of start() was cancelled, nothing to reparent + return + + self._future.set_result(value) + self._spawn_scope._reparent(self._target_scope) + + +if sys.version_info >= (3, 12): + _eager_task_factory_code: CodeType | None = asyncio.eager_task_factory.__code__ +else: + _eager_task_factory_code = None + + +class TaskGroup(abc.TaskGroup): + def __init__(self) -> None: + self.cancel_scope: CancelScope = CancelScope() + self._entered = False + self._exceptions: list[BaseException] = [] + self._tasks: set[asyncio.Task] = set() + self._on_completed_fut: asyncio.Future[None] | None = None + + async def __aenter__(self) -> Self: + if self._entered: + raise RuntimeError("TaskGroup cannot be entered more than once") + + self._entered = True + + self.cancel_scope.__enter__() + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> bool: + try: + if exc_val is not None: + self.cancel_scope.cancel() + if not isinstance(exc_val, CancelledError): + self._exceptions.append(exc_val) + + loop = get_running_loop() + try: + if self._tasks: + with CancelScope() as wait_scope: + while self._tasks: + self._on_completed_fut = loop.create_future() + + try: + await self._on_completed_fut + except CancelledError as exc: + # Shield the scope against further cancellation attempts, + # as they're not productive (#695) + wait_scope.shield = True + self.cancel_scope.cancel() + + # Set exc_val from the cancellation exception if it was + # previously unset. However, we should not replace a native + # cancellation exception with one raise by a cancel scope. + if exc_val is None or ( + isinstance(exc_val, CancelledError) + and not is_anyio_cancellation(exc) + ): + exc_val = exc + + self._on_completed_fut = None + else: + # If there are no child tasks to wait on, run at least one checkpoint + # anyway + await AsyncIOBackend.cancel_shielded_checkpoint() + + if self._exceptions: + # The exception that got us here should already have been + # added to self._exceptions so it's ok to break exception + # chaining and avoid adding a "During handling of above..." + # for each nesting level. + raise BaseExceptionGroup( + "unhandled errors in a TaskGroup", self._exceptions + ) from None + elif exc_val: + raise exc_val + except BaseException as exc: + if self.cancel_scope.__exit__(type(exc), exc, exc.__traceback__): + return True + + raise + + return self.cancel_scope.__exit__(exc_type, exc_val, exc_tb) + finally: + del exc_val, exc_tb, self._exceptions + + def _spawn( + self, + coro: Coroutine[Any, Any, T_co], + name: object, + task_status: _AsyncioTaskStatus | None = None, + ) -> TaskHandle[T_co]: + task_status_future = task_status._future if task_status is not None else None + + def task_done(_task: asyncio.Task) -> None: + if sys.version_info >= (3, 14) and self.cancel_scope._host_task is not None: + asyncio.future_discard_from_awaited_by( + _task, self.cancel_scope._host_task + ) + + task_state = _task_states[_task] + assert task_state.cancel_scope is not None + assert _task in task_state.cancel_scope._tasks + task_state.cancel_scope._tasks.remove(_task) + self._tasks.remove(task) + del _task_states[_task] + + if self._on_completed_fut is not None and not self._tasks: + try: + self._on_completed_fut.set_result(None) + except asyncio.InvalidStateError: + pass + + try: + exc = _task.exception() + except CancelledError as e: + while isinstance(e.__context__, CancelledError): + e = e.__context__ + + exc = e + + if exc is not None: + # The future can only be in the cancelled state if the host task was + # cancelled, so return immediately instead of adding one more + # CancelledError to the exceptions list + if task_status_future is not None and task_status_future.cancelled(): + return + + if task_status_future is None or task_status_future.done(): + if not isinstance(exc, CancelledError): + self._exceptions.append(exc) + + if not self.cancel_scope._effectively_cancelled: + self.cancel_scope.cancel() + else: + task_status_future.set_exception(exc) + elif task_status_future is not None and not task_status_future.done(): + task_status_future.set_exception( + RuntimeError("Child exited without calling task_status.started()") + ) + + if task_status_future is not None: + parent_id = id(current_task()) + caller_state = _task_states.get(cast(asyncio.Task, current_task())) + if caller_state is not None and caller_state.cancel_scope is not None: + initial_scope = caller_state.cancel_scope + else: + # The caller is an unmanaged task (no task state) + initial_scope = self.cancel_scope + else: + parent_id = id(self.cancel_scope._host_task) + initial_scope = self.cancel_scope + + spawn_scope = task_status._spawn_scope if task_status is not None else None + handle = TaskHandle(coro, name, cancel_scope=spawn_scope) + loop = asyncio.get_running_loop() + wrapper_coro = handle._run_coro() + try: + if ( + (factory := loop.get_task_factory()) + and getattr(factory, "__code__", None) is _eager_task_factory_code + and (closure := getattr(factory, "__closure__", None)) + ): + custom_task_constructor = closure[0].cell_contents + task = custom_task_constructor( + wrapper_coro, loop=loop, name=handle.name + ) + else: + task = loop.create_task(wrapper_coro, name=handle.name) + except BaseException: + with suppress(BaseException): + wrapper_coro.close() + + with suppress(BaseException): + coro.close() + + raise + + # Make the spawned task inherit the initial cancel scope + _task_states[task] = TaskState(parent_id=parent_id, cancel_scope=initial_scope) + initial_scope._tasks.add(task) + self._tasks.add(task) + if sys.version_info >= (3, 14) and self.cancel_scope._host_task is not None: + asyncio.future_add_to_awaited_by(task, self.cancel_scope._host_task) + + task.add_done_callback(task_done) + return handle + + def create_task( + self, + coro: Coroutine[Any, Any, T_co], + *, + name: object = None, + context: Context | None = None, + ) -> TaskHandle[T_co]: + if not isinstance(coro, Coroutine): + raise TypeError(f"expected a coroutine, got {coro.__class__.__qualname__}") + + if not self._entered or not self.cancel_scope._active: + coro.close() + raise RuntimeError( + "This task group is not active; no new tasks can be started." + ) + + final_name = get_coro_name(coro, name) + + if context is not None: + return context.run(self._spawn, coro, name=final_name) + else: + return self._spawn(coro, name=final_name) + + async def start( + self, + func: Callable[[Unpack[PosArgsT]], Coroutine[Any, Any, T_co]], + *args: Unpack[PosArgsT], + name: object = None, + return_handle: Literal[False, True] = False, + ) -> Any: + if not self._entered or not self.cancel_scope._active: + raise RuntimeError( + "This task group is not active; no new tasks can be started." + ) + + # Until task_status.started() is called, a task spawned via start() belongs to + # the *caller's* cancel scope, not the target group's, so cancelling the group + # does not cancel a task that hasn't reported startup yet. The + # task_status.started() call moves the task's cancel scope to this task group's + # cancel scope. + # + # The caller may be an unmanaged task (no _task_states entry), in which case + # fall back to the group's own scope. + future: asyncio.Future = asyncio.Future() + final_name = get_callable_name(func, name) + task_status: _AsyncioTaskStatus[Any] = _AsyncioTaskStatus( + future, id(self.cancel_scope._host_task), self.cancel_scope, CancelScope() + ) + coro = call_for_coroutine(func, args, task_status=task_status) + handle = self._spawn(coro, final_name, task_status) + + # If the task raises an exception after sending a start value without a switch + # point between, the task group is cancelled and this method never proceeds to + # process the completed future. That's why we have to have a shielded cancel + # scope here. + try: + await future + except BaseException: + if handle.status is TaskHandle.Status.PENDING: + # Cancel the task and wait for it to exit before returning + handle.cancel() + with CancelScope(shield=True): + await handle.wait() + + raise + + if return_handle: + handle._start_value = future.result() + return handle + else: + return future.result() + + +# +# Threads +# + +_Retval_Queue_Type = tuple[T_Retval | None, BaseException | None] + + +class WorkerThread(Thread): + MAX_IDLE_TIME = 10 # seconds + + def __init__( + self, + root_task: asyncio.Task, + workers: set[WorkerThread], + idle_workers: deque[WorkerThread], + ): + kwargs: dict[str, Any] = {} + if sys.version_info >= (3, 14): + kwargs["context"] = Context() + + super().__init__(name="AnyIO worker thread", **kwargs) + self.root_task = root_task + self.workers = workers + self.idle_workers = idle_workers + self.loop = root_task._loop + self.queue: Queue[ + tuple[Context, Callable, tuple, asyncio.Future, CancelScope] | None + ] = Queue(2) + self.idle_since = AsyncIOBackend.current_time() + self.stopping = False + + def _report_result( + self, future: asyncio.Future, result: Any, exc: BaseException | None + ) -> None: + self.idle_since = AsyncIOBackend.current_time() + if not self.stopping: + self.idle_workers.append(self) + + if not future.cancelled(): + if exc is not None: + if isinstance(exc, StopIteration): + new_exc = RuntimeError("coroutine raised StopIteration") + new_exc.__cause__ = exc + exc = new_exc + + future.set_exception(exc) + else: + future.set_result(result) + + def run(self) -> None: + with claim_worker_thread(AsyncIOBackend, self.loop): + while True: + item = self.queue.get() + if item is None: + # Shutdown command received + return + + context, func, args, future, cancel_scope = item + if not future.cancelled(): + result = None + exception: BaseException | None = None + threadlocals.current_cancel_scope = cancel_scope + try: + result = context.run(func, *args) + except BaseException as exc: + exception = exc + finally: + del threadlocals.current_cancel_scope + + try: + self.loop.call_soon_threadsafe( + self._report_result, future, result, exception + ) + except RuntimeError: + if not self.loop.is_closed(): + raise + + del result, exception + + self.queue.task_done() + del item, context, func, args, future, cancel_scope + + def stop(self, f: asyncio.Task | None = None) -> None: + self.stopping = True + self.queue.put_nowait(None) + self.workers.discard(self) + try: + self.idle_workers.remove(self) + except ValueError: + pass + + +_threadpool_idle_workers: RunVar[deque[WorkerThread]] = RunVar( + "_threadpool_idle_workers" +) +_threadpool_workers: RunVar[set[WorkerThread]] = RunVar("_threadpool_workers") + + +# +# Subprocesses +# + + +@dataclass(eq=False) +class StreamReaderWrapper(abc.ByteReceiveStream): + _stream: asyncio.StreamReader + + async def receive(self, max_bytes: int = 65536) -> bytes: + if max_bytes < 1: + raise ValueError("max_bytes must be a positive integer") + + data = await self._stream.read(max_bytes) + if data: + return data + else: + raise EndOfStream + + async def aclose(self) -> None: + self._stream.set_exception(ClosedResourceError()) + await AsyncIOBackend.checkpoint() + + +@dataclass(eq=False) +class StreamWriterWrapper(abc.ByteSendStream): + _stream: asyncio.StreamWriter + _closed: bool = field(init=False, default=False) + + async def send(self, item: bytes) -> None: + await AsyncIOBackend.checkpoint_if_cancelled() + stream_paused = self._stream._protocol._paused # type: ignore[attr-defined] + try: + self._stream.write(item) + await self._stream.drain() + except (ConnectionResetError, BrokenPipeError, RuntimeError) as exc: + # If closed by us and/or the peer: + # * on stdlib, drain() raises ConnectionResetError or BrokenPipeError + # * on uvloop and Winloop, write() eventually starts raising RuntimeError + if self._closed: + raise ClosedResourceError from exc + elif self._stream.is_closing(): + raise BrokenResourceError from exc + + raise + + if not stream_paused: + await AsyncIOBackend.cancel_shielded_checkpoint() + + async def aclose(self) -> None: + self._closed = True + self._stream.close() + await AsyncIOBackend.checkpoint() + + +@dataclass(eq=False) +class Process(abc.Process): + _process: asyncio.subprocess.Process + _stdin: StreamWriterWrapper | None + _stdout: StreamReaderWrapper | None + _stderr: StreamReaderWrapper | None + _exited: asyncio.Event + _transport: asyncio.SubprocessTransport + + async def aclose(self) -> None: + with CancelScope(shield=True) as scope: + # We need to close the underlying pipe_transports as well to allow a + # process blocking on full buffers to receive SIGPIPE and exit. + if self._stdin: + await self._stdin.aclose() + if pipe := self._transport.get_pipe_transport(0): + pipe.close() + if self._stdout: + await self._stdout.aclose() + if pipe := self._transport.get_pipe_transport(1): + pipe.close() + if self._stderr: + await self._stderr.aclose() + if pipe := self._transport.get_pipe_transport(2): + pipe.close() + + scope.shield = False + try: + await self.wait() + except BaseException: + scope.shield = True + # Closing the transport on asyncio also handles sending kill + self._transport.close() + await self.wait() + raise + + async def wait(self) -> int: + await self._exited.wait() + assert self._process.returncode is not None + return self._process.returncode + + def terminate(self) -> None: + self._process.terminate() + + def kill(self) -> None: + self._process.kill() + + def send_signal(self, signal: int) -> None: + self._process.send_signal(signal) + + @property + def pid(self) -> int: + return self._process.pid + + @property + def returncode(self) -> int | None: + return self._process.returncode + + @property + def stdin(self) -> abc.ByteSendStream | None: + return self._stdin + + @property + def stdout(self) -> abc.ByteReceiveStream | None: + return self._stdout + + @property + def stderr(self) -> abc.ByteReceiveStream | None: + return self._stderr + + +def _forcibly_shutdown_process_pool_on_exit( + workers: set[Process], _task: object +) -> None: + """ + Forcibly shuts down worker processes belonging to this event loop.""" + child_watcher: asyncio.AbstractChildWatcher | None = None # type: ignore[name-defined] + if sys.version_info < (3, 12): + try: + child_watcher = asyncio.get_event_loop_policy().get_child_watcher() + except NotImplementedError: + pass + + # Close as much as possible (w/o async/await) to avoid warnings + for process in workers.copy(): + if process.returncode is not None: + continue + + process._stdin._stream._transport.close() # type: ignore[union-attr] + process._stdout._stream._transport.close() # type: ignore[union-attr] + process._stderr._stream._transport.close() # type: ignore[union-attr] + process.kill() + if child_watcher: + child_watcher.remove_child_handler(process.pid) + + +async def _shutdown_process_pool_on_exit(workers: set[abc.Process]) -> None: + """ + Shuts down worker processes belonging to this event loop. + + NOTE: this only works when the event loop was started using asyncio.run() or + anyio.run(). + + """ + process: abc.Process + try: + await sleep(math.inf) + except asyncio.CancelledError: + workers = workers.copy() + for process in workers: + if process.returncode is None: + process.kill() + + for process in workers: + await process.aclose() + + +# +# Sockets and networking +# + + +class StreamProtocol(asyncio.Protocol): + read_queue: deque[bytes] + read_event: asyncio.Event + write_event: asyncio.Event + exception: Exception | None = None + is_at_eof: bool = False + + def connection_made(self, transport: asyncio.BaseTransport) -> None: + self.read_queue = deque() + self.read_event = asyncio.Event() + self.write_event = asyncio.Event() + self.write_event.set() + cast(asyncio.Transport, transport).set_write_buffer_limits(0) + + def connection_lost(self, exc: Exception | None) -> None: + if exc: + self.exception = exc + + self.read_event.set() + self.write_event.set() + + def data_received(self, data: bytes) -> None: + # ProactorEventloop sometimes sends bytearray instead of bytes + self.read_queue.append(bytes(data)) + self.read_event.set() + + def eof_received(self) -> bool | None: + self.is_at_eof = True + self.read_event.set() + return True + + def pause_writing(self) -> None: + self.write_event = asyncio.Event() + + def resume_writing(self) -> None: + self.write_event.set() + + +class DatagramProtocol(asyncio.DatagramProtocol): + read_queue: deque[tuple[bytes, IPSockAddrType]] + read_event: asyncio.Event + write_event: asyncio.Event + closed_event: asyncio.Event + exception: Exception | None = None + + def connection_made(self, transport: asyncio.BaseTransport) -> None: + self.read_queue = deque(maxlen=100) # arbitrary value + self.read_event = asyncio.Event() + self.write_event = asyncio.Event() + self.closed_event = asyncio.Event() + self.write_event.set() + + def connection_lost(self, exc: Exception | None) -> None: + self.read_event.set() + self.write_event.set() + self.closed_event.set() + + def datagram_received(self, data: bytes, addr: IPSockAddrType) -> None: + addr = convert_ipv6_sockaddr(addr) + self.read_queue.append((data, addr)) + self.read_event.set() + + def error_received(self, exc: Exception) -> None: + self.exception = exc + + def pause_writing(self) -> None: + self.write_event.clear() + + def resume_writing(self) -> None: + self.write_event.set() + + +class SocketStream(abc.SocketStream): + def __init__(self, transport: asyncio.Transport, protocol: StreamProtocol): + self._transport = transport + self._protocol = protocol + self._receive_guard = ResourceGuard("reading from") + self._send_guard = ResourceGuard("writing to") + self._closed = False + + @property + def _raw_socket(self) -> socket.socket: + return self._transport.get_extra_info("socket") + + async def receive(self, max_bytes: int = 65536) -> bytes: + if max_bytes < 1: + raise ValueError("max_bytes must be a positive integer") + + with self._receive_guard: + if ( + not self._protocol.read_event.is_set() + and not self._transport.is_closing() + and not self._protocol.is_at_eof + ): + self._transport.resume_reading() + await self._protocol.read_event.wait() + self._transport.pause_reading() + else: + await AsyncIOBackend.checkpoint() + + try: + chunk = self._protocol.read_queue.popleft() + except IndexError: + if self._closed: + raise ClosedResourceError from None + elif self._protocol.exception: + raise BrokenResourceError from self._protocol.exception + else: + raise EndOfStream from None + + if len(chunk) > max_bytes: + # Split the oversized chunk + chunk, leftover = chunk[:max_bytes], chunk[max_bytes:] + self._protocol.read_queue.appendleft(leftover) + + # If the read queue is empty, clear the flag so that the next call will + # block until data is available + if not self._protocol.read_queue: + self._protocol.read_event.clear() + + return chunk + + async def send(self, item: bytes) -> None: + with self._send_guard: + await AsyncIOBackend.checkpoint() + + if self._closed: + raise ClosedResourceError + elif self._protocol.exception is not None: + raise BrokenResourceError from self._protocol.exception + + try: + self._transport.write(item) + except RuntimeError as exc: + if self._transport.is_closing(): + raise BrokenResourceError from exc + else: + raise + + await self._protocol.write_event.wait() + + async def send_eof(self) -> None: + try: + self._transport.write_eof() + except OSError: + pass + + async def aclose(self) -> None: + self._closed = True + if not self._transport.is_closing(): + try: + self._transport.write_eof() + except OSError: + pass + + self._transport.close() + await sleep(0) + self._transport.abort() + + +class _RawSocketMixin: + _receive_future: asyncio.Future | None = None + _send_future: asyncio.Future | None = None + _closing = False + + def __init__(self, raw_socket: socket.socket): + self.__raw_socket = raw_socket + self._receive_guard = ResourceGuard("reading from") + self._send_guard = ResourceGuard("writing to") + + @property + def _raw_socket(self) -> socket.socket: + return self.__raw_socket + + def _wait_until_readable(self, loop: asyncio.AbstractEventLoop) -> asyncio.Future: + def callback(f: object) -> None: + del self._receive_future + loop.remove_reader(self.__raw_socket) + + f = self._receive_future = asyncio.Future() + loop.add_reader(self.__raw_socket, f.set_result, None) + f.add_done_callback(callback) + return f + + def _wait_until_writable(self, loop: asyncio.AbstractEventLoop) -> asyncio.Future: + def callback(f: object) -> None: + del self._send_future + loop.remove_writer(self.__raw_socket) + + f = self._send_future = asyncio.Future() + loop.add_writer(self.__raw_socket, f.set_result, None) + f.add_done_callback(callback) + return f + + async def aclose(self) -> None: + if not self._closing: + self._closing = True + if self.__raw_socket.fileno() != -1: + self.__raw_socket.close() + + if self._receive_future and not self._receive_future.done(): + self._receive_future.set_result(None) + if self._send_future and not self._send_future.done(): + self._send_future.set_result(None) + + +class UNIXSocketStream(_RawSocketMixin, abc.UNIXSocketStream): + async def send_eof(self) -> None: + with self._send_guard: + self._raw_socket.shutdown(socket.SHUT_WR) + + async def receive(self, max_bytes: int = 65536) -> bytes: + if max_bytes < 1: + raise ValueError("max_bytes must be a positive integer") + + loop = get_running_loop() + await AsyncIOBackend.checkpoint() + with self._receive_guard: + while True: + try: + data = self._raw_socket.recv(max_bytes) + except BlockingIOError: + await self._wait_until_readable(loop) + except OSError as exc: + if self._closing: + raise ClosedResourceError from None + else: + raise BrokenResourceError from exc + else: + if not data: + raise EndOfStream + + return data + + async def send(self, item: bytes) -> None: + loop = get_running_loop() + await AsyncIOBackend.checkpoint() + with self._send_guard: + view = memoryview(item) + while view: + try: + bytes_sent = self._raw_socket.send(view) + except BlockingIOError: + await self._wait_until_writable(loop) + except OSError as exc: + if self._closing: + raise ClosedResourceError from None + else: + raise BrokenResourceError from exc + else: + view = view[bytes_sent:] + + async def receive_fds(self, msglen: int, maxfds: int) -> tuple[bytes, list[int]]: + if not isinstance(msglen, int) or msglen < 0: + raise ValueError("msglen must be a non-negative integer") + if not isinstance(maxfds, int) or maxfds < 1: + raise ValueError("maxfds must be a positive integer") + + loop = get_running_loop() + fds = array.array("i") + await AsyncIOBackend.checkpoint() + with self._receive_guard: + while True: + try: + message, ancdata, _flags, _addr = self._raw_socket.recvmsg( + msglen, socket.CMSG_LEN(maxfds * fds.itemsize) + ) + except BlockingIOError: + await self._wait_until_readable(loop) + except OSError as exc: + if self._closing: + raise ClosedResourceError from None + else: + raise BrokenResourceError from exc + else: + if not message and not ancdata: + raise EndOfStream + + break + + for cmsg_level, cmsg_type, cmsg_data in ancdata: + if cmsg_level != socket.SOL_SOCKET or cmsg_type != socket.SCM_RIGHTS: + raise RuntimeError( + f"Received unexpected ancillary data; message = {message!r}, " + f"cmsg_level = {cmsg_level}, cmsg_type = {cmsg_type}" + ) + + fds.frombytes(cmsg_data[: len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) + + return message, list(fds) + + async def send_fds(self, message: bytes, fds: Collection[int | IOBase]) -> None: + if not message: + raise ValueError("message must not be empty") + if not fds: + raise ValueError("fds must not be empty") + + loop = get_running_loop() + filenos: list[int] = [] + for fd in fds: + if isinstance(fd, int): + filenos.append(fd) + elif isinstance(fd, IOBase): + filenos.append(fd.fileno()) + + fdarray = array.array("i", filenos) + await AsyncIOBackend.checkpoint() + with self._send_guard: + while True: + try: + # The ignore can be removed after mypy picks up + # https://github.com/python/typeshed/pull/5545 + self._raw_socket.sendmsg( + [message], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, fdarray)] + ) + break + except BlockingIOError: + await self._wait_until_writable(loop) + except OSError as exc: + if self._closing: + raise ClosedResourceError from None + else: + raise BrokenResourceError from exc + + +class TCPSocketListener(abc.SocketListener): + _accept_scope: CancelScope | None = None + _closed = False + + def __init__(self, raw_socket: socket.socket): + self.__raw_socket = raw_socket + self._loop = cast(asyncio.BaseEventLoop, get_running_loop()) + self._accept_guard = ResourceGuard("accepting connections from") + + @property + def _raw_socket(self) -> socket.socket: + return self.__raw_socket + + async def accept(self) -> abc.SocketStream: + if self._closed: + raise ClosedResourceError + + with self._accept_guard: + await AsyncIOBackend.checkpoint() + with CancelScope() as self._accept_scope: + try: + client_sock, _addr = await self._loop.sock_accept(self._raw_socket) + except asyncio.CancelledError: + # Workaround for https://bugs.python.org/issue41317 + try: + self._loop.remove_reader(self._raw_socket) + except (ValueError, NotImplementedError): + pass + + if self._closed: + raise ClosedResourceError from None + + raise + finally: + self._accept_scope = None + + client_sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + transport, protocol = await self._loop.connect_accepted_socket( + StreamProtocol, client_sock + ) + return SocketStream(transport, protocol) + + async def aclose(self) -> None: + if self._closed: + return + + self._closed = True + if self._accept_scope: + # Workaround for https://bugs.python.org/issue41317 + try: + self._loop.remove_reader(self._raw_socket) + except (ValueError, NotImplementedError): + pass + + self._accept_scope.cancel() + await sleep(0) + + self._raw_socket.close() + + +class UNIXSocketListener(abc.SocketListener): + def __init__(self, raw_socket: socket.socket): + self.__raw_socket = raw_socket + self._loop = get_running_loop() + self._accept_guard = ResourceGuard("accepting connections from") + self._closed = False + + async def accept(self) -> abc.SocketStream: + await AsyncIOBackend.checkpoint() + with self._accept_guard: + while True: + try: + client_sock, _ = self.__raw_socket.accept() + client_sock.setblocking(False) + return UNIXSocketStream(client_sock) + except BlockingIOError: + f: asyncio.Future = asyncio.Future() + self._loop.add_reader(self.__raw_socket, f.set_result, None) + f.add_done_callback( + lambda _: self._loop.remove_reader(self.__raw_socket) + ) + await f + except OSError as exc: + if self._closed: + raise ClosedResourceError from None + else: + raise BrokenResourceError from exc + + async def aclose(self) -> None: + self._closed = True + self.__raw_socket.close() + + @property + def _raw_socket(self) -> socket.socket: + return self.__raw_socket + + +class UDPSocket(abc.UDPSocket): + def __init__( + self, transport: asyncio.DatagramTransport, protocol: DatagramProtocol + ): + self._transport = transport + self._protocol = protocol + self._receive_guard = ResourceGuard("reading from") + self._send_guard = ResourceGuard("writing to") + self._closed = False + + @property + def _raw_socket(self) -> socket.socket: + return self._transport.get_extra_info("socket") + + async def aclose(self) -> None: + self._closed = True + if not self._transport.is_closing(): + self._transport.close() + + await self._protocol.closed_event.wait() + + async def receive(self) -> tuple[bytes, IPSockAddrType]: + with self._receive_guard: + await AsyncIOBackend.checkpoint() + + # If the buffer is empty, ask for more data + if not self._protocol.read_queue and not self._transport.is_closing(): + self._protocol.read_event.clear() + await self._protocol.read_event.wait() + + try: + return self._protocol.read_queue.popleft() + except IndexError: + if self._closed: + raise ClosedResourceError from None + else: + raise BrokenResourceError from None + + async def send(self, item: UDPPacketType) -> None: + with self._send_guard: + await AsyncIOBackend.checkpoint() + await self._protocol.write_event.wait() + if self._closed: + raise ClosedResourceError + elif self._transport.is_closing(): + raise BrokenResourceError + else: + self._transport.sendto(*item) + + +class ConnectedUDPSocket(abc.ConnectedUDPSocket): + def __init__( + self, transport: asyncio.DatagramTransport, protocol: DatagramProtocol + ): + self._transport = transport + self._protocol = protocol + self._receive_guard = ResourceGuard("reading from") + self._send_guard = ResourceGuard("writing to") + self._closed = False + + @property + def _raw_socket(self) -> socket.socket: + return self._transport.get_extra_info("socket") + + async def aclose(self) -> None: + self._closed = True + if not self._transport.is_closing(): + self._transport.close() + + await self._protocol.closed_event.wait() + + async def receive(self) -> bytes: + with self._receive_guard: + await AsyncIOBackend.checkpoint() + + # If the buffer is empty, ask for more data + if not self._protocol.read_queue and not self._transport.is_closing(): + self._protocol.read_event.clear() + await self._protocol.read_event.wait() + + try: + packet = self._protocol.read_queue.popleft() + except IndexError: + if self._closed: + raise ClosedResourceError from None + else: + raise BrokenResourceError from None + + return packet[0] + + async def send(self, item: bytes) -> None: + with self._send_guard: + await AsyncIOBackend.checkpoint() + await self._protocol.write_event.wait() + if self._closed: + raise ClosedResourceError + elif self._transport.is_closing(): + raise BrokenResourceError + else: + self._transport.sendto(item) + + +class UNIXDatagramSocket(_RawSocketMixin, abc.UNIXDatagramSocket): + async def receive(self) -> UNIXDatagramPacketType: + loop = get_running_loop() + await AsyncIOBackend.checkpoint() + with self._receive_guard: + while True: + try: + data = self._raw_socket.recvfrom(65536) + except BlockingIOError: + await self._wait_until_readable(loop) + except OSError as exc: + if self._closing: + raise ClosedResourceError from None + else: + raise BrokenResourceError from exc + else: + return data + + async def send(self, item: UNIXDatagramPacketType) -> None: + loop = get_running_loop() + await AsyncIOBackend.checkpoint() + with self._send_guard: + while True: + try: + self._raw_socket.sendto(*item) + except BlockingIOError: + await self._wait_until_writable(loop) + except OSError as exc: + if self._closing: + raise ClosedResourceError from None + else: + raise BrokenResourceError from exc + else: + return + + +class ConnectedUNIXDatagramSocket(_RawSocketMixin, abc.ConnectedUNIXDatagramSocket): + async def receive(self) -> bytes: + loop = get_running_loop() + await AsyncIOBackend.checkpoint() + with self._receive_guard: + while True: + try: + data = self._raw_socket.recv(65536) + except BlockingIOError: + await self._wait_until_readable(loop) + except OSError as exc: + if self._closing: + raise ClosedResourceError from None + else: + raise BrokenResourceError from exc + else: + return data + + async def send(self, item: bytes) -> None: + loop = get_running_loop() + await AsyncIOBackend.checkpoint() + with self._send_guard: + while True: + try: + self._raw_socket.send(item) + except BlockingIOError: + await self._wait_until_writable(loop) + except OSError as exc: + if self._closing: + raise ClosedResourceError from None + else: + raise BrokenResourceError from exc + else: + return + + +_read_events: RunVar[dict[int, asyncio.Future[bool]]] = RunVar("read_events") +_write_events: RunVar[dict[int, asyncio.Future[bool]]] = RunVar("write_events") + + +# +# Synchronization +# + + +class Event(BaseEvent): + __slots__ = ("_event",) + + def __new__(cls) -> Self: + return object.__new__(cls) + + def __init__(self) -> None: + self._event = asyncio.Event() + + def set(self) -> None: + self._event.set() + + def is_set(self) -> bool: + return self._event.is_set() + + async def wait(self) -> None: + if self.is_set(): + await AsyncIOBackend.checkpoint() + else: + await self._event.wait() + + def statistics(self) -> EventStatistics: + return EventStatistics(len(self._event._waiters)) + + +class Lock(BaseLock): + __slots__ = "_fast_acquire", "_owner_task", "_waiters" + + def __new__(cls, *, fast_acquire: bool = False) -> Self: + return object.__new__(cls) + + def __init__(self, *, fast_acquire: bool = False) -> None: + self._fast_acquire = fast_acquire + self._owner_task: asyncio.Task | None = None + self._waiters: deque[tuple[asyncio.Task, asyncio.Future]] = deque() + + async def acquire(self) -> None: + task = cast(asyncio.Task, current_task()) + if self._owner_task is None and not self._waiters: + await AsyncIOBackend.checkpoint_if_cancelled() + self._owner_task = task + + # Unless on the "fast path", yield control of the event loop so that other + # tasks can run too + if not self._fast_acquire: + try: + await AsyncIOBackend.cancel_shielded_checkpoint() + except CancelledError: + self.release() + raise + + return + + if self._owner_task == task: + raise RuntimeError("Attempted to acquire an already held Lock") + + fut: asyncio.Future[None] = asyncio.Future() + item = task, fut + self._waiters.append(item) + try: + await fut + except CancelledError: + if fut.cancelled(): + try: + self._waiters.remove(item) + except ValueError: + pass + else: + self.release() + + raise + + def acquire_nowait(self) -> None: + task = cast(asyncio.Task, current_task()) + if self._owner_task is None and not self._waiters: + self._owner_task = task + return + + if self._owner_task is task: + raise RuntimeError("Attempted to acquire an already held Lock") + + raise WouldBlock + + def locked(self) -> bool: + return self._owner_task is not None + + def release(self) -> None: + if self._owner_task != current_task(): + raise RuntimeError("The current task is not holding this lock") + + # A cancelled waiter that already received ownership removes itself from + # _waiters before calling release(); any cancelled waiter still queued here + # was cancelled before being woken, so drop it. + while self._waiters: + task, fut = self._waiters.popleft() + if fut.cancelled(): + continue + + self._owner_task = task + fut.set_result(None) + return + + self._owner_task = None + + def statistics(self) -> LockStatistics: + task_info = AsyncIOTaskInfo(self._owner_task) if self._owner_task else None + return LockStatistics(self.locked(), task_info, len(self._waiters)) + + +class Semaphore(BaseSemaphore): + __slots__ = "_fast_acquire", "_max_value", "_value", "_waiters" + + def __new__( + cls, + initial_value: int, + *, + max_value: int | None = None, + fast_acquire: bool = False, + ) -> Self: + return object.__new__(cls) + + def __init__( + self, + initial_value: int, + *, + max_value: int | None = None, + fast_acquire: bool = False, + ): + super().__init__(initial_value, max_value=max_value) + self._value = initial_value + self._max_value = max_value + self._fast_acquire = fast_acquire + self._waiters: deque[asyncio.Future[None]] = deque() + + async def acquire(self) -> None: + if self._value > 0 and not self._waiters: + await AsyncIOBackend.checkpoint_if_cancelled() + self._value -= 1 + + # Unless on the "fast path", yield control of the event loop so that other + # tasks can run too + if not self._fast_acquire: + try: + await AsyncIOBackend.cancel_shielded_checkpoint() + except CancelledError: + self.release() + raise + + return + + fut: asyncio.Future[None] = asyncio.Future() + self._waiters.append(fut) + try: + await fut + except CancelledError: + if fut.cancelled(): + try: + self._waiters.remove(fut) + except ValueError: + pass + else: + self.release() + + raise + + def acquire_nowait(self) -> None: + if self._value == 0: + raise WouldBlock + + self._value -= 1 + + def release(self) -> None: + if self._max_value is not None and self._value == self._max_value: + raise ValueError("semaphore released too many times") + + while self._waiters: + fut = self._waiters.popleft() + if fut.cancelled(): + continue + + fut.set_result(None) + return + + self._value += 1 + + @property + def value(self) -> int: + return self._value + + @property + def max_value(self) -> int | None: + return self._max_value + + def statistics(self) -> SemaphoreStatistics: + return SemaphoreStatistics(len(self._waiters)) + + +class CapacityLimiter(BaseCapacityLimiter): + __slots__ = "_borrowers", "_total_tokens", "_wait_queue" + + def __new__(cls, total_tokens: float) -> Self: + return object.__new__(cls) + + def __init__(self, total_tokens: float): + self._total_tokens: float = 0 + self._borrowers: set[Any] = set() + self._wait_queue: OrderedDict[Any, asyncio.Event] = OrderedDict() + self.total_tokens = total_tokens + + async def __aenter__(self) -> None: + await self.acquire() + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> None: + self.release() + + @property + def total_tokens(self) -> float: + return self._total_tokens + + @total_tokens.setter + def total_tokens(self, value: float) -> None: + if not isinstance(value, int) and not math.isinf(value): + raise TypeError("total_tokens must be an int or math.inf") + + if value < 0: + raise ValueError("total_tokens must be >= 0") + + self._total_tokens = value + + # Notify waiting tasks that they have acquired the limiter while + # there is spare capacity. + while self._wait_queue and len(self._borrowers) < self._total_tokens: + borrower, event = self._wait_queue.popitem(last=False) + self._borrowers.add(borrower) + event.set() + + @property + def borrowed_tokens(self) -> int: + return len(self._borrowers) + + @property + def available_tokens(self) -> float: + return self._total_tokens - len(self._borrowers) + + def _notify_next_waiter(self) -> None: + """Hand a free token to the next task in line, if any.""" + if self._wait_queue and len(self._borrowers) < self._total_tokens: + borrower, event = self._wait_queue.popitem(last=False) + self._borrowers.add(borrower) + event.set() + + def acquire_nowait(self) -> None: + self.acquire_on_behalf_of_nowait(current_task()) + + def acquire_on_behalf_of_nowait(self, borrower: object) -> None: + if borrower in self._borrowers: + raise RuntimeError( + "this borrower is already holding one of this CapacityLimiter's tokens" + ) + + if self._wait_queue or len(self._borrowers) >= self._total_tokens: + raise WouldBlock + + self._borrowers.add(borrower) + + async def acquire(self) -> None: + return await self.acquire_on_behalf_of(current_task()) + + async def acquire_on_behalf_of(self, borrower: object) -> None: + await AsyncIOBackend.checkpoint_if_cancelled() + try: + self.acquire_on_behalf_of_nowait(borrower) + except WouldBlock: + event = asyncio.Event() + self._wait_queue[borrower] = event + try: + await event.wait() + except BaseException: + self._wait_queue.pop(borrower, None) + if event.is_set(): + self._borrowers.discard(borrower) + self._notify_next_waiter() + + raise + else: + try: + await AsyncIOBackend.cancel_shielded_checkpoint() + except BaseException: + self.release() + raise + + def release(self) -> None: + self.release_on_behalf_of(current_task()) + + def release_on_behalf_of(self, borrower: object) -> None: + try: + self._borrowers.remove(borrower) + except KeyError: + raise RuntimeError( + "this borrower isn't holding any of this CapacityLimiter's tokens" + ) from None + + self._notify_next_waiter() + + def statistics(self) -> CapacityLimiterStatistics: + return CapacityLimiterStatistics( + self.borrowed_tokens, + self.total_tokens, + tuple(self._borrowers), + len(self._wait_queue), + ) + + +_default_thread_limiter: RunVar[CapacityLimiter] = RunVar("_default_thread_limiter") + + +# +# Operating system signals +# + + +class _SignalReceiver: + def __init__(self, signals: tuple[Signals, ...]): + self._signals = signals + self._loop = get_running_loop() + self._signal_queue: deque[Signals] = deque() + self._future: asyncio.Future = asyncio.Future() + self._handled_signals: set[Signals] = set() + + def _deliver(self, signum: Signals) -> None: + self._signal_queue.append(signum) + if not self._future.done(): + self._future.set_result(None) + + def __enter__(self) -> Self: + for sig in set(self._signals): + self._loop.add_signal_handler(sig, self._deliver, sig) + self._handled_signals.add(sig) + + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> None: + for sig in self._handled_signals: + self._loop.remove_signal_handler(sig) + + def __aiter__(self) -> _SignalReceiver: + return self + + async def __anext__(self) -> Signals: + await AsyncIOBackend.checkpoint() + if not self._signal_queue: + self._future = asyncio.Future() + await self._future + + return self._signal_queue.popleft() + + +# +# Testing and debugging +# + + +class AsyncIOTaskInfo(TaskInfo): + def __init__(self, task: asyncio.Task): + task_state = _task_states.get(task) + if task_state is None: + parent_id = None + else: + parent_id = task_state.parent_id + + coro = task.get_coro() + assert coro is not None, "created TaskInfo from a completed Task" + super().__init__(id(task), parent_id, task.get_name(), coro) + self._task = weakref.ref(task) + + def has_pending_cancellation(self) -> bool: + if not (task := self._task()): + # If the task isn't around anymore, it won't have a pending cancellation + return False + + if task._must_cancel or ( # type: ignore[attr-defined] + isinstance(task._fut_waiter, asyncio.Future) # type: ignore[attr-defined] + and task._fut_waiter.cancelled() # type: ignore[attr-defined] + ): + return True + + if task_state := _task_states.get(task): + if cancel_scope := task_state.cancel_scope: + return cancel_scope._effectively_cancelled + + return False + + +class TestRunner(abc.TestRunner): + _send_stream: MemoryObjectSendStream[tuple[Awaitable[Any], asyncio.Future[Any]]] + + def __init__( + self, + *, + debug: bool | None = None, + use_uvloop: bool = False, + loop_factory: Callable[[], AbstractEventLoop] | None = None, + ) -> None: + if use_uvloop and loop_factory is None: + if sys.platform != "win32": + import uvloop + + loop_factory = uvloop.new_event_loop + else: + import winloop + + loop_factory = winloop.new_event_loop + + self._runner = Runner(debug=debug, loop_factory=loop_factory) + self._exceptions: list[BaseException] = [] + self._runner_task: asyncio.Task | None = None + + def __enter__(self) -> Self: + self._runner.__enter__() + self.get_loop().set_exception_handler(self._exception_handler) + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> None: + self._runner.__exit__(exc_type, exc_val, exc_tb) + + def get_loop(self) -> AbstractEventLoop: + return self._runner.get_loop() + + def is_running(self) -> bool: + try: + asyncio.get_running_loop() + return True + except RuntimeError: + return False + + def _exception_handler( + self, loop: asyncio.AbstractEventLoop, context: dict[str, Any] + ) -> None: + if isinstance(context.get("exception"), Exception): + self._exceptions.append(context["exception"]) + else: + loop.default_exception_handler(context) + + def _raise_async_exceptions(self) -> None: + # Re-raise any exceptions raised in asynchronous callbacks + if self._exceptions: + exceptions, self._exceptions = self._exceptions, [] + if len(exceptions) == 1: + raise exceptions[0] + elif exceptions: + raise BaseExceptionGroup( + "Multiple exceptions occurred in asynchronous callbacks", exceptions + ) + + async def _run_tests_and_fixtures( + self, + receive_stream: MemoryObjectReceiveStream[ + tuple[Awaitable[T_Retval], asyncio.Future[T_Retval]] + ], + ) -> None: + from _pytest.outcomes import OutcomeException + + with receive_stream, self._send_stream: + async for coro, future in receive_stream: + try: + retval = await coro + except CancelledError as exc: + if not future.cancelled(): + future.cancel(*exc.args) + + raise + except BaseException as exc: + if not future.cancelled(): + future.set_exception(exc) + + if not isinstance(exc, (Exception, OutcomeException)): + raise + else: + if not future.cancelled(): + future.set_result(retval) + + async def _call_in_runner_task( + self, + func: Callable[P, Awaitable[T_Retval]], + /, + *args: P.args, + **kwargs: P.kwargs, + ) -> T_Retval: + if not self._runner_task: + self._send_stream, receive_stream = create_memory_object_stream[ + tuple[Awaitable[Any], asyncio.Future] + ](1) + self._runner_task = self.get_loop().create_task( + self._run_tests_and_fixtures(receive_stream) + ) + + coro = func(*args, **kwargs) + future: asyncio.Future[T_Retval] = self.get_loop().create_future() + self._send_stream.send_nowait((coro, future)) + return await future + + def run_asyncgen_fixture( + self, + fixture_func: Callable[..., AsyncGenerator[T_Retval, Any]], + kwargs: dict[str, Any], + ) -> Iterable[T_Retval]: + asyncgen = fixture_func(**kwargs) + fixturevalue: T_Retval = self.get_loop().run_until_complete( + self._call_in_runner_task(asyncgen.asend, None) + ) + self._raise_async_exceptions() + + yield fixturevalue + + try: + self.get_loop().run_until_complete( + self._call_in_runner_task(asyncgen.asend, None) + ) + except StopAsyncIteration: + self._raise_async_exceptions() + else: + self.get_loop().run_until_complete(asyncgen.aclose()) + raise RuntimeError("Async generator fixture did not stop") + + def run_fixture( + self, + fixture_func: Callable[..., Coroutine[Any, Any, T_Retval]], + kwargs: dict[str, Any], + ) -> T_Retval: + retval = self.get_loop().run_until_complete( + self._call_in_runner_task(fixture_func, **kwargs) + ) + self._raise_async_exceptions() + return retval + + def run_test( + self, test_func: Callable[..., Coroutine[Any, Any, Any]], kwargs: dict[str, Any] + ) -> None: + from _pytest.outcomes import OutcomeException + + try: + self.get_loop().run_until_complete( + self._call_in_runner_task(test_func, **kwargs) + ) + except Exception as exc: + self._exceptions.append(exc) + except OutcomeException: + raise + except BaseException: + # A BaseException (e.g. KeyboardInterrupt, SystemExit) interrupted the event loop before + # the test completed. Cancel _runner_task so it does not resume when the event + # loop is re-entered during async generator fixture teardown. + if self._runner_task is not None and not self._runner_task.done(): + self._runner_task.cancel() + self._send_stream.close() + try: + self.get_loop().run_until_complete(self._runner_task) + except CancelledError: + pass + finally: + self._runner_task = None + raise + self._raise_async_exceptions() + + +class _ProcessStreamProtocol(asyncio.subprocess.SubprocessStreamProtocol): + """ + A subprocess protocol that allows us to be notified of ``process_exited`` + + asyncio's own ``Process.wait()`` only resolves once every pipe transport has + disconnected so to get same semantics as on trio and uvloop we need this. + """ + + def __init__(self) -> None: + # Match the standard factory for asyncio.create_process + super().__init__(limit=2**16, loop=asyncio.get_running_loop()) + self.exited = asyncio.Event() + + def process_exited(self) -> None: + super().process_exited() + self.exited.set() + + +class AsyncIOBackend(AsyncBackend): + @classmethod + def run( + cls, + func: Callable[[Unpack[PosArgsT]], Awaitable[T_Retval]], + args: tuple[Unpack[PosArgsT]], + kwargs: dict[str, Any], + options: dict[str, Any], + ) -> T_Retval: + @wraps(func) + async def wrapper() -> T_Retval: + task = cast(asyncio.Task, current_task()) + task.set_name(get_callable_name(func)) + _task_states[task] = TaskState(None, None) + + try: + return await func(*args) + finally: + del _task_states[task] + + debug = options.get("debug", None) + loop_factory = options.get("loop_factory", None) + if loop_factory is None and options.get("use_uvloop", False): + if sys.platform != "win32": + import uvloop + + loop_factory = uvloop.new_event_loop + else: + import winloop + + loop_factory = winloop.new_event_loop + + with Runner(debug=debug, loop_factory=loop_factory) as runner: + return runner.run(wrapper()) + + @classmethod + def current_token(cls) -> object: + return get_running_loop() + + @classmethod + def current_time(cls) -> float: + return get_running_loop().time() + + @classmethod + def cancelled_exception_class(cls) -> type[BaseException]: + return CancelledError + + @classmethod + async def checkpoint(cls) -> None: + await sleep(0) + + @classmethod + async def checkpoint_if_cancelled(cls) -> None: + task = current_task() + if task is None: + return + + try: + cancel_scope = _task_states[task].cancel_scope + except KeyError: + return + + while cancel_scope: + if cancel_scope.cancel_called: + await sleep(0) + elif cancel_scope.shield: + break + else: + cancel_scope = cancel_scope._parent_scope + + @classmethod + async def cancel_shielded_checkpoint(cls) -> None: + with CancelScope(shield=True): + await sleep(0) + + @classmethod + async def sleep(cls, delay: float) -> None: + await sleep(delay) + + @classmethod + def create_cancel_scope( + cls, *, deadline: float = math.inf, shield: bool = False + ) -> CancelScope: + return CancelScope(deadline=deadline, shield=shield) + + @classmethod + def current_effective_deadline(cls) -> float: + if (task := current_task()) is None: + return math.inf + + try: + cancel_scope = _task_states[task].cancel_scope + except KeyError: + return math.inf + + deadline = math.inf + while cancel_scope: + deadline = min(deadline, cancel_scope.deadline) + if cancel_scope._cancel_called: + deadline = -math.inf + break + elif cancel_scope.shield: + break + else: + cancel_scope = cancel_scope._parent_scope + + return deadline + + @classmethod + def create_task_group(cls) -> abc.TaskGroup: + return TaskGroup() + + @classmethod + def create_event(cls) -> BaseEvent: + return Event() + + @classmethod + def create_lock(cls, *, fast_acquire: bool) -> BaseLock: + return Lock(fast_acquire=fast_acquire) + + @classmethod + def create_semaphore( + cls, + initial_value: int, + *, + max_value: int | None = None, + fast_acquire: bool = False, + ) -> BaseSemaphore: + return Semaphore(initial_value, max_value=max_value, fast_acquire=fast_acquire) + + @classmethod + def create_capacity_limiter(cls, total_tokens: float) -> BaseCapacityLimiter: + return CapacityLimiter(total_tokens) + + @classmethod + async def run_sync_in_worker_thread( # type: ignore[return] + cls, + func: Callable[[Unpack[PosArgsT]], T_Retval], + args: tuple[Unpack[PosArgsT]], + abandon_on_cancel: bool = False, + limiter: BaseCapacityLimiter | None = None, + ) -> T_Retval: + await cls.checkpoint() + + # If this is the first run in this event loop thread, set up the necessary + # variables + try: + idle_workers = _threadpool_idle_workers.get() + workers = _threadpool_workers.get() + except LookupError: + idle_workers = deque() + workers = set() + _threadpool_idle_workers.set(idle_workers) + _threadpool_workers.set(workers) + + async with limiter or cls.current_default_thread_limiter(): + with CancelScope(shield=not abandon_on_cancel) as scope: + future = asyncio.Future[T_Retval]() + root_task = find_root_task() + if not idle_workers: + worker = WorkerThread(root_task, workers, idle_workers) + worker.start() + workers.add(worker) + root_task.add_done_callback(worker.stop, context=Context()) + else: + worker = idle_workers.pop() + + # Prune any other workers that have been idle for MAX_IDLE_TIME + # seconds or longer + now = cls.current_time() + while idle_workers: + if ( + now - idle_workers[0].idle_since + < WorkerThread.MAX_IDLE_TIME + ): + break + + expired_worker = idle_workers.popleft() + expired_worker.root_task.remove_done_callback( + expired_worker.stop + ) + expired_worker.stop() + + context = copy_context() + context.run(set_current_async_library, None) + if abandon_on_cancel or scope._parent_scope is None: + worker_scope = scope + else: + worker_scope = scope._parent_scope + + worker.queue.put_nowait((context, func, args, future, worker_scope)) + return await future + + @classmethod + def check_cancelled(cls) -> None: + scope: CancelScope | None = threadlocals.current_cancel_scope + while scope is not None: + if scope.cancel_called: + raise CancelledError(f"Cancelled via cancel scope {id(scope):x}") + + if scope.shield: + return + + scope = scope._parent_scope + + @classmethod + def run_async_from_thread( + cls, + func: Callable[[Unpack[PosArgsT]], Coroutine[Any, Any, T_co]], + args: tuple[Unpack[PosArgsT]], + token: object, + ) -> T_co: + async def task_wrapper() -> T_co: + __tracebackhide__ = True + if scope is not None: + task = cast(asyncio.Task, current_task()) + _task_states[task] = TaskState(None, scope) + scope._tasks.add(task) + try: + return await func(*args) + except CancelledError as exc: + raise concurrent.futures.CancelledError(str(exc)) from None + finally: + if scope is not None: + scope._tasks.discard(task) + + loop = cast( + "AbstractEventLoop", token or threadlocals.current_token.native_token + ) + if loop.is_closed(): + raise RunFinishedError + + context = copy_context() + context.run(set_current_async_library, "asyncio") + scope = getattr(threadlocals, "current_cancel_scope", None) + f: concurrent.futures.Future[T_co] = context.run( + asyncio.run_coroutine_threadsafe, task_wrapper(), loop=loop + ) + return f.result() + + @classmethod + def run_sync_from_thread( + cls, + func: Callable[[Unpack[PosArgsT]], T_Retval], + args: tuple[Unpack[PosArgsT]], + token: object, + ) -> T_Retval: + @wraps(func) + def wrapper() -> None: + try: + set_current_async_library("asyncio") + f.set_result(func(*args)) + except BaseException as exc: + f.set_exception(exc) + if not isinstance(exc, Exception): + raise + + loop = cast( + "AbstractEventLoop", token or threadlocals.current_token.native_token + ) + if loop.is_closed(): + raise RunFinishedError + + f: concurrent.futures.Future[T_Retval] = Future() + loop.call_soon_threadsafe(wrapper) + return f.result() + + @classmethod + async def open_process( + cls, + command: StrOrBytesPath | Sequence[StrOrBytesPath], + *, + stdin: int | IO[Any] | None, + stdout: int | IO[Any] | None, + stderr: int | IO[Any] | None, + **kwargs: Any, + ) -> Process: + await cls.checkpoint() + if isinstance(command, PathLike): + command = os.fspath(command) + + # Use loop.subprocess_shell()/subprocess_exec() rather than their + # asyncio.create_subprocess_*() counterparts to get access to + # transport/protocol. + loop = asyncio.get_running_loop() + if isinstance(command, (str, bytes)): + transport, protocol = await loop.subprocess_shell( + _ProcessStreamProtocol, + command, + stdin=stdin, + stdout=stdout, + stderr=stderr, + **kwargs, + ) + else: + transport, protocol = await loop.subprocess_exec( + _ProcessStreamProtocol, + *command, + stdin=stdin, + stdout=stdout, + stderr=stderr, + **kwargs, + ) + + process = asyncio.subprocess.Process(transport, protocol, loop) + stdin_stream = StreamWriterWrapper(process.stdin) if process.stdin else None + stdout_stream = StreamReaderWrapper(process.stdout) if process.stdout else None + stderr_stream = StreamReaderWrapper(process.stderr) if process.stderr else None + return Process( + process, + stdin_stream, + stdout_stream, + stderr_stream, + protocol.exited, + transport, + ) + + @classmethod + def setup_process_pool_exit_at_shutdown(cls, workers: set[abc.Process]) -> None: + create_task( + _shutdown_process_pool_on_exit(workers), + name="AnyIO process pool shutdown task", + ) + find_root_task().add_done_callback( + partial(_forcibly_shutdown_process_pool_on_exit, workers) # type:ignore[arg-type] + ) + + @classmethod + async def connect_tcp( + cls, host: str, port: int, local_address: IPSockAddrType | None = None + ) -> abc.SocketStream: + transport, protocol = cast( + tuple[asyncio.Transport, StreamProtocol], + await get_running_loop().create_connection( + StreamProtocol, host, port, local_addr=local_address + ), + ) + transport.pause_reading() + return SocketStream(transport, protocol) + + @classmethod + async def connect_unix(cls, path: str | bytes) -> abc.UNIXSocketStream: + await cls.checkpoint() + loop = get_running_loop() + raw_socket = socket.socket(socket.AF_UNIX) + raw_socket.setblocking(False) + while True: + try: + raw_socket.connect(path) + except BlockingIOError: + f: asyncio.Future = asyncio.Future() + loop.add_writer(raw_socket, f.set_result, None) + f.add_done_callback(lambda _: loop.remove_writer(raw_socket)) + await f + except BaseException: + raw_socket.close() + raise + else: + return UNIXSocketStream(raw_socket) + + @classmethod + def create_tcp_listener(cls, sock: socket.socket) -> SocketListener: + return TCPSocketListener(sock) + + @classmethod + def create_unix_listener(cls, sock: socket.socket) -> SocketListener: + return UNIXSocketListener(sock) + + @classmethod + async def create_udp_socket( + cls, + family: AddressFamily, + local_address: IPSockAddrType | None, + remote_address: IPSockAddrType | None, + reuse_port: bool, + ) -> UDPSocket | ConnectedUDPSocket: + transport, protocol = await get_running_loop().create_datagram_endpoint( + DatagramProtocol, + local_addr=local_address, + remote_addr=remote_address, + family=family, + reuse_port=reuse_port, + ) + if protocol.exception: + transport.close() + raise protocol.exception + + if not remote_address: + return UDPSocket(transport, protocol) + else: + return ConnectedUDPSocket(transport, protocol) + + @classmethod + async def create_unix_datagram_socket( # type: ignore[override] + cls, raw_socket: socket.socket, remote_path: str | bytes | None + ) -> abc.UNIXDatagramSocket | abc.ConnectedUNIXDatagramSocket: + await cls.checkpoint() + loop = get_running_loop() + + if remote_path: + while True: + try: + raw_socket.connect(remote_path) + except BlockingIOError: + f: asyncio.Future = asyncio.Future() + loop.add_writer(raw_socket, f.set_result, None) + f.add_done_callback(lambda _: loop.remove_writer(raw_socket)) + await f + except BaseException: + raw_socket.close() + raise + else: + return ConnectedUNIXDatagramSocket(raw_socket) + else: + return UNIXDatagramSocket(raw_socket) + + @classmethod + async def getaddrinfo( + cls, + host: bytes | str | None, + port: str | int | None, + *, + family: int | AddressFamily = 0, + type: int | SocketKind = 0, + proto: int = 0, + flags: int = 0, + ) -> Sequence[ + tuple[ + AddressFamily, + SocketKind, + int, + str, + tuple[str, int] | tuple[str, int, int, int] | tuple[int, bytes], + ] + ]: + return await get_running_loop().getaddrinfo( + host, port, family=family, type=type, proto=proto, flags=flags + ) + + @classmethod + async def getnameinfo( + cls, sockaddr: IPSockAddrType, flags: int = 0 + ) -> tuple[str, str]: + return await get_running_loop().getnameinfo(sockaddr, flags) + + @classmethod + async def wait_readable(cls, obj: FileDescriptorLike) -> None: + try: + read_events = _read_events.get() + except LookupError: + read_events = {} + _read_events.set(read_events) + + fd = obj if isinstance(obj, int) else obj.fileno() + if read_events.get(fd): + raise BusyResourceError("reading from") + + loop = get_running_loop() + fut: asyncio.Future[bool] = loop.create_future() + + def cb() -> None: + try: + del read_events[fd] + except KeyError: + pass + else: + remove_reader(fd) + + try: + fut.set_result(True) + except asyncio.InvalidStateError: + pass + + try: + loop.add_reader(fd, cb) + except NotImplementedError: + from anyio._core._asyncio_selector_thread import get_selector + + selector = get_selector() + selector.add_reader(fd, cb) + remove_reader = selector.remove_reader + else: + remove_reader = loop.remove_reader + + read_events[fd] = fut + try: + success = await fut + finally: + try: + del read_events[fd] + except KeyError: + pass + else: + remove_reader(fd) + + if not success: + raise ClosedResourceError + + @classmethod + async def wait_writable(cls, obj: FileDescriptorLike) -> None: + try: + write_events = _write_events.get() + except LookupError: + write_events = {} + _write_events.set(write_events) + + fd = obj if isinstance(obj, int) else obj.fileno() + if write_events.get(fd): + raise BusyResourceError("writing to") + + loop = get_running_loop() + fut: asyncio.Future[bool] = loop.create_future() + + def cb() -> None: + try: + del write_events[fd] + except KeyError: + pass + else: + remove_writer(fd) + + try: + fut.set_result(True) + except asyncio.InvalidStateError: + pass + + try: + loop.add_writer(fd, cb) + except NotImplementedError: + from anyio._core._asyncio_selector_thread import get_selector + + selector = get_selector() + selector.add_writer(fd, cb) + remove_writer = selector.remove_writer + else: + remove_writer = loop.remove_writer + + write_events[fd] = fut + try: + success = await fut + finally: + try: + del write_events[fd] + except KeyError: + pass + else: + remove_writer(fd) + + if not success: + raise ClosedResourceError + + @classmethod + def notify_closing(cls, obj: FileDescriptorLike) -> None: + fd = obj if isinstance(obj, int) else obj.fileno() + loop = get_running_loop() + + try: + write_events = _write_events.get() + except LookupError: + pass + else: + try: + fut = write_events.pop(fd) + except KeyError: + pass + else: + try: + fut.set_result(False) + except asyncio.InvalidStateError: + pass + + try: + loop.remove_writer(fd) + except NotImplementedError: + from anyio._core._asyncio_selector_thread import get_selector + + get_selector().remove_writer(fd) + + try: + read_events = _read_events.get() + except LookupError: + pass + else: + try: + fut = read_events.pop(fd) + except KeyError: + pass + else: + try: + fut.set_result(False) + except asyncio.InvalidStateError: + pass + + try: + loop.remove_reader(fd) + except NotImplementedError: + from anyio._core._asyncio_selector_thread import get_selector + + get_selector().remove_reader(fd) + + @classmethod + async def wrap_listener_socket(cls, sock: socket.socket) -> SocketListener: + if hasattr(socket, "AF_UNIX") and sock.family == socket.AF_UNIX: + return UNIXSocketListener(sock) + + return TCPSocketListener(sock) + + @classmethod + async def wrap_stream_socket(cls, sock: socket.socket) -> SocketStream: + transport, protocol = await get_running_loop().create_connection( + StreamProtocol, sock=sock + ) + return SocketStream(transport, protocol) + + @classmethod + async def wrap_unix_stream_socket(cls, sock: socket.socket) -> UNIXSocketStream: + return UNIXSocketStream(sock) + + @classmethod + async def wrap_udp_socket(cls, sock: socket.socket) -> UDPSocket: + transport, protocol = await get_running_loop().create_datagram_endpoint( + DatagramProtocol, sock=sock + ) + return UDPSocket(transport, protocol) + + @classmethod + async def wrap_connected_udp_socket(cls, sock: socket.socket) -> ConnectedUDPSocket: + transport, protocol = await get_running_loop().create_datagram_endpoint( + DatagramProtocol, sock=sock + ) + return ConnectedUDPSocket(transport, protocol) + + @classmethod + async def wrap_unix_datagram_socket(cls, sock: socket.socket) -> UNIXDatagramSocket: + return UNIXDatagramSocket(sock) + + @classmethod + async def wrap_connected_unix_datagram_socket( + cls, sock: socket.socket + ) -> ConnectedUNIXDatagramSocket: + return ConnectedUNIXDatagramSocket(sock) + + @classmethod + def current_default_thread_limiter(cls) -> CapacityLimiter: + try: + return _default_thread_limiter.get() + except LookupError: + limiter = CapacityLimiter(40) + _default_thread_limiter.set(limiter) + return limiter + + @classmethod + def open_signal_receiver( + cls, *signals: Signals + ) -> AbstractContextManager[AsyncIterator[Signals]]: + return _SignalReceiver(signals) + + @classmethod + def get_current_task(cls) -> TaskInfo: + return AsyncIOTaskInfo(current_task()) # type: ignore[arg-type] + + @classmethod + def get_running_tasks(cls) -> Sequence[TaskInfo]: + return [AsyncIOTaskInfo(task) for task in all_tasks() if not task.done()] + + @classmethod + async def wait_all_tasks_blocked(cls) -> None: + await cls.checkpoint() + this_task = current_task() + while True: + for task in all_tasks(): + if task is this_task: + continue + + waiter = task._fut_waiter # type: ignore[attr-defined] + if waiter is None or waiter.done(): + await sleep(0.1) + break + else: + return + + @classmethod + def create_test_runner(cls, options: dict[str, Any]) -> TestRunner: + return TestRunner(**options) + + +backend_class = AsyncIOBackend diff --git a/.venv/lib/python3.14/site-packages/anyio/_backends/_trio.py b/.venv/lib/python3.14/site-packages/anyio/_backends/_trio.py new file mode 100644 index 0000000000000000000000000000000000000000..2e67786e81a6f41976ad2467f818a205ba5637e9 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/anyio/_backends/_trio.py @@ -0,0 +1,1473 @@ +from __future__ import annotations + +import array +import math +import os +import socket +import sys +import types +import weakref +from collections.abc import ( + AsyncGenerator, + AsyncIterator, + Awaitable, + Callable, + Collection, + Coroutine, + Iterable, + Sequence, +) +from contextlib import AbstractContextManager +from contextvars import Context +from dataclasses import dataclass +from functools import partial, wraps +from io import IOBase +from os import PathLike +from signal import Signals +from socket import AddressFamily, SocketKind +from types import TracebackType +from typing import ( + IO, + TYPE_CHECKING, + Any, + Generic, + Literal, + NoReturn, + ParamSpec, + TypeVar, + cast, + overload, +) + +import trio.from_thread +import trio.lowlevel +from outcome import Error, Outcome, Value +from trio.lowlevel import ( + current_root_task, + current_task, + notify_closing, + wait_readable, + wait_writable, +) +from trio.socket import SocketType as TrioSocketType +from trio.to_thread import run_sync + +from .. import ( + CapacityLimiterStatistics, + EventStatistics, + LockStatistics, + RunFinishedError, + TaskInfo, + WouldBlock, + abc, +) +from .._core._eventloop import claim_worker_thread +from .._core._exceptions import ( + BrokenResourceError, + BusyResourceError, + ClosedResourceError, + EndOfStream, +) +from .._core._sockets import convert_ipv6_sockaddr +from .._core._streams import create_memory_object_stream +from .._core._synchronization import ( + CapacityLimiter as BaseCapacityLimiter, +) +from .._core._synchronization import Event as BaseEvent +from .._core._synchronization import Lock as BaseLock +from .._core._synchronization import ( + ResourceGuard, + SemaphoreStatistics, +) +from .._core._synchronization import Semaphore as BaseSemaphore +from .._core._tasks import CancelScope as BaseCancelScope +from .._core._tasks import TaskHandle +from ..abc import IPSockAddrType, UDPPacketType, UNIXDatagramPacketType +from ..abc._eventloop import AsyncBackend, StrOrBytesPath +from ..abc._tasks import T_contra, call_for_coroutine, get_callable_name, get_coro_name + +if TYPE_CHECKING: + from _typeshed import FileDescriptorLike + + from ..streams.memory import MemoryObjectSendStream + +if sys.version_info < (3, 15): + from typing_extensions import sentinel + +if sys.version_info >= (3, 11): + from typing import Self, TypeVarTuple, Unpack +else: + from exceptiongroup import BaseExceptionGroup + from typing_extensions import Self, TypeVarTuple, Unpack + +T = TypeVar("T") +T_Retval = TypeVar("T_Retval") +T_co = TypeVar("T_co", covariant=True) +T_SockAddr = TypeVar("T_SockAddr", str, IPSockAddrType) +PosArgsT = TypeVarTuple("PosArgsT") +P = ParamSpec("P") + + +def ensure_returns_coro( + func: Callable[P, Awaitable[T_Retval]], +) -> Callable[P, Coroutine[Any, Any, T_Retval]]: + @wraps(func) + def wrapper(*args: P.args, **kwargs: P.kwargs) -> Coroutine[Any, Any, T_Retval]: + awaitable = func(*args, **kwargs) + # Check the common case first. + if isinstance(awaitable, Coroutine): + return awaitable + elif not isinstance(awaitable, Awaitable): + # The user violated the type annotations. Still, we should pass this on to + # Trio so it can raise with an appropriate message. + return awaitable + else: + + @wraps(func) + async def inner_wrapper() -> T_Retval: + return await awaitable + + return inner_wrapper() + + return wrapper + + +# +# Event loop +# + +RunVar = trio.lowlevel.RunVar + + +# +# Timeouts and cancellation +# + + +class CancelScope(BaseCancelScope): + __slots__ = ("__original",) + + def __new__( + cls, original: trio.CancelScope | None = None, **kwargs: object + ) -> Self: + return object.__new__(cls) + + def __init__(self, original: trio.CancelScope | None = None, **kwargs: Any) -> None: + self.__original = original or trio.CancelScope(**kwargs) + + def __enter__(self) -> Self: + self.__original.__enter__() + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> bool: + return self.__original.__exit__(exc_type, exc_val, exc_tb) + + def cancel(self, reason: str | None = None) -> None: + self.__original.cancel(reason) + + @property + def deadline(self) -> float: + return self.__original.deadline + + @deadline.setter + def deadline(self, value: float) -> None: + self.__original.deadline = value + + @property + def cancel_called(self) -> bool: + return self.__original.cancel_called + + @property + def cancelled_caught(self) -> bool: + return self.__original.cancelled_caught + + @property + def shield(self) -> bool: + return self.__original.shield + + @shield.setter + def shield(self, value: bool) -> None: + self.__original.shield = value + + +# +# Task groups +# + +empty_start_value = sentinel("empty_start_value") + + +class _TrioTaskStatus(abc.TaskStatus[T_contra], Generic[T_contra]): + early_start_value: T_contra | object = empty_start_value + real_task_status: trio.TaskStatus[T_contra | None] | None = None + + def started(self, value: T_contra | None = None) -> None: + if self.real_task_status is None: + if self.early_start_value is not empty_start_value: + raise RuntimeError("called 'started' twice on the same task status") + + self.early_start_value = value + else: + self.real_task_status.started(value) + + +class TaskGroup(abc.TaskGroup): + def __init__(self) -> None: + self._entered = False + self._active = False + self._nursery_manager = trio.open_nursery(strict_exception_groups=True) + self.cancel_scope = None # type: ignore[assignment] + + async def __aenter__(self) -> Self: + if self._entered: + raise RuntimeError("TaskGroup cannot be entered more than once") + + self._entered = True + self._active = True + self._nursery = await self._nursery_manager.__aenter__() + self.cancel_scope = CancelScope(self._nursery.cancel_scope) + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> bool: + try: + # trio.Nursery.__exit__ returns bool; .open_nursery has wrong type + return await self._nursery_manager.__aexit__(exc_type, exc_val, exc_tb) # type: ignore[return-value] + except BaseExceptionGroup as exc: + if not exc.split(trio.Cancelled)[1]: + raise trio.Cancelled._create() from exc + + raise + finally: + del exc_val, exc_tb + self._active = False + + def _check_active(self, coro: Coroutine | None = None) -> None: + if not self._active: + if coro is not None: + coro.close() + + raise RuntimeError( + "This task group is not active; no new tasks can be started." + ) + + def create_task( + self, + coro: Coroutine[Any, Any, T_co], + *, + name: object = None, + context: Context | None = None, + ) -> TaskHandle[T_co]: + if not isinstance(coro, Coroutine): + raise TypeError(f"expected a coroutine, got {coro.__class__.__qualname__}") + + self._check_active(coro) + final_name = get_coro_name(coro, name) + handle = TaskHandle(coro, final_name) + if context is not None: + context.run( + partial(self._nursery.start_soon, handle._run_coro, name=handle.name) + ) + else: + self._nursery.start_soon(handle._run_coro, name=handle.name) + + return handle + + async def start( + self, + func: Callable[[Unpack[PosArgsT]], Coroutine[Any, Any, T_co]], + *args: Unpack[PosArgsT], + name: object = None, + return_handle: Literal[False, True] = False, + ) -> Any: + handle: TaskHandle[T_co] + + async def run_coro_with_task_status( + *, task_status: trio.TaskStatus[Any] + ) -> None: + nonlocal handle + wrapper_task_status = _TrioTaskStatus() + coro = call_for_coroutine(func, args, task_status=wrapper_task_status) + if wrapper_task_status.early_start_value is not empty_start_value: + task_status.started(wrapper_task_status.early_start_value) + else: + wrapper_task_status.real_task_status = task_status + + handle = TaskHandle(coro, final_name) + await handle._run_coro() + + self._check_active() + final_name = get_callable_name(func, name) + start_value = await self._nursery.start( + run_coro_with_task_status, name=final_name + ) + if return_handle: + handle._start_value = start_value + return handle + else: + return start_value + + +# +# Subprocesses +# + + +@dataclass(eq=False) +class ReceiveStreamWrapper(abc.ByteReceiveStream): + _stream: trio.abc.ReceiveStream + + async def receive(self, max_bytes: int | None = None) -> bytes: + if max_bytes is not None and max_bytes < 1: + raise ValueError("max_bytes must be a positive integer") + + try: + data = await self._stream.receive_some(max_bytes) + except trio.ClosedResourceError as exc: + raise ClosedResourceError from exc.__cause__ + except trio.BrokenResourceError as exc: + raise BrokenResourceError from exc.__cause__ + + if data: + return bytes(data) + else: + raise EndOfStream + + async def aclose(self) -> None: + await self._stream.aclose() + + +@dataclass(eq=False) +class SendStreamWrapper(abc.ByteSendStream): + _stream: trio.abc.SendStream + + async def send(self, item: bytes) -> None: + try: + await self._stream.send_all(item) + except trio.ClosedResourceError as exc: + raise ClosedResourceError from exc.__cause__ + except trio.BrokenResourceError as exc: + raise BrokenResourceError from exc.__cause__ + + async def aclose(self) -> None: + await self._stream.aclose() + + +@dataclass(eq=False) +class Process(abc.Process): + _process: trio.Process + _stdin: abc.ByteSendStream | None + _stdout: abc.ByteReceiveStream | None + _stderr: abc.ByteReceiveStream | None + + async def aclose(self) -> None: + with CancelScope(shield=True): + if self._stdin: + await self._stdin.aclose() + if self._stdout: + await self._stdout.aclose() + if self._stderr: + await self._stderr.aclose() + + try: + await self.wait() + except BaseException: + self.kill() + with CancelScope(shield=True): + await self.wait() + raise + + async def wait(self) -> int: + return await self._process.wait() + + def terminate(self) -> None: + self._process.terminate() + + def kill(self) -> None: + self._process.kill() + + def send_signal(self, signal: Signals) -> None: + self._process.send_signal(signal) + + @property + def pid(self) -> int: + return self._process.pid + + @property + def returncode(self) -> int | None: + return self._process.returncode + + @property + def stdin(self) -> abc.ByteSendStream | None: + return self._stdin + + @property + def stdout(self) -> abc.ByteReceiveStream | None: + return self._stdout + + @property + def stderr(self) -> abc.ByteReceiveStream | None: + return self._stderr + + +class _ProcessPoolShutdownInstrument(trio.abc.Instrument): + def after_run(self) -> None: + super().after_run() + + +current_default_worker_process_limiter: trio.lowlevel.RunVar = RunVar( + "current_default_worker_process_limiter" +) + + +async def _shutdown_process_pool(workers: set[abc.Process]) -> None: + try: + await trio.sleep(math.inf) + except trio.Cancelled: + for process in workers: + if process.returncode is None: + process.kill() + + with CancelScope(shield=True): + for process in workers: + await process.aclose() + + +# +# Sockets and networking +# + + +class _TrioSocketMixin(Generic[T_SockAddr]): + def __init__(self, trio_socket: TrioSocketType) -> None: + self._trio_socket = trio_socket + self._closed = False + + def _check_closed(self) -> None: + if self._closed: + raise ClosedResourceError + if self._trio_socket.fileno() < 0: + raise BrokenResourceError + + @property + def _raw_socket(self) -> socket.socket: + return self._trio_socket._sock # type: ignore[attr-defined] + + async def aclose(self) -> None: + if self._trio_socket.fileno() >= 0: + self._closed = True + self._trio_socket.close() + + def _convert_socket_error(self, exc: BaseException) -> NoReturn: + if isinstance(exc, trio.ClosedResourceError): + raise ClosedResourceError from exc + elif self._trio_socket.fileno() < 0 and self._closed: + raise ClosedResourceError from None + elif isinstance(exc, OSError): + raise BrokenResourceError from exc + else: + raise exc + + +class SocketStream(_TrioSocketMixin, abc.SocketStream): + def __init__(self, trio_socket: TrioSocketType) -> None: + super().__init__(trio_socket) + self._receive_guard = ResourceGuard("reading from") + self._send_guard = ResourceGuard("writing to") + + async def receive(self, max_bytes: int = 65536) -> bytes: + if max_bytes < 1: + raise ValueError("max_bytes must be a positive integer") + + with self._receive_guard: + try: + data = await self._trio_socket.recv(max_bytes) + except BaseException as exc: + self._convert_socket_error(exc) + + if data: + return data + else: + raise EndOfStream + + async def send(self, item: bytes) -> None: + with self._send_guard: + view = memoryview(item) + while view: + try: + bytes_sent = await self._trio_socket.send(view) + except BaseException as exc: + self._convert_socket_error(exc) + + view = view[bytes_sent:] + + async def send_eof(self) -> None: + self._trio_socket.shutdown(socket.SHUT_WR) + + +class UNIXSocketStream(SocketStream, abc.UNIXSocketStream): + async def receive_fds(self, msglen: int, maxfds: int) -> tuple[bytes, list[int]]: + if not isinstance(msglen, int) or msglen < 0: + raise ValueError("msglen must be a non-negative integer") + if not isinstance(maxfds, int) or maxfds < 1: + raise ValueError("maxfds must be a positive integer") + + fds = array.array("i") + await trio.lowlevel.checkpoint() + with self._receive_guard: + while True: + try: + message, ancdata, _flags, _addr = await self._trio_socket.recvmsg( + msglen, socket.CMSG_LEN(maxfds * fds.itemsize) + ) + except BaseException as exc: + self._convert_socket_error(exc) + else: + if not message and not ancdata: + raise EndOfStream + + break + + for cmsg_level, cmsg_type, cmsg_data in ancdata: + if cmsg_level != socket.SOL_SOCKET or cmsg_type != socket.SCM_RIGHTS: + raise RuntimeError( + f"Received unexpected ancillary data; message = {message!r}, " + f"cmsg_level = {cmsg_level}, cmsg_type = {cmsg_type}" + ) + + fds.frombytes(cmsg_data[: len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) + + return message, list(fds) + + async def send_fds(self, message: bytes, fds: Collection[int | IOBase]) -> None: + if not message: + raise ValueError("message must not be empty") + if not fds: + raise ValueError("fds must not be empty") + + filenos: list[int] = [] + for fd in fds: + if isinstance(fd, int): + filenos.append(fd) + elif isinstance(fd, IOBase): + filenos.append(fd.fileno()) + + fdarray = array.array("i", filenos) + await trio.lowlevel.checkpoint() + with self._send_guard: + while True: + try: + await self._trio_socket.sendmsg( + [message], + [ + ( + socket.SOL_SOCKET, + socket.SCM_RIGHTS, + fdarray, + ) + ], + ) + break + except BaseException as exc: + self._convert_socket_error(exc) + + +class TCPSocketListener(_TrioSocketMixin, abc.SocketListener): + def __init__(self, raw_socket: socket.socket): + super().__init__(trio.socket.from_stdlib_socket(raw_socket)) + self._accept_guard = ResourceGuard("accepting connections from") + + async def accept(self) -> SocketStream: + with self._accept_guard: + try: + trio_socket, _addr = await self._trio_socket.accept() + except BaseException as exc: + self._convert_socket_error(exc) + + trio_socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + return SocketStream(trio_socket) + + +class UNIXSocketListener(_TrioSocketMixin, abc.SocketListener): + def __init__(self, raw_socket: socket.socket): + super().__init__(trio.socket.from_stdlib_socket(raw_socket)) + self._accept_guard = ResourceGuard("accepting connections from") + + async def accept(self) -> UNIXSocketStream: + with self._accept_guard: + try: + trio_socket, _addr = await self._trio_socket.accept() + except BaseException as exc: + self._convert_socket_error(exc) + + return UNIXSocketStream(trio_socket) + + +class UDPSocket(_TrioSocketMixin[IPSockAddrType], abc.UDPSocket): + def __init__(self, trio_socket: TrioSocketType) -> None: + super().__init__(trio_socket) + self._receive_guard = ResourceGuard("reading from") + self._send_guard = ResourceGuard("writing to") + + async def receive(self) -> tuple[bytes, IPSockAddrType]: + with self._receive_guard: + try: + data, addr = await self._trio_socket.recvfrom(65536) + return data, convert_ipv6_sockaddr(addr) + except BaseException as exc: + self._convert_socket_error(exc) + + async def send(self, item: UDPPacketType) -> None: + with self._send_guard: + try: + await self._trio_socket.sendto(*item) + except BaseException as exc: + self._convert_socket_error(exc) + + +class ConnectedUDPSocket(_TrioSocketMixin[IPSockAddrType], abc.ConnectedUDPSocket): + def __init__(self, trio_socket: TrioSocketType) -> None: + super().__init__(trio_socket) + self._receive_guard = ResourceGuard("reading from") + self._send_guard = ResourceGuard("writing to") + + async def receive(self) -> bytes: + with self._receive_guard: + try: + return await self._trio_socket.recv(65536) + except BaseException as exc: + self._convert_socket_error(exc) + + async def send(self, item: bytes) -> None: + with self._send_guard: + try: + await self._trio_socket.send(item) + except BaseException as exc: + self._convert_socket_error(exc) + + +class UNIXDatagramSocket(_TrioSocketMixin[str], abc.UNIXDatagramSocket): + def __init__(self, trio_socket: TrioSocketType) -> None: + super().__init__(trio_socket) + self._receive_guard = ResourceGuard("reading from") + self._send_guard = ResourceGuard("writing to") + + async def receive(self) -> UNIXDatagramPacketType: + with self._receive_guard: + try: + data, addr = await self._trio_socket.recvfrom(65536) + return data, addr + except BaseException as exc: + self._convert_socket_error(exc) + + async def send(self, item: UNIXDatagramPacketType) -> None: + with self._send_guard: + try: + await self._trio_socket.sendto(*item) + except BaseException as exc: + self._convert_socket_error(exc) + + +class ConnectedUNIXDatagramSocket( + _TrioSocketMixin[str], abc.ConnectedUNIXDatagramSocket +): + def __init__(self, trio_socket: TrioSocketType) -> None: + super().__init__(trio_socket) + self._receive_guard = ResourceGuard("reading from") + self._send_guard = ResourceGuard("writing to") + + async def receive(self) -> bytes: + with self._receive_guard: + try: + return await self._trio_socket.recv(65536) + except BaseException as exc: + self._convert_socket_error(exc) + + async def send(self, item: bytes) -> None: + with self._send_guard: + try: + await self._trio_socket.send(item) + except BaseException as exc: + self._convert_socket_error(exc) + + +# +# Synchronization +# + + +class Event(BaseEvent): + __slots__ = ("__original",) + + def __new__(cls) -> Self: + return object.__new__(cls) + + def __init__(self) -> None: + self.__original = trio.Event() + + def is_set(self) -> bool: + return self.__original.is_set() + + async def wait(self) -> None: + return await self.__original.wait() + + def statistics(self) -> EventStatistics: + orig_statistics = self.__original.statistics() + return EventStatistics(tasks_waiting=orig_statistics.tasks_waiting) + + def set(self) -> None: + self.__original.set() + + +class Lock(BaseLock): + __slots__ = "__original", "_fast_acquire" + + def __new__(cls, *, fast_acquire: bool = False) -> Self: + return object.__new__(cls) + + def __init__(self, *, fast_acquire: bool = False) -> None: + self._fast_acquire = fast_acquire + self.__original = trio.Lock() + + @staticmethod + def _convert_runtime_error_msg(exc: RuntimeError) -> None: + if exc.args == ("attempt to re-acquire an already held Lock",): + exc.args = ("Attempted to acquire an already held Lock",) + + async def acquire(self) -> None: + if not self._fast_acquire: + try: + await self.__original.acquire() + except RuntimeError as exc: + self._convert_runtime_error_msg(exc) + raise + + return + + # This is the "fast path" where we don't let other tasks run + await trio.lowlevel.checkpoint_if_cancelled() + try: + self.__original.acquire_nowait() + except trio.WouldBlock: + await self.__original._lot.park() + except RuntimeError as exc: + self._convert_runtime_error_msg(exc) + raise + + def acquire_nowait(self) -> None: + try: + self.__original.acquire_nowait() + except trio.WouldBlock: + raise WouldBlock from None + except RuntimeError as exc: + self._convert_runtime_error_msg(exc) + raise + + def locked(self) -> bool: + return self.__original.locked() + + def release(self) -> None: + self.__original.release() + + def statistics(self) -> LockStatistics: + orig_statistics = self.__original.statistics() + owner = TrioTaskInfo(orig_statistics.owner) if orig_statistics.owner else None + return LockStatistics( + orig_statistics.locked, owner, orig_statistics.tasks_waiting + ) + + +class Semaphore(BaseSemaphore): + __slots__ = ("__original",) + + def __new__( + cls, + initial_value: int, + *, + max_value: int | None = None, + fast_acquire: bool = False, + ) -> Self: + return object.__new__(cls) + + def __init__( + self, + initial_value: int, + *, + max_value: int | None = None, + fast_acquire: bool = False, + ) -> None: + super().__init__(initial_value, max_value=max_value, fast_acquire=fast_acquire) + self.__original = trio.Semaphore(initial_value, max_value=max_value) + + async def acquire(self) -> None: + if not self._fast_acquire: + await self.__original.acquire() + return + + # This is the "fast path" where we don't let other tasks run + await trio.lowlevel.checkpoint_if_cancelled() + try: + self.__original.acquire_nowait() + except trio.WouldBlock: + await self.__original._lot.park() + + def acquire_nowait(self) -> None: + try: + self.__original.acquire_nowait() + except trio.WouldBlock: + raise WouldBlock from None + + @property + def max_value(self) -> int | None: + return self.__original.max_value + + @property + def value(self) -> int: + return self.__original.value + + def release(self) -> None: + self.__original.release() + + def statistics(self) -> SemaphoreStatistics: + orig_statistics = self.__original.statistics() + return SemaphoreStatistics(orig_statistics.tasks_waiting) + + +class CapacityLimiter(BaseCapacityLimiter): + __slots__ = ("__original",) + + def __new__( + cls, + total_tokens: float | None = None, + *, + original: trio.CapacityLimiter | None = None, + ) -> Self: + return object.__new__(cls) + + def __init__( + self, + total_tokens: float | None = None, + *, + original: trio.CapacityLimiter | None = None, + ) -> None: + if original is not None: + self.__original = original + else: + assert total_tokens is not None + self.__original = trio.CapacityLimiter(total_tokens) + + async def __aenter__(self) -> None: + return await self.__original.__aenter__() + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> None: + await self.__original.__aexit__(exc_type, exc_val, exc_tb) + + @property + def total_tokens(self) -> float: + return self.__original.total_tokens + + @total_tokens.setter + def total_tokens(self, value: float) -> None: + self.__original.total_tokens = value + + @property + def borrowed_tokens(self) -> int: + return self.__original.borrowed_tokens + + @property + def available_tokens(self) -> float: + return self.__original.available_tokens + + def acquire_nowait(self) -> None: + try: + self.__original.acquire_nowait() + except trio.WouldBlock: + raise WouldBlock from None + + def acquire_on_behalf_of_nowait(self, borrower: object) -> None: + try: + self.__original.acquire_on_behalf_of_nowait(borrower) + except trio.WouldBlock: + raise WouldBlock from None + + async def acquire(self) -> None: + await self.__original.acquire() + + async def acquire_on_behalf_of(self, borrower: object) -> None: + await self.__original.acquire_on_behalf_of(borrower) + + def release(self) -> None: + return self.__original.release() + + def release_on_behalf_of(self, borrower: object) -> None: + return self.__original.release_on_behalf_of(borrower) + + def statistics(self) -> CapacityLimiterStatistics: + orig = self.__original.statistics() + return CapacityLimiterStatistics( + borrowed_tokens=orig.borrowed_tokens, + total_tokens=orig.total_tokens, + borrowers=tuple(orig.borrowers), + tasks_waiting=orig.tasks_waiting, + ) + + +_capacity_limiter_wrapper: trio.lowlevel.RunVar = RunVar("_capacity_limiter_wrapper") + + +# +# Signal handling +# + + +class _SignalReceiver: + _iterator: AsyncIterator[int] + + def __init__(self, signals: tuple[Signals, ...]): + self._signals = signals + + def __enter__(self) -> Self: + self._cm = trio.open_signal_receiver(*self._signals) + self._iterator = self._cm.__enter__() + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> bool | None: + return self._cm.__exit__(exc_type, exc_val, exc_tb) + + def __aiter__(self) -> _SignalReceiver: + return self + + async def __anext__(self) -> Signals: + signum = await self._iterator.__anext__() + return Signals(signum) + + +# +# Testing and debugging +# + + +class TestRunner(abc.TestRunner): + def __init__(self, **options: Any) -> None: + from queue import Queue + + self._call_queue: Queue[Callable[[], object]] = Queue() + self._send_stream: ( + MemoryObjectSendStream[tuple[Awaitable[Any], list[Outcome]]] | None + ) = None + self._options = options + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: types.TracebackType | None, + ) -> None: + if self._send_stream: + self._send_stream.close() + while self._send_stream is not None: + self._call_queue.get()() + + def is_running(self) -> bool: + return trio.lowlevel.in_trio_task() + + async def _run_tests_and_fixtures(self) -> None: + self._send_stream, receive_stream = create_memory_object_stream[ + tuple[Awaitable[Any], list[Outcome]] + ](1) + with receive_stream: + async for awaitable, outcome_holder in receive_stream: + try: + retval = await awaitable + except BaseException as exc: + outcome_holder.append(Error(exc)) + else: + outcome_holder.append(Value(retval)) + + def _main_task_finished(self, outcome: object) -> None: + self._send_stream = None + + def _call_in_runner_task( + self, + func: Callable[P, Awaitable[T_Retval]], + /, + *args: P.args, + **kwargs: P.kwargs, + ) -> T_Retval: + if self._send_stream is None: + trio.lowlevel.start_guest_run( + self._run_tests_and_fixtures, + run_sync_soon_threadsafe=self._call_queue.put, + done_callback=self._main_task_finished, + **self._options, + ) + while self._send_stream is None: + self._call_queue.get()() + + outcome_holder: list[Outcome] = [] + self._send_stream.send_nowait((func(*args, **kwargs), outcome_holder)) + while not outcome_holder: + self._call_queue.get()() + + return outcome_holder[0].unwrap() + + def run_asyncgen_fixture( + self, + fixture_func: Callable[..., AsyncGenerator[T_Retval, Any]], + kwargs: dict[str, Any], + ) -> Iterable[T_Retval]: + asyncgen = fixture_func(**kwargs) + fixturevalue: T_Retval = self._call_in_runner_task(asyncgen.asend, None) + + yield fixturevalue + + try: + self._call_in_runner_task(asyncgen.asend, None) + except StopAsyncIteration: + pass + else: + self._call_in_runner_task(asyncgen.aclose) + raise RuntimeError("Async generator fixture did not stop") + + def run_fixture( + self, + fixture_func: Callable[..., Coroutine[Any, Any, T_Retval]], + kwargs: dict[str, Any], + ) -> T_Retval: + return self._call_in_runner_task(fixture_func, **kwargs) + + def run_test( + self, test_func: Callable[..., Coroutine[Any, Any, Any]], kwargs: dict[str, Any] + ) -> None: + self._call_in_runner_task(test_func, **kwargs) + + +class TrioTaskInfo(TaskInfo): + def __init__(self, task: trio.lowlevel.Task): + parent_id = None + if task.parent_nursery and task.parent_nursery.parent_task: + parent_id = id(task.parent_nursery.parent_task) + + super().__init__(id(task), parent_id, task.name, task.coro) + self._task = weakref.proxy(task) + + def has_pending_cancellation(self) -> bool: + try: + return self._task._cancel_status.effectively_cancelled + except ReferenceError: + # If the task is no longer around, it surely doesn't have a cancellation + # pending + return False + + +class TrioBackend(AsyncBackend): + @classmethod + def run( + cls, + func: Callable[[Unpack[PosArgsT]], Awaitable[T_Retval]], + args: tuple[Unpack[PosArgsT]], + kwargs: dict[str, Any], + options: dict[str, Any], + ) -> T_Retval: + assert not kwargs, "unreachable, and not supported by Trio" + return trio.run(ensure_returns_coro(func), *args, **options) + + @classmethod + def current_token(cls) -> object: + return trio.lowlevel.current_trio_token() + + @classmethod + def current_time(cls) -> float: + return trio.current_time() + + @classmethod + def cancelled_exception_class(cls) -> type[BaseException]: + return trio.Cancelled + + @classmethod + async def checkpoint(cls) -> None: + await trio.lowlevel.checkpoint() + + @classmethod + async def checkpoint_if_cancelled(cls) -> None: + await trio.lowlevel.checkpoint_if_cancelled() + + @classmethod + async def cancel_shielded_checkpoint(cls) -> None: + await trio.lowlevel.cancel_shielded_checkpoint() + + @classmethod + async def sleep(cls, delay: float) -> None: + await trio.sleep(delay) + + @classmethod + def create_cancel_scope( + cls, *, deadline: float = math.inf, shield: bool = False + ) -> BaseCancelScope: + return CancelScope(deadline=deadline, shield=shield) + + @classmethod + def current_effective_deadline(cls) -> float: + return trio.current_effective_deadline() + + @classmethod + def create_task_group(cls) -> abc.TaskGroup: + return TaskGroup() + + @classmethod + def create_event(cls) -> BaseEvent: + return Event() + + @classmethod + def create_lock(cls, *, fast_acquire: bool) -> Lock: + return Lock(fast_acquire=fast_acquire) + + @classmethod + def create_semaphore( + cls, + initial_value: int, + *, + max_value: int | None = None, + fast_acquire: bool = False, + ) -> BaseSemaphore: + return Semaphore(initial_value, max_value=max_value, fast_acquire=fast_acquire) + + @classmethod + def create_capacity_limiter(cls, total_tokens: float) -> CapacityLimiter: + return CapacityLimiter(total_tokens) + + @classmethod + async def run_sync_in_worker_thread( + cls, + func: Callable[[Unpack[PosArgsT]], T_Retval], + args: tuple[Unpack[PosArgsT]], + abandon_on_cancel: bool = False, + limiter: BaseCapacityLimiter | None = None, + ) -> T_Retval: + def wrapper() -> T_Retval: + with claim_worker_thread(TrioBackend, token): + return func(*args) + + token = TrioBackend.current_token() + return await run_sync( + wrapper, + abandon_on_cancel=abandon_on_cancel, + limiter=cast(trio.CapacityLimiter, limiter), + ) + + @classmethod + def check_cancelled(cls) -> None: + trio.from_thread.check_cancelled() + + @classmethod + def run_async_from_thread( + cls, + func: Callable[[Unpack[PosArgsT]], Coroutine[Any, Any, T_co]], + args: tuple[Unpack[PosArgsT]], + token: object, + ) -> T_co: + trio_token = cast("trio.lowlevel.TrioToken | None", token) + try: + return trio.from_thread.run(func, *args, trio_token=trio_token) + except trio.RunFinishedError: + raise RunFinishedError from None + + @classmethod + def run_sync_from_thread( + cls, + func: Callable[[Unpack[PosArgsT]], T_Retval], + args: tuple[Unpack[PosArgsT]], + token: object, + ) -> T_Retval: + trio_token = cast("trio.lowlevel.TrioToken | None", token) + try: + return trio.from_thread.run_sync(func, *args, trio_token=trio_token) + except trio.RunFinishedError: + raise RunFinishedError from None + + @classmethod + async def open_process( + cls, + command: StrOrBytesPath | Sequence[StrOrBytesPath], + *, + stdin: int | IO[Any] | None, + stdout: int | IO[Any] | None, + stderr: int | IO[Any] | None, + **kwargs: Any, + ) -> Process: + def convert_item(item: StrOrBytesPath) -> str: + str_or_bytes = os.fspath(item) + if isinstance(str_or_bytes, str): + return str_or_bytes + else: + return os.fsdecode(str_or_bytes) + + if isinstance(command, (str, bytes, PathLike)): + process = await trio.lowlevel.open_process( + convert_item(command), + stdin=stdin, + stdout=stdout, + stderr=stderr, + shell=True, + **kwargs, + ) + else: + process = await trio.lowlevel.open_process( + [convert_item(item) for item in command], + stdin=stdin, + stdout=stdout, + stderr=stderr, + shell=False, + **kwargs, + ) + + stdin_stream = SendStreamWrapper(process.stdin) if process.stdin else None + stdout_stream = ReceiveStreamWrapper(process.stdout) if process.stdout else None + stderr_stream = ReceiveStreamWrapper(process.stderr) if process.stderr else None + return Process(process, stdin_stream, stdout_stream, stderr_stream) + + @classmethod + def setup_process_pool_exit_at_shutdown(cls, workers: set[abc.Process]) -> None: + trio.lowlevel.spawn_system_task(_shutdown_process_pool, workers) + + @classmethod + async def connect_tcp( + cls, host: str, port: int, local_address: IPSockAddrType | None = None + ) -> SocketStream: + family = socket.AF_INET6 if ":" in host else socket.AF_INET + trio_socket = trio.socket.socket(family) + trio_socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + if local_address: + await trio_socket.bind(local_address) + + try: + await trio_socket.connect((host, port)) + except BaseException: + trio_socket.close() + raise + + return SocketStream(trio_socket) + + @classmethod + async def connect_unix(cls, path: str | bytes) -> abc.UNIXSocketStream: + trio_socket = trio.socket.socket(socket.AF_UNIX) + try: + await trio_socket.connect(path) + except BaseException: + trio_socket.close() + raise + + return UNIXSocketStream(trio_socket) + + @classmethod + def create_tcp_listener(cls, sock: socket.socket) -> abc.SocketListener: + return TCPSocketListener(sock) + + @classmethod + def create_unix_listener(cls, sock: socket.socket) -> abc.SocketListener: + return UNIXSocketListener(sock) + + @classmethod + async def create_udp_socket( + cls, + family: socket.AddressFamily, + local_address: IPSockAddrType | None, + remote_address: IPSockAddrType | None, + reuse_port: bool, + ) -> UDPSocket | ConnectedUDPSocket: + trio_socket = trio.socket.socket(family=family, type=socket.SOCK_DGRAM) + + if reuse_port: + trio_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + + if local_address: + await trio_socket.bind(local_address) + + if remote_address: + await trio_socket.connect(remote_address) + return ConnectedUDPSocket(trio_socket) + else: + return UDPSocket(trio_socket) + + @classmethod + @overload + async def create_unix_datagram_socket( + cls, raw_socket: socket.socket, remote_path: None + ) -> abc.UNIXDatagramSocket: ... + + @classmethod + @overload + async def create_unix_datagram_socket( + cls, raw_socket: socket.socket, remote_path: str | bytes + ) -> abc.ConnectedUNIXDatagramSocket: ... + + @classmethod + async def create_unix_datagram_socket( + cls, raw_socket: socket.socket, remote_path: str | bytes | None + ) -> abc.UNIXDatagramSocket | abc.ConnectedUNIXDatagramSocket: + trio_socket = trio.socket.from_stdlib_socket(raw_socket) + + if remote_path: + await trio_socket.connect(remote_path) + return ConnectedUNIXDatagramSocket(trio_socket) + else: + return UNIXDatagramSocket(trio_socket) + + @classmethod + async def getaddrinfo( + cls, + host: bytes | str | None, + port: str | int | None, + *, + family: int | AddressFamily = 0, + type: int | SocketKind = 0, + proto: int = 0, + flags: int = 0, + ) -> Sequence[ + tuple[ + AddressFamily, + SocketKind, + int, + str, + tuple[str, int] | tuple[str, int, int, int] | tuple[int, bytes], + ] + ]: + return await trio.socket.getaddrinfo(host, port, family, type, proto, flags) + + @classmethod + async def getnameinfo( + cls, sockaddr: IPSockAddrType, flags: int = 0 + ) -> tuple[str, str]: + return await trio.socket.getnameinfo(sockaddr, flags) + + @classmethod + async def wait_readable(cls, obj: FileDescriptorLike) -> None: + try: + await wait_readable(obj) + except trio.ClosedResourceError as exc: + raise ClosedResourceError().with_traceback(exc.__traceback__) from None + except trio.BusyResourceError: + raise BusyResourceError("reading from") from None + + @classmethod + async def wait_writable(cls, obj: FileDescriptorLike) -> None: + try: + await wait_writable(obj) + except trio.ClosedResourceError as exc: + raise ClosedResourceError().with_traceback(exc.__traceback__) from None + except trio.BusyResourceError: + raise BusyResourceError("writing to") from None + + @classmethod + def notify_closing(cls, obj: FileDescriptorLike) -> None: + notify_closing(obj) + + @classmethod + async def wrap_listener_socket(cls, sock: socket.socket) -> abc.SocketListener: + if hasattr(socket, "AF_UNIX") and sock.family == socket.AF_UNIX: + return UNIXSocketListener(sock) + + return TCPSocketListener(sock) + + @classmethod + async def wrap_stream_socket(cls, sock: socket.socket) -> SocketStream: + trio_sock = trio.socket.from_stdlib_socket(sock) + return SocketStream(trio_sock) + + @classmethod + async def wrap_unix_stream_socket(cls, sock: socket.socket) -> UNIXSocketStream: + trio_sock = trio.socket.from_stdlib_socket(sock) + return UNIXSocketStream(trio_sock) + + @classmethod + async def wrap_udp_socket(cls, sock: socket.socket) -> UDPSocket: + trio_sock = trio.socket.from_stdlib_socket(sock) + return UDPSocket(trio_sock) + + @classmethod + async def wrap_connected_udp_socket(cls, sock: socket.socket) -> ConnectedUDPSocket: + trio_sock = trio.socket.from_stdlib_socket(sock) + return ConnectedUDPSocket(trio_sock) + + @classmethod + async def wrap_unix_datagram_socket(cls, sock: socket.socket) -> UNIXDatagramSocket: + trio_sock = trio.socket.from_stdlib_socket(sock) + return UNIXDatagramSocket(trio_sock) + + @classmethod + async def wrap_connected_unix_datagram_socket( + cls, sock: socket.socket + ) -> ConnectedUNIXDatagramSocket: + trio_sock = trio.socket.from_stdlib_socket(sock) + return ConnectedUNIXDatagramSocket(trio_sock) + + @classmethod + def current_default_thread_limiter(cls) -> CapacityLimiter: + try: + return _capacity_limiter_wrapper.get() + except LookupError: + limiter = CapacityLimiter( + original=trio.to_thread.current_default_thread_limiter() + ) + _capacity_limiter_wrapper.set(limiter) + return limiter + + @classmethod + def open_signal_receiver( + cls, *signals: Signals + ) -> AbstractContextManager[AsyncIterator[Signals]]: + return _SignalReceiver(signals) + + @classmethod + def get_current_task(cls) -> TaskInfo: + task = current_task() + return TrioTaskInfo(task) + + @classmethod + def get_running_tasks(cls) -> Sequence[TaskInfo]: + root_task = current_root_task() + assert root_task + task_infos = [TrioTaskInfo(root_task)] + nurseries = root_task.child_nurseries + while nurseries: + new_nurseries: list[trio.Nursery] = [] + for nursery in nurseries: + for task in nursery.child_tasks: + task_infos.append(TrioTaskInfo(task)) + new_nurseries.extend(task.child_nurseries) + + nurseries = new_nurseries + + return task_infos + + @classmethod + async def wait_all_tasks_blocked(cls) -> None: + from trio.testing import wait_all_tasks_blocked + + await wait_all_tasks_blocked() + + @classmethod + def create_test_runner(cls, options: dict[str, Any]) -> TestRunner: + return TestRunner(**options) + + +backend_class = TrioBackend diff --git a/.venv/lib/python3.14/site-packages/anyio/_core/__init__.py b/.venv/lib/python3.14/site-packages/anyio/_core/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.venv/lib/python3.14/site-packages/anyio/_core/__pycache__/__init__.cpython-314.pyc b/.venv/lib/python3.14/site-packages/anyio/_core/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b1c0e01d2d7189b033310d40495935a7a22d4bf7 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/anyio/_core/__pycache__/__init__.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/anyio/_core/__pycache__/_asyncio_selector_thread.cpython-314.pyc b/.venv/lib/python3.14/site-packages/anyio/_core/__pycache__/_asyncio_selector_thread.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..dfd151b7fdc76d7ab66412ad48e744ba24caf38c Binary files /dev/null and b/.venv/lib/python3.14/site-packages/anyio/_core/__pycache__/_asyncio_selector_thread.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/anyio/_core/__pycache__/_concurrency_utils.cpython-314.pyc b/.venv/lib/python3.14/site-packages/anyio/_core/__pycache__/_concurrency_utils.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..37d9fa9d8f750a8e8339ba516225cb3950130168 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/anyio/_core/__pycache__/_concurrency_utils.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/anyio/_core/__pycache__/_contextmanagers.cpython-314.pyc b/.venv/lib/python3.14/site-packages/anyio/_core/__pycache__/_contextmanagers.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c78eef6460437f03c8108bddd51637f2a6cd5a05 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/anyio/_core/__pycache__/_contextmanagers.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/anyio/_core/__pycache__/_eventloop.cpython-314.pyc b/.venv/lib/python3.14/site-packages/anyio/_core/__pycache__/_eventloop.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b63f76742ae42d4a080d1230aac00ef0ecf867d4 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/anyio/_core/__pycache__/_eventloop.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/anyio/_core/__pycache__/_exceptions.cpython-314.pyc b/.venv/lib/python3.14/site-packages/anyio/_core/__pycache__/_exceptions.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0191a9a287b0d5a5e8588473af358da2bb73f571 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/anyio/_core/__pycache__/_exceptions.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/anyio/_core/__pycache__/_fileio.cpython-314.pyc b/.venv/lib/python3.14/site-packages/anyio/_core/__pycache__/_fileio.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..479fda79890290d501b9cf787b547fdd4199c295 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/anyio/_core/__pycache__/_fileio.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/anyio/_core/__pycache__/_futures.cpython-314.pyc b/.venv/lib/python3.14/site-packages/anyio/_core/__pycache__/_futures.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..71d326bd51e7eaf95efda71ec59738a8f1188415 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/anyio/_core/__pycache__/_futures.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/anyio/_core/__pycache__/_resources.cpython-314.pyc b/.venv/lib/python3.14/site-packages/anyio/_core/__pycache__/_resources.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e3ce4a224955ce583c8dc166b9dd1833ad7a6953 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/anyio/_core/__pycache__/_resources.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/anyio/_core/__pycache__/_signals.cpython-314.pyc b/.venv/lib/python3.14/site-packages/anyio/_core/__pycache__/_signals.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c1d24bae92fe175a8ff4f7df5e7a391c766edfdf Binary files /dev/null and b/.venv/lib/python3.14/site-packages/anyio/_core/__pycache__/_signals.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/anyio/_core/__pycache__/_sockets.cpython-314.pyc b/.venv/lib/python3.14/site-packages/anyio/_core/__pycache__/_sockets.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..95e69e690827f527281afe3a225975156e582b16 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/anyio/_core/__pycache__/_sockets.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/anyio/_core/__pycache__/_streams.cpython-314.pyc b/.venv/lib/python3.14/site-packages/anyio/_core/__pycache__/_streams.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c1bcd4e5ebf91e3c9cbcfc4996615736a9f448c6 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/anyio/_core/__pycache__/_streams.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/anyio/_core/__pycache__/_subprocesses.cpython-314.pyc b/.venv/lib/python3.14/site-packages/anyio/_core/__pycache__/_subprocesses.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..73c25bee973655483499b4bc64c92b37b1ce1aa1 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/anyio/_core/__pycache__/_subprocesses.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/anyio/_core/__pycache__/_synchronization.cpython-314.pyc b/.venv/lib/python3.14/site-packages/anyio/_core/__pycache__/_synchronization.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c7872dc5277a03408c7553d87ad01f4038ea0c7e Binary files /dev/null and b/.venv/lib/python3.14/site-packages/anyio/_core/__pycache__/_synchronization.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/anyio/_core/__pycache__/_tasks.cpython-314.pyc b/.venv/lib/python3.14/site-packages/anyio/_core/__pycache__/_tasks.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..03875030330bf098e31803ac914bbd49b83e219b Binary files /dev/null and b/.venv/lib/python3.14/site-packages/anyio/_core/__pycache__/_tasks.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/anyio/_core/__pycache__/_tempfile.cpython-314.pyc b/.venv/lib/python3.14/site-packages/anyio/_core/__pycache__/_tempfile.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2efcb583c160525eb3d3e2bb1829787cf07cb50d Binary files /dev/null and b/.venv/lib/python3.14/site-packages/anyio/_core/__pycache__/_tempfile.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/anyio/_core/__pycache__/_testing.cpython-314.pyc b/.venv/lib/python3.14/site-packages/anyio/_core/__pycache__/_testing.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2f5ad7b472d83fe58712b55d1a8187cfaef4ef81 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/anyio/_core/__pycache__/_testing.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/anyio/_core/__pycache__/_typedattr.cpython-314.pyc b/.venv/lib/python3.14/site-packages/anyio/_core/__pycache__/_typedattr.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0db49605d30e757b3784d79622d0d94b9fb4941b Binary files /dev/null and b/.venv/lib/python3.14/site-packages/anyio/_core/__pycache__/_typedattr.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/anyio/_core/_asyncio_selector_thread.py b/.venv/lib/python3.14/site-packages/anyio/_core/_asyncio_selector_thread.py new file mode 100644 index 0000000000000000000000000000000000000000..9f35bae568e33e6a9e1219761c83cc8350fa0532 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/anyio/_core/_asyncio_selector_thread.py @@ -0,0 +1,167 @@ +from __future__ import annotations + +import asyncio +import socket +import threading +from collections.abc import Callable +from selectors import EVENT_READ, EVENT_WRITE, DefaultSelector +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from _typeshed import FileDescriptorLike + +_selector_lock = threading.Lock() +_selector: Selector | None = None + + +class Selector: + def __init__(self) -> None: + self._thread = threading.Thread(target=self.run, name="AnyIO socket selector") + self._selector = DefaultSelector() + self._send, self._receive = socket.socketpair() + self._send.setblocking(False) + self._receive.setblocking(False) + # This somewhat reduces the amount of memory wasted queueing up data + # for wakeups. With these settings, maximum number of 1-byte sends + # before getting BlockingIOError: + # Linux 4.8: 6 + # macOS (darwin 15.5): 1 + # Windows 10: 525347 + # Windows you're weird. (And on Windows setting SNDBUF to 0 makes send + # blocking, even on non-blocking sockets, so don't do that.) + self._receive.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 1) + self._send.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 1) + # On Windows this is a TCP socket so this might matter. On other + # platforms this fails b/c AF_UNIX sockets aren't actually TCP. + try: + self._send.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + except OSError: + pass + + self._selector.register(self._receive, EVENT_READ) + self._closed = False + + def start(self) -> None: + self._thread.start() + threading._register_atexit(self._stop) # type: ignore[attr-defined] + + def _stop(self) -> None: + global _selector + self._closed = True + self._notify_self() + self._send.close() + self._thread.join() + self._selector.unregister(self._receive) + self._receive.close() + self._selector.close() + _selector = None + assert not self._selector.get_map(), ( + "selector still has registered file descriptors after shutdown" + ) + + def _notify_self(self) -> None: + try: + self._send.send(b"\x00") + except BlockingIOError: + pass + + def add_reader(self, fd: FileDescriptorLike, callback: Callable[[], Any]) -> None: + loop = asyncio.get_running_loop() + try: + key = self._selector.get_key(fd) + except KeyError: + self._selector.register(fd, EVENT_READ, {EVENT_READ: (loop, callback)}) + else: + if EVENT_READ in key.data: + raise ValueError( + "this file descriptor is already registered for reading" + ) + + key.data[EVENT_READ] = loop, callback + self._selector.modify(fd, key.events | EVENT_READ, key.data) + + self._notify_self() + + def add_writer(self, fd: FileDescriptorLike, callback: Callable[[], Any]) -> None: + loop = asyncio.get_running_loop() + try: + key = self._selector.get_key(fd) + except KeyError: + self._selector.register(fd, EVENT_WRITE, {EVENT_WRITE: (loop, callback)}) + else: + if EVENT_WRITE in key.data: + raise ValueError( + "this file descriptor is already registered for writing" + ) + + key.data[EVENT_WRITE] = loop, callback + self._selector.modify(fd, key.events | EVENT_WRITE, key.data) + + self._notify_self() + + def remove_reader(self, fd: FileDescriptorLike) -> bool: + try: + key = self._selector.get_key(fd) + except KeyError: + return False + + if new_events := key.events ^ EVENT_READ: + del key.data[EVENT_READ] + self._selector.modify(fd, new_events, key.data) + else: + self._selector.unregister(fd) + + return True + + def remove_writer(self, fd: FileDescriptorLike) -> bool: + try: + key = self._selector.get_key(fd) + except KeyError: + return False + + if new_events := key.events ^ EVENT_WRITE: + del key.data[EVENT_WRITE] + self._selector.modify(fd, new_events, key.data) + else: + self._selector.unregister(fd) + + return True + + def run(self) -> None: + while not self._closed: + for key, events in self._selector.select(): + if key.fileobj is self._receive: + try: + while self._receive.recv(4096): + pass + except BlockingIOError: + pass + + continue + + if events & EVENT_READ: + loop, callback = key.data[EVENT_READ] + self.remove_reader(key.fd) + try: + loop.call_soon_threadsafe(callback) + except RuntimeError: + pass # the loop was already closed + + if events & EVENT_WRITE: + loop, callback = key.data[EVENT_WRITE] + self.remove_writer(key.fd) + try: + loop.call_soon_threadsafe(callback) + except RuntimeError: + pass # the loop was already closed + + +def get_selector() -> Selector: + global _selector + + with _selector_lock: + if _selector is None: + _selector = Selector() + _selector.start() + + return _selector diff --git a/.venv/lib/python3.14/site-packages/anyio/_core/_concurrency_utils.py b/.venv/lib/python3.14/site-packages/anyio/_core/_concurrency_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..f66229a57d7b0d5f793bdfe568a751770939c924 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/anyio/_core/_concurrency_utils.py @@ -0,0 +1,156 @@ +from __future__ import annotations + +from inspect import iscoroutine + +__all__ = ("amap", "as_completed", "gather") + +from collections.abc import AsyncGenerator, Awaitable, Callable, Coroutine, Iterable +from contextlib import asynccontextmanager +from typing import TYPE_CHECKING, Any, TypeVar, overload + +from ..abc._tasks import get_coro_name +from ._exceptions import BrokenResourceError +from ._streams import create_memory_object_stream +from ._tasks import TaskHandle, create_task_group + +if TYPE_CHECKING: + from anyio.streams.memory import MemoryObjectReceiveStream, MemoryObjectSendStream + +R = TypeVar("R") +S = TypeVar("S") +T = TypeVar("T") +U = TypeVar("U") +V = TypeVar("V") +W = TypeVar("W") + + +@overload +async def gather( + coro1: Coroutine[Any, Any, R], coro2: Coroutine[Any, Any, S], / +) -> tuple[R, S]: ... + + +@overload +async def gather( + coro1: Coroutine[Any, Any, R], + coro2: Coroutine[Any, Any, S], + coro3: Coroutine[Any, Any, T], + /, +) -> tuple[R, S, T]: ... + + +@overload +async def gather( + coro1: Coroutine[Any, Any, R], + coro2: Coroutine[Any, Any, S], + coro3: Coroutine[Any, Any, T], + coro4: Coroutine[Any, Any, U], + /, +) -> tuple[R, S, T, U]: ... + + +@overload +async def gather( + coro1: Coroutine[Any, Any, R], + coro2: Coroutine[Any, Any, S], + coro3: Coroutine[Any, Any, T], + coro4: Coroutine[Any, Any, U], + coro5: Coroutine[Any, Any, V], + /, +) -> tuple[R, S, T, U, V]: ... + + +@overload +async def gather( + coro1: Coroutine[Any, Any, R], + coro2: Coroutine[Any, Any, S], + coro3: Coroutine[Any, Any, T], + coro4: Coroutine[Any, Any, U], + coro5: Coroutine[Any, Any, V], + coro6: Coroutine[Any, Any, W], + /, +) -> tuple[R, S, T, U, V, W]: ... + + +# handle arbitrary length if awaitables are all of the same type +@overload +async def gather(*coros: Coroutine[Any, Any, R]) -> tuple[R, ...]: ... + + +async def gather(*coros: Coroutine[Any, Any, Any]) -> tuple[Any, ...]: + """ + Run coroutines concurrently in a task group. The order of result values corresponds + to the order of coroutines passed. + + :param coros: coroutine objects to run as tasks + :return: task results for each argument in the same order as they were passed + + """ + handles: list[TaskHandle[Any]] = [] + async with create_task_group() as tg: + handles.extend(tg.create_task(coro) for coro in coros) + + return tuple(h.return_value for h in handles) + + +@asynccontextmanager +async def as_completed( + *awaitables: Awaitable[R], +) -> AsyncGenerator[MemoryObjectReceiveStream[TaskHandle[R]]]: + """ + Run awaitable objects concurrently in a task group, returning an iterator which can + be used to get finished task handles in the order they complete. + + :param awaitables: awaitable objects to run as tasks + :return: MemoryObjectReceiveStream for iterating over task handles as tasks complete. + + """ + if not awaitables: + raise ValueError("as_completed() takes at least one awaitable") + + send, recv = create_memory_object_stream[TaskHandle[R]](len(awaitables)) + task_handles: list[TaskHandle[R]] = [] + + async def runner( + awaitable: Awaitable[R], + index: int, + _send: MemoryObjectSendStream[TaskHandle[R]], + ) -> R: + async with _send: + try: + return await awaitable + finally: + try: + _send.send_nowait(task_handles[index]) + except BrokenResourceError: + pass + + async with recv, create_task_group() as tg: + async with send: + for i, awaitable in enumerate(awaitables): + name = get_coro_name(awaitable) if iscoroutine(awaitable) else None + task_handles.append( + tg.start_soon(runner, awaitable, i, send.clone(), name=name) + ) + try: + yield recv + finally: + tg.cancel_scope.cancel() + + +async def amap( + func: Callable[[T], Coroutine[Any, Any, R]], args: Iterable[T] +) -> list[R]: + """ + Run the given coroutine function concurrently for multiple argument values. + + :param func: a coroutine function that takes a single argument + :param args: a sequence of argument values to pass to ``func`` + :return: task results for each argument in the same order as they were passed + + """ + handles: list[TaskHandle[R]] = [] + async with create_task_group() as tg: + handles.extend(tg.start_soon(func, arg) for arg in args) + + return [h.return_value for h in handles] diff --git a/.venv/lib/python3.14/site-packages/anyio/_core/_contextmanagers.py b/.venv/lib/python3.14/site-packages/anyio/_core/_contextmanagers.py new file mode 100644 index 0000000000000000000000000000000000000000..302f32b0c78a7071605b195c55054cfdb0b55f37 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/anyio/_core/_contextmanagers.py @@ -0,0 +1,200 @@ +from __future__ import annotations + +from abc import abstractmethod +from contextlib import AbstractAsyncContextManager, AbstractContextManager +from inspect import isasyncgen, iscoroutine, isgenerator +from types import TracebackType +from typing import Protocol, TypeVar, cast, final + +_T_co = TypeVar("_T_co", covariant=True) +_ExitT_co = TypeVar("_ExitT_co", covariant=True, bound="bool | None") + + +class _SupportsCtxMgr(Protocol[_T_co, _ExitT_co]): + def __contextmanager__(self) -> AbstractContextManager[_T_co, _ExitT_co]: ... + + +class _SupportsAsyncCtxMgr(Protocol[_T_co, _ExitT_co]): + def __asynccontextmanager__( + self, + ) -> AbstractAsyncContextManager[_T_co, _ExitT_co]: ... + + +class ContextManagerMixin: + """ + Mixin class providing context manager functionality via a generator-based + implementation. + + This class allows you to implement a context manager via :meth:`__contextmanager__` + which should return a generator. The mechanics are meant to mirror those of + :func:`@contextmanager `. + + .. note:: Classes using this mix-in are not reentrant as context managers, meaning + that once you enter it, you can't re-enter before first exiting it. + + .. seealso:: :doc:`contextmanagers` + """ + + __cm: AbstractContextManager[object, bool | None] | None = None + + @final + def __enter__(self: _SupportsCtxMgr[_T_co, bool | None]) -> _T_co: + # Needed for mypy to assume self still has the __cm member + assert isinstance(self, ContextManagerMixin) + if self.__cm is not None: + raise RuntimeError( + f"this {self.__class__.__qualname__} has already been entered" + ) + + cm = self.__contextmanager__() + if not isinstance(cm, AbstractContextManager): + if isgenerator(cm): + raise TypeError( + "__contextmanager__() returned a generator object instead of " + "a context manager. Did you forget to add the @contextmanager " + "decorator?" + ) + + raise TypeError( + f"__contextmanager__() did not return a context manager object, " + f"but {cm.__class__!r}" + ) + + if cm is self: + raise TypeError( + f"{self.__class__.__qualname__}.__contextmanager__() returned " + f"self. Did you forget to add the @contextmanager decorator and a " + f"'yield' statement?" + ) + + value = cm.__enter__() + self.__cm = cm + return value + + @final + def __exit__( + self: _SupportsCtxMgr[object, _ExitT_co], + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> _ExitT_co: + # Needed for mypy to assume self still has the __cm member + assert isinstance(self, ContextManagerMixin) + if self.__cm is None: + raise RuntimeError( + f"this {self.__class__.__qualname__} has not been entered yet" + ) + + # Prevent circular references + cm = self.__cm + del self.__cm + + return cast(_ExitT_co, cm.__exit__(exc_type, exc_val, exc_tb)) + + @abstractmethod + def __contextmanager__(self) -> AbstractContextManager[object, bool | None]: + """ + Implement your context manager logic here. + + This method **must** be decorated with + :func:`@contextmanager `. + + .. note:: Remember that the ``yield`` will raise any exception raised in the + enclosed context block, so use a ``finally:`` block to clean up resources! + + :return: a context manager object + """ + + +class AsyncContextManagerMixin: + """ + Mixin class providing async context manager functionality via a generator-based + implementation. + + This class allows you to implement a context manager via + :meth:`__asynccontextmanager__`. The mechanics are meant to mirror those of + :func:`@asynccontextmanager `. + + .. note:: Classes using this mix-in are not reentrant as context managers, meaning + that once you enter it, you can't re-enter before first exiting it. + + .. seealso:: :doc:`contextmanagers` + """ + + __cm: AbstractAsyncContextManager[object, bool | None] | None = None + + @final + async def __aenter__(self: _SupportsAsyncCtxMgr[_T_co, bool | None]) -> _T_co: + # Needed for mypy to assume self still has the __cm member + assert isinstance(self, AsyncContextManagerMixin) + if self.__cm is not None: + raise RuntimeError( + f"this {self.__class__.__qualname__} has already been entered" + ) + + cm = self.__asynccontextmanager__() + if not isinstance(cm, AbstractAsyncContextManager): + if isasyncgen(cm): + raise TypeError( + "__asynccontextmanager__() returned an async generator instead of " + "an async context manager. Did you forget to add the " + "@asynccontextmanager decorator?" + ) + elif iscoroutine(cm): + cm.close() + raise TypeError( + "__asynccontextmanager__() returned a coroutine object instead of " + "an async context manager. Did you forget to add the " + "@asynccontextmanager decorator and a 'yield' statement?" + ) + + raise TypeError( + f"__asynccontextmanager__() did not return an async context manager, " + f"but {cm.__class__!r}" + ) + + if cm is self: + raise TypeError( + f"{self.__class__.__qualname__}.__asynccontextmanager__() returned " + f"self. Did you forget to add the @asynccontextmanager decorator and a " + f"'yield' statement?" + ) + + value = await cm.__aenter__() + self.__cm = cm + return value + + @final + async def __aexit__( + self: _SupportsAsyncCtxMgr[object, _ExitT_co], + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> _ExitT_co: + assert isinstance(self, AsyncContextManagerMixin) + if self.__cm is None: + raise RuntimeError( + f"this {self.__class__.__qualname__} has not been entered yet" + ) + + # Prevent circular references + cm = self.__cm + del self.__cm + + return cast(_ExitT_co, await cm.__aexit__(exc_type, exc_val, exc_tb)) + + @abstractmethod + def __asynccontextmanager__( + self, + ) -> AbstractAsyncContextManager[object, bool | None]: + """ + Implement your async context manager logic here. + + This method **must** be decorated with + :func:`@asynccontextmanager `. + + .. note:: Remember that the ``yield`` will raise any exception raised in the + enclosed context block, so use a ``finally:`` block to clean up resources! + + :return: an async context manager object + """ diff --git a/.venv/lib/python3.14/site-packages/anyio/_core/_eventloop.py b/.venv/lib/python3.14/site-packages/anyio/_core/_eventloop.py new file mode 100644 index 0000000000000000000000000000000000000000..a3e2ab1cc172f2fbd94f244ab110cb16aa7eb8eb --- /dev/null +++ b/.venv/lib/python3.14/site-packages/anyio/_core/_eventloop.py @@ -0,0 +1,240 @@ +from __future__ import annotations + +import math +import sys +import threading +from collections.abc import Awaitable, Callable, Generator +from contextlib import contextmanager +from contextvars import Token +from importlib import import_module +from typing import TYPE_CHECKING, Any, TypeVar + +from ._exceptions import NoEventLoopError + +if sys.version_info >= (3, 11): + from typing import TypeVarTuple, Unpack +else: + from typing_extensions import TypeVarTuple, Unpack + +sniffio: Any +try: + import sniffio +except ModuleNotFoundError: + sniffio = None + +if TYPE_CHECKING: + from ..abc import AsyncBackend + +# This must be updated when new backends are introduced +BACKENDS = "asyncio", "trio" + +T_Retval = TypeVar("T_Retval") +PosArgsT = TypeVarTuple("PosArgsT") + +threadlocals = threading.local() +loaded_backends: dict[str, type[AsyncBackend]] = {} + + +def run( + func: Callable[[Unpack[PosArgsT]], Awaitable[T_Retval]], + *args: Unpack[PosArgsT], + backend: str = "asyncio", + backend_options: dict[str, Any] | None = None, +) -> T_Retval: + """ + Run the given coroutine function in an asynchronous event loop. + + The current thread must not be already running an event loop. + + :param func: a coroutine function + :param args: positional arguments to ``func`` + :param backend: name of the asynchronous event loop implementation – currently + either ``asyncio`` or ``trio`` + :param backend_options: keyword arguments to call the backend ``run()`` + implementation with (documented :ref:`here `) + :return: the return value of the coroutine function + :raises RuntimeError: if an asynchronous event loop is already running in this + thread + :raises LookupError: if the named backend is not found + + """ + if asynclib_name := current_async_library(): + raise RuntimeError(f"Already running {asynclib_name} in this thread") + + try: + async_backend = get_async_backend(backend) + except ImportError as exc: + if backend in BACKENDS: + raise LookupError( + f"Backend {backend!r} is not available. " + f"Install it with: pip install anyio[{backend}]" + ) from exc + + raise LookupError(f"No such backend: {backend}") from exc + + token = None + if asynclib_name is None: + # Since we're in control of the event loop, we can cache the name of the async + # library + token = set_current_async_library(backend) + + try: + backend_options = backend_options or {} + return async_backend.run(func, args, {}, backend_options) + finally: + reset_current_async_library(token) + + +async def sleep(delay: float) -> None: + """ + Pause the current task for the specified duration. + + :param delay: the duration, in seconds + + """ + return await get_async_backend().sleep(delay) + + +async def sleep_forever() -> None: + """ + Pause the current task until it's cancelled. + + This is a shortcut for ``sleep(math.inf)``. + + .. versionadded:: 3.1 + + """ + await sleep(math.inf) + + +async def sleep_until(deadline: float) -> None: + """ + Pause the current task until the given time. + + :param deadline: the absolute time to wake up at (according to the internal + monotonic clock of the event loop) + + .. versionadded:: 3.1 + + """ + now = current_time() + await sleep(max(deadline - now, 0)) + + +def current_time() -> float: + """ + Return the current value of the event loop's internal clock. + + :return: the clock value (seconds) + :raises NoEventLoopError: if no supported asynchronous event loop is running in the + current thread + + """ + return get_async_backend().current_time() + + +def get_all_backends() -> tuple[str, ...]: + """Return a tuple of the names of all built-in backends.""" + return BACKENDS + + +def get_available_backends() -> tuple[str, ...]: + """ + Test for the availability of built-in backends. + + :return a tuple of the built-in backend names that were successfully imported + + .. versionadded:: 4.12 + + """ + available_backends: list[str] = [] + for backend_name in get_all_backends(): + try: + get_async_backend(backend_name) + except ImportError: + continue + + available_backends.append(backend_name) + + return tuple(available_backends) + + +def get_cancelled_exc_class() -> type[BaseException]: + """ + Return the current async library's cancellation exception class. + + :raises NoEventLoopError: if no supported asynchronous event loop is running in the + current thread + + """ + return get_async_backend().cancelled_exception_class() + + +# +# Private API +# + + +@contextmanager +def claim_worker_thread( + backend_class: type[AsyncBackend], token: object +) -> Generator[Any, None, None]: + from ..lowlevel import EventLoopToken + + threadlocals.current_token = EventLoopToken(backend_class, token) + try: + yield + finally: + del threadlocals.current_token + + +def get_async_backend(asynclib_name: str | None = None) -> type[AsyncBackend]: + if asynclib_name is None: + asynclib_name = current_async_library() + if not asynclib_name: + raise NoEventLoopError( + f"Not currently running on any asynchronous event loop. " + f"Available async backends: {', '.join(get_all_backends())}" + ) + + # We use our own dict instead of sys.modules to get the already imported back-end + # class because the appropriate modules in sys.modules could potentially be only + # partially initialized + try: + return loaded_backends[asynclib_name] + except KeyError: + module = import_module(f"anyio._backends._{asynclib_name}") + loaded_backends[asynclib_name] = module.backend_class + return module.backend_class + + +def current_async_library() -> str | None: + if sniffio is None: + # If sniffio is not installed, we assume we're either running asyncio or nothing + import asyncio + + try: + asyncio.get_running_loop() + return "asyncio" + except RuntimeError: + pass + else: + try: + return sniffio.current_async_library() + except sniffio.AsyncLibraryNotFoundError: + pass + + return None + + +def set_current_async_library(asynclib_name: str | None) -> Token | None: + # no-op if sniffio is not installed + if sniffio is None: + return None + + return sniffio.current_async_library_cvar.set(asynclib_name) + + +def reset_current_async_library(token: Token | None) -> None: + if token is not None: + sniffio.current_async_library_cvar.reset(token) diff --git a/.venv/lib/python3.14/site-packages/anyio/_core/_exceptions.py b/.venv/lib/python3.14/site-packages/anyio/_core/_exceptions.py new file mode 100644 index 0000000000000000000000000000000000000000..7d9165f53ab239182f9158f8eeec7ea27b417c92 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/anyio/_core/_exceptions.py @@ -0,0 +1,205 @@ +from __future__ import annotations + +import sys +from collections.abc import Generator +from textwrap import dedent +from typing import Any + +if sys.version_info < (3, 11): + from exceptiongroup import BaseExceptionGroup + + +class BrokenResourceError(Exception): + """ + Raised when trying to use a resource that has been rendered unusable due to external + causes (e.g. a send stream whose peer has disconnected). + """ + + +class BrokenWorkerProcess(Exception): + """ + Raised by :meth:`~anyio.to_process.run_sync` if the worker process terminates abruptly or + otherwise misbehaves. + """ + + +class BrokenWorkerInterpreter(Exception): + """ + Raised by :meth:`~anyio.to_interpreter.run_sync` if an unexpected exception is + raised in the subinterpreter. + """ + + def __init__(self, excinfo: Any): + # This was adapted from concurrent.futures.interpreter.ExecutionFailed + msg = excinfo.formatted + if not msg: + if excinfo.type and excinfo.msg: + msg = f"{excinfo.type.__name__}: {excinfo.msg}" + else: + msg = excinfo.type.__name__ or excinfo.msg + + super().__init__(msg) + self.excinfo = excinfo + + def __str__(self) -> str: + try: + formatted = self.excinfo.errdisplay + except Exception: + return super().__str__() + else: + return dedent( + f""" + {super().__str__()} + + Uncaught in the interpreter: + + {formatted} + """.strip() + ) + + +class BusyResourceError(Exception): + """ + Raised when two tasks are trying to read from or write to the same resource + concurrently. + """ + + def __init__(self, action: str): + super().__init__(f"Another task is already {action} this resource") + + +class ClosedResourceError(Exception): + """Raised when trying to use a resource that has been closed.""" + + +class ConnectionFailed(OSError): + """ + Raised when a connection attempt fails. + + .. note:: This class inherits from :exc:`OSError` for backwards compatibility. + """ + + +def iterate_exceptions( + exception: BaseException, +) -> Generator[BaseException, None, None]: + if isinstance(exception, BaseExceptionGroup): + for exc in exception.exceptions: + yield from iterate_exceptions(exc) + else: + yield exception + + +class DelimiterNotFound(Exception): + """ + Raised during + :meth:`~anyio.streams.buffered.BufferedByteReceiveStream.receive_until` if the + maximum number of bytes has been read without the delimiter being found. + """ + + def __init__(self, max_bytes: int) -> None: + super().__init__( + f"The delimiter was not found among the first {max_bytes} bytes" + ) + + +class EndOfStream(Exception): + """ + Raised when trying to read from a stream that has been closed from the other end. + """ + + +class IncompleteRead(Exception): + """ + Raised during + :meth:`~anyio.streams.buffered.BufferedByteReceiveStream.receive_exactly` or + :meth:`~anyio.streams.buffered.BufferedByteReceiveStream.receive_until` if the + connection is closed before the requested amount of bytes has been read. + """ + + def __init__(self) -> None: + super().__init__( + "The stream was closed before the read operation could be completed" + ) + + +class TypedAttributeLookupError(LookupError): + """ + Raised by :meth:`~anyio.TypedAttributeProvider.extra` when the given typed attribute + is not found and no default value has been given. + """ + + +class WouldBlock(Exception): + """Raised by ``X_nowait`` functions if ``X()`` would block.""" + + +class NoEventLoopError(RuntimeError): + """ + Raised by several functions that require an event loop to be running in the current + thread when there is no running event loop. + + This is also raised by :func:`.from_thread.run` and :func:`.from_thread.run_sync` + if not calling from an AnyIO worker thread, and no ``token`` was passed. + """ + + +class RunFinishedError(RuntimeError): + """ + Raised by :func:`.from_thread.run` and :func:`.from_thread.run_sync` if the event + loop associated with the explicitly passed token has already finished. + """ + + def __init__(self) -> None: + super().__init__( + "The event loop associated with the given token has already finished" + ) + + +class TaskFailed(Exception): + """ + Raised when awaiting on, or attempting to access the return value of, a + :class:`.TaskHandle` that raised an exception. + """ + + +class TaskCancelled(TaskFailed): + """ + Raised when awaiting on, or attempting to access the return value of, a + :class:`.TaskHandle` that was cancelled. + """ + + +class TaskNotFinished(Exception): + """ + Raised when attempting to access the return value or exception of a + :class:`.TaskHandle` that is still pending completion. + """ + + +class FutureFailed(Exception): + """ + Raised when awaiting on, or attempting to access the return value of, a + :class:`.Future` that raised an exception. + """ + + +class FutureCancelled(FutureFailed): + """ + Raised when attempting to access the return value or exception of a + :class:`.Future` that was cancelled. + """ + + +class FutureNotFinished(Exception): + """ + Raised when attempting to access the return value or exception of a + :class:`.Future` that is still pending completion. + """ + + +class FutureAlreadyFinished(Exception): + """ + Raised when attempting set a result of or await a + :class:`.Future` that has already completed. + """ diff --git a/.venv/lib/python3.14/site-packages/anyio/_core/_fileio.py b/.venv/lib/python3.14/site-packages/anyio/_core/_fileio.py new file mode 100644 index 0000000000000000000000000000000000000000..498eef41d19fde3991f05472e0bc385c520f49ec --- /dev/null +++ b/.venv/lib/python3.14/site-packages/anyio/_core/_fileio.py @@ -0,0 +1,1045 @@ +from __future__ import annotations + +import os +import pathlib +import sys +from collections.abc import ( + AsyncIterator, + Callable, + Iterable, + Iterator, + Sequence, +) +from dataclasses import dataclass +from functools import partial +from os import PathLike +from typing import ( + IO, + TYPE_CHECKING, + Any, + AnyStr, + ClassVar, + Final, + Generic, + TypeVar, + overload, +) + +from .. import to_thread +from ..abc import AsyncResource +from ._synchronization import CapacityLimiter + +if sys.version_info >= (3, 11): + from typing import Self +else: + from typing_extensions import Self + +if sys.version_info >= (3, 14): + from pathlib.types import PathInfo + +if TYPE_CHECKING: + from types import ModuleType + + from _typeshed import OpenBinaryMode, OpenTextMode, ReadableBuffer, WriteableBuffer +else: + ReadableBuffer = OpenBinaryMode = OpenTextMode = WriteableBuffer = object + + +T = TypeVar("T", bound="Path") + + +class AsyncFile(AsyncResource, Generic[AnyStr]): + """ + An asynchronous file object. + + This class wraps a standard file object and provides async friendly versions of the + following blocking methods (where available on the original file object): + + * read + * read1 + * readline + * readlines + * readinto + * readinto1 + * write + * writelines + * truncate + * seek + * tell + * flush + + All other methods are directly passed through. + + This class supports the asynchronous context manager protocol which closes the + underlying file at the end of the context block. + + This class also supports asynchronous iteration:: + + async with await open_file(...) as f: + async for line in f: + print(line) + """ + + def __init__( + self, fp: IO[AnyStr], *, limiter: CapacityLimiter | None = None + ) -> None: + if limiter is not None and not isinstance(limiter, CapacityLimiter): + raise TypeError( + f"limiter must be a CapacityLimiter or None, not " + f"{limiter.__class__.__name__}" + ) + + self._fp: Any = fp + self._limiter = limiter + + def __getattr__(self, name: str) -> object: + return getattr(self._fp, name) + + @property + def limiter(self) -> CapacityLimiter | None: + """The capacity limiter used by this file object, if not the global limiter.""" + return self._limiter + + @property + def wrapped(self) -> IO[AnyStr]: + """The wrapped file object.""" + return self._fp + + async def __aiter__(self) -> AsyncIterator[AnyStr]: + while True: + line = await self.readline() + if line: + yield line + else: + break + + async def aclose(self) -> None: + return await to_thread.run_sync(self._fp.close, limiter=self._limiter) + + async def read(self, size: int = -1) -> AnyStr: + return await to_thread.run_sync(self._fp.read, size, limiter=self._limiter) + + async def read1(self: AsyncFile[bytes], size: int = -1) -> bytes: + return await to_thread.run_sync(self._fp.read1, size, limiter=self._limiter) + + async def readline(self) -> AnyStr: + return await to_thread.run_sync(self._fp.readline, limiter=self._limiter) + + async def readlines(self) -> list[AnyStr]: + return await to_thread.run_sync(self._fp.readlines, limiter=self._limiter) + + async def readinto(self: AsyncFile[bytes], b: WriteableBuffer) -> int: + return await to_thread.run_sync(self._fp.readinto, b, limiter=self._limiter) + + async def readinto1(self: AsyncFile[bytes], b: WriteableBuffer) -> int: + return await to_thread.run_sync(self._fp.readinto1, b, limiter=self._limiter) + + @overload + async def write(self: AsyncFile[bytes], b: ReadableBuffer) -> int: ... + + @overload + async def write(self: AsyncFile[str], b: str) -> int: ... + + async def write(self, b: ReadableBuffer | str) -> int: + return await to_thread.run_sync(self._fp.write, b, limiter=self._limiter) + + @overload + async def writelines( + self: AsyncFile[bytes], lines: Iterable[ReadableBuffer] + ) -> None: ... + + @overload + async def writelines(self: AsyncFile[str], lines: Iterable[str]) -> None: ... + + async def writelines(self, lines: Iterable[ReadableBuffer] | Iterable[str]) -> None: + return await to_thread.run_sync( + self._fp.writelines, lines, limiter=self._limiter + ) + + async def truncate(self, size: int | None = None) -> int: + return await to_thread.run_sync(self._fp.truncate, size, limiter=self._limiter) + + async def seek(self, offset: int, whence: int | None = os.SEEK_SET) -> int: + return await to_thread.run_sync( + self._fp.seek, offset, whence, limiter=self._limiter + ) + + async def tell(self) -> int: + return await to_thread.run_sync(self._fp.tell, limiter=self._limiter) + + async def flush(self) -> None: + return await to_thread.run_sync(self._fp.flush, limiter=self._limiter) + + +@overload +async def open_file( + file: str | PathLike[str] | int, + mode: OpenBinaryMode, + buffering: int = ..., + encoding: str | None = ..., + errors: str | None = ..., + newline: str | None = ..., + closefd: bool = ..., + opener: Callable[[str, int], int] | None = ..., + *, + limiter: CapacityLimiter | None = ..., +) -> AsyncFile[bytes]: ... + + +@overload +async def open_file( + file: str | PathLike[str] | int, + mode: OpenTextMode = ..., + buffering: int = ..., + encoding: str | None = ..., + errors: str | None = ..., + newline: str | None = ..., + closefd: bool = ..., + opener: Callable[[str, int], int] | None = ..., + *, + limiter: CapacityLimiter | None = ..., +) -> AsyncFile[str]: ... + + +async def open_file( + file: str | PathLike[str] | int, + mode: str = "r", + buffering: int = -1, + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, + closefd: bool = True, + opener: Callable[[str, int], int] | None = None, + *, + limiter: CapacityLimiter | None = None, +) -> AsyncFile[Any]: + """ + Open a file asynchronously. + + Except for ``limiter``, the arguments are exactly the same as for the builtin :func:`open`. + + :param limiter: an optional capacity limiter to use with the file + instead of the default one + :return: an asynchronous file object + + .. versionchanged:: 4.14.0 + Added the ``limiter`` keyword argument. + + """ + fp = await to_thread.run_sync( + open, + file, + mode, + buffering, + encoding, + errors, + newline, + closefd, + opener, + limiter=limiter, + ) + return AsyncFile(fp, limiter=limiter) + + +def wrap_file( + file: IO[AnyStr], *, limiter: CapacityLimiter | None = None +) -> AsyncFile[AnyStr]: + """ + Wrap an existing file as an asynchronous file. + + :param file: an existing file-like object + :param limiter: an optional capacity limiter to use with the file + instead of the default one + :return: an asynchronous file object + + .. versionchanged:: 4.14.0 + Added the ``limiter`` keyword argument. + + """ + return AsyncFile(file, limiter=limiter) + + +@dataclass(eq=False) +class _PathIterator(AsyncIterator[T]): + iterator: Iterator[PathLike[str]] + limiter: CapacityLimiter | None + # This was added to ensure that iterating over a subclass of Path yields instances + # of that subclass rather than the base Path class. + path_cls: type[T] + + async def __anext__(self) -> T: + nextval = await to_thread.run_sync( + next, self.iterator, None, abandon_on_cancel=True, limiter=self.limiter + ) + if nextval is None: + raise StopAsyncIteration from None + + return self.path_cls(nextval, limiter=self.limiter) + + +class Path: + """ + An asynchronous version of :class:`pathlib.Path`. + + This class cannot be substituted for :class:`pathlib.Path` or + :class:`pathlib.PurePath`, but it is compatible with the :class:`os.PathLike` + interface. + + It implements the Python 3.10 version of :class:`pathlib.Path` interface, except for + the deprecated :meth:`~pathlib.Path.link_to` method. + + Some methods may be unavailable or have limited functionality, based on the Python + version: + + * :meth:`~pathlib.Path.copy` (available on Python 3.14 or later) + * :meth:`~pathlib.Path.copy_into` (available on Python 3.14 or later) + * :meth:`~pathlib.Path.from_uri` (available on Python 3.13 or later) + * :meth:`~pathlib.PurePath.full_match` (available on Python 3.13 or later) + * :attr:`~pathlib.Path.info` (available on Python 3.14 or later) + * :meth:`~pathlib.Path.is_junction` (available on Python 3.12 or later) + * :meth:`~pathlib.PurePath.match` (the ``case_sensitive`` parameter is only + available on Python 3.13 or later) + * :meth:`~pathlib.Path.move` (available on Python 3.14 or later) + * :meth:`~pathlib.Path.move_into` (available on Python 3.14 or later) + * :meth:`~pathlib.PurePath.relative_to` (the ``walk_up`` parameter is only available + on Python 3.12 or later) + * :meth:`~pathlib.Path.walk` (available on Python 3.12 or later) + + Any methods that do disk I/O need to be awaited on. These methods are: + + * :meth:`~pathlib.Path.absolute` + * :meth:`~pathlib.Path.chmod` + * :meth:`~pathlib.Path.cwd` + * :meth:`~pathlib.Path.exists` + * :meth:`~pathlib.Path.expanduser` + * :meth:`~pathlib.Path.group` + * :meth:`~pathlib.Path.hardlink_to` + * :meth:`~pathlib.Path.home` + * :meth:`~pathlib.Path.is_block_device` + * :meth:`~pathlib.Path.is_char_device` + * :meth:`~pathlib.Path.is_dir` + * :meth:`~pathlib.Path.is_fifo` + * :meth:`~pathlib.Path.is_file` + * :meth:`~pathlib.Path.is_junction` + * :meth:`~pathlib.Path.is_mount` + * :meth:`~pathlib.Path.is_socket` + * :meth:`~pathlib.Path.is_symlink` + * :meth:`~pathlib.Path.lchmod` + * :meth:`~pathlib.Path.lstat` + * :meth:`~pathlib.Path.mkdir` + * :meth:`~pathlib.Path.open` + * :meth:`~pathlib.Path.owner` + * :meth:`~pathlib.Path.read_bytes` + * :meth:`~pathlib.Path.read_text` + * :meth:`~pathlib.Path.readlink` + * :meth:`~pathlib.Path.rename` + * :meth:`~pathlib.Path.replace` + * :meth:`~pathlib.Path.resolve` + * :meth:`~pathlib.Path.rmdir` + * :meth:`~pathlib.Path.samefile` + * :meth:`~pathlib.Path.stat` + * :meth:`~pathlib.Path.symlink_to` + * :meth:`~pathlib.Path.touch` + * :meth:`~pathlib.Path.unlink` + * :meth:`~pathlib.Path.walk` + * :meth:`~pathlib.Path.write_bytes` + * :meth:`~pathlib.Path.write_text` + + Additionally, the following methods return an async iterator yielding + :class:`~.Path` objects: + + * :meth:`~pathlib.Path.glob` + * :meth:`~pathlib.Path.iterdir` + * :meth:`~pathlib.Path.rglob` + + .. versionchanged:: 4.14.0 + Added the ``limiter`` keyword argument. + """ + + __slots__ = "__weakref__", "_limiter", "_path" + + __weakref__: Any + + def __init__( + self, *args: str | PathLike[str], limiter: CapacityLimiter | None = None + ) -> None: + if limiter is not None and not isinstance(limiter, CapacityLimiter): + raise TypeError( + f"limiter must be a CapacityLimiter or None, not " + f"{limiter.__class__.__name__}" + ) + + self._path: Final[pathlib.Path] = pathlib.Path(*args) + self._limiter = limiter + + def __fspath__(self) -> str: + return self._path.__fspath__() + + if sys.version_info >= (3, 15): + + def __vfspath__(self) -> str: + return self._path.__vfspath__() + + def __str__(self) -> str: + return self._path.__str__() + + def __repr__(self) -> str: + return f"{self.__class__.__name__}({self.as_posix()!r})" + + def __bytes__(self) -> bytes: + return self._path.__bytes__() + + def __hash__(self) -> int: + return self._path.__hash__() + + def __eq__(self, other: object) -> bool: + target = other._path if isinstance(other, Path) else other + return self._path.__eq__(target) + + def __lt__(self, other: pathlib.PurePath | Path) -> bool: + target = other._path if isinstance(other, Path) else other + return self._path.__lt__(target) + + def __le__(self, other: pathlib.PurePath | Path) -> bool: + target = other._path if isinstance(other, Path) else other + return self._path.__le__(target) + + def __gt__(self, other: pathlib.PurePath | Path) -> bool: + target = other._path if isinstance(other, Path) else other + return self._path.__gt__(target) + + def __ge__(self, other: pathlib.PurePath | Path) -> bool: + target = other._path if isinstance(other, Path) else other + return self._path.__ge__(target) + + def __truediv__(self, other: str | PathLike[str]) -> Self: + return type(self)(self._path / other, limiter=self._limiter) + + def __rtruediv__(self, other: str | PathLike[str]) -> Self: + return type(self)(other, limiter=self._limiter) / self + + @property + def limiter(self) -> CapacityLimiter | None: + """The capacity limiter used by this path, if not the global limiter.""" + return self._limiter + + @property + def parts(self) -> tuple[str, ...]: + return self._path.parts + + @property + def drive(self) -> str: + return self._path.drive + + @property + def root(self) -> str: + return self._path.root + + @property + def anchor(self) -> str: + return self._path.anchor + + @property + def parents(self) -> Sequence[Self]: + return tuple(type(self)(p, limiter=self._limiter) for p in self._path.parents) + + @property + def parent(self) -> Self: + return type(self)(self._path.parent, limiter=self._limiter) + + @property + def name(self) -> str: + return self._path.name + + @property + def suffix(self) -> str: + return self._path.suffix + + @property + def suffixes(self) -> list[str]: + return self._path.suffixes + + @property + def stem(self) -> str: + return self._path.stem + + async def absolute(self) -> Self: + path = await to_thread.run_sync(self._path.absolute, limiter=self._limiter) + return type(self)(path, limiter=self._limiter) + + def as_posix(self) -> str: + return self._path.as_posix() + + def as_uri(self) -> str: + return self._path.as_uri() + + if sys.version_info >= (3, 13): + parser: ClassVar[ModuleType] = pathlib.Path.parser + + @classmethod + def from_uri(cls, uri: str, *, limiter: CapacityLimiter | None = None) -> Self: + return cls(pathlib.Path.from_uri(uri), limiter=limiter) + + def full_match( + self, path_pattern: str, *, case_sensitive: bool | None = None + ) -> bool: + return self._path.full_match(path_pattern, case_sensitive=case_sensitive) + + def match( + self, path_pattern: str, *, case_sensitive: bool | None = None + ) -> bool: + return self._path.match(path_pattern, case_sensitive=case_sensitive) + else: + + def match(self, path_pattern: str) -> bool: + return self._path.match(path_pattern) + + if sys.version_info >= (3, 14): + + @property + def info(self) -> PathInfo: + return self._path.info + + async def copy( + self, + target: str | os.PathLike[str], + *, + follow_symlinks: bool = True, + preserve_metadata: bool = False, + ) -> Self: + func = partial( + self._path.copy, + follow_symlinks=follow_symlinks, + preserve_metadata=preserve_metadata, + ) + return type(self)( + await to_thread.run_sync( + func, pathlib.Path(target), limiter=self._limiter + ), + limiter=self._limiter, + ) + + async def copy_into( + self, + target_dir: str | os.PathLike[str], + *, + follow_symlinks: bool = True, + preserve_metadata: bool = False, + ) -> Self: + func = partial( + self._path.copy_into, + follow_symlinks=follow_symlinks, + preserve_metadata=preserve_metadata, + ) + return type(self)( + await to_thread.run_sync( + func, pathlib.Path(target_dir), limiter=self._limiter + ), + limiter=self._limiter, + ) + + async def move(self, target: str | os.PathLike[str]) -> Self: + # Upstream does not handle anyio.Path properly as a PathLike + target = pathlib.Path(target) + return type(self)( + await to_thread.run_sync( + self._path.move, target, limiter=self._limiter + ), + limiter=self._limiter, + ) + + async def move_into( + self, + target_dir: str | os.PathLike[str], + ) -> Self: + return type(self)( + await to_thread.run_sync( + self._path.move_into, target_dir, limiter=self._limiter + ), + limiter=self._limiter, + ) + + def is_relative_to(self, other: str | PathLike[str]) -> bool: + try: + self.relative_to(other) + return True + except ValueError: + return False + + async def chmod(self, mode: int, *, follow_symlinks: bool = True) -> None: + func = partial(os.chmod, follow_symlinks=follow_symlinks) + return await to_thread.run_sync(func, self._path, mode, limiter=self._limiter) + + @classmethod + async def cwd(cls, *, limiter: CapacityLimiter | None = None) -> Self: + path = await to_thread.run_sync(pathlib.Path.cwd, limiter=limiter) + return cls(path, limiter=limiter) + + if sys.version_info >= (3, 12): + + async def exists(self, *, follow_symlinks: bool = True) -> bool: + return await to_thread.run_sync( + partial(self._path.exists, follow_symlinks=follow_symlinks), + abandon_on_cancel=True, + limiter=self._limiter, + ) + + else: + + async def exists(self) -> bool: + return await to_thread.run_sync( + self._path.exists, abandon_on_cancel=True, limiter=self._limiter + ) + + async def expanduser(self) -> Self: + return type(self)( + await to_thread.run_sync( + self._path.expanduser, abandon_on_cancel=True, limiter=self._limiter + ), + limiter=self._limiter, + ) + + if sys.version_info < (3, 12): + # Python 3.11 and earlier + def glob(self, pattern: str) -> AsyncIterator[Self]: + gen = self._path.glob(pattern) + return _PathIterator(gen, self._limiter, type(self)) + elif (3, 12) <= sys.version_info < (3, 13): + # changed in Python 3.12: + # - The case_sensitive parameter was added. + def glob( + self, + pattern: str, + *, + case_sensitive: bool | None = None, + ) -> AsyncIterator[Self]: + gen = self._path.glob(pattern, case_sensitive=case_sensitive) + return _PathIterator(gen, self._limiter, type(self)) + elif sys.version_info >= (3, 13): + # Changed in Python 3.13: + # - The recurse_symlinks parameter was added. + # - The pattern parameter accepts a path-like object. + def glob( # type: ignore[misc] # mypy doesn't allow for differing signatures in a conditional block + self, + pattern: str | PathLike[str], + *, + case_sensitive: bool | None = None, + recurse_symlinks: bool = False, + ) -> AsyncIterator[Self]: + gen = self._path.glob( + pattern, # type: ignore[arg-type] + case_sensitive=case_sensitive, + recurse_symlinks=recurse_symlinks, + ) + return _PathIterator(gen, self._limiter, type(self)) + + if sys.version_info >= (3, 13): + + async def group(self, *, follow_symlinks: bool = True) -> str: + return await to_thread.run_sync( + partial(self._path.group, follow_symlinks=follow_symlinks), + abandon_on_cancel=True, + limiter=self._limiter, + ) + + else: + + async def group(self) -> str: + return await to_thread.run_sync( + self._path.group, abandon_on_cancel=True, limiter=self._limiter + ) + + async def hardlink_to( + self, target: str | bytes | PathLike[str] | PathLike[bytes] + ) -> None: + if isinstance(target, Path): + target = target._path + + await to_thread.run_sync(os.link, target, self, limiter=self._limiter) + + @classmethod + async def home(cls, *, limiter: CapacityLimiter | None = None) -> Self: + home_path = await to_thread.run_sync(pathlib.Path.home, limiter=limiter) + return cls(home_path, limiter=limiter) + + def is_absolute(self) -> bool: + return self._path.is_absolute() + + async def is_block_device(self) -> bool: + return await to_thread.run_sync( + self._path.is_block_device, abandon_on_cancel=True, limiter=self._limiter + ) + + async def is_char_device(self) -> bool: + return await to_thread.run_sync( + self._path.is_char_device, abandon_on_cancel=True, limiter=self._limiter + ) + + if sys.version_info >= (3, 13): + + async def is_dir(self, *, follow_symlinks: bool = True) -> bool: + return await to_thread.run_sync( + partial(self._path.is_dir, follow_symlinks=follow_symlinks), + abandon_on_cancel=True, + limiter=self._limiter, + ) + + else: + + async def is_dir(self) -> bool: + return await to_thread.run_sync( + self._path.is_dir, abandon_on_cancel=True, limiter=self._limiter + ) + + async def is_fifo(self) -> bool: + return await to_thread.run_sync( + self._path.is_fifo, abandon_on_cancel=True, limiter=self._limiter + ) + + if sys.version_info >= (3, 13): + + async def is_file(self, *, follow_symlinks: bool = True) -> bool: + return await to_thread.run_sync( + partial(self._path.is_file, follow_symlinks=follow_symlinks), + abandon_on_cancel=True, + limiter=self._limiter, + ) + + else: + + async def is_file(self) -> bool: + return await to_thread.run_sync( + self._path.is_file, abandon_on_cancel=True, limiter=self._limiter + ) + + if sys.version_info >= (3, 12): + + async def is_junction(self) -> bool: + return await to_thread.run_sync( + self._path.is_junction, limiter=self._limiter + ) + + async def is_mount(self) -> bool: + return await to_thread.run_sync( + os.path.ismount, self._path, abandon_on_cancel=True, limiter=self._limiter + ) + + if sys.version_info < (3, 15): + + def is_reserved(self) -> bool: + return self._path.is_reserved() + + async def is_socket(self) -> bool: + return await to_thread.run_sync( + self._path.is_socket, abandon_on_cancel=True, limiter=self._limiter + ) + + async def is_symlink(self) -> bool: + return await to_thread.run_sync( + self._path.is_symlink, abandon_on_cancel=True, limiter=self._limiter + ) + + async def iterdir(self) -> AsyncIterator[Self]: + gen = ( + self._path.iterdir() + if sys.version_info < (3, 13) + else await to_thread.run_sync( + self._path.iterdir, abandon_on_cancel=True, limiter=self._limiter + ) + ) + async for path in _PathIterator(gen, self._limiter, type(self)): + yield path + + def joinpath(self, *args: str | PathLike[str]) -> Self: + return type(self)(self._path.joinpath(*args), limiter=self._limiter) + + async def lchmod(self, mode: int) -> None: + await to_thread.run_sync(self._path.lchmod, mode, limiter=self._limiter) + + async def lstat(self) -> os.stat_result: + return await to_thread.run_sync( + self._path.lstat, abandon_on_cancel=True, limiter=self._limiter + ) + + async def mkdir( + self, mode: int = 0o777, parents: bool = False, exist_ok: bool = False + ) -> None: + await to_thread.run_sync( + self._path.mkdir, mode, parents, exist_ok, limiter=self._limiter + ) + + @overload + async def open( + self, + mode: OpenBinaryMode, + buffering: int = ..., + encoding: str | None = ..., + errors: str | None = ..., + newline: str | None = ..., + ) -> AsyncFile[bytes]: ... + + @overload + async def open( + self, + mode: OpenTextMode = ..., + buffering: int = ..., + encoding: str | None = ..., + errors: str | None = ..., + newline: str | None = ..., + ) -> AsyncFile[str]: ... + + async def open( + self, + mode: str = "r", + buffering: int = -1, + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, + ) -> AsyncFile[Any]: + fp = await to_thread.run_sync( + self._path.open, + mode, + buffering, + encoding, + errors, + newline, + limiter=self._limiter, + ) + return AsyncFile(fp, limiter=self._limiter) + + if sys.version_info >= (3, 13): + + async def owner(self, *, follow_symlinks: bool = True) -> str: + return await to_thread.run_sync( + partial(self._path.owner, follow_symlinks=follow_symlinks), + abandon_on_cancel=True, + limiter=self._limiter, + ) + + else: + + async def owner(self) -> str: + return await to_thread.run_sync( + self._path.owner, abandon_on_cancel=True, limiter=self._limiter + ) + + async def read_bytes(self) -> bytes: + return await to_thread.run_sync(self._path.read_bytes, limiter=self._limiter) + + if sys.version_info >= (3, 13): + + async def read_text( + self, + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, + ) -> str: + return await to_thread.run_sync( + self._path.read_text, + encoding, + errors, + newline, + limiter=self._limiter, + ) + + else: + + async def read_text( + self, encoding: str | None = None, errors: str | None = None + ) -> str: + return await to_thread.run_sync( + self._path.read_text, encoding, errors, limiter=self._limiter + ) + + if sys.version_info >= (3, 12): + + def relative_to( + self, *other: str | PathLike[str], walk_up: bool = False + ) -> Self: + # relative_to() should work with any PathLike but it doesn't + others = [pathlib.Path(other) for other in other] + return type(self)( + self._path.relative_to(*others, walk_up=walk_up), limiter=self._limiter + ) + + else: + + def relative_to(self, *other: str | PathLike[str]) -> Self: + return type(self)(self._path.relative_to(*other), limiter=self._limiter) + + async def readlink(self) -> Self: + target = await to_thread.run_sync( + os.readlink, self._path, limiter=self._limiter + ) + return type(self)(target, limiter=self._limiter) + + async def rename(self, target: str | pathlib.PurePath | Path) -> Self: + if isinstance(target, Path): + target = target._path + + await to_thread.run_sync(self._path.rename, target, limiter=self._limiter) + return type(self)(target, limiter=self._limiter) + + async def replace(self, target: str | pathlib.PurePath | Path) -> Self: + if isinstance(target, Path): + target = target._path + + await to_thread.run_sync(self._path.replace, target, limiter=self._limiter) + return type(self)(target, limiter=self._limiter) + + async def resolve(self, strict: bool = False) -> Self: + func = partial(self._path.resolve, strict=strict) + return type(self)( + await to_thread.run_sync( + func, abandon_on_cancel=True, limiter=self._limiter + ), + limiter=self._limiter, + ) + + if sys.version_info < (3, 12): + # Pre Python 3.12 + def rglob(self, pattern: str) -> AsyncIterator[Self]: + gen = self._path.rglob(pattern) + return _PathIterator(gen, self._limiter, type(self)) + elif (3, 12) <= sys.version_info < (3, 13): + # Changed in Python 3.12: + # - The case_sensitive parameter was added. + def rglob( + self, pattern: str, *, case_sensitive: bool | None = None + ) -> AsyncIterator[Self]: + gen = self._path.rglob(pattern, case_sensitive=case_sensitive) + return _PathIterator(gen, self._limiter, type(self)) + elif sys.version_info >= (3, 13): + # Changed in Python 3.13: + # - The recurse_symlinks parameter was added. + # - The pattern parameter accepts a path-like object. + def rglob( # type: ignore[misc] # mypy doesn't allow for differing signatures in a conditional block + self, + pattern: str | PathLike[str], + *, + case_sensitive: bool | None = None, + recurse_symlinks: bool = False, + ) -> AsyncIterator[Self]: + gen = self._path.rglob( + pattern, # type: ignore[arg-type] + case_sensitive=case_sensitive, + recurse_symlinks=recurse_symlinks, + ) + return _PathIterator(gen, self._limiter, type(self)) + + async def rmdir(self) -> None: + await to_thread.run_sync(self._path.rmdir, limiter=self._limiter) + + async def samefile(self, other_path: str | PathLike[str]) -> bool: + if isinstance(other_path, Path): + other_path = other_path._path + + return await to_thread.run_sync( + self._path.samefile, + other_path, + abandon_on_cancel=True, + limiter=self._limiter, + ) + + async def stat(self, *, follow_symlinks: bool = True) -> os.stat_result: + func = partial(os.stat, follow_symlinks=follow_symlinks) + return await to_thread.run_sync( + func, self._path, abandon_on_cancel=True, limiter=self._limiter + ) + + async def symlink_to( + self, + target: str | bytes | PathLike[str] | PathLike[bytes], + target_is_directory: bool = False, + ) -> None: + if isinstance(target, Path): + target = target._path + + await to_thread.run_sync( + self._path.symlink_to, target, target_is_directory, limiter=self._limiter + ) + + async def touch(self, mode: int = 0o666, exist_ok: bool = True) -> None: + await to_thread.run_sync( + self._path.touch, mode, exist_ok, limiter=self._limiter + ) + + async def unlink(self, missing_ok: bool = False) -> None: + try: + await to_thread.run_sync(self._path.unlink, limiter=self._limiter) + except FileNotFoundError: + if not missing_ok: + raise + + if sys.version_info >= (3, 12): + + async def walk( + self, + top_down: bool = True, + on_error: Callable[[OSError], object] | None = None, + follow_symlinks: bool = False, + ) -> AsyncIterator[tuple[Self, list[str], list[str]]]: + def get_next_value() -> tuple[pathlib.Path, list[str], list[str]] | None: + try: + return next(gen) + except StopIteration: + return None + + gen = self._path.walk(top_down, on_error, follow_symlinks) + while True: + value = await to_thread.run_sync(get_next_value, limiter=self._limiter) + if value is None: + return + + root, dirs, paths = value + yield type(self)(root, limiter=self._limiter), dirs, paths + + def with_name(self, name: str) -> Self: + return type(self)(self._path.with_name(name), limiter=self._limiter) + + if sys.version_info < (3, 13): + # Backport pathlib's Python>=3.13 behavior for empty stems on paths with non-empty suffixes. + # See: https://github.com/python/cpython/pull/114612 + def with_stem(self, stem: str) -> Self: + suffix = self._path.suffix + if not suffix: + return self.with_name(stem) + elif not stem: + # If the suffix is non-empty, we can't make the stem empty. + raise ValueError(f"{self!r} has a non-empty suffix") + else: + return self.with_name(stem + suffix) + else: + + def with_stem(self, stem: str) -> Self: + return type(self)(self._path.with_stem(stem), limiter=self._limiter) + + def with_suffix(self, suffix: str) -> Self: + return type(self)(self._path.with_suffix(suffix), limiter=self._limiter) + + def with_segments(self, *pathsegments: str | PathLike[str]) -> Self: + return type(self)(*pathsegments, limiter=self._limiter) + + async def write_bytes(self, data: ReadableBuffer) -> int: + return await to_thread.run_sync( + self._path.write_bytes, data, limiter=self._limiter + ) + + async def write_text( + self, + data: str, + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, + ) -> int: + return await to_thread.run_sync( + self._path.write_text, + data, + encoding, + errors, + newline, + limiter=self._limiter, + ) + + +PathLike.register(Path) diff --git a/.venv/lib/python3.14/site-packages/anyio/_core/_futures.py b/.venv/lib/python3.14/site-packages/anyio/_core/_futures.py new file mode 100644 index 0000000000000000000000000000000000000000..7730c53c7f9bf55742396fa26fa7dd054cdfcb18 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/anyio/_core/_futures.py @@ -0,0 +1,191 @@ +from __future__ import annotations + +from collections.abc import Generator +from enum import Enum, auto +from typing import Any, Generic, TypeVar + +from ._exceptions import ( + FutureAlreadyFinished, + FutureCancelled, + FutureFailed, + FutureNotFinished, +) +from ._synchronization import Event + +T = TypeVar("T") + + +class Future(Generic[T]): + """ + An awaitable object that works similar to a :class:`asyncio.Future` but + with similar characteristics to a :class:`.TaskHandle`. + """ + + class Status(Enum): + """ + The status of a future handle. + + .. attribute:: PENDING + + The future has not finished yet. + .. attribute:: FINISHED + + The future has finished with a return value. + .. attribute:: CANCELLED + + The future was cancelled and has finished since. + .. attribute:: FAILED + + The future raised an exception. + """ + + PENDING = auto() + FINISHED = auto() + CANCELLED = auto() + FAILED = auto() + + __slots__ = ( + "_cancelled", + "_exception", + "_finished_event", + "_name", + "_return_value", + ) + _return_value: T + + def __init__(self, *, name: str | None = None) -> None: + self._finished_event = Event() + self._exception: BaseException | None = None + self._cancelled: bool = False + self._name = name + + def _check_pending(self) -> None: + """Shortcut for checking if a Future is pending + + :raises FutureAlreadyFinished: if future was already given a result or exception. + :raises FutureCancelled: if future has been cancelled previously. + """ + match self.status: + case Future.Status.PENDING: + return + case Future.Status.FINISHED: + raise FutureAlreadyFinished("future has already finished") + case Future.Status.FAILED: + raise FutureAlreadyFinished("future already failed") + case Future.Status.CANCELLED: + raise FutureCancelled("future was cancelled") + + async def wait(self) -> None: + """ + Waits for the future to finish. + + This method will attempt to wait for a result or exception + """ + await self._finished_event.wait() + + def cancel(self) -> None: + """Cancels a pending `.Future` object + + Does nothing if the Future was already finished. + """ + if self.status is Future.Status.PENDING: + self._cancelled = True + self._finished_event.set() + + @property + def exception(self) -> BaseException | None: + """ + The exception value of a `.Future` + + :raises FutureNotFinished: if future is still pending + :raises FutureCancelled: if future was cancelled + :returns: None if future succeeds with a result sent instead + otherwise this will be an exception + """ + + match self.status: + case Future.Status.PENDING: + raise FutureNotFinished("the future has not finished yet") + case Future.Status.FINISHED: + return None + case Future.Status.CANCELLED: + raise FutureCancelled("the future was cancelled") + case Future.Status.FAILED: + return self._exception + + @exception.setter + def exception(self, exception: BaseException) -> None: + """Send exception for a `.Future` object + + :raises FutureAlreadyFinished: if future was already given a result or exception. + :raises FutureCancelled: if future has been cancelled previously. + """ + self._check_pending() + self._exception = exception + self._finished_event.set() + + @property + def return_value(self) -> T: + """ + The return value of the future. + + :raises FutureNotFinished: if the future has not finished yet + :raises FutureCancelled: if the future was cancelled + :raises FutureFailed: if the future raised an exception + + """ + match self.status: + case Future.Status.PENDING: + raise FutureNotFinished("the future has not finished yet") + case Future.Status.FINISHED: + return self._return_value + case Future.Status.CANCELLED: + raise FutureCancelled("the future was cancelled") + case Future.Status.FAILED: + raise FutureFailed( + "the future raised an exception" + ) from self._exception + + @return_value.setter + def return_value(self, value: T) -> None: + """ + Send pending result for a `.Future` object + + :raises FutureAlreadyFinished: if future was already given a result or exception. + :raises FutureCancelled: if future has been cancelled previously. + """ + self._check_pending() + self._return_value = value + self._finished_event.set() + + @property + def status(self) -> Future.Status: + """ + The current status of a future. + + Every future starts in the :attr:`~Future.Status.PENDING` state. + If a future is cancelled while in this state, it will transition to the + :attr:`Future.Status.CANCELLED` state. When the task finishes, it will + transition to one of the three final states ( + :attr:`Future.Status.FINISHED`, :attr:`Future.Status.FAILED`, or + :attr:`Future.Status.CANCELLED`) depending on the exception the task + raised, if any. No other status transitions will happen. + """ + if not self._finished_event.is_set(): + return Future.Status.PENDING + elif self._cancelled: + return Future.Status.CANCELLED + elif self._exception is not None: + return Future.Status.FAILED + else: + return Future.Status.FINISHED + + def __await__(self) -> Generator[Any, Any, T]: + yield from self.wait().__await__() + return self.return_value + + def __repr__(self) -> str: + return ( + f"<{self.__class__.__name__} {self.status.name.lower()} " + f"name={self._name!r}>" + ) diff --git a/.venv/lib/python3.14/site-packages/anyio/_core/_resources.py b/.venv/lib/python3.14/site-packages/anyio/_core/_resources.py new file mode 100644 index 0000000000000000000000000000000000000000..6ca35d94a930661c4c976c204cfb6b31cc15be37 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/anyio/_core/_resources.py @@ -0,0 +1,18 @@ +from __future__ import annotations + +from ..abc import AsyncResource # noqa: TC001 +from ._tasks import CancelScope + + +async def aclose_forcefully(resource: AsyncResource) -> None: + """ + Close an asynchronous resource in a cancelled scope. + + Doing this closes the resource without waiting on anything. + + :param resource: the resource to close + + """ + with CancelScope() as scope: + scope.cancel() + await resource.aclose() diff --git a/.venv/lib/python3.14/site-packages/anyio/_core/_signals.py b/.venv/lib/python3.14/site-packages/anyio/_core/_signals.py new file mode 100644 index 0000000000000000000000000000000000000000..e24c79e10d4b76775679f7dd0dbe3f5860150451 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/anyio/_core/_signals.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +from collections.abc import AsyncIterator +from contextlib import AbstractContextManager +from signal import Signals + +from ._eventloop import get_async_backend + + +def open_signal_receiver( + *signals: Signals, +) -> AbstractContextManager[AsyncIterator[Signals]]: + """ + Start receiving operating system signals. + + :param signals: signals to receive (e.g. ``signal.SIGINT``) + :return: an asynchronous context manager for an asynchronous iterator which yields + signal numbers + :raises NoEventLoopError: if no supported asynchronous event loop is running in the + current thread + + .. warning:: Windows does not support signals natively so it is best to avoid + relying on this in cross-platform applications. + + .. warning:: On asyncio, this permanently replaces any previous signal handler for + the given signals, as set via :meth:`~asyncio.loop.add_signal_handler`. + + """ + return get_async_backend().open_signal_receiver(*signals) diff --git a/.venv/lib/python3.14/site-packages/anyio/_core/_sockets.py b/.venv/lib/python3.14/site-packages/anyio/_core/_sockets.py new file mode 100644 index 0000000000000000000000000000000000000000..14ee6e41e79f5a839b7bcf97dd3ca5e3ae656f49 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/anyio/_core/_sockets.py @@ -0,0 +1,1011 @@ +from __future__ import annotations + +import errno +import os +import socket +import ssl +import stat +import sys +from collections.abc import Awaitable +from dataclasses import dataclass +from ipaddress import IPv4Address, IPv6Address, ip_address +from os import PathLike, chmod +from socket import AddressFamily, SocketKind +from typing import TYPE_CHECKING, Any, Literal, cast, overload + +from .. import ConnectionFailed, to_thread +from ..abc import ( + ByteStreamConnectable, + ConnectedUDPSocket, + ConnectedUNIXDatagramSocket, + IPAddressType, + IPSockAddrType, + SocketListener, + SocketStream, + UDPSocket, + UNIXDatagramSocket, + UNIXSocketStream, +) +from ..streams.stapled import MultiListener +from ..streams.tls import TLSConnectable, TLSStream +from ._eventloop import get_async_backend +from ._resources import aclose_forcefully +from ._synchronization import Event +from ._tasks import create_task_group, move_on_after + +if TYPE_CHECKING: + from _typeshed import FileDescriptorLike +else: + FileDescriptorLike = object + +if sys.version_info < (3, 11): + from exceptiongroup import ExceptionGroup + +if sys.version_info >= (3, 12): + from typing import override +else: + from typing_extensions import override + +if sys.version_info < (3, 13): + from typing_extensions import deprecated +else: + from warnings import deprecated + +IPPROTO_IPV6 = getattr(socket, "IPPROTO_IPV6", 41) # https://bugs.python.org/issue29515 + +AnyIPAddressFamily = Literal[ + AddressFamily.AF_UNSPEC, AddressFamily.AF_INET, AddressFamily.AF_INET6 +] +IPAddressFamily = Literal[AddressFamily.AF_INET, AddressFamily.AF_INET6] + + +def idna2008_resolve(host: str) -> bytes: + try: + return host.encode("ascii") + except UnicodeEncodeError: + import idna + + return idna.encode(host, uts46=True) + + +# tls_hostname given +@overload +async def connect_tcp( + remote_host: IPAddressType, + remote_port: int, + *, + local_host: IPAddressType | None = ..., + local_port: int | None = ..., + ssl_context: ssl.SSLContext | None = ..., + tls_standard_compatible: bool = ..., + tls_hostname: str, + happy_eyeballs_delay: float = ..., +) -> TLSStream: ... + + +# ssl_context given +@overload +async def connect_tcp( + remote_host: IPAddressType, + remote_port: int, + *, + local_host: IPAddressType | None = ..., + local_port: int | None = ..., + ssl_context: ssl.SSLContext, + tls_standard_compatible: bool = ..., + tls_hostname: str | None = ..., + happy_eyeballs_delay: float = ..., +) -> TLSStream: ... + + +# tls=True +@overload +async def connect_tcp( + remote_host: IPAddressType, + remote_port: int, + *, + local_host: IPAddressType | None = ..., + local_port: int | None = ..., + tls: Literal[True], + ssl_context: ssl.SSLContext | None = ..., + tls_standard_compatible: bool = ..., + tls_hostname: str | None = ..., + happy_eyeballs_delay: float = ..., +) -> TLSStream: ... + + +# tls=False +@overload +async def connect_tcp( + remote_host: IPAddressType, + remote_port: int, + *, + local_host: IPAddressType | None = ..., + local_port: int | None = ..., + tls: Literal[False], + ssl_context: ssl.SSLContext | None = ..., + tls_standard_compatible: bool = ..., + tls_hostname: str | None = ..., + happy_eyeballs_delay: float = ..., +) -> SocketStream: ... + + +# No TLS arguments +@overload +async def connect_tcp( + remote_host: IPAddressType, + remote_port: int, + *, + local_host: IPAddressType | None = ..., + local_port: int | None = ..., + happy_eyeballs_delay: float = ..., +) -> SocketStream: ... + + +async def connect_tcp( + remote_host: IPAddressType, + remote_port: int, + *, + local_host: IPAddressType | None = None, + local_port: int | None = None, + tls: bool = False, + ssl_context: ssl.SSLContext | None = None, + tls_standard_compatible: bool = True, + tls_hostname: str | None = None, + happy_eyeballs_delay: float = 0.25, +) -> SocketStream | TLSStream: + """ + Connect to a host using the TCP protocol. + + This function implements the stateless version of the Happy Eyeballs algorithm (RFC + 6555). If ``remote_host`` is a host name that resolves to multiple IP addresses, + each one is tried until one connection attempt succeeds. If the first attempt does + not connected within 250 milliseconds, a second attempt is started using the next + address in the list, and so on. On IPv6 enabled systems, an IPv6 address (if + available) is tried first. + + When the connection has been established, a TLS handshake will be done if either + ``ssl_context`` or ``tls_hostname`` is not ``None``, or if ``tls`` is ``True``. + + :param remote_host: the IP address or host name to connect to + :param remote_port: port on the target host to connect to + :param local_host: the interface address or name to bind the socket to before + connecting + :param local_port: the local port to bind to (requires ``local_host`` to also be + set) + :param tls: ``True`` to do a TLS handshake with the connected stream and return a + :class:`~anyio.streams.tls.TLSStream` instead + :param ssl_context: the SSL context object to use (if omitted, a default context is + created) + :param tls_standard_compatible: If ``True``, performs the TLS shutdown handshake + before closing the stream and requires that the server does this as well. + Otherwise, :exc:`~ssl.SSLEOFError` may be raised during reads from the stream. + Some protocols, such as HTTP, require this option to be ``False``. + See :meth:`~ssl.SSLContext.wrap_socket` for details. + :param tls_hostname: host name to check the server certificate against (defaults to + the value of ``remote_host``) + :param happy_eyeballs_delay: delay (in seconds) before starting the next connection + attempt + :return: a socket stream object if no TLS handshake was done, otherwise a TLS stream + :raises ConnectionFailed: if the connection fails + + """ + # Placed here due to https://github.com/python/mypy/issues/7057 + connected_stream: SocketStream | None = None + + async def try_connect(remote_host: str, event: Event) -> None: + nonlocal connected_stream + try: + stream = await asynclib.connect_tcp(remote_host, remote_port, local_address) + except OSError as exc: + oserrors.append(exc) + return + else: + if connected_stream is None: + connected_stream = stream + tg.cancel_scope.cancel() + else: + await stream.aclose() + finally: + event.set() + + asynclib = get_async_backend() + local_address: IPSockAddrType | None = None + family = socket.AF_UNSPEC + if local_host: + gai_res = await getaddrinfo(str(local_host), local_port) + family, *_, local_address = gai_res[0] + + target_host = str(remote_host) + try: + addr_obj = ip_address(remote_host) + except ValueError: + addr_obj = None + + if addr_obj is not None: + if isinstance(addr_obj, IPv6Address): + target_addrs = [(socket.AF_INET6, addr_obj.compressed)] + else: + target_addrs = [(socket.AF_INET, addr_obj.compressed)] + else: + # getaddrinfo() will raise an exception if name resolution fails + gai_res = await getaddrinfo( + target_host, remote_port, family=family, type=socket.SOCK_STREAM + ) + + # Organize the list so that the first address is an IPv6 address (if available) + # and the second one is an IPv4 addresses. The rest can be in whatever order. + v6_found = v4_found = False + target_addrs = [] + for af, *_, sa in gai_res: + if af == socket.AF_INET6 and not v6_found: + v6_found = True + target_addrs.insert(0, (af, sa[0])) + elif af == socket.AF_INET and not v4_found and v6_found: + v4_found = True + target_addrs.insert(1, (af, sa[0])) + else: + target_addrs.append((af, sa[0])) + + oserrors: list[OSError] = [] + try: + async with create_task_group() as tg: + for _af, addr in target_addrs: + event = Event() + tg.start_soon(try_connect, addr, event) + with move_on_after(happy_eyeballs_delay): + await event.wait() + + if connected_stream is None: + cause = ( + oserrors[0] + if len(oserrors) == 1 + else ExceptionGroup("multiple connection attempts failed", oserrors) + ) + raise OSError("All connection attempts failed") from cause + finally: + oserrors.clear() + + if tls or tls_hostname or ssl_context: + try: + return await TLSStream.wrap( + connected_stream, + server_side=False, + hostname=tls_hostname or str(remote_host), + ssl_context=ssl_context, + standard_compatible=tls_standard_compatible, + ) + except BaseException: + await aclose_forcefully(connected_stream) + raise + + return connected_stream + + +async def connect_unix(path: str | bytes | PathLike[Any]) -> UNIXSocketStream: + """ + Connect to the given UNIX socket. + + Not available on Windows. + + :param path: path to the socket + :return: a socket stream object + :raises ConnectionFailed: if the connection fails + + """ + path = os.fspath(path) + return await get_async_backend().connect_unix(path) + + +async def create_tcp_listener( + *, + local_host: IPAddressType | None = None, + local_port: int = 0, + family: AnyIPAddressFamily = socket.AddressFamily.AF_UNSPEC, + backlog: int = 65536, + reuse_port: bool = False, +) -> MultiListener[SocketStream]: + """ + Create a TCP socket listener. + + :param local_port: port number to listen on + :param local_host: IP address of the interface to listen on. If omitted, listen on + all IPv4 and IPv6 interfaces. To listen on all interfaces on a specific address + family, use ``0.0.0.0`` for IPv4 or ``::`` for IPv6. + :param family: address family (used if ``local_host`` was omitted) + :param backlog: maximum number of queued incoming connections (up to a maximum of + 2**16, or 65536) + :param reuse_port: ``True`` to allow multiple sockets to bind to the same + address/port (not supported on Windows) + :return: a multi-listener object containing one or more socket listeners + :raises OSError: if there's an error creating a socket, or binding to one or more + interfaces failed + + """ + asynclib = get_async_backend() + backlog = min(backlog, 65536) + local_host = str(local_host) if local_host is not None else None + + def setup_raw_socket( + fam: AddressFamily, + bind_addr: tuple[str, int] | tuple[str, int, int, int], + *, + v6only: bool = True, + ) -> socket.socket: + sock = socket.socket(fam) + try: + sock.setblocking(False) + + if fam == AddressFamily.AF_INET6: + sock.setsockopt(IPPROTO_IPV6, socket.IPV6_V6ONLY, v6only) + + # For Windows, enable exclusive address use. For others, enable address + # reuse. + if sys.platform == "win32": + sock.setsockopt(socket.SOL_SOCKET, socket.SO_EXCLUSIVEADDRUSE, 1) + else: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + + if reuse_port: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + + # Workaround for #554 + if fam == socket.AF_INET6 and "%" in bind_addr[0]: + addr, scope_id = bind_addr[0].split("%", 1) + bind_addr = (addr, bind_addr[1], 0, int(scope_id)) + + sock.bind(bind_addr) + sock.listen(backlog) + except BaseException: + sock.close() + raise + + return sock + + # We passing type=0 on non-Windows platforms as a workaround for a uvloop bug + # where we don't get the correct scope ID for IPv6 link-local addresses when passing + # type=socket.SOCK_STREAM to getaddrinfo(): + # https://github.com/MagicStack/uvloop/issues/539 + gai_res = await getaddrinfo( + local_host, + local_port, + family=family, + type=socket.SOCK_STREAM if sys.platform == "win32" else 0, + flags=socket.AI_PASSIVE | socket.AI_ADDRCONFIG, + ) + + # The set comprehension is here to work around a glibc bug: + # https://sourceware.org/bugzilla/show_bug.cgi?id=14969 + sockaddrs = sorted({res for res in gai_res if res[1] == SocketKind.SOCK_STREAM}) + + # Special case for dual-stack binding on the "any" interface + if ( + local_host is None + and family == AddressFamily.AF_UNSPEC + and socket.has_dualstack_ipv6() + and any(fam == AddressFamily.AF_INET6 for fam, *_ in gai_res) + ): + raw_socket = setup_raw_socket( + AddressFamily.AF_INET6, ("::", local_port), v6only=False + ) + listener = asynclib.create_tcp_listener(raw_socket) + return MultiListener([listener]) + + errors: list[OSError] = [] + try: + for _ in range(len(sockaddrs)): + listeners: list[SocketListener] = [] + bound_ephemeral_port = local_port + try: + for fam, *_, sockaddr in sockaddrs: + sockaddr = sockaddr[0], bound_ephemeral_port, *sockaddr[2:] + raw_socket = setup_raw_socket(fam, sockaddr) + + # Store the assigned port if an ephemeral port was requested, so + # we'll bind to the same port on all interfaces + if local_port == 0 and len(gai_res) > 1: + bound_ephemeral_port = raw_socket.getsockname()[1] + + listeners.append(asynclib.create_tcp_listener(raw_socket)) + except BaseException as exc: + for listener in listeners: + await listener.aclose() + + # If an ephemeral port was requested but binding the assigned port + # failed for another interface, rotate the address list and try again + if ( + isinstance(exc, OSError) + and exc.errno == errno.EADDRINUSE + and local_port == 0 + and bound_ephemeral_port + ): + errors.append(exc) + sockaddrs.append(sockaddrs.pop(0)) + continue + + raise + + return MultiListener(listeners) + + raise OSError( + f"Could not create {len(sockaddrs)} listeners with a consistent port" + ) from ExceptionGroup("Several bind attempts failed", errors) + finally: + del errors # Prevent reference cycles + + +async def create_unix_listener( + path: str | bytes | PathLike[Any], + *, + mode: int | None = None, + backlog: int = 65536, +) -> SocketListener: + """ + Create a UNIX socket listener. + + Not available on Windows. + + :param path: path of the socket + :param mode: permissions to set on the socket + :param backlog: maximum number of queued incoming connections (up to a maximum of + 2**16, or 65536) + :return: a listener object + + .. versionchanged:: 3.0 + If a socket already exists on the file system in the given path, it will be + removed first. + + """ + backlog = min(backlog, 65536) + raw_socket = await setup_unix_local_socket(path, mode, socket.SOCK_STREAM) + try: + raw_socket.listen(backlog) + return get_async_backend().create_unix_listener(raw_socket) + except BaseException: + raw_socket.close() + raise + + +async def create_udp_socket( + family: AnyIPAddressFamily = AddressFamily.AF_UNSPEC, + *, + local_host: IPAddressType | None = None, + local_port: int = 0, + reuse_port: bool = False, +) -> UDPSocket: + """ + Create a UDP socket. + + If ``port`` has been given, the socket will be bound to this port on the local + machine, making this socket suitable for providing UDP based services. + + :param family: address family (``AF_INET`` or ``AF_INET6``) – automatically + determined from ``local_host`` if omitted + :param local_host: IP address or host name of the local interface to bind to + :param local_port: local port to bind to + :param reuse_port: ``True`` to allow multiple sockets to bind to the same + address/port (not supported on Windows) + :return: a UDP socket + + """ + if family is AddressFamily.AF_UNSPEC and not local_host: + raise ValueError('Either "family" or "local_host" must be given') + + if local_host: + gai_res = await getaddrinfo( + str(local_host), + local_port, + family=family, + type=socket.SOCK_DGRAM, + flags=socket.AI_PASSIVE | socket.AI_ADDRCONFIG, + ) + family = cast(AnyIPAddressFamily, gai_res[0][0]) + local_address = gai_res[0][-1] + elif family is AddressFamily.AF_INET6: + local_address = ("::", 0) + else: + local_address = ("0.0.0.0", 0) + + sock = await get_async_backend().create_udp_socket( + family, local_address, None, reuse_port + ) + return cast(UDPSocket, sock) + + +async def create_connected_udp_socket( + remote_host: IPAddressType, + remote_port: int, + *, + family: AnyIPAddressFamily = AddressFamily.AF_UNSPEC, + local_host: IPAddressType | None = None, + local_port: int = 0, + reuse_port: bool = False, +) -> ConnectedUDPSocket: + """ + Create a connected UDP socket. + + Connected UDP sockets can only communicate with the specified remote host/port, an + any packets sent from other sources are dropped. + + :param remote_host: remote host to set as the default target + :param remote_port: port on the remote host to set as the default target + :param family: address family (``AF_INET`` or ``AF_INET6``) – automatically + determined from ``local_host`` or ``remote_host`` if omitted + :param local_host: IP address or host name of the local interface to bind to + :param local_port: local port to bind to + :param reuse_port: ``True`` to allow multiple sockets to bind to the same + address/port (not supported on Windows) + :return: a connected UDP socket + + """ + local_address = None + if local_host: + gai_res = await getaddrinfo( + str(local_host), + local_port, + family=family, + type=socket.SOCK_DGRAM, + flags=socket.AI_PASSIVE | socket.AI_ADDRCONFIG, + ) + family = cast(AnyIPAddressFamily, gai_res[0][0]) + local_address = gai_res[0][-1] + + gai_res = await getaddrinfo( + str(remote_host), remote_port, family=family, type=socket.SOCK_DGRAM + ) + family = cast(AnyIPAddressFamily, gai_res[0][0]) + remote_address = gai_res[0][-1] + + sock = await get_async_backend().create_udp_socket( + family, local_address, remote_address, reuse_port + ) + return cast(ConnectedUDPSocket, sock) + + +async def create_unix_datagram_socket( + *, + local_path: None | str | bytes | PathLike[Any] = None, + local_mode: int | None = None, +) -> UNIXDatagramSocket: + """ + Create a UNIX datagram socket. + + Not available on Windows. + + If ``local_path`` has been given, the socket will be bound to this path, making this + socket suitable for receiving datagrams from other processes. Other processes can + send datagrams to this socket only if ``local_path`` is set. + + If a socket already exists on the file system in the ``local_path``, it will be + removed first. + + :param local_path: the path on which to bind to + :param local_mode: permissions to set on the local socket + :return: a UNIX datagram socket + + """ + raw_socket = await setup_unix_local_socket( + local_path, local_mode, socket.SOCK_DGRAM + ) + return await get_async_backend().create_unix_datagram_socket(raw_socket, None) + + +async def create_connected_unix_datagram_socket( + remote_path: str | bytes | PathLike[Any], + *, + local_path: None | str | bytes | PathLike[Any] = None, + local_mode: int | None = None, +) -> ConnectedUNIXDatagramSocket: + """ + Create a connected UNIX datagram socket. + + Connected datagram sockets can only communicate with the specified remote path. + + If ``local_path`` has been given, the socket will be bound to this path, making + this socket suitable for receiving datagrams from other processes. Other processes + can send datagrams to this socket only if ``local_path`` is set. + + If a socket already exists on the file system in the ``local_path``, it will be + removed first. + + :param remote_path: the path to set as the default target + :param local_path: the path on which to bind to + :param local_mode: permissions to set on the local socket + :return: a connected UNIX datagram socket + + """ + remote_path = os.fspath(remote_path) + raw_socket = await setup_unix_local_socket( + local_path, local_mode, socket.SOCK_DGRAM + ) + return await get_async_backend().create_unix_datagram_socket( + raw_socket, remote_path + ) + + +async def getaddrinfo( + host: bytes | str | None, + port: str | int | None, + *, + family: int | AddressFamily = 0, + type: int | SocketKind = 0, + proto: int = 0, + flags: int = 0, +) -> list[tuple[AddressFamily, SocketKind, int, str, tuple[str, int]]]: + """ + Look up a numeric IP address given a host name. + + Internationalized domain names are translated according to the (non-transitional) + IDNA 2008 standard. + + .. note:: 4-tuple IPv6 socket addresses are automatically converted to 2-tuples of + (host, port), unlike what :func:`socket.getaddrinfo` does. + + :param host: host name + :param port: port number + :param family: socket family (`'AF_INET``, ...) + :param type: socket type (``SOCK_STREAM``, ...) + :param proto: protocol number + :param flags: flags to pass to upstream ``getaddrinfo()`` + :return: list of tuples containing (family, type, proto, canonname, sockaddr) + + .. seealso:: :func:`socket.getaddrinfo` + + """ + # Handle unicode hostnames + encoded_host = idna2008_resolve(host) if isinstance(host, str) else host + gai_res = await get_async_backend().getaddrinfo( + encoded_host, port, family=family, type=type, proto=proto, flags=flags + ) + return [ + (family, type, proto, canonname, convert_ipv6_sockaddr(sockaddr)) + for family, type, proto, canonname, sockaddr in gai_res + # filter out IPv6 results when IPv6 is disabled + if not isinstance(sockaddr[0], int) + ] + + +def getnameinfo(sockaddr: IPSockAddrType, flags: int = 0) -> Awaitable[tuple[str, str]]: + """ + Look up the host name of an IP address. + + :param sockaddr: socket address (e.g. (ipaddress, port) for IPv4) + :param flags: flags to pass to upstream ``getnameinfo()`` + :return: a tuple of (host name, service name) + :raises NoEventLoopError: if no supported asynchronous event loop is running in the + current thread + + .. seealso:: :func:`socket.getnameinfo` + + """ + return get_async_backend().getnameinfo(sockaddr, flags) + + +@deprecated("This function is deprecated; use `wait_readable` instead") +def wait_socket_readable(sock: socket.socket) -> Awaitable[None]: + """ + .. deprecated:: 4.7.0 + Use :func:`wait_readable` instead. + + Wait until the given socket has data to be read. + + .. warning:: Only use this on raw sockets that have not been wrapped by any higher + level constructs like socket streams! + + :param sock: a socket object + :raises ~anyio.ClosedResourceError: if the socket was closed while waiting for the + socket to become readable + :raises ~anyio.BusyResourceError: if another task is already waiting for the socket + to become readable + :raises NoEventLoopError: if no supported asynchronous event loop is running in the + current thread + + """ + return get_async_backend().wait_readable(sock.fileno()) + + +@deprecated("This function is deprecated; use `wait_writable` instead") +def wait_socket_writable(sock: socket.socket) -> Awaitable[None]: + """ + .. deprecated:: 4.7.0 + Use :func:`wait_writable` instead. + + Wait until the given socket can be written to. + + This does **NOT** work on Windows when using the asyncio backend with a proactor + event loop (default on py3.8+). + + .. warning:: Only use this on raw sockets that have not been wrapped by any higher + level constructs like socket streams! + + :param sock: a socket object + :raises ~anyio.ClosedResourceError: if the socket was closed while waiting for the + socket to become writable + :raises ~anyio.BusyResourceError: if another task is already waiting for the socket + to become writable + :raises NoEventLoopError: if no supported asynchronous event loop is running in the + current thread + + """ + return get_async_backend().wait_writable(sock.fileno()) + + +def wait_readable(obj: FileDescriptorLike) -> Awaitable[None]: + """ + Wait until the given object has data to be read. + + On Unix systems, ``obj`` must either be an integer file descriptor, or else an + object with a ``.fileno()`` method which returns an integer file descriptor. Any + kind of file descriptor can be passed, though the exact semantics will depend on + your kernel. For example, this probably won't do anything useful for on-disk files. + + On Windows systems, ``obj`` must either be an integer ``SOCKET`` handle, or else an + object with a ``.fileno()`` method which returns an integer ``SOCKET`` handle. File + descriptors aren't supported, and neither are handles that refer to anything besides + a ``SOCKET``. + + On backends where this functionality is not natively provided (asyncio + ``ProactorEventLoop`` on Windows), it is provided using a separate selector thread + which is set to shut down when the interpreter shuts down. + + .. warning:: Don't use this on raw sockets that have been wrapped by any higher + level constructs like socket streams! + + :param obj: an object with a ``.fileno()`` method or an integer handle + :raises ~anyio.ClosedResourceError: if the object was closed while waiting for the + object to become readable + :raises ~anyio.BusyResourceError: if another task is already waiting for the object + to become readable + :raises NoEventLoopError: if no supported asynchronous event loop is running in the + current thread + + """ + return get_async_backend().wait_readable(obj) + + +def wait_writable(obj: FileDescriptorLike) -> Awaitable[None]: + """ + Wait until the given object can be written to. + + :param obj: an object with a ``.fileno()`` method or an integer handle + :raises ~anyio.ClosedResourceError: if the object was closed while waiting for the + object to become writable + :raises ~anyio.BusyResourceError: if another task is already waiting for the object + to become writable + :raises NoEventLoopError: if no supported asynchronous event loop is running in the + current thread + + .. seealso:: See the documentation of :func:`wait_readable` for the definition of + ``obj`` and notes on backend compatibility. + + .. warning:: Don't use this on raw sockets that have been wrapped by any higher + level constructs like socket streams! + + """ + return get_async_backend().wait_writable(obj) + + +def notify_closing(obj: FileDescriptorLike) -> None: + """ + Call this before closing a file descriptor (on Unix) or socket (on + Windows). This will cause any `wait_readable` or `wait_writable` + calls on the given object to immediately wake up and raise + `~anyio.ClosedResourceError`. + + This doesn't actually close the object – you still have to do that + yourself afterwards. Also, you want to be careful to make sure no + new tasks start waiting on the object in between when you call this + and when it's actually closed. So to close something properly, you + usually want to do these steps in order: + + 1. Explicitly mark the object as closed, so that any new attempts + to use it will abort before they start. + 2. Call `notify_closing` to wake up any already-existing users. + 3. Actually close the object. + + It's also possible to do them in a different order if that's more + convenient, *but only if* you make sure not to have any checkpoints in + between the steps. This way they all happen in a single atomic + step, so other tasks won't be able to tell what order they happened + in anyway. + + :param obj: an object with a ``.fileno()`` method or an integer handle + :raises NoEventLoopError: if no supported asynchronous event loop is running in the + current thread + + """ + get_async_backend().notify_closing(obj) + + +# +# Private API +# + + +def convert_ipv6_sockaddr( + sockaddr: tuple[str, int, int, int] | tuple[str, int], +) -> tuple[str, int]: + """ + Convert a 4-tuple IPv6 socket address to a 2-tuple (address, port) format. + + If the scope ID is nonzero, it is added to the address, separated with ``%``. + Otherwise the flow id and scope id are simply cut off from the tuple. + Any other kinds of socket addresses are returned as-is. + + :param sockaddr: the result of :meth:`~socket.socket.getsockname` + :return: the converted socket address + + """ + # This is more complicated than it should be because of MyPy + if isinstance(sockaddr, tuple) and len(sockaddr) == 4: + host, port, _flowinfo, scope_id = sockaddr + if scope_id: + # PyPy (as of v7.3.11) leaves the interface name in the result, so + # we discard it and only get the scope ID from the end + # (https://foss.heptapod.net/pypy/pypy/-/issues/3938) + host = host.split("%")[0] + + # Add scope_id to the address + return f"{host}%{scope_id}", port + else: + return host, port + else: + return sockaddr + + +async def setup_unix_local_socket( + path: None | str | bytes | PathLike[Any], + mode: int | None, + socktype: int, +) -> socket.socket: + """ + Create a UNIX local socket object, deleting the socket at the given path if it + exists. + + Not available on Windows. + + :param path: path of the socket + :param mode: permissions to set on the socket + :param socktype: socket.SOCK_STREAM or socket.SOCK_DGRAM + + """ + path_str: str | None + if path is not None: + path_str = os.fsdecode(path) + + # Linux abstract namespace sockets aren't backed by a concrete file so skip stat call + if not path_str.startswith("\0"): + # Copied from pathlib... + try: + stat_result = os.stat(path) + except OSError as e: + if e.errno not in ( + errno.ENOENT, + errno.ENOTDIR, + errno.EBADF, + errno.ELOOP, + ): + raise + else: + if stat.S_ISSOCK(stat_result.st_mode): + os.unlink(path) + else: + path_str = None + + raw_socket = socket.socket(socket.AF_UNIX, socktype) + raw_socket.setblocking(False) + + if path_str is not None: + try: + await to_thread.run_sync(raw_socket.bind, path_str, abandon_on_cancel=True) + if mode is not None: + await to_thread.run_sync(chmod, path_str, mode, abandon_on_cancel=True) + except BaseException: + raw_socket.close() + raise + + return raw_socket + + +@dataclass +class TCPConnectable(ByteStreamConnectable): + """ + Connects to a TCP server at the given host and port. + + :param host: host name or IP address of the server + :param port: TCP port number of the server + """ + + host: str | IPv4Address | IPv6Address + port: int + + def __post_init__(self) -> None: + if self.port < 1 or self.port > 65535: + raise ValueError("TCP port number out of range") + + @override + async def connect(self) -> SocketStream: + try: + return await connect_tcp(self.host, self.port) + except OSError as exc: + raise ConnectionFailed( + f"error connecting to {self.host}:{self.port}: {exc}" + ) from exc + + +@dataclass +class UNIXConnectable(ByteStreamConnectable): + """ + Connects to a UNIX domain socket at the given path. + + :param path: the file system path of the socket + """ + + path: str | bytes | PathLike[str] | PathLike[bytes] + + @override + async def connect(self) -> UNIXSocketStream: + try: + return await connect_unix(self.path) + except OSError as exc: + raise ConnectionFailed(f"error connecting to {self.path!r}: {exc}") from exc + + +def as_connectable( + remote: ByteStreamConnectable + | tuple[str | IPv4Address | IPv6Address, int] + | str + | bytes + | PathLike[str], + /, + *, + tls: bool = False, + ssl_context: ssl.SSLContext | None = None, + tls_hostname: str | None = None, + tls_standard_compatible: bool = True, +) -> ByteStreamConnectable: + """ + Return a byte stream connectable from the given object. + + If a bytestream connectable is given, it is returned unchanged. + If a tuple of (host, port) is given, a TCP connectable is returned. + If a string or bytes path is given, a UNIX connectable is returned. + + If ``tls=True``, the connectable will be wrapped in a + :class:`~.streams.tls.TLSConnectable`. + + :param remote: a connectable, a tuple of (host, port) or a path to a UNIX socket + :param tls: if ``True``, wrap the plaintext connectable in a + :class:`~.streams.tls.TLSConnectable`, using the provided TLS settings) + :param ssl_context: if ``tls=True``, the SSLContext object to use (if not provided, + a secure default will be created) + :param tls_hostname: if ``tls=True``, host name of the server to use for checking + the server certificate (defaults to the host portion of the address for TCP + connectables) + :param tls_standard_compatible: if ``False`` and ``tls=True``, makes the TLS stream + skip the closing handshake when closing the connection, so it won't raise an + exception if the server does the same + + """ + connectable: TCPConnectable | UNIXConnectable | TLSConnectable + if isinstance(remote, ByteStreamConnectable): + return remote + elif isinstance(remote, tuple) and len(remote) == 2: + connectable = TCPConnectable(*remote) + elif isinstance(remote, (str, bytes, PathLike)): + connectable = UNIXConnectable(remote) + else: + raise TypeError(f"cannot convert {remote!r} to a connectable") + + if tls: + if not tls_hostname and isinstance(connectable, TCPConnectable): + tls_hostname = str(connectable.host) + + connectable = TLSConnectable( + connectable, + ssl_context=ssl_context, + hostname=tls_hostname, + standard_compatible=tls_standard_compatible, + ) + + return connectable diff --git a/.venv/lib/python3.14/site-packages/anyio/_core/_streams.py b/.venv/lib/python3.14/site-packages/anyio/_core/_streams.py new file mode 100644 index 0000000000000000000000000000000000000000..2b9c7df200f9520357503c754bcdea1c047bdda3 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/anyio/_core/_streams.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +import math +from typing import TypeVar +from warnings import warn + +from ..streams.memory import ( + MemoryObjectReceiveStream, + MemoryObjectSendStream, + _MemoryObjectStreamState, +) + +T_Item = TypeVar("T_Item") + + +class create_memory_object_stream( + tuple[MemoryObjectSendStream[T_Item], MemoryObjectReceiveStream[T_Item]], +): + """ + Create a memory object stream. + + The stream's item type can be annotated like + :func:`create_memory_object_stream[T_Item]`. + + :param max_buffer_size: number of items held in the buffer until ``send()`` starts + blocking + :param item_type: old way of marking the streams with the right generic type for + static typing (does nothing on AnyIO 4) + + .. deprecated:: 4.0 + Use ``create_memory_object_stream[YourItemType](...)`` instead. + :return: a tuple of (send stream, receive stream) + + """ + + def __new__( # type: ignore[misc] + cls, max_buffer_size: float = 0, item_type: object = None + ) -> tuple[MemoryObjectSendStream[T_Item], MemoryObjectReceiveStream[T_Item]]: + if max_buffer_size != math.inf and not isinstance(max_buffer_size, int): + raise ValueError("max_buffer_size must be either an integer or math.inf") + if max_buffer_size < 0: + raise ValueError("max_buffer_size cannot be negative") + if item_type is not None: + warn( + "The item_type argument has been deprecated in AnyIO 4.0. " + "Use create_memory_object_stream[YourItemType](...) instead.", + DeprecationWarning, + stacklevel=2, + ) + + state = _MemoryObjectStreamState[T_Item](max_buffer_size) + return (MemoryObjectSendStream(state), MemoryObjectReceiveStream(state)) diff --git a/.venv/lib/python3.14/site-packages/anyio/_core/_subprocesses.py b/.venv/lib/python3.14/site-packages/anyio/_core/_subprocesses.py new file mode 100644 index 0000000000000000000000000000000000000000..53184a2e6cd238fcc56e9eb5b3374b25eeaf7741 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/anyio/_core/_subprocesses.py @@ -0,0 +1,196 @@ +from __future__ import annotations + +from collections.abc import AsyncIterable, Iterable, Mapping, Sequence +from io import BytesIO +from os import PathLike +from subprocess import PIPE, CalledProcessError, CompletedProcess +from typing import IO, Any, TypeAlias, cast + +from ..abc import Process # noqa: TC001 +from ._eventloop import get_async_backend +from ._tasks import create_task_group + +StrOrBytesPath: TypeAlias = str | bytes | PathLike[str] | PathLike[bytes] + + +async def run_process( + command: StrOrBytesPath | Sequence[StrOrBytesPath], + *, + input: bytes | None = None, + stdin: int | IO[Any] | None = None, + stdout: int | IO[Any] | None = PIPE, + stderr: int | IO[Any] | None = PIPE, + check: bool = True, + cwd: StrOrBytesPath | None = None, + env: Mapping[str, str] | None = None, + startupinfo: Any = None, + creationflags: int = 0, + start_new_session: bool = False, + pass_fds: Sequence[int] = (), + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, +) -> CompletedProcess[bytes]: + """ + Run an external command in a subprocess and wait until it completes. + + .. seealso:: :func:`subprocess.run` + + :param command: either a string to pass to the shell, or an iterable of strings + containing the executable name or path and its arguments + :param input: bytes passed to the standard input of the subprocess + :param stdin: one of :data:`subprocess.PIPE`, :data:`subprocess.DEVNULL`, + a file-like object, or `None`; ``input`` overrides this + :param stdout: one of :data:`subprocess.PIPE`, :data:`subprocess.DEVNULL`, + a file-like object, or `None` + :param stderr: one of :data:`subprocess.PIPE`, :data:`subprocess.DEVNULL`, + :data:`subprocess.STDOUT`, a file-like object, or `None` + :param check: if ``True``, raise :exc:`~subprocess.CalledProcessError` if the + process terminates with a return code other than 0 + :param cwd: If not ``None``, change the working directory to this before running the + command + :param env: if not ``None``, this mapping replaces the inherited environment + variables from the parent process + :param startupinfo: an instance of :class:`subprocess.STARTUPINFO` that can be used + to specify process startup parameters (Windows only) + :param creationflags: flags that can be used to control the creation of the + subprocess (see :class:`subprocess.Popen` for the specifics) + :param start_new_session: if ``true`` the setsid() system call will be made in the + child process prior to the execution of the subprocess. (POSIX only) + :param pass_fds: sequence of file descriptors to keep open between the parent and + child processes. (POSIX only) + :param user: effective user to run the process as (Python >= 3.9, POSIX only) + :param group: effective group to run the process as (Python >= 3.9, POSIX only) + :param extra_groups: supplementary groups to set in the subprocess (Python >= 3.9, + POSIX only) + :param umask: if not negative, this umask is applied in the child process before + running the given command (Python >= 3.9, POSIX only) + :return: an object representing the completed process + :raises ~subprocess.CalledProcessError: if ``check`` is ``True`` and the process + exits with a nonzero return code + + """ + + async def drain_stream(stream: AsyncIterable[bytes], index: int) -> None: + buffer = BytesIO() + async for chunk in stream: + buffer.write(chunk) + + stream_contents[index] = buffer.getvalue() + + if stdin is not None and input is not None: + raise ValueError("only one of stdin and input is allowed") + + async with await open_process( + command, + stdin=PIPE if input else stdin, + stdout=stdout, + stderr=stderr, + cwd=cwd, + env=env, + startupinfo=startupinfo, + creationflags=creationflags, + start_new_session=start_new_session, + pass_fds=pass_fds, + user=user, + group=group, + extra_groups=extra_groups, + umask=umask, + ) as process: + stream_contents: list[bytes | None] = [None, None] + async with create_task_group() as tg: + if process.stdout: + tg.start_soon(drain_stream, process.stdout, 0) + + if process.stderr: + tg.start_soon(drain_stream, process.stderr, 1) + + if process.stdin and input: + await process.stdin.send(input) + await process.stdin.aclose() + + await process.wait() + + output, errors = stream_contents + if check and process.returncode != 0: + raise CalledProcessError(cast(int, process.returncode), command, output, errors) + + return CompletedProcess(command, cast(int, process.returncode), output, errors) + + +async def open_process( + command: StrOrBytesPath | Sequence[StrOrBytesPath], + *, + stdin: int | IO[Any] | None = PIPE, + stdout: int | IO[Any] | None = PIPE, + stderr: int | IO[Any] | None = PIPE, + cwd: StrOrBytesPath | None = None, + env: Mapping[str, str] | None = None, + startupinfo: Any = None, + creationflags: int = 0, + start_new_session: bool = False, + pass_fds: Sequence[int] = (), + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, +) -> Process: + """ + Start an external command in a subprocess. + + .. seealso:: :class:`subprocess.Popen` + + :param command: either a string to pass to the shell, or an iterable of strings + containing the executable name or path and its arguments + :param stdin: one of :data:`subprocess.PIPE`, :data:`subprocess.DEVNULL`, a + file-like object, or ``None`` + :param stdout: one of :data:`subprocess.PIPE`, :data:`subprocess.DEVNULL`, + a file-like object, or ``None`` + :param stderr: one of :data:`subprocess.PIPE`, :data:`subprocess.DEVNULL`, + :data:`subprocess.STDOUT`, a file-like object, or ``None`` + :param cwd: If not ``None``, the working directory is changed before executing + :param env: If env is not ``None``, it must be a mapping that defines the + environment variables for the new process + :param creationflags: flags that can be used to control the creation of the + subprocess (see :class:`subprocess.Popen` for the specifics) + :param startupinfo: an instance of :class:`subprocess.STARTUPINFO` that can be used + to specify process startup parameters (Windows only) + :param start_new_session: if ``true`` the setsid() system call will be made in the + child process prior to the execution of the subprocess. (POSIX only) + :param pass_fds: sequence of file descriptors to keep open between the parent and + child processes. (POSIX only) + :param user: effective user to run the process as (POSIX only) + :param group: effective group to run the process as (POSIX only) + :param extra_groups: supplementary groups to set in the subprocess (POSIX only) + :param umask: if not negative, this umask is applied in the child process before + running the given command (POSIX only) + :return: an asynchronous process object + + """ + kwargs: dict[str, Any] = {} + if user is not None: + kwargs["user"] = user + + if group is not None: + kwargs["group"] = group + + if extra_groups is not None: + kwargs["extra_groups"] = extra_groups + + if umask >= 0: + kwargs["umask"] = umask + + return await get_async_backend().open_process( + command, + stdin=stdin, + stdout=stdout, + stderr=stderr, + cwd=cwd, + env=env, + startupinfo=startupinfo, + creationflags=creationflags, + start_new_session=start_new_session, + pass_fds=pass_fds, + **kwargs, + ) diff --git a/.venv/lib/python3.14/site-packages/anyio/_core/_synchronization.py b/.venv/lib/python3.14/site-packages/anyio/_core/_synchronization.py new file mode 100644 index 0000000000000000000000000000000000000000..cb988c60db56f412ae0e051e08d68380e8b6a413 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/anyio/_core/_synchronization.py @@ -0,0 +1,778 @@ +from __future__ import annotations + +import math +import sys +from collections import deque +from collections.abc import Callable +from dataclasses import dataclass +from types import TracebackType +from typing import TypeVar + +from ..lowlevel import checkpoint_if_cancelled +from ._eventloop import get_async_backend +from ._exceptions import BusyResourceError, NoEventLoopError +from ._tasks import CancelScope +from ._testing import TaskInfo, get_current_task + +if sys.version_info >= (3, 11): + from typing import Self +else: + from typing_extensions import Self + +T = TypeVar("T") + + +@dataclass(frozen=True) +class EventStatistics: + """ + :ivar int tasks_waiting: number of tasks waiting on :meth:`~.Event.wait` + """ + + tasks_waiting: int + + +@dataclass(frozen=True) +class CapacityLimiterStatistics: + """ + :ivar int borrowed_tokens: number of tokens currently borrowed by tasks + :ivar float total_tokens: total number of available tokens + :ivar tuple borrowers: tasks or other objects currently holding tokens borrowed from + this limiter + :ivar int tasks_waiting: number of tasks waiting on + :meth:`~.CapacityLimiter.acquire` or + :meth:`~.CapacityLimiter.acquire_on_behalf_of` + """ + + borrowed_tokens: int + total_tokens: float + borrowers: tuple[object, ...] + tasks_waiting: int + + +@dataclass(frozen=True) +class LockStatistics: + """ + :ivar bool locked: flag indicating if this lock is locked or not + :ivar ~anyio.TaskInfo owner: task currently holding the lock (or ``None`` if the + lock is not held by any task) + :ivar int tasks_waiting: number of tasks waiting on :meth:`~.Lock.acquire` + """ + + locked: bool + owner: TaskInfo | None + tasks_waiting: int + + +@dataclass(frozen=True) +class ConditionStatistics: + """ + :ivar int tasks_waiting: number of tasks blocked on :meth:`~.Condition.wait` + :ivar ~anyio.LockStatistics lock_statistics: statistics of the underlying + :class:`~.Lock` + """ + + tasks_waiting: int + lock_statistics: LockStatistics + + +@dataclass(frozen=True) +class SemaphoreStatistics: + """ + :ivar int tasks_waiting: number of tasks waiting on :meth:`~.Semaphore.acquire` + + """ + + tasks_waiting: int + + +class Event: + __slots__ = ("__weakref__",) + + def __new__(cls) -> Event: + try: + return get_async_backend().create_event() + except NoEventLoopError: + return EventAdapter() + + def set(self) -> None: + """Set the flag, notifying all listeners.""" + raise NotImplementedError + + def is_set(self) -> bool: + """Return ``True`` if the flag is set, ``False`` if not.""" + raise NotImplementedError + + async def wait(self) -> None: + """ + Wait until the flag has been set. + + If the flag has already been set when this method is called, it returns + immediately. + + """ + raise NotImplementedError + + def statistics(self) -> EventStatistics: + """Return statistics about the current state of this event.""" + raise NotImplementedError + + +class EventAdapter(Event): + __slots__ = "_internal_event", "_is_set" + + def __new__(cls) -> Self: + return object.__new__(cls) + + def __init__(self) -> None: + self._internal_event: Event | None = None + self._is_set = False + + @property + def _event(self) -> Event: + if self._internal_event is None: + self._internal_event = get_async_backend().create_event() + if self._is_set: + self._internal_event.set() + + return self._internal_event + + def set(self) -> None: + if self._internal_event is None: + self._is_set = True + else: + self._event.set() + + def is_set(self) -> bool: + if self._internal_event is None: + return self._is_set + + return self._internal_event.is_set() + + async def wait(self) -> None: + await self._event.wait() + + def statistics(self) -> EventStatistics: + if self._internal_event is None: + return EventStatistics(tasks_waiting=0) + + return self._internal_event.statistics() + + +class Lock: + __slots__ = ("__weakref__",) + + def __new__(cls, *, fast_acquire: bool = False) -> Lock: + try: + return get_async_backend().create_lock(fast_acquire=fast_acquire) + except NoEventLoopError: + return LockAdapter(fast_acquire=fast_acquire) + + async def __aenter__(self) -> None: + await self.acquire() + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> None: + self.release() + + async def acquire(self) -> None: + """Acquire the lock.""" + raise NotImplementedError + + def acquire_nowait(self) -> None: + """ + Acquire the lock, without blocking. + + :raises ~anyio.WouldBlock: if the operation would block + + """ + raise NotImplementedError + + def release(self) -> None: + """Release the lock.""" + raise NotImplementedError + + def locked(self) -> bool: + """Return True if the lock is currently held.""" + raise NotImplementedError + + def statistics(self) -> LockStatistics: + """ + Return statistics about the current state of this lock. + + .. versionadded:: 3.0 + """ + raise NotImplementedError + + +class LockAdapter(Lock): + __slots__ = "_fast_acquire", "_internal_lock" + + def __new__(cls, *, fast_acquire: bool = False) -> Self: + return object.__new__(cls) + + def __init__(self, *, fast_acquire: bool = False): + self._internal_lock: Lock | None = None + self._fast_acquire = fast_acquire + + @property + def _lock(self) -> Lock: + if self._internal_lock is None: + self._internal_lock = get_async_backend().create_lock( + fast_acquire=self._fast_acquire + ) + + return self._internal_lock + + async def __aenter__(self) -> None: + await self._lock.acquire() + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> None: + if self._internal_lock is not None: + self._internal_lock.release() + + async def acquire(self) -> None: + """Acquire the lock.""" + await self._lock.acquire() + + def acquire_nowait(self) -> None: + """ + Acquire the lock, without blocking. + + :raises ~anyio.WouldBlock: if the operation would block + + """ + self._lock.acquire_nowait() + + def release(self) -> None: + """Release the lock.""" + self._lock.release() + + def locked(self) -> bool: + """Return True if the lock is currently held.""" + return self._lock.locked() + + def statistics(self) -> LockStatistics: + """ + Return statistics about the current state of this lock. + + .. versionadded:: 3.0 + + """ + if self._internal_lock is None: + return LockStatistics(False, None, 0) + + return self._internal_lock.statistics() + + +class Condition: + __slots__ = "__weakref__", "_lock", "_owner_task", "_waiters" + + def __init__(self, lock: Lock | None = None): + self._owner_task: TaskInfo | None = None + self._lock = lock or Lock() + self._waiters: deque[Event] = deque() + + async def __aenter__(self) -> None: + await self.acquire() + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> None: + self.release() + + def _check_acquired(self) -> None: + if self._owner_task != get_current_task(): + raise RuntimeError("The current task is not holding the underlying lock") + + async def acquire(self) -> None: + """Acquire the underlying lock.""" + await self._lock.acquire() + self._owner_task = get_current_task() + + def acquire_nowait(self) -> None: + """ + Acquire the underlying lock, without blocking. + + :raises ~anyio.WouldBlock: if the operation would block + + """ + self._lock.acquire_nowait() + self._owner_task = get_current_task() + + def release(self) -> None: + """Release the underlying lock.""" + self._lock.release() + + def locked(self) -> bool: + """Return True if the lock is set.""" + return self._lock.locked() + + def notify(self, n: int = 1) -> None: + """Notify exactly n listeners.""" + self._check_acquired() + for _ in range(n): + try: + event = self._waiters.popleft() + except IndexError: + break + + event.set() + + def notify_all(self) -> None: + """Notify all the listeners.""" + self._check_acquired() + for event in self._waiters: + event.set() + + self._waiters.clear() + + async def wait(self) -> None: + """Wait for a notification.""" + await checkpoint_if_cancelled() + self._check_acquired() + event = Event() + self._waiters.append(event) + self.release() + try: + await event.wait() + except BaseException: + if not event.is_set(): + self._waiters.remove(event) + elif self._waiters: + # This task was notified by could not act on it, so pass + # it on to the next task + self._waiters.popleft().set() + + raise + finally: + with CancelScope(shield=True): + await self.acquire() + + async def wait_for(self, predicate: Callable[[], T]) -> T: + """ + Wait until a predicate becomes true. + + :param predicate: a callable that returns a truthy value when the condition is + met + :return: the result of the predicate + + .. versionadded:: 4.11.0 + + """ + while not (result := predicate()): + await self.wait() + + return result + + def statistics(self) -> ConditionStatistics: + """ + Return statistics about the current state of this condition. + + .. versionadded:: 3.0 + """ + return ConditionStatistics(len(self._waiters), self._lock.statistics()) + + +class Semaphore: + __slots__ = "__weakref__", "_fast_acquire" + + def __new__( + cls, + initial_value: int, + *, + max_value: int | None = None, + fast_acquire: bool = False, + ) -> Semaphore: + try: + return get_async_backend().create_semaphore( + initial_value, max_value=max_value, fast_acquire=fast_acquire + ) + except NoEventLoopError: + return SemaphoreAdapter(initial_value, max_value=max_value) + + def __init__( + self, + initial_value: int, + *, + max_value: int | None = None, + fast_acquire: bool = False, + ): + if not isinstance(initial_value, int): + raise TypeError("initial_value must be an integer") + if initial_value < 0: + raise ValueError("initial_value must be >= 0") + if max_value is not None: + if not isinstance(max_value, int): + raise TypeError("max_value must be an integer or None") + if max_value < initial_value: + raise ValueError( + "max_value must be equal to or higher than initial_value" + ) + + self._fast_acquire = fast_acquire + + async def __aenter__(self) -> Self: + await self.acquire() + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> None: + self.release() + + async def acquire(self) -> None: + """Decrement the semaphore value, blocking if necessary.""" + raise NotImplementedError + + def acquire_nowait(self) -> None: + """ + Acquire the underlying lock, without blocking. + + :raises ~anyio.WouldBlock: if the operation would block + + """ + raise NotImplementedError + + def release(self) -> None: + """Increment the semaphore value.""" + raise NotImplementedError + + @property + def value(self) -> int: + """The current value of the semaphore.""" + raise NotImplementedError + + @property + def max_value(self) -> int | None: + """The maximum value of the semaphore.""" + raise NotImplementedError + + def statistics(self) -> SemaphoreStatistics: + """ + Return statistics about the current state of this semaphore. + + .. versionadded:: 3.0 + """ + raise NotImplementedError + + +class SemaphoreAdapter(Semaphore): + __slots__ = "_initial_value", "_internal_semaphore", "_max_value" + + def __new__( + cls, + initial_value: int, + *, + max_value: int | None = None, + fast_acquire: bool = False, + ) -> Self: + return object.__new__(cls) + + def __init__( + self, + initial_value: int, + *, + max_value: int | None = None, + fast_acquire: bool = False, + ) -> None: + super().__init__(initial_value, max_value=max_value, fast_acquire=fast_acquire) + self._internal_semaphore: Semaphore | None = None + self._initial_value = initial_value + self._max_value = max_value + + @property + def _semaphore(self) -> Semaphore: + if self._internal_semaphore is None: + self._internal_semaphore = get_async_backend().create_semaphore( + self._initial_value, max_value=self._max_value + ) + + return self._internal_semaphore + + async def acquire(self) -> None: + await self._semaphore.acquire() + + def acquire_nowait(self) -> None: + self._semaphore.acquire_nowait() + + def release(self) -> None: + self._semaphore.release() + + @property + def value(self) -> int: + if self._internal_semaphore is None: + return self._initial_value + + return self._semaphore.value + + @property + def max_value(self) -> int | None: + return self._max_value + + def statistics(self) -> SemaphoreStatistics: + if self._internal_semaphore is None: + return SemaphoreStatistics(tasks_waiting=0) + + return self._semaphore.statistics() + + +class CapacityLimiter: + __slots__ = ("__weakref__",) + + def __new__(cls, total_tokens: float) -> CapacityLimiter: + try: + return get_async_backend().create_capacity_limiter(total_tokens) + except NoEventLoopError: + return CapacityLimiterAdapter(total_tokens) + + async def __aenter__(self) -> None: + raise NotImplementedError + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> None: + raise NotImplementedError + + @property + def total_tokens(self) -> float: + """ + The total number of tokens available for borrowing. + + This is a read-write property. If the total number of tokens is increased, the + proportionate number of tasks waiting on this limiter will be granted their + tokens. + + .. versionchanged:: 3.0 + The property is now writable. + .. versionchanged:: 4.12 + The value can now be set to 0. + + """ + raise NotImplementedError + + @total_tokens.setter + def total_tokens(self, value: float) -> None: + raise NotImplementedError + + @property + def borrowed_tokens(self) -> int: + """The number of tokens that have currently been borrowed.""" + raise NotImplementedError + + @property + def available_tokens(self) -> float: + """The number of tokens currently available to be borrowed""" + raise NotImplementedError + + def acquire_nowait(self) -> None: + """ + Acquire a token for the current task without waiting for one to become + available. + + :raises ~anyio.WouldBlock: if there are no tokens available for borrowing + + """ + raise NotImplementedError + + def acquire_on_behalf_of_nowait(self, borrower: object) -> None: + """ + Acquire a token without waiting for one to become available. + + :param borrower: the entity borrowing a token + :raises ~anyio.WouldBlock: if there are no tokens available for borrowing + + """ + raise NotImplementedError + + async def acquire(self) -> None: + """ + Acquire a token for the current task, waiting if necessary for one to become + available. + + """ + raise NotImplementedError + + async def acquire_on_behalf_of(self, borrower: object) -> None: + """ + Acquire a token, waiting if necessary for one to become available. + + :param borrower: the entity borrowing a token + + """ + raise NotImplementedError + + def release(self) -> None: + """ + Release the token held by the current task. + + :raises RuntimeError: if the current task has not borrowed a token from this + limiter. + + """ + raise NotImplementedError + + def release_on_behalf_of(self, borrower: object) -> None: + """ + Release the token held by the given borrower. + + :raises RuntimeError: if the borrower has not borrowed a token from this + limiter. + + """ + raise NotImplementedError + + def statistics(self) -> CapacityLimiterStatistics: + """ + Return statistics about the current state of this limiter. + + .. versionadded:: 3.0 + + """ + raise NotImplementedError + + +class CapacityLimiterAdapter(CapacityLimiter): + __slots__ = "_internal_limiter", "_total_tokens" + + def __new__(cls, total_tokens: float) -> Self: + return object.__new__(cls) + + def __init__(self, total_tokens: float) -> None: + self._internal_limiter: CapacityLimiter | None = None + self.total_tokens = total_tokens + + @property + def _limiter(self) -> CapacityLimiter: + if self._internal_limiter is None: + self._internal_limiter = get_async_backend().create_capacity_limiter( + self._total_tokens + ) + + return self._internal_limiter + + async def __aenter__(self) -> None: + await self._limiter.__aenter__() + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> None: + return await self._limiter.__aexit__(exc_type, exc_val, exc_tb) + + @property + def total_tokens(self) -> float: + if self._internal_limiter is None: + return self._total_tokens + + return self._internal_limiter.total_tokens + + @total_tokens.setter + def total_tokens(self, value: float) -> None: + if not isinstance(value, int) and not math.isinf(value): + raise TypeError("total_tokens must be an int or math.inf") + elif value < 0: + raise ValueError("total_tokens must be >= 0") + + if self._internal_limiter is None: + self._total_tokens = value + return + + self._limiter.total_tokens = value + + @property + def borrowed_tokens(self) -> int: + if self._internal_limiter is None: + return 0 + + return self._internal_limiter.borrowed_tokens + + @property + def available_tokens(self) -> float: + if self._internal_limiter is None: + return self._total_tokens + + return self._internal_limiter.available_tokens + + def acquire_nowait(self) -> None: + self._limiter.acquire_nowait() + + def acquire_on_behalf_of_nowait(self, borrower: object) -> None: + self._limiter.acquire_on_behalf_of_nowait(borrower) + + async def acquire(self) -> None: + await self._limiter.acquire() + + async def acquire_on_behalf_of(self, borrower: object) -> None: + await self._limiter.acquire_on_behalf_of(borrower) + + def release(self) -> None: + self._limiter.release() + + def release_on_behalf_of(self, borrower: object) -> None: + self._limiter.release_on_behalf_of(borrower) + + def statistics(self) -> CapacityLimiterStatistics: + if self._internal_limiter is None: + return CapacityLimiterStatistics( + borrowed_tokens=0, + total_tokens=self.total_tokens, + borrowers=(), + tasks_waiting=0, + ) + + return self._internal_limiter.statistics() + + +class ResourceGuard: + """ + A context manager for ensuring that a resource is only used by a single task at a + time. + + Entering this context manager while the previous has not exited it yet will trigger + :exc:`BusyResourceError`. + + :param action: the action to guard against (visible in the :exc:`BusyResourceError` + when triggered, e.g. "Another task is already {action} this resource") + + .. versionadded:: 4.1 + """ + + __slots__ = "__weakref__", "_guarded", "action" + + def __init__(self, action: str = "using"): + self.action: str = action + self._guarded = False + + def __enter__(self) -> None: + if self._guarded: + raise BusyResourceError(self.action) + + self._guarded = True + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> None: + self._guarded = False diff --git a/.venv/lib/python3.14/site-packages/anyio/_core/_tasks.py b/.venv/lib/python3.14/site-packages/anyio/_core/_tasks.py new file mode 100644 index 0000000000000000000000000000000000000000..044eda6dc90ab446b5b6e1dc1626cb6dd510e7db --- /dev/null +++ b/.venv/lib/python3.14/site-packages/anyio/_core/_tasks.py @@ -0,0 +1,464 @@ +from __future__ import annotations + +import math +import sys +from collections.abc import ( + Coroutine, + Generator, +) +from contextlib import contextmanager +from enum import Enum, auto +from inspect import iscoroutine +from types import TracebackType +from typing import Any, Generic, final + +from ..abc import TaskGroup, TaskStatus +from ._eventloop import get_async_backend, get_cancelled_exc_class +from ._exceptions import TaskCancelled, TaskFailed, TaskNotFinished + +if sys.version_info >= (3, 13): + from typing import TypeVar +else: + from typing_extensions import TypeVar + +if sys.version_info >= (3, 11): + from typing import Never, Self, TypeVarTuple +else: + from typing_extensions import Never, Self, TypeVarTuple + +T = TypeVar("T") +T_co = TypeVar("T_co", covariant=True) +T_startval_co = TypeVar("T_startval_co", covariant=True, default=Never) +PosArgsT = TypeVarTuple("PosArgsT") + + +class _IgnoredTaskStatus(TaskStatus[object]): + def started(self, value: object = None) -> None: + pass + + +TASK_STATUS_IGNORED = _IgnoredTaskStatus() + + +class CancelScope: + """ + Wraps a unit of work that can be made separately cancellable. + + :param deadline: The time (clock value) when this scope is cancelled automatically + :param shield: ``True`` to shield the cancel scope from external cancellation + :raises NoEventLoopError: if no supported asynchronous event loop is running in the + current thread + """ + + __slots__ = ("__weakref__",) + + def __new__( + cls, *, deadline: float = math.inf, shield: bool = False + ) -> CancelScope: + return get_async_backend().create_cancel_scope(shield=shield, deadline=deadline) + + def cancel(self, reason: str | None = None) -> None: + """ + Cancel this scope immediately. + + :param reason: a message describing the reason for the cancellation + + """ + raise NotImplementedError + + @property + def deadline(self) -> float: + """ + The time (clock value) when this scope is cancelled automatically. + + Will be ``float('inf')`` if no timeout has been set. + + """ + raise NotImplementedError + + @deadline.setter + def deadline(self, value: float) -> None: + raise NotImplementedError + + @property + def cancel_called(self) -> bool: + """``True`` if :meth:`cancel` has been called.""" + raise NotImplementedError + + @property + def cancelled_caught(self) -> bool: + """ + ``True`` if this scope suppressed a cancellation exception it itself raised. + + This is typically used to check if any work was interrupted, or to see if the + scope was cancelled due to its deadline being reached. The value will, however, + only be ``True`` if the cancellation was triggered by the scope itself (and not + an outer scope). + + """ + raise NotImplementedError + + @property + def shield(self) -> bool: + """ + ``True`` if this scope is shielded from external cancellation. + + While a scope is shielded, it will not receive cancellations from outside. + + """ + raise NotImplementedError + + @shield.setter + def shield(self, value: bool) -> None: + raise NotImplementedError + + def __enter__(self) -> Self: + raise NotImplementedError + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> bool: + raise NotImplementedError + + +@contextmanager +def fail_at( + deadline: float | None, shield: bool = False, reason: str | None = None +) -> Generator[CancelScope, None, None]: + """ + Create a context manager which raises a :class:`TimeoutError` if the code in the + enclosing context does not finish by the given deadline. + + :param deadline: the deadline before raising the exception, or + ``None`` to disable the timeout + :param shield: ``True`` to shield the cancel scope from external cancellation + :param reason: explanation for timeout to add to the message of a raised `TimeoutError` + :return: a context manager that yields a cancel scope + :rtype: :class:`~typing.ContextManager`\\[:class:`~anyio.CancelScope`\\] + :raises NoEventLoopError: if no supported asynchronous event loop is running in the + current thread + + .. versionadded:: 4.15.0 + + """ + current_time = get_async_backend().current_time + effective_deadline = math.inf if deadline is None else deadline + with get_async_backend().create_cancel_scope( + deadline=effective_deadline, shield=shield + ) as cancel_scope: + yield cancel_scope + + if cancel_scope.cancelled_caught and current_time() >= cancel_scope.deadline: + raise TimeoutError(reason) if reason else TimeoutError + + +@contextmanager +def fail_after( + delay: float | None, shield: bool = False, reason: str | None = None +) -> Generator[CancelScope, None, None]: + """ + Create a context manager which raises a :class:`TimeoutError` if the code in the + enclosing context block does not finish in time. + + :param delay: maximum allowed time (in seconds) before raising the exception, or + ``None`` to disable the timeout + :param shield: ``True`` to shield the cancel scope from external cancellation + :param reason: explanation for timeout to add to the message of a raised `TimeoutError` + :return: a context manager that yields a cancel scope + :rtype: :class:`~typing.ContextManager`\\[:class:`~anyio.CancelScope`\\] + :raises NoEventLoopError: if no supported asynchronous event loop is running in the + current thread + + """ + current_time = get_async_backend().current_time + deadline = (current_time() + delay) if delay is not None else math.inf + with fail_at(deadline, shield=shield, reason=reason) as scope: + yield scope + + +def move_on_at(deadline: float | None, shield: bool = False) -> CancelScope: + """ + Create a cancel scope with a deadline that expires after the given delay. + + :param deadline: the deadline before exiting the context block, or + ``None`` to disable the timeout + :param shield: ``True`` to shield the cancel scope from external cancellation + :return: a cancel scope + :raises NoEventLoopError: if no supported asynchronous event loop is running in the + current thread + + .. versionadded:: 4.15.0 + + """ + return get_async_backend().create_cancel_scope( + deadline=deadline if deadline is not None else math.inf, shield=shield + ) + + +def move_on_after(delay: float | None, shield: bool = False) -> CancelScope: + """ + Create a cancel scope with a deadline that expires after the given delay. + + :param delay: maximum allowed time (in seconds) before exiting the context block, or + ``None`` to disable the timeout + :param shield: ``True`` to shield the cancel scope from external cancellation + :return: a cancel scope + :raises NoEventLoopError: if no supported asynchronous event loop is running in the + current thread + + .. note:: Unlike with :func:`fail_after`, the timer starts when this function is + called, not when the context manager is entered. This will be changed in v5.0. + + """ + deadline = ( + (get_async_backend().current_time() + delay) if delay is not None else math.inf + ) + return get_async_backend().create_cancel_scope(deadline=deadline, shield=shield) + + +def current_effective_deadline() -> float: + """ + Return the nearest deadline among all the cancel scopes effective for the current + task. + + :return: a clock value from the event loop's internal clock (or ``float('inf')`` if + there is no deadline in effect, or ``float('-inf')`` if the current scope has + been cancelled) + :rtype: float + :raises NoEventLoopError: if no supported asynchronous event loop is running in the + current thread + + """ + return get_async_backend().current_effective_deadline() + + +def create_task_group() -> TaskGroup: + """ + Create a task group. + + :return: a task group + :raises NoEventLoopError: if no supported asynchronous event loop is running in the + current thread + + """ + return get_async_backend().create_task_group() + + +@final +class TaskHandle(Generic[T_co, T_startval_co]): + """ + Returned from the task-spawning methods of :class:`TaskGroup`. Can be awaited on to + get the return value of the task (or the raised exception). If the task was + terminated by a :exc:`BaseException`, :exc:`TaskFailed` will be raised (or its + subclass :exc:`TaskCancelled` if the task was cancelled). + + .. versionadded:: 4.14.0 + """ + + class Status(Enum): + """ + The status of a task handle. + + .. attribute:: PENDING + + The task has not finished yet. + .. attribute:: FINISHED + + The task has finished with a return value. + .. attribute:: CANCELLING + + The task has been cancelled but has not finished yet. + .. attribute:: CANCELLED + + The task was cancelled and has finished since. + .. attribute:: FAILED + + The task raised an exception. + """ + + PENDING = auto() + FINISHED = auto() + CANCELLING = auto() + CANCELLED = auto() + FAILED = auto() + + __slots__ = ( + "__weakref__", + "_cancel_scope", + "_coro", + "_exception", + "_finished_event", + "_name", + "_return_value", + "_start_value", + ) + + _return_value: T_co + _start_value: T_startval_co + + def __init__( + self, + coro: Coroutine[Any, Any, T_co], + name: object, + cancel_scope: CancelScope | None = None, + ) -> None: + from ._synchronization import Event + + self._coro = coro + self._cancel_scope = cancel_scope if cancel_scope is not None else CancelScope() + self._finished_event = Event() + self._exception: BaseException | None = None + + if name is not None: + self._name = str(name) + elif iscoroutine(coro): + self._name = coro.__qualname__ + else: + self._name = str(coro) # coroutine-like object (e.g. asend() objects) + + async def _run_coro(self) -> None: + __tracebackhide__ = True + + with self._cancel_scope: + try: + retval = await self._coro + except BaseException as exc: + self._exception = exc + raise + else: + self._return_value = retval + finally: + self._finished_event.set() + del self # Break the reference cycle + + def cancel(self) -> None: + """ + Set the task to a cancelled state. + + This will interrupt any interruptible asynchronous operation, and will cause + any further awaits on this task to get immediately cancelled, unless done in + a shielded cancel scope. + + If the task has already finished, this method has no effect. + """ + if not self._finished_event.is_set(): + self._cancel_scope.cancel() + + @property + def coro(self) -> Coroutine[Any, Any, T_co]: + """ + The coroutine object that was passed to one of the task-spawning methods in + :class:`TaskGroup`. + """ + return self._coro + + @property + def status(self) -> TaskHandle.Status: + """ + The current status of the task. + + Every task starts in the :attr:`~TaskHandle.Status.PENDING` state. + If a task is cancelled while in this state, it will transition to the + :attr:`~TaskHandle.Status.CANCELLING` state. When the task finishes, it will + transition to one of the three final states ( + :attr:`~TaskHandle.Status.FINISHED`, :attr:`~TaskHandle.Status.FAILED`, or + :attr:`~TaskHandle.Status.CANCELLING`) depending on the exception the task + raised, if any. No other status transitions will happen. + """ + if not self._finished_event.is_set(): + if self._cancel_scope.cancel_called: + return TaskHandle.Status.CANCELLING + else: + return TaskHandle.Status.PENDING + elif self._exception is not None: + if isinstance(self._exception, get_cancelled_exc_class()): + return TaskHandle.Status.CANCELLED + else: + return TaskHandle.Status.FAILED + else: + return TaskHandle.Status.FINISHED + + @property + def name(self) -> str: + """The name of the task.""" + return self._name + + @property + def exception(self) -> BaseException | None: + """ + The exception raised by the task, or ``None`` if it finished without raising. + + :raises TaskNotFinished: if the task has not finished yet + :raises TaskCancelled: if the task was cancelled + + """ + match self.status: + case TaskHandle.Status.PENDING: + raise TaskNotFinished("the task has not finished yet") + case TaskHandle.Status.FINISHED: + return None + case TaskHandle.Status.CANCELLING: + raise TaskCancelled("the task was cancelled") + case TaskHandle.Status.CANCELLED: + raise TaskCancelled("the task was cancelled") from self._exception + case TaskHandle.Status.FAILED: + return self._exception + + @property + def return_value(self) -> T_co: + """ + The return value of the task. + + :raises TaskNotFinished: if the task has not finished yet + :raises TaskCancelled: if the task was cancelled + :raises TaskFailed: if the task raised an exception + + """ + match self.status: + case TaskHandle.Status.PENDING: + raise TaskNotFinished("the task has not finished yet") + case TaskHandle.Status.FINISHED: + return self._return_value + case TaskHandle.Status.CANCELLING: + raise TaskCancelled("the task was cancelled") + case TaskHandle.Status.CANCELLED: + raise TaskCancelled("the task was cancelled") from self._exception + case TaskHandle.Status.FAILED: + raise TaskFailed("the task raised an exception") from self._exception + + @property + def start_value(self) -> T_startval_co: + """ + The value passed to :meth:`task_status.started() <.abc.TaskStatus.started>`, + + :raises RuntimeError: if the task was not started with :meth:`TaskGroup.start() + <.abc.TaskGroup.start>` + """ + try: + return self._start_value + except AttributeError: + raise RuntimeError( + "the task was not started with TaskGroup.start()" + ) from None + + async def wait(self) -> None: + """ + Wait for the task to finish. + + This method will return as soon as the task has finished, no matter how it + happened. + """ + await self._finished_event.wait() + + def __await__(self) -> Generator[Any, Any, T_co]: + yield from self._finished_event.wait().__await__() + return self.return_value + + def __repr__(self) -> str: + return ( + f"<{self.__class__.__name__} {self.status.name.lower()} " + f"name={self._name!r} coro={self._coro!r}>" + ) diff --git a/.venv/lib/python3.14/site-packages/anyio/_core/_tempfile.py b/.venv/lib/python3.14/site-packages/anyio/_core/_tempfile.py new file mode 100644 index 0000000000000000000000000000000000000000..96030867befb177e754f596efcc3e9ee1f234442 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/anyio/_core/_tempfile.py @@ -0,0 +1,615 @@ +from __future__ import annotations + +import os +import sys +import tempfile +from collections.abc import Iterable +from io import BytesIO, TextIOWrapper +from types import TracebackType +from typing import ( + TYPE_CHECKING, + Any, + AnyStr, + Generic, + overload, +) + +from .. import to_thread +from .._core._fileio import AsyncFile +from ..lowlevel import checkpoint_if_cancelled +from ._tasks import CancelScope + +if TYPE_CHECKING: + from _typeshed import OpenBinaryMode, OpenTextMode, ReadableBuffer, WriteableBuffer + + +class TemporaryFile(Generic[AnyStr]): + """ + An asynchronous temporary file that is automatically created and cleaned up. + + This class provides an asynchronous context manager interface to a temporary file. + The file is created using Python's standard `tempfile.TemporaryFile` function in a + background thread, and is wrapped as an asynchronous file using `AsyncFile`. + + :param mode: The mode in which the file is opened. Defaults to "w+b". + :param buffering: The buffering policy (-1 means the default buffering). + :param encoding: The encoding used to decode or encode the file. Only applicable in + text mode. + :param newline: Controls how universal newlines mode works (only applicable in text + mode). + :param suffix: The suffix for the temporary file name. + :param prefix: The prefix for the temporary file name. + :param dir: The directory in which the temporary file is created. + :param errors: The error handling scheme used for encoding/decoding errors. + """ + + _async_file: AsyncFile[AnyStr] + + @overload + def __init__( + self: TemporaryFile[bytes], + mode: OpenBinaryMode = ..., + buffering: int = ..., + encoding: str | None = ..., + newline: str | None = ..., + suffix: str | None = ..., + prefix: str | None = ..., + dir: str | None = ..., + *, + errors: str | None = ..., + ): ... + @overload + def __init__( + self: TemporaryFile[str], + mode: OpenTextMode, + buffering: int = ..., + encoding: str | None = ..., + newline: str | None = ..., + suffix: str | None = ..., + prefix: str | None = ..., + dir: str | None = ..., + *, + errors: str | None = ..., + ): ... + + def __init__( + self, + mode: OpenTextMode | OpenBinaryMode = "w+b", + buffering: int = -1, + encoding: str | None = None, + newline: str | None = None, + suffix: str | None = None, + prefix: str | None = None, + dir: str | None = None, + *, + errors: str | None = None, + ) -> None: + self.mode = mode + self.buffering = buffering + self.encoding = encoding + self.newline = newline + self.suffix: str | None = suffix + self.prefix: str | None = prefix + self.dir: str | None = dir + self.errors = errors + + async def __aenter__(self) -> AsyncFile[AnyStr]: + fp = await to_thread.run_sync( + lambda: tempfile.TemporaryFile( + self.mode, + self.buffering, + self.encoding, + self.newline, + self.suffix, + self.prefix, + self.dir, + errors=self.errors, + ) + ) + self._async_file = AsyncFile(fp) + return self._async_file + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: + await self._async_file.aclose() + + +class NamedTemporaryFile(Generic[AnyStr]): + """ + An asynchronous named temporary file that is automatically created and cleaned up. + + This class provides an asynchronous context manager for a temporary file with a + visible name in the file system. It uses Python's standard + :func:`~tempfile.NamedTemporaryFile` function and wraps the file object with + :class:`AsyncFile` for asynchronous operations. + + :param mode: The mode in which the file is opened. Defaults to "w+b". + :param buffering: The buffering policy (-1 means the default buffering). + :param encoding: The encoding used to decode or encode the file. Only applicable in + text mode. + :param newline: Controls how universal newlines mode works (only applicable in text + mode). + :param suffix: The suffix for the temporary file name. + :param prefix: The prefix for the temporary file name. + :param dir: The directory in which the temporary file is created. + :param delete: Whether to delete the file when it is closed. + :param errors: The error handling scheme used for encoding/decoding errors. + :param delete_on_close: (Python 3.12+) Whether to delete the file on close. + """ + + _async_file: AsyncFile[AnyStr] + + @overload + def __init__( + self: NamedTemporaryFile[bytes], + mode: OpenBinaryMode = ..., + buffering: int = ..., + encoding: str | None = ..., + newline: str | None = ..., + suffix: str | None = ..., + prefix: str | None = ..., + dir: str | None = ..., + delete: bool = ..., + *, + errors: str | None = ..., + delete_on_close: bool = ..., + ): ... + @overload + def __init__( + self: NamedTemporaryFile[str], + mode: OpenTextMode, + buffering: int = ..., + encoding: str | None = ..., + newline: str | None = ..., + suffix: str | None = ..., + prefix: str | None = ..., + dir: str | None = ..., + delete: bool = ..., + *, + errors: str | None = ..., + delete_on_close: bool = ..., + ): ... + + def __init__( + self, + mode: OpenBinaryMode | OpenTextMode = "w+b", + buffering: int = -1, + encoding: str | None = None, + newline: str | None = None, + suffix: str | None = None, + prefix: str | None = None, + dir: str | None = None, + delete: bool = True, + *, + errors: str | None = None, + delete_on_close: bool = True, + ) -> None: + self._params: dict[str, Any] = { + "mode": mode, + "buffering": buffering, + "encoding": encoding, + "newline": newline, + "suffix": suffix, + "prefix": prefix, + "dir": dir, + "delete": delete, + "errors": errors, + } + if sys.version_info >= (3, 12): + self._params["delete_on_close"] = delete_on_close + + async def __aenter__(self) -> AsyncFile[AnyStr]: + fp = await to_thread.run_sync( + lambda: tempfile.NamedTemporaryFile(**self._params) + ) + self._async_file = AsyncFile(fp) + return self._async_file + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: + await self._async_file.aclose() + + +class SpooledTemporaryFile(AsyncFile[AnyStr]): + """ + An asynchronous spooled temporary file that starts in memory and is spooled to disk. + + This class provides an asynchronous interface to a spooled temporary file, much like + Python's standard :class:`~tempfile.SpooledTemporaryFile`. It supports asynchronous + write operations and provides a method to force a rollover to disk. + + :param max_size: Maximum size in bytes before the file is rolled over to disk. + :param mode: The mode in which the file is opened. Defaults to "w+b". + :param buffering: The buffering policy (-1 means the default buffering). + :param encoding: The encoding used to decode or encode the file (text mode only). + :param newline: Controls how universal newlines mode works (text mode only). + :param suffix: The suffix for the temporary file name. + :param prefix: The prefix for the temporary file name. + :param dir: The directory in which the temporary file is created. + :param errors: The error handling scheme used for encoding/decoding errors. + """ + + _rolled: bool = False + + @overload + def __init__( + self: SpooledTemporaryFile[bytes], + max_size: int = ..., + mode: OpenBinaryMode = ..., + buffering: int = ..., + encoding: str | None = ..., + newline: str | None = ..., + suffix: str | None = ..., + prefix: str | None = ..., + dir: str | None = ..., + *, + errors: str | None = ..., + ): ... + @overload + def __init__( + self: SpooledTemporaryFile[str], + max_size: int = ..., + mode: OpenTextMode = ..., + buffering: int = ..., + encoding: str | None = ..., + newline: str | None = ..., + suffix: str | None = ..., + prefix: str | None = ..., + dir: str | None = ..., + *, + errors: str | None = ..., + ): ... + + def __init__( + self, + max_size: int = 0, + mode: OpenBinaryMode | OpenTextMode = "w+b", + buffering: int = -1, + encoding: str | None = None, + newline: str | None = None, + suffix: str | None = None, + prefix: str | None = None, + dir: str | None = None, + *, + errors: str | None = None, + ) -> None: + self._tempfile_params: dict[str, Any] = { + "mode": mode, + "buffering": buffering, + "encoding": encoding, + "newline": newline, + "suffix": suffix, + "prefix": prefix, + "dir": dir, + "errors": errors, + } + self._max_size = max_size + if "b" in mode: + super().__init__(BytesIO()) # type: ignore[arg-type] + else: + super().__init__( + TextIOWrapper( # type: ignore[arg-type] + BytesIO(), + encoding=encoding, + errors=errors, + newline=newline, + write_through=True, + ) + ) + + async def aclose(self) -> None: + if not self._rolled: + self._fp.close() + return + + await super().aclose() + + async def _check(self) -> None: + if self._rolled or self._fp.tell() <= self._max_size: + return + + await self.rollover() + + async def rollover(self) -> None: + if self._rolled: + return + + self._rolled = True + buffer = self._fp + buffer.seek(0) + self._fp = await to_thread.run_sync( + lambda: tempfile.TemporaryFile(**self._tempfile_params) + ) + await self.write(buffer.read()) + buffer.close() + + @property + def closed(self) -> bool: + return self._fp.closed + + async def read(self, size: int = -1) -> AnyStr: + if not self._rolled: + await checkpoint_if_cancelled() + return self._fp.read(size) + + return await super().read(size) # type: ignore[return-value] + + async def read1(self: SpooledTemporaryFile[bytes], size: int = -1) -> bytes: + if not self._rolled: + await checkpoint_if_cancelled() + return self._fp.read1(size) + + return await super().read1(size) + + async def readline(self) -> AnyStr: + if not self._rolled: + await checkpoint_if_cancelled() + return self._fp.readline() + + return await super().readline() # type: ignore[return-value] + + async def readlines(self) -> list[AnyStr]: + if not self._rolled: + await checkpoint_if_cancelled() + return self._fp.readlines() + + return await super().readlines() # type: ignore[return-value] + + async def readinto(self: SpooledTemporaryFile[bytes], b: WriteableBuffer) -> int: + if not self._rolled: + await checkpoint_if_cancelled() + return self._fp.readinto(b) + + return await super().readinto(b) + + async def readinto1(self: SpooledTemporaryFile[bytes], b: WriteableBuffer) -> int: + if not self._rolled: + await checkpoint_if_cancelled() + return self._fp.readinto1(b) + + return await super().readinto1(b) + + async def seek(self, offset: int, whence: int | None = os.SEEK_SET) -> int: + if not self._rolled: + await checkpoint_if_cancelled() + return self._fp.seek(offset, whence) + + return await super().seek(offset, whence) + + async def tell(self) -> int: + if not self._rolled: + await checkpoint_if_cancelled() + return self._fp.tell() + + return await super().tell() + + async def truncate(self, size: int | None = None) -> int: + if not self._rolled: + await checkpoint_if_cancelled() + return self._fp.truncate(size) + + return await super().truncate(size) + + @overload + async def write(self: SpooledTemporaryFile[bytes], b: ReadableBuffer) -> int: ... + @overload + async def write(self: SpooledTemporaryFile[str], b: str) -> int: ... + + async def write(self, b: ReadableBuffer | str) -> int: + """ + Asynchronously write data to the spooled temporary file. + + If the file has not yet been rolled over, the data is written synchronously, + and a rollover is triggered if the size exceeds the maximum size. + + :param s: The data to write. + :return: The number of bytes written. + :raises RuntimeError: If the underlying file is not initialized. + + """ + if not self._rolled: + await checkpoint_if_cancelled() + result = self._fp.write(b) + await self._check() + return result + + return await super().write(b) # type: ignore[misc] + + @overload + async def writelines( + self: SpooledTemporaryFile[bytes], lines: Iterable[ReadableBuffer] + ) -> None: ... + @overload + async def writelines( + self: SpooledTemporaryFile[str], lines: Iterable[str] + ) -> None: ... + + async def writelines(self, lines: Iterable[str] | Iterable[ReadableBuffer]) -> None: + """ + Asynchronously write a list of lines to the spooled temporary file. + + If the file has not yet been rolled over, the lines are written synchronously, + and a rollover is triggered if the size exceeds the maximum size. + + :param lines: An iterable of lines to write. + :raises RuntimeError: If the underlying file is not initialized. + + """ + if not self._rolled: + await checkpoint_if_cancelled() + result = self._fp.writelines(lines) + await self._check() + return result + + return await super().writelines(lines) # type: ignore[misc] + + +class TemporaryDirectory(Generic[AnyStr]): + """ + An asynchronous temporary directory that is created and cleaned up automatically. + + This class provides an asynchronous context manager for creating a temporary + directory. It wraps Python's standard :class:`~tempfile.TemporaryDirectory` to + perform directory creation and cleanup operations in a background thread. + + :param suffix: Suffix to be added to the temporary directory name. + :param prefix: Prefix to be added to the temporary directory name. + :param dir: The parent directory where the temporary directory is created. + :param ignore_cleanup_errors: Whether to ignore errors during cleanup + :param delete: Whether to delete the directory upon closing (Python 3.12+). + """ + + def __init__( + self, + suffix: AnyStr | None = None, + prefix: AnyStr | None = None, + dir: AnyStr | None = None, + *, + ignore_cleanup_errors: bool = False, + delete: bool = True, + ) -> None: + self.suffix: AnyStr | None = suffix + self.prefix: AnyStr | None = prefix + self.dir: AnyStr | None = dir + self.ignore_cleanup_errors = ignore_cleanup_errors + self.delete = delete + + self._tempdir: tempfile.TemporaryDirectory | None = None + + async def __aenter__(self) -> str: + params: dict[str, Any] = { + "suffix": self.suffix, + "prefix": self.prefix, + "dir": self.dir, + "ignore_cleanup_errors": self.ignore_cleanup_errors, + } + if sys.version_info >= (3, 12): + params["delete"] = self.delete + + self._tempdir = await to_thread.run_sync( + lambda: tempfile.TemporaryDirectory(**params) + ) + return await to_thread.run_sync(self._tempdir.__enter__) + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: + if self._tempdir is not None: + with CancelScope(shield=True): + await to_thread.run_sync( + self._tempdir.__exit__, exc_type, exc_value, traceback + ) + + async def cleanup(self) -> None: + if self._tempdir is not None: + await to_thread.run_sync(self._tempdir.cleanup) + + +@overload +async def mkstemp( + suffix: str | None = None, + prefix: str | None = None, + dir: str | None = None, + text: bool = False, +) -> tuple[int, str]: ... + + +@overload +async def mkstemp( + suffix: bytes | None = None, + prefix: bytes | None = None, + dir: bytes | None = None, + text: bool = False, +) -> tuple[int, bytes]: ... + + +async def mkstemp( + suffix: AnyStr | None = None, + prefix: AnyStr | None = None, + dir: AnyStr | None = None, + text: bool = False, +) -> tuple[int, str | bytes]: + """ + Asynchronously create a temporary file and return an OS-level handle and the file + name. + + This function wraps `tempfile.mkstemp` and executes it in a background thread. + + :param suffix: Suffix to be added to the file name. + :param prefix: Prefix to be added to the file name. + :param dir: Directory in which the temporary file is created. + :param text: Whether the file is opened in text mode. + :return: A tuple containing the file descriptor and the file name. + + """ + return await to_thread.run_sync(tempfile.mkstemp, suffix, prefix, dir, text) + + +@overload +async def mkdtemp( + suffix: str | None = None, + prefix: str | None = None, + dir: str | None = None, +) -> str: ... + + +@overload +async def mkdtemp( + suffix: bytes | None = None, + prefix: bytes | None = None, + dir: bytes | None = None, +) -> bytes: ... + + +async def mkdtemp( + suffix: AnyStr | None = None, + prefix: AnyStr | None = None, + dir: AnyStr | None = None, +) -> str | bytes: + """ + Asynchronously create a temporary directory and return its path. + + This function wraps `tempfile.mkdtemp` and executes it in a background thread. + + :param suffix: Suffix to be added to the directory name. + :param prefix: Prefix to be added to the directory name. + :param dir: Parent directory where the temporary directory is created. + :return: The path of the created temporary directory. + + """ + return await to_thread.run_sync(tempfile.mkdtemp, suffix, prefix, dir) + + +async def gettempdir() -> str: + """ + Asynchronously return the name of the directory used for temporary files. + + This function wraps `tempfile.gettempdir` and executes it in a background thread. + + :return: The path of the temporary directory as a string. + + """ + return await to_thread.run_sync(tempfile.gettempdir) + + +async def gettempdirb() -> bytes: + """ + Asynchronously return the name of the directory used for temporary files in bytes. + + This function wraps `tempfile.gettempdirb` and executes it in a background thread. + + :return: The path of the temporary directory as bytes. + + """ + return await to_thread.run_sync(tempfile.gettempdirb) diff --git a/.venv/lib/python3.14/site-packages/anyio/_core/_testing.py b/.venv/lib/python3.14/site-packages/anyio/_core/_testing.py new file mode 100644 index 0000000000000000000000000000000000000000..fa2c3913709d4143d1967fc2857bb67be361f6db --- /dev/null +++ b/.venv/lib/python3.14/site-packages/anyio/_core/_testing.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +from collections.abc import Awaitable, Generator +from typing import Any, cast + +from ._eventloop import get_async_backend + + +class TaskInfo: + """ + Represents an asynchronous task. + + :ivar int id: the unique identifier of the task + :ivar parent_id: the identifier of the parent task, if any + :vartype parent_id: Optional[int] + :ivar str name: the description of the task (if any) + :ivar ~collections.abc.Coroutine coro: the coroutine object of the task + """ + + __slots__ = "_name", "coro", "id", "name", "parent_id" + + def __init__( + self, + id: int, + parent_id: int | None, + name: str | None, + coro: Generator[Any, Any, Any] | Awaitable[Any], + ): + func = get_current_task + self._name = f"{func.__module__}.{func.__qualname__}" + self.id: int = id + self.parent_id: int | None = parent_id + self.name: str | None = name + self.coro: Generator[Any, Any, Any] | Awaitable[Any] = coro + + def __eq__(self, other: object) -> bool: + if isinstance(other, TaskInfo): + return self.id == other.id + + return NotImplemented + + def __hash__(self) -> int: + return hash(self.id) + + def __repr__(self) -> str: + return f"{self.__class__.__name__}(id={self.id!r}, name={self.name!r})" + + def has_pending_cancellation(self) -> bool: + """ + Return ``True`` if the task has a cancellation pending, ``False`` otherwise. + + """ + return False + + +def get_current_task() -> TaskInfo: + """ + Return the current task. + + :return: a representation of the current task + :raises NoEventLoopError: if no supported asynchronous event loop is running in the + current thread + + """ + return get_async_backend().get_current_task() + + +def get_running_tasks() -> list[TaskInfo]: + """ + Return a list of running tasks in the current event loop. + + :return: a list of task info objects + :raises NoEventLoopError: if no supported asynchronous event loop is running in the + current thread + + """ + return cast("list[TaskInfo]", get_async_backend().get_running_tasks()) + + +async def wait_all_tasks_blocked() -> None: + """Wait until all other tasks are waiting for something.""" + await get_async_backend().wait_all_tasks_blocked() diff --git a/.venv/lib/python3.14/site-packages/anyio/_core/_typedattr.py b/.venv/lib/python3.14/site-packages/anyio/_core/_typedattr.py new file mode 100644 index 0000000000000000000000000000000000000000..59d3da9d8787d65295cffc3a1f43275f27d71772 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/anyio/_core/_typedattr.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +import sys +from collections.abc import Callable, Mapping +from typing import Any, TypeVar, final, overload + +from ._exceptions import TypedAttributeLookupError + +if sys.version_info < (3, 15): + from typing_extensions import sentinel + +T_Attr = TypeVar("T_Attr") +T_Default = TypeVar("T_Default") +undefined = sentinel("undefined") + + +def typed_attribute() -> Any: + """Return a unique object, used to mark typed attributes.""" + return object() + + +class TypedAttributeSet: + """ + Superclass for typed attribute collections. + + Checks that every public attribute of every subclass has a type annotation. + """ + + def __init_subclass__(cls) -> None: + annotations: dict[str, Any] = getattr(cls, "__annotations__", {}) + for attrname in dir(cls): + if not attrname.startswith("_") and attrname not in annotations: + raise TypeError( + f"Attribute {attrname!r} is missing its type annotation" + ) + + super().__init_subclass__() + + +class TypedAttributeProvider: + """Base class for classes that wish to provide typed extra attributes.""" + + @property + def extra_attributes(self) -> Mapping[T_Attr, Callable[[], T_Attr]]: + """ + A mapping of the extra attributes to callables that return the corresponding + values. + + If the provider wraps another provider, the attributes from that wrapper should + also be included in the returned mapping (but the wrapper may override the + callables from the wrapped instance). + + """ + return {} + + @overload + def extra(self, attribute: T_Attr) -> T_Attr: ... + + @overload + def extra(self, attribute: T_Attr, default: T_Default) -> T_Attr | T_Default: ... + + @final + def extra(self, attribute: Any, default: object = undefined) -> object: + """ + extra(attribute, default=undefined) + + Return the value of the given typed extra attribute. + + :param attribute: the attribute (member of a :class:`~TypedAttributeSet`) to + look for + :param default: the value that should be returned if no value is found for the + attribute + :raises ~anyio.TypedAttributeLookupError: if the search failed and no default + value was given + + """ + try: + getter = self.extra_attributes[attribute] + except KeyError: + if default is undefined: + raise TypedAttributeLookupError("Attribute not found") from None + else: + return default + + return getter() diff --git a/.venv/lib/python3.14/site-packages/anyio/_lazyimport.py b/.venv/lib/python3.14/site-packages/anyio/_lazyimport.py new file mode 100644 index 0000000000000000000000000000000000000000..dc0a4faba7009d3fb542842cce1c0eec1c568440 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/anyio/_lazyimport.py @@ -0,0 +1,176 @@ +from __future__ import annotations + +__all__ = ( + "fix_package_names", + "install_lazy_importer", + "set_deprecated_aliases", +) + +import ast +import inspect +import sys +import warnings +from importlib import import_module +from types import ModuleType +from typing import Any + + +def install_lazy_importer() -> bool: + module_globals = sys._getframe(1).f_globals + module_name = module_globals["__name__"] + module_prefix = module_name + "." + module = sys.modules[module_name] + lazy_map, deprecated_aliases, submodule_names = _build_lazy_map(module) + names = sorted(lazy_map) + + # Delete symbols that are not part of the API + del module_globals["TYPE_CHECKING"] + del module_globals["install_lazy_importer"] + + if not lazy_map and not deprecated_aliases: + return False + + def __getattr__(name: str) -> Any: + if new_name := deprecated_aliases.get(name): + emit_deprecation_warning(module_name, name, new_name) + target_mod, target_attr = new_name.rsplit(".", 1) + elif name in submodule_names: + target_mod, target_attr = "." + name, "" + else: + try: + target_mod, target_attr = lazy_map[name] + except KeyError: + raise AttributeError( + f"module {module_name!r} has no attribute {name!r}" + ) from None + + imported = import_module(target_mod, module_name) + value = getattr(imported, target_attr) if target_attr else imported + + # patch the module name to match + if ( + getattr(value, "__module__", "").startswith(module_prefix) + and name not in deprecated_aliases + ): + value.__module__ = module_name + + module_globals[name] = value + return value + + def __dir__() -> list[str]: + return names + + module_globals["__dir__"] = __dir__ + module_globals["__getattr__"] = __getattr__ + module_globals.pop("fix_package_names", None) + module_globals.pop("set_deprecated_aliases", None) + return True + + +def fix_package_names() -> None: + module_globals = sys._getframe(1).f_globals + module_prefix = module_globals["__name__"] + "." + del module_globals[fix_package_names.__name__] + for value in module_globals.values(): + if modname := getattr(value, "__module__", ""): + if modname.startswith(module_prefix): + parts = modname.split(".") + value.__module__ = ".".join( + part for part in parts if not part.startswith("_") + ) + + +def emit_deprecation_warning(module_name: str, name: str, target: str) -> None: + warnings.warn( + f"The {module_name}.{name} alias is deprecated, use {target} instead.", + DeprecationWarning, + stacklevel=3, + ) + + +def set_deprecated_aliases(aliases: dict[str, str]) -> None: + module_globals = sys._getframe(1).f_globals + module_name = module_globals["__name__"] + del module_globals[set_deprecated_aliases.__name__] + + def __getattr__(name: str) -> Any: + try: + target = aliases[name] + except KeyError: + raise AttributeError( + f"module {module_name!r} has no attribute {name!r}" + ) from None + + emit_deprecation_warning(module_name, name, target) + target_modname, attrname = target.rsplit(".", 1) + module = import_module(target_modname) + return getattr(module, attrname) + + sys.modules[module_name].__dict__["__getattr__"] = __getattr__ + + +def _build_lazy_map( + module: ModuleType, +) -> tuple[dict[str, tuple[str, str]], dict[str, str], list[str]]: + try: + source = inspect.getsource(module) + except OSError: + return {}, {}, [] + + tree = compile(source, module.__file__ or "", "exec", ast.PyCF_ONLY_AST) + assert isinstance(tree, ast.Module) + out: dict[str, tuple[str, str]] = {} + deprecated_aliases: dict[str, str] = {} + submodule_names: list[str] = [] + + for node in tree.body: + if not isinstance(node, ast.If) or not _is_type_checking_block(node.test): + continue + + for stmt in node.body: + match stmt: + case ast.ImportFrom(): + if stmt.module is None: + submodule_names.extend(alias.name for alias in stmt.names) + else: + base = "." * stmt.level + (stmt.module or "") + for alias in stmt.names: + if alias.name == "*": + raise RuntimeError("star imports not supported") + + exported = alias.asname or alias.name + out[exported] = (base, alias.name) + case ast.Expr() if isinstance(stmt.value, ast.Call): + call = stmt.value + if ( + isinstance(call.func, ast.Name) + and call.func.id == "set_deprecated_aliases" + ): + arg0 = call.args[0] + assert isinstance(arg0, ast.Dict) + for key, value in zip(arg0.keys, arg0.values, strict=True): + assert isinstance(key, ast.Constant) + assert isinstance(key.value, str) + assert isinstance(value, ast.Constant) + assert isinstance(value.value, str) + deprecated_aliases[key.value] = value.value + + return out, deprecated_aliases, submodule_names + + +def _is_type_checking_block(test: ast.AST) -> bool: + if not isinstance(test, ast.BoolOp): + return False + + subtest = test.values[0] + match subtest: + case ast.Name(): + return subtest.id == "TYPE_CHECKING" + case ast.Attribute(): + return ( + isinstance(subtest.value, ast.Name) + and subtest.value.id == "typing" + and subtest.attr == "TYPE_CHECKING" + ) + case _: + return False diff --git a/.venv/lib/python3.14/site-packages/anyio/abc/__init__.py b/.venv/lib/python3.14/site-packages/anyio/abc/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ba738f87f2fc778f5bb4c87fa1251a42afd22529 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/anyio/abc/__init__.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from .._lazyimport import ( + fix_package_names, + install_lazy_importer, + set_deprecated_aliases, +) + +if TYPE_CHECKING or not install_lazy_importer(): + from ._eventloop import AsyncBackend as AsyncBackend + from ._resources import AsyncResource as AsyncResource + from ._sockets import ConnectedUDPSocket as ConnectedUDPSocket + from ._sockets import ConnectedUNIXDatagramSocket as ConnectedUNIXDatagramSocket + from ._sockets import IPAddressType as IPAddressType + from ._sockets import IPSockAddrType as IPSockAddrType + from ._sockets import SocketAttribute as SocketAttribute + from ._sockets import SocketListener as SocketListener + from ._sockets import SocketStream as SocketStream + from ._sockets import UDPPacketType as UDPPacketType + from ._sockets import UDPSocket as UDPSocket + from ._sockets import UNIXDatagramPacketType as UNIXDatagramPacketType + from ._sockets import UNIXDatagramSocket as UNIXDatagramSocket + from ._sockets import UNIXSocketStream as UNIXSocketStream + from ._streams import AnyByteReceiveStream as AnyByteReceiveStream + from ._streams import AnyByteSendStream as AnyByteSendStream + from ._streams import AnyByteStream as AnyByteStream + from ._streams import AnyByteStreamConnectable as AnyByteStreamConnectable + from ._streams import ( + AnyUnreliableByteReceiveStream as AnyUnreliableByteReceiveStream, + ) + from ._streams import AnyUnreliableByteSendStream as AnyUnreliableByteSendStream + from ._streams import AnyUnreliableByteStream as AnyUnreliableByteStream + from ._streams import ByteReceiveStream as ByteReceiveStream + from ._streams import ByteSendStream as ByteSendStream + from ._streams import ByteStream as ByteStream + from ._streams import ByteStreamConnectable as ByteStreamConnectable + from ._streams import Listener as Listener + from ._streams import ObjectReceiveStream as ObjectReceiveStream + from ._streams import ObjectSendStream as ObjectSendStream + from ._streams import ObjectStream as ObjectStream + from ._streams import ObjectStreamConnectable as ObjectStreamConnectable + from ._streams import UnreliableObjectReceiveStream as UnreliableObjectReceiveStream + from ._streams import UnreliableObjectSendStream as UnreliableObjectSendStream + from ._streams import UnreliableObjectStream as UnreliableObjectStream + from ._subprocesses import Process as Process + from ._tasks import TaskGroup as TaskGroup + from ._tasks import TaskStatus as TaskStatus + from ._testing import TestRunner as TestRunner + + fix_package_names() + set_deprecated_aliases( + { + "CapacityLimiter": "anyio.CapacityLimiter", + "Condition": "anyio.Condition", + "Event": "anyio.Event", + "Lock": "anyio.Lock", + "Semaphore": "anyio.Semaphore", + "CancelScope": "anyio.CancelScope", + "BlockingPortal": "anyio.from_thread.BlockingPortal", + } + ) diff --git a/.venv/lib/python3.14/site-packages/anyio/abc/__pycache__/__init__.cpython-314.pyc b/.venv/lib/python3.14/site-packages/anyio/abc/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..245f3e72b0e3fdb5f4871cb082b38e8fb6b45576 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/anyio/abc/__pycache__/__init__.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/anyio/abc/__pycache__/_eventloop.cpython-314.pyc b/.venv/lib/python3.14/site-packages/anyio/abc/__pycache__/_eventloop.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3b22e5b41fbe3418bcd8030fdf13786f5308c234 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/anyio/abc/__pycache__/_eventloop.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/anyio/abc/__pycache__/_resources.cpython-314.pyc b/.venv/lib/python3.14/site-packages/anyio/abc/__pycache__/_resources.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1e16e0e763b57dcbe909bd4aa0ffa6ece29b9719 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/anyio/abc/__pycache__/_resources.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/anyio/abc/__pycache__/_sockets.cpython-314.pyc b/.venv/lib/python3.14/site-packages/anyio/abc/__pycache__/_sockets.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7bc7ff16b2f35753ef9de3a821deb17f3b6ee1bb Binary files /dev/null and b/.venv/lib/python3.14/site-packages/anyio/abc/__pycache__/_sockets.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/anyio/abc/__pycache__/_streams.cpython-314.pyc b/.venv/lib/python3.14/site-packages/anyio/abc/__pycache__/_streams.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..03d747147c14e2556ef5487f0fbe3b81709cb95a Binary files /dev/null and b/.venv/lib/python3.14/site-packages/anyio/abc/__pycache__/_streams.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/anyio/abc/__pycache__/_subprocesses.cpython-314.pyc b/.venv/lib/python3.14/site-packages/anyio/abc/__pycache__/_subprocesses.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..adb582e79fdaf8df66160052dadeccb0bc7e4fb6 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/anyio/abc/__pycache__/_subprocesses.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/anyio/abc/__pycache__/_tasks.cpython-314.pyc b/.venv/lib/python3.14/site-packages/anyio/abc/__pycache__/_tasks.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1c15ad5fd4d359f123eb21029894150ba5623ef4 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/anyio/abc/__pycache__/_tasks.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/anyio/abc/__pycache__/_testing.cpython-314.pyc b/.venv/lib/python3.14/site-packages/anyio/abc/__pycache__/_testing.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..de1d3699f48596b5973e80e58174258a1f190852 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/anyio/abc/__pycache__/_testing.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/anyio/abc/_eventloop.py b/.venv/lib/python3.14/site-packages/anyio/abc/_eventloop.py new file mode 100644 index 0000000000000000000000000000000000000000..a3cbed2febe41bbff583c80bc1442daee3862e8b --- /dev/null +++ b/.venv/lib/python3.14/site-packages/anyio/abc/_eventloop.py @@ -0,0 +1,428 @@ +from __future__ import annotations + +import math +import sys +from abc import ABCMeta, abstractmethod +from collections.abc import ( + AsyncIterator, + Awaitable, + Callable, + Coroutine, + Iterable, + Mapping, + Sequence, +) +from contextlib import AbstractContextManager +from os import PathLike +from signal import Signals +from socket import AddressFamily, SocketKind, socket +from typing import ( + IO, + TYPE_CHECKING, + Any, + TypeAlias, + TypeVar, + overload, +) + +if sys.version_info >= (3, 11): + from typing import TypeVarTuple, Unpack +else: + from typing_extensions import TypeVarTuple, Unpack + +if TYPE_CHECKING: + from _typeshed import FileDescriptorLike + + from .._core._synchronization import CapacityLimiter, Event, Lock, Semaphore + from .._core._tasks import CancelScope + from .._core._testing import TaskInfo + from ._sockets import ( + ConnectedUDPSocket, + ConnectedUNIXDatagramSocket, + IPSockAddrType, + SocketListener, + SocketStream, + UDPSocket, + UNIXDatagramSocket, + UNIXSocketStream, + ) + from ._subprocesses import Process + from ._tasks import TaskGroup + from ._testing import TestRunner + +T_Retval = TypeVar("T_Retval") +T_co = TypeVar("T_co", covariant=True) +PosArgsT = TypeVarTuple("PosArgsT") +StrOrBytesPath: TypeAlias = str | bytes | PathLike[str] | PathLike[bytes] + + +class AsyncBackend(metaclass=ABCMeta): + @classmethod + @abstractmethod + def run( + cls, + func: Callable[[Unpack[PosArgsT]], Awaitable[T_Retval]], + args: tuple[Unpack[PosArgsT]], + kwargs: dict[str, Any], + options: dict[str, Any], + ) -> T_Retval: + """ + Run the given coroutine function in an asynchronous event loop. + + The current thread must not be already running an event loop. + + :param func: a coroutine function + :param args: positional arguments to ``func`` + :param kwargs: positional arguments to ``func`` + :param options: keyword arguments to call the backend ``run()`` implementation + with + :return: the return value of the coroutine function + """ + + @classmethod + @abstractmethod + def current_token(cls) -> object: + """ + Return an object that allows other threads to run code inside the event loop. + + :return: a token object, specific to the event loop running in the current + thread + """ + + @classmethod + @abstractmethod + def current_time(cls) -> float: + """ + Return the current value of the event loop's internal clock. + + :return: the clock value (seconds) + """ + + @classmethod + @abstractmethod + def cancelled_exception_class(cls) -> type[BaseException]: + """Return the exception class that is raised in a task if it's cancelled.""" + + @classmethod + @abstractmethod + async def checkpoint(cls) -> None: + """ + Check if the task has been cancelled, and allow rescheduling of other tasks. + + This is effectively the same as running :meth:`checkpoint_if_cancelled` and then + :meth:`cancel_shielded_checkpoint`. + """ + + @classmethod + async def checkpoint_if_cancelled(cls) -> None: + """ + Check if the current task group has been cancelled. + + This will check if the task has been cancelled, but will not allow other tasks + to be scheduled if not. + + """ + if cls.current_effective_deadline() == -math.inf: + await cls.checkpoint() + + @classmethod + async def cancel_shielded_checkpoint(cls) -> None: + """ + Allow the rescheduling of other tasks. + + This will give other tasks the opportunity to run, but without checking if the + current task group has been cancelled, unlike with :meth:`checkpoint`. + + """ + with cls.create_cancel_scope(shield=True): + await cls.sleep(0) + + @classmethod + @abstractmethod + async def sleep(cls, delay: float) -> None: + """ + Pause the current task for the specified duration. + + :param delay: the duration, in seconds + """ + + @classmethod + @abstractmethod + def create_cancel_scope( + cls, *, deadline: float = math.inf, shield: bool = False + ) -> CancelScope: + pass + + @classmethod + @abstractmethod + def current_effective_deadline(cls) -> float: + """ + Return the nearest deadline among all the cancel scopes effective for the + current task. + + :return: + - a clock value from the event loop's internal clock + - ``inf`` if there is no deadline in effect + - ``-inf`` if the current scope has been cancelled + :rtype: float + """ + + @classmethod + @abstractmethod + def create_task_group(cls) -> TaskGroup: + pass + + @classmethod + @abstractmethod + def create_event(cls) -> Event: + pass + + @classmethod + @abstractmethod + def create_lock(cls, *, fast_acquire: bool) -> Lock: + pass + + @classmethod + @abstractmethod + def create_semaphore( + cls, + initial_value: int, + *, + max_value: int | None = None, + fast_acquire: bool = False, + ) -> Semaphore: + pass + + @classmethod + @abstractmethod + def create_capacity_limiter(cls, total_tokens: float) -> CapacityLimiter: + pass + + @classmethod + @abstractmethod + async def run_sync_in_worker_thread( + cls, + func: Callable[[Unpack[PosArgsT]], T_Retval], + args: tuple[Unpack[PosArgsT]], + abandon_on_cancel: bool = False, + limiter: CapacityLimiter | None = None, + ) -> T_Retval: + pass + + @classmethod + @abstractmethod + def check_cancelled(cls) -> None: + pass + + @classmethod + @abstractmethod + def run_async_from_thread( + cls, + func: Callable[[Unpack[PosArgsT]], Coroutine[Any, Any, T_co]], + args: tuple[Unpack[PosArgsT]], + token: object, + ) -> T_co: + pass + + @classmethod + @abstractmethod + def run_sync_from_thread( + cls, + func: Callable[[Unpack[PosArgsT]], T_Retval], + args: tuple[Unpack[PosArgsT]], + token: object, + ) -> T_Retval: + pass + + @classmethod + @abstractmethod + async def open_process( + cls, + command: StrOrBytesPath | Sequence[StrOrBytesPath], + *, + stdin: int | IO[Any] | None, + stdout: int | IO[Any] | None, + stderr: int | IO[Any] | None, + cwd: StrOrBytesPath | None = None, + env: Mapping[str, str] | None = None, + startupinfo: Any = None, + creationflags: int = 0, + start_new_session: bool = False, + pass_fds: Sequence[int] = (), + user: str | int | None = None, + group: str | int | None = None, + extra_groups: Iterable[str | int] | None = None, + umask: int = -1, + **kwargs: Any, + ) -> Process: + pass + + @classmethod + @abstractmethod + def setup_process_pool_exit_at_shutdown(cls, workers: set[Process]) -> None: + pass + + @classmethod + @abstractmethod + async def connect_tcp( + cls, host: str, port: int, local_address: IPSockAddrType | None = None + ) -> SocketStream: + pass + + @classmethod + @abstractmethod + async def connect_unix(cls, path: str | bytes) -> UNIXSocketStream: + pass + + @classmethod + @abstractmethod + def create_tcp_listener(cls, sock: socket) -> SocketListener: + pass + + @classmethod + @abstractmethod + def create_unix_listener(cls, sock: socket) -> SocketListener: + pass + + @classmethod + @abstractmethod + async def create_udp_socket( + cls, + family: AddressFamily, + local_address: IPSockAddrType | None, + remote_address: IPSockAddrType | None, + reuse_port: bool, + ) -> UDPSocket | ConnectedUDPSocket: + pass + + @classmethod + @overload + async def create_unix_datagram_socket( + cls, raw_socket: socket, remote_path: None + ) -> UNIXDatagramSocket: ... + + @classmethod + @overload + async def create_unix_datagram_socket( + cls, raw_socket: socket, remote_path: str | bytes + ) -> ConnectedUNIXDatagramSocket: ... + + @classmethod + @abstractmethod + async def create_unix_datagram_socket( + cls, raw_socket: socket, remote_path: str | bytes | None + ) -> UNIXDatagramSocket | ConnectedUNIXDatagramSocket: + pass + + @classmethod + @abstractmethod + async def getaddrinfo( + cls, + host: bytes | str | None, + port: str | int | None, + *, + family: int | AddressFamily = 0, + type: int | SocketKind = 0, + proto: int = 0, + flags: int = 0, + ) -> Sequence[ + tuple[ + AddressFamily, + SocketKind, + int, + str, + tuple[str, int] | tuple[str, int, int, int] | tuple[int, bytes], + ] + ]: + pass + + @classmethod + @abstractmethod + async def getnameinfo( + cls, sockaddr: IPSockAddrType, flags: int = 0 + ) -> tuple[str, str]: + pass + + @classmethod + @abstractmethod + async def wait_readable(cls, obj: FileDescriptorLike) -> None: + pass + + @classmethod + @abstractmethod + async def wait_writable(cls, obj: FileDescriptorLike) -> None: + pass + + @classmethod + @abstractmethod + def notify_closing(cls, obj: FileDescriptorLike) -> None: + pass + + @classmethod + @abstractmethod + async def wrap_listener_socket(cls, sock: socket) -> SocketListener: + pass + + @classmethod + @abstractmethod + async def wrap_stream_socket(cls, sock: socket) -> SocketStream: + pass + + @classmethod + @abstractmethod + async def wrap_unix_stream_socket(cls, sock: socket) -> UNIXSocketStream: + pass + + @classmethod + @abstractmethod + async def wrap_udp_socket(cls, sock: socket) -> UDPSocket: + pass + + @classmethod + @abstractmethod + async def wrap_connected_udp_socket(cls, sock: socket) -> ConnectedUDPSocket: + pass + + @classmethod + @abstractmethod + async def wrap_unix_datagram_socket(cls, sock: socket) -> UNIXDatagramSocket: + pass + + @classmethod + @abstractmethod + async def wrap_connected_unix_datagram_socket( + cls, sock: socket + ) -> ConnectedUNIXDatagramSocket: + pass + + @classmethod + @abstractmethod + def current_default_thread_limiter(cls) -> CapacityLimiter: + pass + + @classmethod + @abstractmethod + def open_signal_receiver( + cls, *signals: Signals + ) -> AbstractContextManager[AsyncIterator[Signals]]: + pass + + @classmethod + @abstractmethod + def get_current_task(cls) -> TaskInfo: + pass + + @classmethod + @abstractmethod + def get_running_tasks(cls) -> Sequence[TaskInfo]: + pass + + @classmethod + @abstractmethod + async def wait_all_tasks_blocked(cls) -> None: + pass + + @classmethod + @abstractmethod + def create_test_runner(cls, options: dict[str, Any]) -> TestRunner: + pass diff --git a/.venv/lib/python3.14/site-packages/anyio/abc/_resources.py b/.venv/lib/python3.14/site-packages/anyio/abc/_resources.py new file mode 100644 index 0000000000000000000000000000000000000000..9373f71a3fc4ccfb2002d03328cf62f78802375a --- /dev/null +++ b/.venv/lib/python3.14/site-packages/anyio/abc/_resources.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +import sys +from abc import ABCMeta, abstractmethod +from types import TracebackType +from typing import TypeVar + +if sys.version_info >= (3, 11): + from typing import Self +else: + from typing_extensions import Self + +T = TypeVar("T") + + +class AsyncResource(metaclass=ABCMeta): + """ + Abstract base class for all closeable asynchronous resources. + + Works as an asynchronous context manager which returns the instance itself on enter, + and calls :meth:`aclose` on exit. + """ + + __slots__ = () + + async def __aenter__(self) -> Self: + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> None: + await self.aclose() + + @abstractmethod + async def aclose(self) -> None: + """Close the resource.""" diff --git a/.venv/lib/python3.14/site-packages/anyio/abc/_sockets.py b/.venv/lib/python3.14/site-packages/anyio/abc/_sockets.py new file mode 100644 index 0000000000000000000000000000000000000000..d6a1139dfd9d31776c581b25db1309d9bfbb080e --- /dev/null +++ b/.venv/lib/python3.14/site-packages/anyio/abc/_sockets.py @@ -0,0 +1,401 @@ +from __future__ import annotations + +import errno +import socket +from abc import abstractmethod +from collections.abc import Callable, Collection, Mapping +from contextlib import AsyncExitStack +from io import IOBase +from ipaddress import IPv4Address, IPv6Address +from socket import AddressFamily +from typing import TYPE_CHECKING, Any, TypeAlias, TypeVar + +from .._core._eventloop import get_async_backend +from .._core._typedattr import ( + TypedAttributeProvider, + TypedAttributeSet, + typed_attribute, +) +from ._streams import ByteStream, Listener, UnreliableObjectStream + +if TYPE_CHECKING: + from ._tasks import TaskGroup + +IPAddressType: TypeAlias = str | IPv4Address | IPv6Address +IPSockAddrType: TypeAlias = tuple[str, int] +SockAddrType: TypeAlias = IPSockAddrType | str +UDPPacketType: TypeAlias = tuple[bytes, IPSockAddrType] +UNIXDatagramPacketType: TypeAlias = tuple[bytes, str] +T_Retval = TypeVar("T_Retval") + + +def _validate_socket( + sock_or_fd: socket.socket | int, + sock_type: socket.SocketKind, + addr_family: socket.AddressFamily = socket.AF_UNSPEC, + *, + require_connected: bool = False, + require_bound: bool = False, +) -> socket.socket: + if isinstance(sock_or_fd, int): + try: + sock = socket.socket(fileno=sock_or_fd) + except OSError as exc: + if exc.errno == errno.ENOTSOCK: + raise ValueError( + "the file descriptor does not refer to a socket" + ) from exc + elif require_connected: + raise ValueError("the socket must be connected") from exc + elif require_bound: + raise ValueError("the socket must be bound to a local address") from exc + else: + raise + elif isinstance(sock_or_fd, socket.socket): + sock = sock_or_fd + else: + raise TypeError( + f"expected an int or socket, got {type(sock_or_fd).__qualname__} instead" + ) + + try: + if require_connected: + try: + sock.getpeername() + except OSError as exc: + raise ValueError("the socket must be connected") from exc + + if require_bound: + try: + if sock.family in (socket.AF_INET, socket.AF_INET6): + bound_addr = sock.getsockname()[1] + else: + bound_addr = sock.getsockname() + except OSError: + bound_addr = None + + if not bound_addr: + raise ValueError("the socket must be bound to a local address") + + if addr_family != socket.AF_UNSPEC and sock.family != addr_family: + raise ValueError( + f"address family mismatch: expected {addr_family.name}, got " + f"{sock.family.name}" + ) + + if sock.type != sock_type: + raise ValueError( + f"socket type mismatch: expected {sock_type.name}, got {sock.type.name}" + ) + except BaseException: + # Avoid ResourceWarning from the locally constructed socket object + if isinstance(sock_or_fd, int): + sock.detach() + + raise + + sock.setblocking(False) + return sock + + +class SocketAttribute(TypedAttributeSet): + """ + .. attribute:: family + :type: socket.AddressFamily + + the address family of the underlying socket + + .. attribute:: local_address + :type: tuple[str, int] | str + + the local address the underlying socket is connected to + + .. attribute:: local_port + :type: int + + for IP based sockets, the local port the underlying socket is bound to + + .. attribute:: raw_socket + :type: socket.socket + + the underlying stdlib socket object + + .. attribute:: remote_address + :type: tuple[str, int] | str + + the remote address the underlying socket is connected to + + .. attribute:: remote_port + :type: int + + for IP based sockets, the remote port the underlying socket is connected to + """ + + family: AddressFamily = typed_attribute() + local_address: SockAddrType = typed_attribute() + local_port: int = typed_attribute() + raw_socket: socket.socket = typed_attribute() + remote_address: SockAddrType = typed_attribute() + remote_port: int = typed_attribute() + + +class _SocketProvider(TypedAttributeProvider): + @property + def extra_attributes(self) -> Mapping[Any, Callable[[], Any]]: + from .._core._sockets import convert_ipv6_sockaddr as convert + + attributes: dict[Any, Callable[[], Any]] = { + SocketAttribute.family: lambda: self._raw_socket.family, + SocketAttribute.local_address: lambda: convert( + self._raw_socket.getsockname() + ), + SocketAttribute.raw_socket: lambda: self._raw_socket, + } + try: + peername: tuple[str, int] | None = convert(self._raw_socket.getpeername()) + except OSError: + peername = None + + # Provide the remote address for connected sockets + if peername is not None: + attributes[SocketAttribute.remote_address] = lambda: peername + + # Provide local and remote ports for IP based sockets + if self._raw_socket.family in (AddressFamily.AF_INET, AddressFamily.AF_INET6): + attributes[SocketAttribute.local_port] = lambda: ( + self._raw_socket.getsockname()[1] + ) + if peername is not None: + remote_port = peername[1] + attributes[SocketAttribute.remote_port] = lambda: remote_port + + return attributes + + @property + @abstractmethod + def _raw_socket(self) -> socket.socket: + pass + + +class SocketStream(ByteStream, _SocketProvider): + """ + Transports bytes over a socket. + + Supports all relevant extra attributes from :class:`~SocketAttribute`. + """ + + @classmethod + async def from_socket(cls, sock_or_fd: socket.socket | int) -> SocketStream: + """ + Wrap an existing socket object or file descriptor as a socket stream. + + The newly created socket wrapper takes ownership of the socket being passed in. + The existing socket must already be connected. + + :param sock_or_fd: a socket object or file descriptor + :return: a socket stream + + """ + sock = _validate_socket(sock_or_fd, socket.SOCK_STREAM, require_connected=True) + return await get_async_backend().wrap_stream_socket(sock) + + +class UNIXSocketStream(SocketStream): + @classmethod + async def from_socket(cls, sock_or_fd: socket.socket | int) -> UNIXSocketStream: + """ + Wrap an existing socket object or file descriptor as a UNIX socket stream. + + The newly created socket wrapper takes ownership of the socket being passed in. + The existing socket must already be connected. + + :param sock_or_fd: a socket object or file descriptor + :return: a UNIX socket stream + + """ + sock = _validate_socket( + sock_or_fd, socket.SOCK_STREAM, socket.AF_UNIX, require_connected=True + ) + return await get_async_backend().wrap_unix_stream_socket(sock) + + @abstractmethod + async def send_fds(self, message: bytes, fds: Collection[int | IOBase]) -> None: + """ + Send file descriptors along with a message to the peer. + + :param message: a non-empty bytestring + :param fds: a collection of files (either numeric file descriptors or open file + or socket objects) + """ + + @abstractmethod + async def receive_fds(self, msglen: int, maxfds: int) -> tuple[bytes, list[int]]: + """ + Receive file descriptors along with a message from the peer. + + :param msglen: length of the message to expect from the peer + :param maxfds: maximum number of file descriptors to expect from the peer + :return: a tuple of (message, file descriptors) + """ + + +class SocketListener(Listener[SocketStream], _SocketProvider): + """ + Listens to incoming socket connections. + + Supports all relevant extra attributes from :class:`~SocketAttribute`. + """ + + @classmethod + async def from_socket( + cls, + sock_or_fd: socket.socket | int, + ) -> SocketListener: + """ + Wrap an existing socket object or file descriptor as a socket listener. + + The newly created listener takes ownership of the socket being passed in. + + :param sock_or_fd: a socket object or file descriptor + :return: a socket listener + + """ + sock = _validate_socket(sock_or_fd, socket.SOCK_STREAM, require_bound=True) + return await get_async_backend().wrap_listener_socket(sock) + + @abstractmethod + async def accept(self) -> SocketStream: + """Accept an incoming connection.""" + + async def serve( + self, + handler: Callable[[SocketStream], Any], + task_group: TaskGroup | None = None, + ) -> None: + from .. import create_task_group + + async with AsyncExitStack() as stack: + if task_group is None: + task_group = await stack.enter_async_context(create_task_group()) + + while True: + stream = await self.accept() + task_group.start_soon(handler, stream) + + +class UDPSocket(UnreliableObjectStream[UDPPacketType], _SocketProvider): + """ + Represents an unconnected UDP socket. + + Supports all relevant extra attributes from :class:`~SocketAttribute`. + """ + + @classmethod + async def from_socket(cls, sock_or_fd: socket.socket | int) -> UDPSocket: + """ + Wrap an existing socket object or file descriptor as a UDP socket. + + The newly created socket wrapper takes ownership of the socket being passed in. + The existing socket must be bound to a local address. + + :param sock_or_fd: a socket object or file descriptor + :return: a UDP socket + + """ + sock = _validate_socket(sock_or_fd, socket.SOCK_DGRAM, require_bound=True) + return await get_async_backend().wrap_udp_socket(sock) + + async def sendto(self, data: bytes, host: str, port: int) -> None: + """ + Alias for :meth:`~.UnreliableObjectSendStream.send` ((data, (host, port))). + + """ + return await self.send((data, (host, port))) + + +class ConnectedUDPSocket(UnreliableObjectStream[bytes], _SocketProvider): + """ + Represents an connected UDP socket. + + Supports all relevant extra attributes from :class:`~SocketAttribute`. + """ + + @classmethod + async def from_socket(cls, sock_or_fd: socket.socket | int) -> ConnectedUDPSocket: + """ + Wrap an existing socket object or file descriptor as a connected UDP socket. + + The newly created socket wrapper takes ownership of the socket being passed in. + The existing socket must already be connected. + + :param sock_or_fd: a socket object or file descriptor + :return: a connected UDP socket + + """ + sock = _validate_socket( + sock_or_fd, + socket.SOCK_DGRAM, + require_connected=True, + ) + return await get_async_backend().wrap_connected_udp_socket(sock) + + +class UNIXDatagramSocket( + UnreliableObjectStream[UNIXDatagramPacketType], _SocketProvider +): + """ + Represents an unconnected Unix datagram socket. + + Supports all relevant extra attributes from :class:`~SocketAttribute`. + """ + + @classmethod + async def from_socket( + cls, + sock_or_fd: socket.socket | int, + ) -> UNIXDatagramSocket: + """ + Wrap an existing socket object or file descriptor as a UNIX datagram + socket. + + The newly created socket wrapper takes ownership of the socket being passed in. + + :param sock_or_fd: a socket object or file descriptor + :return: a UNIX datagram socket + + """ + sock = _validate_socket(sock_or_fd, socket.SOCK_DGRAM, socket.AF_UNIX) + return await get_async_backend().wrap_unix_datagram_socket(sock) + + async def sendto(self, data: bytes, path: str) -> None: + """Alias for :meth:`~.UnreliableObjectSendStream.send` ((data, path)).""" + return await self.send((data, path)) + + +class ConnectedUNIXDatagramSocket(UnreliableObjectStream[bytes], _SocketProvider): + """ + Represents a connected Unix datagram socket. + + Supports all relevant extra attributes from :class:`~SocketAttribute`. + """ + + @classmethod + async def from_socket( + cls, + sock_or_fd: socket.socket | int, + ) -> ConnectedUNIXDatagramSocket: + """ + Wrap an existing socket object or file descriptor as a connected UNIX datagram + socket. + + The newly created socket wrapper takes ownership of the socket being passed in. + The existing socket must already be connected. + + :param sock_or_fd: a socket object or file descriptor + :return: a connected UNIX datagram socket + + """ + sock = _validate_socket( + sock_or_fd, socket.SOCK_DGRAM, socket.AF_UNIX, require_connected=True + ) + return await get_async_backend().wrap_connected_unix_datagram_socket(sock) diff --git a/.venv/lib/python3.14/site-packages/anyio/abc/_streams.py b/.venv/lib/python3.14/site-packages/anyio/abc/_streams.py new file mode 100644 index 0000000000000000000000000000000000000000..daf335dc3235ff8e4d5253610cd5e68de17c3f6d --- /dev/null +++ b/.venv/lib/python3.14/site-packages/anyio/abc/_streams.py @@ -0,0 +1,237 @@ +from __future__ import annotations + +from abc import ABCMeta, abstractmethod +from collections.abc import Callable +from typing import TYPE_CHECKING, Any, Generic, TypeAlias, TypeVar + +from .._core._exceptions import EndOfStream +from .._core._typedattr import TypedAttributeProvider +from ._resources import AsyncResource + +if TYPE_CHECKING: + from ._tasks import TaskGroup + +T_Item = TypeVar("T_Item") +T_co = TypeVar("T_co", covariant=True) +T_contra = TypeVar("T_contra", contravariant=True) + + +class UnreliableObjectReceiveStream( + AsyncResource, TypedAttributeProvider, Generic[T_co] +): + """ + An interface for receiving objects. + + This interface makes no guarantees that the received messages arrive in the order in + which they were sent, or that no messages are missed. + + Asynchronously iterating over objects of this type will yield objects matching the + given type parameter. + """ + + def __aiter__(self) -> UnreliableObjectReceiveStream[T_co]: + return self + + async def __anext__(self) -> T_co: + try: + return await self.receive() + except EndOfStream: + raise StopAsyncIteration from None + + @abstractmethod + async def receive(self) -> T_co: + """ + Receive the next item. + + :raises ~anyio.ClosedResourceError: if the receive stream has been explicitly + closed + :raises ~anyio.EndOfStream: if this stream has been closed from the other end + :raises ~anyio.BrokenResourceError: if this stream has been rendered unusable + due to external causes + """ + + +class UnreliableObjectSendStream( + AsyncResource, TypedAttributeProvider, Generic[T_contra] +): + """ + An interface for sending objects. + + This interface makes no guarantees that the messages sent will reach the + recipient(s) in the same order in which they were sent, or at all. + """ + + @abstractmethod + async def send(self, item: T_contra) -> None: + """ + Send an item to the peer(s). + + :param item: the item to send + :raises ~anyio.ClosedResourceError: if the send stream has been explicitly + closed + :raises ~anyio.BrokenResourceError: if this stream has been rendered unusable + due to external causes + """ + + +class UnreliableObjectStream( + UnreliableObjectReceiveStream[T_Item], UnreliableObjectSendStream[T_Item] +): + """ + A bidirectional message stream which does not guarantee the order or reliability of + message delivery. + """ + + +class ObjectReceiveStream(UnreliableObjectReceiveStream[T_co]): + """ + A receive message stream which guarantees that messages are received in the same + order in which they were sent, and that no messages are missed. + """ + + +class ObjectSendStream(UnreliableObjectSendStream[T_contra]): + """ + A send message stream which guarantees that messages are delivered in the same order + in which they were sent, without missing any messages in the middle. + """ + + +class ObjectStream( + ObjectReceiveStream[T_Item], + ObjectSendStream[T_Item], + UnreliableObjectStream[T_Item], +): + """ + A bidirectional message stream which guarantees the order and reliability of message + delivery. + """ + + @abstractmethod + async def send_eof(self) -> None: + """ + Send an end-of-file indication to the peer. + + You should not try to send any further data to this stream after calling this + method. This method is idempotent (does nothing on successive calls). + """ + + +class ByteReceiveStream(AsyncResource, TypedAttributeProvider): + """ + An interface for receiving bytes from a single peer. + + Iterating this byte stream will yield a byte string of arbitrary length, but no more + than 65536 bytes. + """ + + def __aiter__(self) -> ByteReceiveStream: + return self + + async def __anext__(self) -> bytes: + try: + return await self.receive() + except EndOfStream: + raise StopAsyncIteration from None + + @abstractmethod + async def receive(self, max_bytes: int = 65536) -> bytes: + """ + Receive at most ``max_bytes`` bytes from the peer. + + .. note:: Implementers of this interface should not return an empty + :class:`bytes` object, and users should ignore them. + + :param max_bytes: maximum number of bytes to receive (must be a positive + integer) + :return: the received bytes + :raises ValueError: if ``max_bytes`` is less than 1 + :raises ~anyio.EndOfStream: if this stream has been closed from the other end + """ + + +class ByteSendStream(AsyncResource, TypedAttributeProvider): + """An interface for sending bytes to a single peer.""" + + @abstractmethod + async def send(self, item: bytes) -> None: + """ + Send the given bytes to the peer. + + :param item: the bytes to send + """ + + +class ByteStream(ByteReceiveStream, ByteSendStream): + """A bidirectional byte stream.""" + + @abstractmethod + async def send_eof(self) -> None: + """ + Send an end-of-file indication to the peer. + + You should not try to send any further data to this stream after calling this + method. This method is idempotent (does nothing on successive calls). + """ + + +#: Type alias for all unreliable bytes-oriented receive streams. +AnyUnreliableByteReceiveStream: TypeAlias = ( + UnreliableObjectReceiveStream[bytes] | ByteReceiveStream +) +#: Type alias for all unreliable bytes-oriented send streams. +AnyUnreliableByteSendStream: TypeAlias = ( + UnreliableObjectSendStream[bytes] | ByteSendStream +) +#: Type alias for all unreliable bytes-oriented streams. +AnyUnreliableByteStream: TypeAlias = UnreliableObjectStream[bytes] | ByteStream +#: Type alias for all bytes-oriented receive streams. +AnyByteReceiveStream: TypeAlias = ObjectReceiveStream[bytes] | ByteReceiveStream +#: Type alias for all bytes-oriented send streams. +AnyByteSendStream: TypeAlias = ObjectSendStream[bytes] | ByteSendStream +#: Type alias for all bytes-oriented streams. +AnyByteStream: TypeAlias = ObjectStream[bytes] | ByteStream + + +class Listener(AsyncResource, TypedAttributeProvider, Generic[T_co]): + """An interface for objects that let you accept incoming connections.""" + + @abstractmethod + async def serve( + self, handler: Callable[[T_co], Any], task_group: TaskGroup | None = None + ) -> None: + """ + Accept incoming connections as they come in and start tasks to handle them. + + :param handler: a callable that will be used to handle each accepted connection + :param task_group: the task group that will be used to start tasks for handling + each accepted connection (if omitted, an ad-hoc task group will be created) + """ + + +class ObjectStreamConnectable(Generic[T_co], metaclass=ABCMeta): + @abstractmethod + async def connect(self) -> ObjectStream[T_co]: + """ + Connect to the remote endpoint. + + :return: an object stream connected to the remote end + :raises ConnectionFailed: if the connection fails + """ + + +class ByteStreamConnectable(metaclass=ABCMeta): + @abstractmethod + async def connect(self) -> ByteStream: + """ + Connect to the remote endpoint. + + :return: a bytestream connected to the remote end + :raises ConnectionFailed: if the connection fails + """ + + +#: Type alias for all connectables returning bytestreams or bytes-oriented object streams +AnyByteStreamConnectable: TypeAlias = ( + ObjectStreamConnectable[bytes] | ByteStreamConnectable +) diff --git a/.venv/lib/python3.14/site-packages/anyio/abc/_subprocesses.py b/.venv/lib/python3.14/site-packages/anyio/abc/_subprocesses.py new file mode 100644 index 0000000000000000000000000000000000000000..555f15f10d2b8375a791a620eafd87795d8e9107 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/anyio/abc/_subprocesses.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +from abc import abstractmethod +from signal import Signals +from typing import TYPE_CHECKING + +from ._resources import AsyncResource + +if TYPE_CHECKING: + from ._streams import ByteReceiveStream, ByteSendStream + + +class Process(AsyncResource): + """An asynchronous version of :class:`subprocess.Popen`.""" + + @abstractmethod + async def wait(self) -> int: + """ + Wait until the process exits. + + :return: the exit code of the process + """ + + @abstractmethod + def terminate(self) -> None: + """ + Terminates the process, gracefully if possible. + + On Windows, this calls ``TerminateProcess()``. + On POSIX systems, this sends ``SIGTERM`` to the process. + + .. seealso:: :meth:`subprocess.Popen.terminate` + """ + + @abstractmethod + def kill(self) -> None: + """ + Kills the process. + + On Windows, this calls ``TerminateProcess()``. + On POSIX systems, this sends ``SIGKILL`` to the process. + + .. seealso:: :meth:`subprocess.Popen.kill` + """ + + @abstractmethod + def send_signal(self, signal: Signals) -> None: + """ + Send a signal to the subprocess. + + .. seealso:: :meth:`subprocess.Popen.send_signal` + + :param signal: the signal number (e.g. :data:`signal.SIGHUP`) + """ + + @property + @abstractmethod + def pid(self) -> int: + """The process ID of the process.""" + + @property + @abstractmethod + def returncode(self) -> int | None: + """ + The return code of the process. If the process has not yet terminated, this will + be ``None``. + """ + + @property + @abstractmethod + def stdin(self) -> ByteSendStream | None: + """The stream for the standard input of the process.""" + + @property + @abstractmethod + def stdout(self) -> ByteReceiveStream | None: + """The stream for the standard output of the process.""" + + @property + @abstractmethod + def stderr(self) -> ByteReceiveStream | None: + """The stream for the standard error output of the process.""" diff --git a/.venv/lib/python3.14/site-packages/anyio/abc/_tasks.py b/.venv/lib/python3.14/site-packages/anyio/abc/_tasks.py new file mode 100644 index 0000000000000000000000000000000000000000..0a019e2ed3bd6bc36c97f87f3a8b9eee2ba0f657 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/anyio/abc/_tasks.py @@ -0,0 +1,223 @@ +from __future__ import annotations + +import sys +from abc import ABCMeta, abstractmethod +from collections.abc import Callable, Coroutine +from contextvars import Context +from types import TracebackType +from typing import TYPE_CHECKING, Any, Literal, Protocol, final, overload + +if sys.version_info >= (3, 13): + from typing import TypeVar +else: + from typing_extensions import TypeVar + +if sys.version_info >= (3, 11): + from typing import TypeVarTuple, Unpack +else: + from typing_extensions import TypeVarTuple, Unpack + +if TYPE_CHECKING: + from .._core._tasks import CancelScope, TaskHandle + +T_co = TypeVar("T_co", covariant=True) +T_contra = TypeVar("T_contra", contravariant=True, default=None) +PosArgsT = TypeVarTuple("PosArgsT") + + +def get_coro_name(coro: Coroutine[Any, Any, object], override: object = None) -> str: + if override is not None: + return str(override) + + try: + cr_frame = coro.cr_frame # type: ignore[attr-defined] + module = cr_frame.f_globals["__name__"] + except (AttributeError, KeyError): + module = None + + qualname = getattr(coro, "__qualname__", None) + return ".".join([x for x in (module, qualname) if x]) + + +def get_callable_name(func: Callable, override: object = None) -> str: + if override is not None: + return str(override) + + module = getattr(func, "__module__", None) + qualname = getattr(func, "__qualname__", None) + return ".".join([x for x in (module, qualname) if x]) + + +def call_for_coroutine( + func: Callable[[Unpack[PosArgsT]], Coroutine[Any, Any, T_co]], + args: tuple[Unpack[PosArgsT]], + **kwargs: Any, +) -> Coroutine[Any, Any, T_co]: + """ + Call the given function with the given positional and keyword arguments. + + :return: the resulting coroutine + :raises TypeError: if the return value was not a coroutine object + + """ + coro = func(*args, **kwargs) + if not isinstance(coro, Coroutine): + prefix = f"{func.__module__}." if hasattr(func, "__module__") else "" + raise TypeError( + f"Expected {prefix}{func.__qualname__}() to return a coroutine, but " + f"the return value ({coro!r}) is not a coroutine object" + ) + + return coro + + +class TaskStatus(Protocol[T_contra]): + @overload + def started(self: TaskStatus[None]) -> None: ... + + @overload + def started(self, value: T_contra) -> None: ... + + def started(self, value: T_contra | None = None) -> None: + """ + Signal that the task has started. + + :param value: object passed back to the starter of the task + """ + + +class TaskGroup(metaclass=ABCMeta): + """ + Groups several asynchronous tasks together. + + :ivar cancel_scope: the cancel scope inherited by all child tasks + :vartype cancel_scope: CancelScope + + .. note:: On asyncio, support for eager task factories is considered to be + **experimental**. In particular, they don't follow the usual semantics of new + tasks being scheduled on the next iteration of the event loop, and may thus + cause unexpected behavior in code that wasn't written with such semantics in + mind. + """ + + cancel_scope: CancelScope + + def cancel(self, reason: str | None = None) -> None: + """ + Cancel this task group's cancel scope immediately. + + This is a shortcut for calling ``.cancel_scope.cancel()`` on the task group. + + :param reason: a message describing the reason for the cancellation + + .. versionadded:: 4.14.0 + + """ + self.cancel_scope.cancel(reason) + + @abstractmethod + def create_task( + self, + coro: Coroutine[Any, Any, T_co], + *, + name: object = None, + context: Context | None = None, + ) -> TaskHandle[T_co]: + """ + Create a new task from a coroutine object and schedule it to run. + + :param coro: a coroutine object + :param name: optional name to give the task + :param context: optional context to run the task in + :return: a task handle + + .. versionadded:: 4.14.0 + """ + + @final + def start_soon( + self, + func: Callable[[Unpack[PosArgsT]], Coroutine[Any, Any, T_co]], + *args: Unpack[PosArgsT], + name: object = None, + ) -> TaskHandle[T_co]: + """ + Start a new task in this task group. + + :param func: a coroutine function + :param args: positional arguments to call the function with + :param name: name of the task, for the purposes of introspection and debugging + :return: a task handle + + .. versionadded:: 3.0 + .. versionchanged:: 4.14.0 + This method now returns a task handle. + + """ + final_name = get_callable_name(func, name) + return self.create_task(call_for_coroutine(func, args), name=final_name) + + @overload + async def start( + self, + func: Callable[..., Coroutine[Any, Any, T_co]], + *args: object, + name: object = None, + return_handle: Literal[False] = ..., + ) -> Any: ... + + @overload + async def start( + self, + func: Callable[..., Coroutine[Any, Any, T_co]], + *args: object, + name: object = None, + return_handle: Literal[True], + ) -> TaskHandle[T_co, Any]: ... + + @abstractmethod + async def start( + self, + func: Callable[..., Coroutine[Any, Any, T_co]], + *args: object, + name: object = None, + return_handle: Literal[False, True] = False, + ) -> Any: + """ + Start a new task and wait until it signals for readiness. + + The target callable must accept a keyword argument ``task_status`` (of type + :class:`TaskStatus`). Awaiting on this method will return whatever was passed to + ``task_status.started()`` (``None`` by default). + + .. note:: The :class:`TaskStatus` class is generic, and the type argument should + indicate the type of the value that will be passed to + ``task_status.started()``. + + :param func: a coroutine function that accepts the ``task_status`` keyword + argument + :param args: positional arguments to call the function with + :param name: an optional name for the task, for introspection and debugging + :param return_handle: if ``True``, return a :class:`TaskHandle` which also + contains the start value in ``start_value`` + :return: the value passed to ``task_status.started()`` + :raises RuntimeError: if the task finishes without calling + ``task_status.started()`` + + .. seealso:: :ref:`start_initialize` + + .. versionadded:: 3.0 + """ + + @abstractmethod + async def __aenter__(self) -> TaskGroup: + """Enter the task group context and allow starting new tasks.""" + + @abstractmethod + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> bool: + """Exit the task group context waiting for all tasks to finish.""" diff --git a/.venv/lib/python3.14/site-packages/anyio/abc/_testing.py b/.venv/lib/python3.14/site-packages/anyio/abc/_testing.py new file mode 100644 index 0000000000000000000000000000000000000000..b18d8357e7a71d228877584e98b1286d273928f4 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/anyio/abc/_testing.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +import sys +import types +from abc import ABCMeta, abstractmethod +from collections.abc import AsyncGenerator, Callable, Coroutine, Iterable +from typing import Any, TypeVar + +if sys.version_info >= (3, 11): + from typing import Self +else: + from typing_extensions import Self + +_T = TypeVar("_T") + + +class TestRunner(metaclass=ABCMeta): + """ + Encapsulates a running event loop. Every call made through this object will use the + same event loop. + """ + + def __enter__(self) -> Self: + return self + + @abstractmethod + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: types.TracebackType | None, + ) -> bool | None: ... + + @abstractmethod + def run_asyncgen_fixture( + self, + fixture_func: Callable[..., AsyncGenerator[_T, Any]], + kwargs: dict[str, Any], + ) -> Iterable[_T]: + """ + Run an async generator fixture. + + :param fixture_func: the fixture function + :param kwargs: keyword arguments to call the fixture function with + :return: an iterator yielding the value yielded from the async generator + """ + + @abstractmethod + def run_fixture( + self, + fixture_func: Callable[..., Coroutine[Any, Any, _T]], + kwargs: dict[str, Any], + ) -> _T: + """ + Run an async fixture. + + :param fixture_func: the fixture function + :param kwargs: keyword arguments to call the fixture function with + :return: the return value of the fixture function + """ + + @abstractmethod + def run_test( + self, test_func: Callable[..., Coroutine[Any, Any, Any]], kwargs: dict[str, Any] + ) -> None: + """ + Run an async test function. + + :param test_func: the test function + :param kwargs: keyword arguments to call the test function with + """ + + @abstractmethod + def is_running(self) -> bool: + """ + Check if the test runner is running. + + :return: ``True`` if the coroutine is currently being run, ``False`` otherwise. + """ diff --git a/.venv/lib/python3.14/site-packages/anyio/from_thread.py b/.venv/lib/python3.14/site-packages/anyio/from_thread.py new file mode 100644 index 0000000000000000000000000000000000000000..f163d818b030f752350558e805988bf8c558b262 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/anyio/from_thread.py @@ -0,0 +1,588 @@ +from __future__ import annotations + +__all__ = ( + "BlockingPortal", + "BlockingPortalProvider", + "check_cancelled", + "run", + "run_sync", + "start_blocking_portal", +) + +import sys +from collections.abc import Awaitable, Callable, Coroutine, Generator +from concurrent.futures import Future +from contextlib import ( + AbstractAsyncContextManager, + AbstractContextManager, + contextmanager, +) +from contextvars import Context +from dataclasses import dataclass, field +from functools import partial +from inspect import isawaitable +from threading import Lock, Thread, current_thread, get_ident +from types import TracebackType +from typing import ( + Any, + Generic, + TypeVar, + cast, + overload, +) + +from ._core._eventloop import ( + get_cancelled_exc_class, + threadlocals, +) +from ._core._eventloop import run as run_eventloop +from ._core._exceptions import NoEventLoopError +from ._core._synchronization import Event +from ._core._tasks import CancelScope, create_task_group +from .abc._tasks import TaskStatus +from .lowlevel import EventLoopToken, current_token + +if sys.version_info >= (3, 11): + from typing import Self, TypeVarTuple, Unpack +else: + from typing_extensions import Self, TypeVarTuple, Unpack + +T_Retval = TypeVar("T_Retval") +T_co = TypeVar("T_co", covariant=True) +PosArgsT = TypeVarTuple("PosArgsT") + + +def _token_or_error(token: EventLoopToken | None) -> EventLoopToken: + if token is not None: + return token + + try: + return threadlocals.current_token + except AttributeError: + raise NoEventLoopError( + "Not running inside an AnyIO worker thread, and no event loop token was " + "provided" + ) from None + + +def run( + func: Callable[[Unpack[PosArgsT]], Coroutine[Any, Any, T_co]], + *args: Unpack[PosArgsT], + token: EventLoopToken | None = None, +) -> T_co: + """ + Call a coroutine function from a worker thread. + + :param func: a coroutine function + :param args: positional arguments for the callable + :param token: an event loop token to use to get back to the event loop thread + (required if calling this function from outside an AnyIO worker thread) + :return: the return value of the coroutine function + :raises MissingTokenError: if no token was provided and called from outside an + AnyIO worker thread + :raises RunFinishedError: if the event loop tied to ``token`` is no longer running + + .. versionchanged:: 4.11.0 + Added the ``token`` parameter. + + """ + explicit_token = token is not None + token = _token_or_error(token) + return token.backend_class.run_async_from_thread( + func, args, token=token.native_token if explicit_token else None + ) + + +def run_sync( + func: Callable[[Unpack[PosArgsT]], T_Retval], + *args: Unpack[PosArgsT], + token: EventLoopToken | None = None, +) -> T_Retval: + """ + Call a function in the event loop thread from a worker thread. + + :param func: a callable + :param args: positional arguments for the callable + :param token: an event loop token to use to get back to the event loop thread + (required if calling this function from outside an AnyIO worker thread) + :return: the return value of the callable + :raises MissingTokenError: if no token was provided and called from outside an + AnyIO worker thread + :raises RunFinishedError: if the event loop tied to ``token`` is no longer running + + .. versionchanged:: 4.11.0 + Added the ``token`` parameter. + + """ + explicit_token = token is not None + token = _token_or_error(token) + return token.backend_class.run_sync_from_thread( + func, args, token=token.native_token if explicit_token else None + ) + + +class _BlockingAsyncContextManager(AbstractContextManager, Generic[T_co]): + _enter_future: Future[T_co] + _exit_future: Future[bool | None] + _exit_event: Event + _exit_exc_info: tuple[ + type[BaseException] | None, BaseException | None, TracebackType | None + ] = (None, None, None) + + def __init__( + self, async_cm: AbstractAsyncContextManager[T_co], portal: BlockingPortal + ): + self._async_cm = async_cm + self._portal = portal + + async def run_async_cm(self) -> bool | None: + try: + self._exit_event = Event() + value = await self._async_cm.__aenter__() + except BaseException as exc: + self._enter_future.set_exception(exc) + raise + else: + self._enter_future.set_result(value) + + try: + # Wait for the sync context manager to exit. + # This next statement can raise `get_cancelled_exc_class()` if + # something went wrong in a task group in this async context + # manager. + await self._exit_event.wait() + finally: + # In case of cancellation, it could be that we end up here before + # `_BlockingAsyncContextManager.__exit__` is called, and an + # `_exit_exc_info` has been set. + result = await self._async_cm.__aexit__(*self._exit_exc_info) + + return result + + def __enter__(self) -> T_co: + self._enter_future = Future() + self._exit_future = self._portal.start_task_soon(self.run_async_cm) + return self._enter_future.result() + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + /, + ) -> bool | None: + self._exit_exc_info = exc_type, exc_value, traceback + self._portal.call(self._exit_event.set) + return self._exit_future.result() + + +class _BlockingPortalTaskStatus(TaskStatus): + def __init__(self, future: Future): + self._future = future + + def started(self, value: object = None) -> None: + self._future.set_result(value) + + +class BlockingPortal: + """ + An object that lets external threads run code in an asynchronous event loop. + + :raises NoEventLoopError: if no supported asynchronous event loop is running in the + current thread + """ + + def __init__(self) -> None: + self._token = current_token() + self._event_loop_thread_id: int | None = get_ident() + self._stop_event = Event() + self._task_group = create_task_group() + + async def __aenter__(self) -> Self: + await self._task_group.__aenter__() + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> bool: + await self.stop() + return await self._task_group.__aexit__(exc_type, exc_val, exc_tb) + + def _check_running(self) -> None: + if self._event_loop_thread_id is None: + raise RuntimeError("This portal is not running") + if self._event_loop_thread_id == get_ident(): + raise RuntimeError( + "This method cannot be called from the event loop thread" + ) + + async def sleep_until_stopped(self) -> None: + """Sleep until :meth:`stop` is called.""" + await self._stop_event.wait() + + async def stop(self, cancel_remaining: bool = False) -> None: + """ + Signal the portal to shut down. + + This marks the portal as no longer accepting new calls and exits from + :meth:`sleep_until_stopped`. + + :param cancel_remaining: ``True`` to cancel all the remaining tasks, ``False`` + to let them finish before returning + + """ + self._event_loop_thread_id = None + self._stop_event.set() + if cancel_remaining: + self._task_group.cancel_scope.cancel("the blocking portal is shutting down") + + async def _call_func( + self, + func: Callable[[Unpack[PosArgsT]], Awaitable[T_Retval] | T_Retval], + args: tuple[Unpack[PosArgsT]], + kwargs: dict[str, Any], + future: Future[T_Retval], + ) -> None: + event_loop_thread_id = self._event_loop_thread_id + + def callback(f: Future[T_Retval]) -> None: + if f.cancelled(): + if event_loop_thread_id == get_ident(): + scope.cancel("the future was cancelled") + elif event_loop_thread_id is not None: + run_sync( + scope.cancel, "the future was cancelled", token=self._token + ) + + try: + retval_or_awaitable = func(*args, **kwargs) + if isawaitable(retval_or_awaitable): + with CancelScope() as scope: + future.add_done_callback(callback) + retval = await retval_or_awaitable + else: + retval = retval_or_awaitable + except get_cancelled_exc_class(): + future.cancel() + future.set_running_or_notify_cancel() + except BaseException as exc: + if not future.cancelled(): + future.set_exception(exc) + + # Let base exceptions fall through + if not isinstance(exc, Exception): + raise + else: + if not future.cancelled(): + future.set_result(retval) + finally: + scope = None # type: ignore[assignment] + + def _spawn_task_from_thread( + self, + func: Callable[[Unpack[PosArgsT]], Awaitable[T_Retval] | T_Retval], + args: tuple[Unpack[PosArgsT]], + kwargs: dict[str, Any], + name: object, + future: Future[T_Retval], + ) -> None: + """ + Spawn a new task using the given callable. + + :param func: a callable + :param args: positional arguments to be passed to the callable + :param kwargs: keyword arguments to be passed to the callable + :param name: name of the task (will be coerced to a string if not ``None``) + :param future: a future that will resolve to the return value of the callable, + or the exception raised during its execution + + """ + run_sync( + partial(self._task_group.start_soon, name=name), + self._call_func, + func, + args, + kwargs, + future, + token=self._token, + ) + + @overload + def call( + self, + func: Callable[[Unpack[PosArgsT]], Awaitable[T_Retval]], + *args: Unpack[PosArgsT], + ) -> T_Retval: ... + + @overload + def call( + self, func: Callable[[Unpack[PosArgsT]], T_Retval], *args: Unpack[PosArgsT] + ) -> T_Retval: ... + + def call( + self, + func: Callable[[Unpack[PosArgsT]], Awaitable[T_Retval] | T_Retval], + *args: Unpack[PosArgsT], + ) -> T_Retval: + """ + Call the given function in the event loop thread. + + If the callable returns a coroutine object, it is awaited on. + + :param func: any callable + :raises RuntimeError: if the portal is not running or if this method is called + from within the event loop thread + + """ + return cast(T_Retval, self.start_task_soon(func, *args).result()) + + @overload + def start_task_soon( + self, + func: Callable[[Unpack[PosArgsT]], Awaitable[T_Retval]], + *args: Unpack[PosArgsT], + name: object = None, + ) -> Future[T_Retval]: ... + + @overload + def start_task_soon( + self, + func: Callable[[Unpack[PosArgsT]], T_Retval], + *args: Unpack[PosArgsT], + name: object = None, + ) -> Future[T_Retval]: ... + + def start_task_soon( + self, + func: Callable[[Unpack[PosArgsT]], Awaitable[T_Retval] | T_Retval], + *args: Unpack[PosArgsT], + name: object = None, + ) -> Future[T_Retval]: + """ + Start a task in the portal's task group. + + The task will be run inside a cancel scope which can be cancelled by cancelling + the returned future. + + :param func: the target function + :param args: positional arguments passed to ``func`` + :param name: name of the task (will be coerced to a string if not ``None``) + :return: a future that resolves with the return value of the callable if the + task completes successfully, or with the exception raised in the task + :raises RuntimeError: if the portal is not running or if this method is called + from within the event loop thread + :rtype: concurrent.futures.Future[T_Retval] + + .. versionadded:: 3.0 + + """ + self._check_running() + f: Future[T_Retval] = Future() + self._spawn_task_from_thread(func, args, {}, name, f) + return f + + def start_task( + self, + func: Callable[..., Awaitable[T_Retval]], + *args: object, + name: object = None, + ) -> tuple[Future[T_Retval], Any]: + """ + Start a task in the portal's task group and wait until it signals for readiness. + + This method works the same way as :meth:`.abc.TaskGroup.start`. + + :param func: the target function + :param args: positional arguments passed to ``func`` + :param name: name of the task (will be coerced to a string if not ``None``) + :return: a tuple of (future, task_status_value) where the ``task_status_value`` + is the value passed to ``task_status.started()`` from within the target + function + :rtype: tuple[concurrent.futures.Future[T_Retval], Any] + + .. versionadded:: 3.0 + + """ + + def task_done(future: Future[T_Retval]) -> None: + if not task_status_future.done(): + if future.cancelled(): + task_status_future.cancel() + elif future.exception(): + task_status_future.set_exception(future.exception()) + else: + exc = RuntimeError( + "Task exited without calling task_status.started()" + ) + task_status_future.set_exception(exc) + + self._check_running() + task_status_future: Future = Future() + task_status = _BlockingPortalTaskStatus(task_status_future) + f: Future = Future() + f.add_done_callback(task_done) + self._spawn_task_from_thread(func, args, {"task_status": task_status}, name, f) + return f, task_status_future.result() + + def wrap_async_context_manager( + self, cm: AbstractAsyncContextManager[T_co] + ) -> AbstractContextManager[T_co]: + """ + Wrap an async context manager as a synchronous context manager via this portal. + + Spawns a task that will call both ``__aenter__()`` and ``__aexit__()``, stopping + in the middle until the synchronous context manager exits. + + :param cm: an asynchronous context manager + :return: a synchronous context manager + + .. versionadded:: 2.1 + + """ + return _BlockingAsyncContextManager(cm, self) + + +@dataclass +class BlockingPortalProvider: + """ + A manager for a blocking portal. Used as a context manager. The first thread to + enter this context manager causes a blocking portal to be started with the specific + parameters, and the last thread to exit causes the portal to be shut down. Thus, + there will be exactly one blocking portal running in this context as long as at + least one thread has entered this context manager. + + The parameters are the same as for :func:`~anyio.run`. + + :param backend: name of the backend + :param backend_options: backend options + + .. versionadded:: 4.4 + """ + + backend: str = "asyncio" + backend_options: dict[str, Any] | None = None + _lock: Lock = field(init=False, default_factory=Lock) + _leases: int = field(init=False, default=0) + _portal: BlockingPortal = field(init=False) + _portal_cm: AbstractContextManager[BlockingPortal] | None = field( + init=False, default=None + ) + + def __enter__(self) -> BlockingPortal: + with self._lock: + if self._portal_cm is None: + self._portal_cm = start_blocking_portal( + self.backend, self.backend_options + ) + self._portal = self._portal_cm.__enter__() + + self._leases += 1 + return self._portal + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> None: + portal_cm: AbstractContextManager[BlockingPortal] | None = None + with self._lock: + assert self._portal_cm + assert self._leases > 0 + self._leases -= 1 + if not self._leases: + portal_cm = self._portal_cm + self._portal_cm = None + del self._portal + + if portal_cm: + portal_cm.__exit__(None, None, None) + + +@contextmanager +def start_blocking_portal( + backend: str = "asyncio", + backend_options: dict[str, Any] | None = None, + *, + name: str | None = None, +) -> Generator[BlockingPortal, Any, None]: + """ + Start a new event loop in a new thread and run a blocking portal in its main task. + + The parameters are the same as for :func:`~anyio.run`. + + :param backend: name of the backend + :param backend_options: backend options + :param name: name of the thread + :return: a context manager that yields a blocking portal + + .. versionchanged:: 3.0 + Usage as a context manager is now required. + + """ + + async def run_portal() -> None: + async with BlockingPortal() as portal_: + if name is None: + current_thread().name = f"{backend}-portal-{id(portal_):x}" + + future.set_result(portal_) + await portal_.sleep_until_stopped() + + def run_blocking_portal() -> None: + if future.set_running_or_notify_cancel(): + try: + run_eventloop( + run_portal, backend=backend, backend_options=backend_options + ) + except BaseException as exc: + if not future.done(): + future.set_exception(exc) + + future: Future[BlockingPortal] = Future() + kwargs: dict[str, Any] = {} + if sys.version_info >= (3, 14): + kwargs["context"] = Context() + + thread = Thread(target=run_blocking_portal, daemon=True, name=name, **kwargs) + thread.start() + try: + cancel_remaining_tasks = False + portal = future.result() + try: + yield portal + except BaseException: + cancel_remaining_tasks = True + raise + finally: + try: + portal.call(portal.stop, cancel_remaining_tasks) + except RuntimeError: + pass + finally: + thread.join() + + +def check_cancelled() -> None: + """ + Check if the cancel scope of the host task's running the current worker thread has + been cancelled. + + If the host task's current cancel scope has indeed been cancelled, the + backend-specific cancellation exception will be raised. + + :raises RuntimeError: if the current thread was not spawned by + :func:`.to_thread.run_sync` + + """ + try: + token: EventLoopToken = threadlocals.current_token + except AttributeError: + raise NoEventLoopError( + "This function can only be called inside an AnyIO worker thread" + ) from None + + token.backend_class.check_cancelled() diff --git a/.venv/lib/python3.14/site-packages/anyio/functools.py b/.venv/lib/python3.14/site-packages/anyio/functools.py new file mode 100644 index 0000000000000000000000000000000000000000..b0bdfb4585efc4e4799388547668fba86fb5c687 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/anyio/functools.py @@ -0,0 +1,400 @@ +from __future__ import annotations + +__all__ = ( + "AsyncCacheInfo", + "AsyncCacheParameters", + "AsyncLRUCacheWrapper", + "cache", + "lru_cache", + "reduce", +) + +import functools +from collections import OrderedDict +from collections.abc import ( + AsyncIterable, + Awaitable, + Callable, + Coroutine, + Hashable, + Iterable, +) +from functools import update_wrapper +from inspect import iscoroutinefunction +from typing import ( + Any, + Generic, + NamedTuple, + ParamSpec, + TypedDict, + TypeVar, + cast, + final, + overload, +) +from weakref import WeakKeyDictionary + +from ._core._eventloop import current_time +from ._core._synchronization import Lock +from .lowlevel import RunVar, checkpoint + +T = TypeVar("T") +S = TypeVar("S") +P = ParamSpec("P") +lru_cache_items: RunVar[ + WeakKeyDictionary[ + AsyncLRUCacheWrapper[Any, Any], + OrderedDict[ + Hashable, + tuple[_InitialMissingType, Lock, float | None] + | tuple[Any, None, float | None], + ], + ] +] = RunVar("lru_cache_items") + + +class _InitialMissingType: + pass + + +initial_missing: _InitialMissingType = _InitialMissingType() + + +class AsyncCacheInfo(NamedTuple): + hits: int + misses: int + maxsize: int | None + currsize: int + ttl: int | None + + +class AsyncCacheParameters(TypedDict): + maxsize: int | None + typed: bool + always_checkpoint: bool + ttl: int | None + + +class _LRUMethodWrapper(Generic[T]): + def __init__(self, wrapper: AsyncLRUCacheWrapper[..., T], instance: object): + self.__wrapper = wrapper + self.__instance = instance + + def cache_info(self) -> AsyncCacheInfo: + return self.__wrapper.cache_info() + + def cache_parameters(self) -> AsyncCacheParameters: + return self.__wrapper.cache_parameters() + + def cache_clear(self) -> None: + self.__wrapper.cache_clear() + + async def __call__(self, *args: Any, **kwargs: Any) -> T: + if self.__instance is None: + return await self.__wrapper(*args, **kwargs) + + return await self.__wrapper(self.__instance, *args, **kwargs) + + +@final +class AsyncLRUCacheWrapper(Generic[P, T]): + def __init__( + self, + func: Callable[P, Awaitable[T]], + maxsize: int | None, + typed: bool, + always_checkpoint: bool, + ttl: int | None, + ): + self.__wrapped__ = func + self._hits: int = 0 + self._misses: int = 0 + self._maxsize = max(maxsize, 0) if maxsize is not None else None + self._currsize: int = 0 + self._typed = typed + self._always_checkpoint = always_checkpoint + self._ttl = ttl + update_wrapper(self, func) + + def cache_info(self) -> AsyncCacheInfo: + return AsyncCacheInfo( + self._hits, self._misses, self._maxsize, self._currsize, self._ttl + ) + + def cache_parameters(self) -> AsyncCacheParameters: + return { + "maxsize": self._maxsize, + "typed": self._typed, + "always_checkpoint": self._always_checkpoint, + "ttl": self._ttl, + } + + def cache_clear(self) -> None: + if cache := lru_cache_items.get(None): + cache.pop(self, None) + self._hits = self._misses = self._currsize = 0 + + async def __call__(self, *args: P.args, **kwargs: P.kwargs) -> T: + # Easy case first: if maxsize == 0, no caching is done + if self._maxsize == 0: + value = await self.__wrapped__(*args, **kwargs) + self._misses += 1 + return value + + # The key is constructed as a flat tuple to avoid memory overhead + key: tuple[Any, ...] = args + if kwargs: + # initial_missing is used as a separator + key += (initial_missing,) + sum(kwargs.items(), ()) + + if self._typed: + key += tuple(type(arg) for arg in args) + if kwargs: + key += (initial_missing,) + tuple(type(val) for val in kwargs.values()) + + try: + cache = lru_cache_items.get() + except LookupError: + cache = WeakKeyDictionary() + lru_cache_items.set(cache) + + try: + cache_entry = cache[self] + except KeyError: + cache_entry = cache[self] = OrderedDict() + + cached_value: T | _InitialMissingType + try: + cached_value, lock, expires_at = cache_entry[key] + except KeyError: + # We're the first task to call this function + cached_value, lock, expires_at = ( + initial_missing, + Lock(fast_acquire=not self._always_checkpoint), + None, + ) + cache_entry[key] = cached_value, lock, expires_at + + if lock is None: + if expires_at is not None and current_time() >= expires_at: + self._currsize -= 1 + cached_value, lock, expires_at = ( + initial_missing, + Lock(fast_acquire=not self._always_checkpoint), + None, + ) + cache_entry[key] = cached_value, lock, expires_at + else: + # The value was already cached + self._hits += 1 + cache_entry.move_to_end(key) + if self._always_checkpoint: + await checkpoint() + + return cast(T, cached_value) + + async with lock: + # Check if another task filled the cache while we acquired the lock + if (cached_value := cache_entry[key][0]) is initial_missing: + self._misses += 1 + if self._maxsize is not None and self._currsize >= self._maxsize: + cache_entry.popitem(last=False) + else: + self._currsize += 1 + + value = await self.__wrapped__(*args, **kwargs) + expires_at = ( + current_time() + self._ttl if self._ttl is not None else None + ) + cache_entry[key] = value, None, expires_at + else: + # Another task filled the cache while we were waiting for the lock + self._hits += 1 + cache_entry.move_to_end(key) + value = cast(T, cached_value) + + return value + + def __get__( + self, instance: object, owner: type | None = None + ) -> _LRUMethodWrapper[T]: + wrapper = _LRUMethodWrapper(self, instance) + update_wrapper(wrapper, self.__wrapped__) + return wrapper + + +class _LRUCacheWrapper: + def __init__( + self, maxsize: int | None, typed: bool, always_checkpoint: bool, ttl: int | None + ): + self._maxsize = maxsize + self._typed = typed + self._always_checkpoint = always_checkpoint + self._ttl = ttl + + @overload + def __call__( # type: ignore[overload-overlap] + self, func: Callable[P, Coroutine[Any, Any, T]], / + ) -> AsyncLRUCacheWrapper[P, T]: ... + + @overload + def __call__( + self, func: Callable[..., T], / + ) -> functools._lru_cache_wrapper[T]: ... + + def __call__( + self, f: Callable[P, Coroutine[Any, Any, T]] | Callable[..., T], / + ) -> AsyncLRUCacheWrapper[P, T] | functools._lru_cache_wrapper[T]: + if iscoroutinefunction(f): + return AsyncLRUCacheWrapper( + f, self._maxsize, self._typed, self._always_checkpoint, self._ttl + ) + + return functools.lru_cache(maxsize=self._maxsize, typed=self._typed)(f) # type: ignore[arg-type] + + +@overload +def cache( # type: ignore[overload-overlap] + func: Callable[P, Coroutine[Any, Any, T]], / +) -> AsyncLRUCacheWrapper[P, T]: ... + + +@overload +def cache(func: Callable[..., T], /) -> functools._lru_cache_wrapper[T]: ... + + +def cache(func: Callable[..., Any] | Callable[P, Coroutine[Any, Any, Any]], /) -> Any: + """ + A convenient shortcut for :func:`lru_cache` with ``maxsize=None``. + + This is the asynchronous equivalent to :func:`functools.cache`. + + """ + return lru_cache(maxsize=None)(func) + + +@overload +def lru_cache( + *, + maxsize: int | None = ..., + typed: bool = ..., + always_checkpoint: bool = ..., + ttl: int | None = ..., +) -> _LRUCacheWrapper: ... + + +@overload +def lru_cache( # type: ignore[overload-overlap] + func: Callable[P, Coroutine[Any, Any, T]], / +) -> AsyncLRUCacheWrapper[P, T]: ... + + +@overload +def lru_cache(func: Callable[..., T], /) -> functools._lru_cache_wrapper[T]: ... + + +def lru_cache( + func: Callable[..., Coroutine[Any, Any, Any]] | Callable[..., Any] | None = None, + /, + *, + maxsize: int | None = 128, + typed: bool = False, + always_checkpoint: bool = False, + ttl: int | None = None, +) -> Any: + """ + An asynchronous version of :func:`functools.lru_cache`. + + If a synchronous function is passed, the standard library + :func:`functools.lru_cache` is applied instead. + + :param always_checkpoint: if ``True``, every call to the cached function will be + guaranteed to yield control to the event loop at least once + :param ttl: time in seconds after which to invalidate cache entries + + .. note:: Caches and locks are managed on a per-event loop basis. + + """ + if func is None: + return _LRUCacheWrapper(maxsize, typed, always_checkpoint, ttl) + + if not callable(func): + raise TypeError("the first argument must be callable") + + return _LRUCacheWrapper(maxsize, typed, always_checkpoint, ttl)(func) + + +@overload +async def reduce( + function: Callable[[T, S], Awaitable[T]], + iterable: Iterable[S] | AsyncIterable[S], + /, + initial: T, +) -> T: ... + + +@overload +async def reduce( + function: Callable[[T, T], Awaitable[T]], + iterable: Iterable[T] | AsyncIterable[T], + /, +) -> T: ... + + +async def reduce( # type: ignore[misc] + function: Callable[[T, T], Awaitable[T]] | Callable[[T, S], Awaitable[T]], + iterable: Iterable[T] | Iterable[S] | AsyncIterable[T] | AsyncIterable[S], + /, + initial: T | _InitialMissingType = initial_missing, +) -> T: + """ + Asynchronous version of :func:`functools.reduce`. + + :param function: a coroutine function that takes two arguments: the accumulated + value and the next element from the iterable + :param iterable: an iterable or async iterable + :param initial: the initial value (if missing, the first element of the iterable is + used as the initial value) + + """ + element: Any + function_called = False + if isinstance(iterable, AsyncIterable): + async_it = iterable.__aiter__() + if initial is initial_missing: + try: + value = cast(T, await async_it.__anext__()) + except StopAsyncIteration: + raise TypeError( + "reduce() of empty sequence with no initial value" + ) from None + else: + value = cast(T, initial) + + async for element in async_it: + value = await function(value, element) + function_called = True + elif isinstance(iterable, Iterable): + it = iter(iterable) + if initial is initial_missing: + try: + value = cast(T, next(it)) + except StopIteration: + raise TypeError( + "reduce() of empty sequence with no initial value" + ) from None + else: + value = cast(T, initial) + + for element in it: + value = await function(value, element) + function_called = True + else: + raise TypeError("reduce() argument 2 must be an iterable or async iterable") + + # Make sure there is at least one checkpoint, even if an empty iterable and an + # initial value were given + if not function_called: + await checkpoint() + + return value diff --git a/.venv/lib/python3.14/site-packages/anyio/itertools.py b/.venv/lib/python3.14/site-packages/anyio/itertools.py new file mode 100644 index 0000000000000000000000000000000000000000..2a10cfd3ba01b22f424d18accaf9758233a67cca --- /dev/null +++ b/.venv/lib/python3.14/site-packages/anyio/itertools.py @@ -0,0 +1,629 @@ +from __future__ import annotations + +__all__ = ( + "Chain", + "accumulate", + "batched", + "combinations", + "combinations_with_replacement", + "compress", + "count", + "cycle", + "dropwhile", + "filterfalse", + "groupby", + "islice", + "pairwise", + "permutations", + "product", + "repeat", + "starmap", + "takewhile", + "tee", + "zip_longest", +) + +import itertools +import operator +import sys +from collections.abc import ( + AsyncGenerator, + AsyncIterable, + AsyncIterator, + Awaitable, + Callable, + Iterable, + Iterator, +) +from dataclasses import dataclass, field +from typing import Any, Generic, TypeVar, cast, overload + +from ._core._synchronization import Lock +from ._core._tasks import CancelScope +from .lowlevel import cancel_shielded_checkpoint, checkpoint, checkpoint_if_cancelled + +if sys.version_info < (3, 15): + from typing_extensions import sentinel + +T = TypeVar("T") +R = TypeVar("R") +_tee_end = sentinel("_tee_end") + + +@dataclass(eq=False) +class _IterableAsyncIterator(AsyncIterator[T]): + iterator: Iterator[T] + + async def __anext__(self) -> T: + await checkpoint_if_cancelled() + try: + result = next(self.iterator) + except StopIteration: + await cancel_shielded_checkpoint() + raise StopAsyncIteration from None + + await cancel_shielded_checkpoint() + return result + + +def _iterate(iterable: Iterable[T] | AsyncIterable[T]) -> AsyncIterator[T]: + if isinstance(iterable, AsyncIterator): + return iterable + + if isinstance(iterable, AsyncIterable): + return iterable.__aiter__() + + return _IterableAsyncIterator(iter(iterable)) + + +@dataclass(eq=False) +class _TeeLink(Generic[T]): + value: object | None = None + next: _TeeLink[T] | None = None + filled: bool = False + + +@dataclass(eq=False) +class _TeeState(Generic[T]): + iterator: AsyncIterator[T] + lock: Lock = field(default_factory=Lock) + + async def fill(self, link: _TeeLink[T]) -> bool: + if link.filled: + return False + + async with self.lock: + if link.filled: + return True + + link.value = await anext(self.iterator, _tee_end) + if link.value is not _tee_end: + link.next = _TeeLink() + + link.filled = True + return True + + +class _TeeAsyncIterator(AsyncIterator[T]): + _state: _TeeState[T] + _link: _TeeLink[T] + _element_yielded: bool + + def __init__( + self, iterable: Iterable[T] | AsyncIterable[T] | _TeeAsyncIterator[T] + ) -> None: + if isinstance(iterable, _TeeAsyncIterator): + self._state = iterable._state + self._link = iterable._link + else: + self._state = _TeeState(_iterate(iterable)) + self._link = _TeeLink() + + self._element_yielded = False + + async def __anext__(self) -> T: + had_yieldpoint = await self._state.fill(self._link) + if self._link.value is _tee_end: + if not self._element_yielded: + await checkpoint() + + raise StopAsyncIteration + + if not had_yieldpoint: + await checkpoint_if_cancelled() + + self._element_yielded = True + value = cast(T, self._link.value) + next_link = self._link.next + assert next_link is not None + self._link = next_link + if not had_yieldpoint: + await cancel_shielded_checkpoint() + + return value + + +async def _operator_add(x: T, y: T) -> T: + return operator.add(x, y) + + +async def accumulate( + iterable: Iterable[T] | AsyncIterable[T], + function: Callable[[T, T], Awaitable[T]] = _operator_add, + *, + initial: T | None = None, +) -> AsyncGenerator[T, None]: + iterator = _iterate(iterable) + if initial is None: + try: + total = await anext(iterator) + except StopAsyncIteration: + await checkpoint() + return + else: + await checkpoint_if_cancelled() + total = initial + await cancel_shielded_checkpoint() + + yield total + + async for element in iterator: + total = await function(total, element) + yield total + + +async def batched( + iterable: Iterable[T] | AsyncIterable[T], n: int, *, strict: bool = False +) -> AsyncGenerator[tuple[T, ...], None]: + if n < 1: + raise ValueError("n must be at least one") + + iterator = _iterate(iterable) + + while True: + batch: list[T] = [] + for _ in range(n): + try: + batch.append(await anext(iterator)) + except StopAsyncIteration: + if not batch: + await checkpoint() + return + if strict: + raise ValueError("batched(): incomplete batch") from None + + yield tuple(batch) + return + + yield tuple(batch) + + +class Chain: + def __call__( + self, *iterables: Iterable[T] | AsyncIterable[T] + ) -> AsyncGenerator[T, None]: + return self.from_iterable(iterables) + + async def from_iterable( + self, + iterables: ( + Iterable[Iterable[T] | AsyncIterable[T]] + | AsyncIterable[Iterable[T] | AsyncIterable[T]] + ), + ) -> AsyncGenerator[T, None]: + element_yielded = False + outer_iter = _iterate(iterables) + + try: + async for iterable in outer_iter: + async for element in _iterate(iterable): + element_yielded = True + yield element + finally: + aclose = getattr(outer_iter, "aclose", None) + if aclose is not None: + with CancelScope(shield=True): + await aclose() + + if not element_yielded: + await checkpoint() + + +chain: Chain = Chain() + + +async def combinations( + iterable: Iterable[T] | AsyncIterable[T], r: int +) -> AsyncGenerator[tuple[T, ...], None]: + pool: list[T] = [element async for element in _iterate(iterable)] + async for combination in _iterate(itertools.combinations(pool, r)): + yield combination + + +async def combinations_with_replacement( + iterable: Iterable[T] | AsyncIterable[T], r: int +) -> AsyncGenerator[tuple[T, ...], None]: + pool: list[T] = [element async for element in _iterate(iterable)] + async for combination in _iterate(itertools.combinations_with_replacement(pool, r)): + yield combination + + +async def compress( + data: Iterable[T] | AsyncIterable[T], + selectors: Iterable[object] | AsyncIterable[object], +) -> AsyncGenerator[T, None]: + data_iterator = _iterate(data) + selector_iterator = _iterate(selectors) + element_yielded = False + + while True: + try: + datum = await anext(data_iterator) + selector = await anext(selector_iterator) + except StopAsyncIteration: + if not element_yielded: + await checkpoint() + + return + + if selector: + element_yielded = True + yield datum + + +async def count(start: int = 0, step: int = 1) -> AsyncGenerator[int, None]: + n = start + while True: + await checkpoint_if_cancelled() + value = n + n += step + await cancel_shielded_checkpoint() + yield value + + +async def cycle( + iterable: Iterable[T] | AsyncIterable[T], +) -> AsyncGenerator[T, None]: + saved: list[T] = [] + async for element in _iterate(iterable): + saved.append(element) + yield element + + if not saved: + await checkpoint() + return + + while True: + for element in saved: + await checkpoint() + yield element + + +async def dropwhile( + predicate: Callable[[T], Awaitable[object]], + iterable: Iterable[T] | AsyncIterable[T], +) -> AsyncGenerator[T, None]: + element_yielded = False + dropping = True + + async for element in _iterate(iterable): + if dropping and await predicate(element): + continue + + dropping = False + element_yielded = True + yield element + + if not element_yielded: + await checkpoint() + + +async def filterfalse( + predicate: Callable[[T], Awaitable[object]], + iterable: Iterable[T] | AsyncIterable[T], +) -> AsyncGenerator[T, None]: + element_yielded = False + + async for element in _iterate(iterable): + if not await predicate(element): + element_yielded = True + yield element + + if not element_yielded: + await checkpoint() + + +@overload +def groupby( + iterable: Iterable[T] | AsyncIterable[T], +) -> AsyncGenerator[tuple[T, list[T]], None]: ... + + +@overload +def groupby( + iterable: Iterable[T] | AsyncIterable[T], + key: Callable[[T], Awaitable[R]], +) -> AsyncGenerator[tuple[R, list[T]], None]: ... + + +async def groupby( + iterable: Iterable[T] | AsyncIterable[T], + key: Callable[[T], Awaitable[object]] | None = None, +) -> AsyncGenerator[tuple[object, list[T]], None]: + iterator = _iterate(iterable) + try: + element = await anext(iterator) + except StopAsyncIteration: + await checkpoint() + return + + group_key = element if key is None else await key(element) + values = [element] + + async for element in iterator: + next_key = element if key is None else await key(element) + if next_key != group_key: + completed_group = group_key, values + group_key = next_key + values = [element] + yield completed_group + else: + values.append(element) + + yield group_key, values + + +@overload +def islice( + iterable: Iterable[T] | AsyncIterable[T], + stop: int | None, + /, +) -> AsyncGenerator[T, None]: ... + + +@overload +def islice( + iterable: Iterable[T] | AsyncIterable[T], + start: int | None, + stop: int | None, + step: int | None = 1, + /, +) -> AsyncGenerator[T, None]: ... + + +async def islice( + iterable: Iterable[T] | AsyncIterable[T], + *args: int | None, +) -> AsyncGenerator[T, None]: + if not args: + raise TypeError("islice expected at least 2 arguments, got 1") + if len(args) > 3: + raise TypeError(f"islice expected at most 4 arguments, got {len(args) + 1}") + + slice_args = slice(*args) + + start_message = ( + "Indices for islice() must be None or an integer: 0 <= x <= sys.maxsize." + ) + stop_message = ( + "Stop argument for islice() must be None or an integer: 0 <= x <= sys.maxsize." + ) + step_message = "Step for islice() must be a positive integer or None." + + def normalize_index(value: object, message: str) -> int: + try: + index = operator.index(cast(Any, value)) + except TypeError: + raise ValueError(message) from None + + if index < 0 or index > sys.maxsize: + raise ValueError(message) + + return index + + start = ( + 0 + if slice_args.start is None + else normalize_index(slice_args.start, start_message) + ) + stop = ( + None + if slice_args.stop is None + else normalize_index(slice_args.stop, stop_message) + ) + step = ( + 1 if slice_args.step is None else normalize_index(slice_args.step, step_message) + ) + + if step <= 0: + raise ValueError(step_message) + + if stop == 0 or start == stop: + await checkpoint() + return + + iterator = _iterate(iterable) + index = 0 + element_yielded = False + + while stop is None or index < stop: + try: + element = await anext(iterator) + except StopAsyncIteration: + if not element_yielded: + await checkpoint() + + return + + if index >= start and (index - start) % step == 0: + index += 1 + element_yielded = True + yield element + else: + index += 1 + + if not element_yielded: + await checkpoint() + + +async def pairwise( + iterable: Iterable[T] | AsyncIterable[T], +) -> AsyncGenerator[tuple[T, T], None]: + iterator = _iterate(iterable) + try: + previous = await anext(iterator) + except StopAsyncIteration: + await checkpoint() + return + + element_yielded = False + async for element in iterator: + element_yielded = True + pair = (previous, element) + previous = element + yield pair + + if not element_yielded: + await checkpoint() + + +async def permutations( + iterable: Iterable[T] | AsyncIterable[T], r: int | None = None +) -> AsyncGenerator[tuple[T, ...], None]: + pool: list[T] = [element async for element in _iterate(iterable)] + n = len(pool) + if r is None: + r = n + elif not isinstance(r, int): + raise TypeError("Expected int as r") + elif r < 0: + raise ValueError("r must be non-negative") + + async for permutation in _iterate(itertools.permutations(pool, r)): + yield permutation + + +async def product( + *iterables: Iterable[T] | AsyncIterable[T], repeat: int = 1 +) -> AsyncGenerator[tuple[T, ...], None]: + repeat = operator.index(repeat) + if repeat < 0: + raise ValueError("repeat argument cannot be negative") + + pools: list[tuple[T, ...]] = [] + for iterable in iterables: + pool: list[T] = [element async for element in _iterate(iterable)] + pools.append(tuple(pool)) + + async for value in _iterate(itertools.product(*pools, repeat=repeat)): + yield value + + +async def repeat(element: T, times: int | None = None) -> AsyncGenerator[T, None]: + if times is None: + while True: + await checkpoint() + yield element + + remaining = operator.index(cast(Any, times)) + if remaining <= 0: + await checkpoint() + return + + while remaining > 0: + await checkpoint_if_cancelled() + remaining -= 1 + await cancel_shielded_checkpoint() + yield element + + +async def starmap( + function: Callable[..., Awaitable[R]], + iterable: ( + Iterable[Iterable[object] | AsyncIterable[object]] + | AsyncIterable[Iterable[object] | AsyncIterable[object]] + ), +) -> AsyncGenerator[R, None]: + result_yielded = False + + async for args_iterable in _iterate(iterable): + args = [element async for element in _iterate(args_iterable)] + result_yielded = True + yield await function(*args) + + if not result_yielded: + await checkpoint() + + +def tee( + iterable: Iterable[T] | AsyncIterable[T], n: int = 2 +) -> tuple[AsyncIterator[T], ...]: + n = operator.index(cast(Any, n)) + if n < 0: + raise ValueError("n must be >= 0") + if n == 0: + return () + + iterator = _TeeAsyncIterator(iterable) + iterators: list[AsyncIterator[T]] = [iterator] + iterators.extend(_TeeAsyncIterator(iterator) for _ in range(n - 1)) + return tuple(iterators) + + +async def takewhile( + predicate: Callable[[T], Awaitable[object]], + iterable: Iterable[T] | AsyncIterable[T], +) -> AsyncGenerator[T, None]: + element_yielded = False + + async for element in _iterate(iterable): + if not await predicate(element): + if not element_yielded: + await checkpoint() + + return + + element_yielded = True + yield element + + if not element_yielded: + await checkpoint() + + +async def zip_longest( + *iterables: Iterable[object] | AsyncIterable[object], + fillvalue: object = None, +) -> AsyncGenerator[tuple[object, ...], None]: + iterators = [_iterate(iterable) for iterable in iterables] + num_active = len(iterators) + if not num_active: + await checkpoint() + return + + active = [True] * num_active + tuple_yielded = False + + while True: + values: list[object] = [] + for index, iterator in enumerate(iterators): + if not active[index]: + values.append(fillvalue) + continue + + try: + value = await anext(iterator) + except StopAsyncIteration: + active[index] = False + num_active -= 1 + if not num_active: + if not tuple_yielded: + await checkpoint() + + return + + value = fillvalue + + values.append(value) + + tuple_yielded = True + yield tuple(values) diff --git a/.venv/lib/python3.14/site-packages/anyio/lowlevel.py b/.venv/lib/python3.14/site-packages/anyio/lowlevel.py new file mode 100644 index 0000000000000000000000000000000000000000..361b2eb84d3600e8c64afe9b010825fd2b8c5288 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/anyio/lowlevel.py @@ -0,0 +1,228 @@ +from __future__ import annotations + +__all__ = ( + "EventLoopToken", + "RunVar", + "RunvarToken", + "cancel_shielded_checkpoint", + "checkpoint", + "checkpoint_if_cancelled", + "current_token", +) + +import enum +from dataclasses import dataclass +from types import TracebackType +from typing import TYPE_CHECKING, Any, Generic, Literal, TypeVar, final, overload +from weakref import WeakKeyDictionary + +from ._core._eventloop import get_async_backend + +if TYPE_CHECKING: + from .abc import AsyncBackend + +T = TypeVar("T") +D = TypeVar("D") + + +async def checkpoint() -> None: + """ + Check for cancellation and allow the scheduler to switch to another task. + + Equivalent to (but more efficient than):: + + await checkpoint_if_cancelled() + await cancel_shielded_checkpoint() + + .. versionadded:: 3.0 + + """ + await get_async_backend().checkpoint() + + +async def checkpoint_if_cancelled() -> None: + """ + Enter a checkpoint if the enclosing cancel scope has been cancelled. + + This does not allow the scheduler to switch to a different task. + + .. versionadded:: 3.0 + + """ + await get_async_backend().checkpoint_if_cancelled() + + +async def cancel_shielded_checkpoint() -> None: + """ + Allow the scheduler to switch to another task but without checking for cancellation. + + Equivalent to (but potentially more efficient than):: + + with CancelScope(shield=True): + await checkpoint() + + .. versionadded:: 3.0 + + """ + await get_async_backend().cancel_shielded_checkpoint() + + +@final +@dataclass(frozen=True, repr=False) +class EventLoopToken: + """ + An opaque object that holds a reference to an event loop. + + .. versionadded:: 4.11.0 + """ + + backend_class: type[AsyncBackend] + native_token: object + + +def current_token() -> EventLoopToken: + """ + Return a token object that can be used to call code in the current event loop from + another thread. + + :raises NoEventLoopError: if no supported asynchronous event loop is running in the + current thread + + .. versionadded:: 4.11.0 + + """ + backend_class = get_async_backend() + raw_token = backend_class.current_token() + return EventLoopToken(backend_class, raw_token) + + +_run_vars: WeakKeyDictionary[object, dict[RunVar[Any], Any]] = WeakKeyDictionary() + + +class _NoValueSet(enum.Enum): + NO_VALUE_SET = enum.auto() + + +class RunvarToken(Generic[T]): + """ + A token that can be used to restore a :class:`RunVar` to its previous value. + + Returned by :meth:`RunVar.set`. Can be used as a context manager to automatically + reset the variable on exit, or passed directly to :meth:`RunVar.reset`. + """ + + __slots__ = "_redeemed", "_value", "_var" + + def __init__(self, var: RunVar[T], value: T | Literal[_NoValueSet.NO_VALUE_SET]): + self._var = var + self._value: T | Literal[_NoValueSet.NO_VALUE_SET] = value + self._redeemed = False + + def __enter__(self) -> RunvarToken[T]: + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> None: + self._var.reset(self) + + +class RunVar(Generic[T]): + """ + Like a :class:`~contextvars.ContextVar`, except scoped to the running event loop. + + Can be used as a context manager, Just like :class:`~contextvars.ContextVar`, that + will reset the variable to its previous value when the context block is exited. + """ + + __slots__ = "_default", "_name" + + NO_VALUE_SET: Literal[_NoValueSet.NO_VALUE_SET] = _NoValueSet.NO_VALUE_SET + + def __init__( + self, name: str, default: T | Literal[_NoValueSet.NO_VALUE_SET] = NO_VALUE_SET + ): + self._name = name + self._default = default + + @property + def _current_vars(self) -> dict[RunVar[T], T]: + native_token = current_token().native_token + try: + return _run_vars[native_token] + except KeyError: + run_vars = _run_vars[native_token] = {} + return run_vars + + @overload + def get(self, default: D) -> T | D: ... + + @overload + def get(self) -> T: ... + + def get( + self, default: D | Literal[_NoValueSet.NO_VALUE_SET] = NO_VALUE_SET + ) -> T | D: + """ + Return the current value of this run variable. + + :param default: a fallback value to return if no value has been set + :return: the current value, the provided default, or the variable's own default + :raises LookupError: if no value is set and no default is available + + """ + try: + return self._current_vars[self] + except KeyError: + if default is not RunVar.NO_VALUE_SET: + return default + elif self._default is not RunVar.NO_VALUE_SET: + return self._default + + raise LookupError( + f'Run variable "{self._name}" has no value and no default set' + ) + + def set(self, value: T) -> RunvarToken[T]: + """ + Set the value of this run variable for the current event loop. + + :param value: the new value + :return: a token that can be used to restore the previous value + + """ + current_vars = self._current_vars + token = RunvarToken(self, current_vars.get(self, RunVar.NO_VALUE_SET)) + current_vars[self] = value + return token + + def reset(self, token: RunvarToken[T]) -> None: + """ + Restore this run variable to the value it held before the matching :meth:`set`. + + :param token: the token returned by :meth:`set` + :raises ValueError: if the token belongs to a different :class:`RunVar` or the token + has already been used + + """ + if token._var is not self: + raise ValueError("This token does not belong to this RunVar") + + if token._redeemed: + raise ValueError("This token has already been used") + + if token._value is _NoValueSet.NO_VALUE_SET: + try: + del self._current_vars[self] + except KeyError: + pass + else: + self._current_vars[self] = token._value + + token._redeemed = True + + def __repr__(self) -> str: + return f"" diff --git a/.venv/lib/python3.14/site-packages/anyio/py.typed b/.venv/lib/python3.14/site-packages/anyio/py.typed new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.venv/lib/python3.14/site-packages/anyio/pytest_plugin.py b/.venv/lib/python3.14/site-packages/anyio/pytest_plugin.py new file mode 100644 index 0000000000000000000000000000000000000000..d6238cd110ab793ccccb9671fc7eccc33c5415e5 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/anyio/pytest_plugin.py @@ -0,0 +1,404 @@ +from __future__ import annotations + +import dataclasses +import socket +import sys +from collections.abc import Callable, Generator, Iterator +from contextlib import ExitStack, contextmanager +from inspect import isasyncgenfunction, iscoroutinefunction, ismethod +from typing import TYPE_CHECKING, Any, cast + +import pytest +from _pytest.fixtures import FuncFixtureInfo, SubRequest +from _pytest.outcomes import Exit +from _pytest.scope import Scope + +from . import get_available_backends +from ._core._eventloop import ( + current_async_library, + get_async_backend, + reset_current_async_library, + set_current_async_library, +) +from ._core._exceptions import iterate_exceptions + +if TYPE_CHECKING: + from .abc import TestRunner + +if sys.version_info < (3, 11): + from exceptiongroup import ExceptionGroup + +if TYPE_CHECKING: + # pytest >= 9.2 keeps CallSpec2 as a TYPE_CHECKING-only alias of CallSpec + from _pytest.python import CallSpec2 as CallSpec +else: + try: + # c.f. https://github.com/pytest-dev/pytest/pull/14742 + # pytest >= 9.2 + from _pytest.python import CallSpec + except ImportError: + from _pytest.python import CallSpec2 as CallSpec + +_current_runner: TestRunner | None = None +_runner_stack: ExitStack | None = None +_runner_leases = 0 + + +def extract_backend_and_options(backend: object) -> tuple[str, dict[str, Any]]: + if isinstance(backend, str): + return backend, {} + elif isinstance(backend, tuple) and len(backend) == 2: + if isinstance(backend[0], str) and isinstance(backend[1], dict): + return cast(tuple[str, dict[str, Any]], backend) + + raise TypeError("anyio_backend must be either a string or tuple of (string, dict)") + + +@contextmanager +def get_runner( + backend_name: str, backend_options: dict[str, Any] +) -> Iterator[TestRunner]: + global _current_runner, _runner_leases, _runner_stack + if _current_runner is None: + asynclib = get_async_backend(backend_name) + _runner_stack = ExitStack() + if current_async_library() is None: + # Since we're in control of the event loop, we can cache the name of the + # async library + token = set_current_async_library(backend_name) + _runner_stack.callback(reset_current_async_library, token) + + backend_options = backend_options or {} + _current_runner = _runner_stack.enter_context( + asynclib.create_test_runner(backend_options) + ) + + _runner_leases += 1 + try: + yield _current_runner + finally: + _runner_leases -= 1 + if not _runner_leases: + assert _runner_stack is not None + _runner_stack.close() + _runner_stack = _current_runner = None + + +def pytest_addoption(parser: pytest.Parser) -> None: + group = parser.getgroup("anyio") + parser.addini( + "anyio_mode", + default="strict", + help='AnyIO plugin mode (either "strict" or "auto")', + ) + group.addoption( + "--anyio-mode", + default=None, + dest="anyio_mode", + help=""" + 'auto' - All async test functions will be handled by AnyIO pytest plugin + 'strict' - Disabling autoprocessing(useful when anyio tests + need to coexist with other async test plugins) + """, + ) + + +def _is_auto_mode(config: pytest.Config, mode: str) -> bool: + return ( + config.getoption(mode, default=None) == "auto" or config.getini(mode) == "auto" + ) + + +def pytest_configure(config: pytest.Config) -> None: + config.addinivalue_line( + "markers", + "anyio: mark the (coroutine function) test to be run asynchronously via anyio.", + ) + if ( + _is_auto_mode(config, "anyio_mode") + and config.pluginmanager.has_plugin("asyncio") + and _is_auto_mode(config, "asyncio_mode") + ): + config.issue_config_time_warning( + pytest.PytestConfigWarning( + "AnyIO auto mode has been enabled together with pytest-asyncio auto " + "mode. This may cause unexpected behavior." + ), + 1, + ) + + +@pytest.hookimpl(hookwrapper=True) +def pytest_fixture_setup(fixturedef: Any, request: Any) -> Generator[Any]: + def wrapper(anyio_backend: Any, request: SubRequest, **kwargs: Any) -> Any: + # Rebind any fixture methods to the request instance + if ( + request.instance + and ismethod(func) + and type(func.__self__) is type(request.instance) + ): + local_func = func.__func__.__get__(request.instance) + else: + local_func = func + + backend_name, backend_options = extract_backend_and_options(anyio_backend) + if has_backend_arg: + kwargs["anyio_backend"] = anyio_backend + + if has_request_arg: + kwargs["request"] = request + + with get_runner(backend_name, backend_options) as runner: + # re-entrant call into the test runner detected. this happens when an async fixture + # is dynamically requested via request.getfixturevalue() from inside a running async + # test or fixture. on asyncio this raises RuntimeError: This event loop is already + # running, on trio the runner deadlocks - the host loop blocks waiting for the + # coroutine to return, but the coroutine is waiting for the host loop. raising here + # prevents the hang and gives a consistent error across backends. + if runner.is_running(): + raise RuntimeError( + "Cannot schedule a coroutine in the test runner while another is already running; " + "likely caused by request.getfixturevalue() on an async fixture." + ) + + if isasyncgenfunction(local_func): + yield from runner.run_asyncgen_fixture(local_func, kwargs) + else: + yield runner.run_fixture(local_func, kwargs) + + # Only apply this to coroutine functions and async generator functions in requests + # that involve the anyio_backend fixture + func = fixturedef.func + if isasyncgenfunction(func) or iscoroutinefunction(func): + if "anyio_backend" in request.fixturenames: + fixturedef.func = wrapper + original_argname = fixturedef.argnames + + if not (has_backend_arg := "anyio_backend" in fixturedef.argnames): + fixturedef.argnames += ("anyio_backend",) + + if not (has_request_arg := "request" in fixturedef.argnames): + fixturedef.argnames += ("request",) + + try: + return (yield) + finally: + fixturedef.func = func + fixturedef.argnames = original_argname + + return (yield) + + +@pytest.hookimpl(tryfirst=True) +def pytest_pycollect_makeitem( + collector: pytest.Module | pytest.Class, name: str, obj: object +) -> None: + if collector.istestfunction(obj, name): + inner_func = obj.hypothesis.inner_test if hasattr(obj, "hypothesis") else obj + if iscoroutinefunction(inner_func): + anyio_auto_mode = _is_auto_mode(collector.config, "anyio_mode") + marker = collector.get_closest_marker("anyio") + own_markers = getattr(obj, "pytestmark", ()) + if ( + anyio_auto_mode + or marker + or any(marker.name == "anyio" for marker in own_markers) + ): + pytest.mark.usefixtures("anyio_backend")(obj) + + +def pytest_collection_finish(session: pytest.Session) -> None: + for i, item in reversed(list(enumerate(session.items))): + if ( + isinstance(item, pytest.Function) + and iscoroutinefunction(item.function) + and item.get_closest_marker("anyio") is not None + and "anyio_backend" not in item.fixturenames + ): + new_items = [] + try: + cs_fields = {f.name for f in dataclasses.fields(CallSpec)} + except TypeError: + cs_fields = set() + + for param_index, backend in enumerate(get_available_backends()): + if "_arg2scope" in cs_fields: # pytest >= 8 + callspec = CallSpec( + params={"anyio_backend": backend}, + indices={"anyio_backend": param_index}, + _arg2scope={"anyio_backend": Scope.Module}, + _idlist=[backend], + marks=[], + ) + else: # pytest 7.x + callspec = CallSpec( # type: ignore[call-arg] + funcargs={}, + params={"anyio_backend": backend}, + indices={"anyio_backend": param_index}, + arg2scope={"anyio_backend": Scope.Module}, + idlist=[backend], + marks=[], + ) + + fi = item._fixtureinfo + new_names_closure = list(fi.names_closure) + if "anyio_backend" not in new_names_closure: + new_names_closure.append("anyio_backend") + + new_fixtureinfo = FuncFixtureInfo( + argnames=fi.argnames, + initialnames=fi.initialnames, + names_closure=new_names_closure, + name2fixturedefs=fi.name2fixturedefs, + ) + new_item = pytest.Function.from_parent( + item.parent, + name=f"{item.originalname}[{backend}]", + callspec=callspec, + callobj=item.obj, + fixtureinfo=new_fixtureinfo, + keywords=item.keywords, + originalname=item.originalname, + ) + new_items.append(new_item) + + session.items[i : i + 1] = new_items + + +@pytest.hookimpl(tryfirst=True) +def pytest_pyfunc_call(pyfuncitem: Any) -> bool | None: + def run_with_hypothesis(**kwargs: Any) -> None: + with get_runner(backend_name, backend_options) as runner: + runner.run_test(original_func, kwargs) + + backend = pyfuncitem.funcargs.get("anyio_backend") + if backend: + backend_name, backend_options = extract_backend_and_options(backend) + + if hasattr(pyfuncitem.obj, "hypothesis"): + # Wrap the inner test function unless it's already wrapped + original_func = pyfuncitem.obj.hypothesis.inner_test + if original_func.__qualname__ != run_with_hypothesis.__qualname__: + if iscoroutinefunction(original_func): + pyfuncitem.obj.hypothesis.inner_test = run_with_hypothesis + + return None + + if iscoroutinefunction(pyfuncitem.obj): + funcargs = pyfuncitem.funcargs + testargs = {arg: funcargs[arg] for arg in pyfuncitem._fixtureinfo.argnames} + with get_runner(backend_name, backend_options) as runner: + try: + runner.run_test(pyfuncitem.obj, testargs) + except ExceptionGroup as excgrp: + for exc in iterate_exceptions(excgrp): + if isinstance(exc, (Exit, KeyboardInterrupt, SystemExit)): + raise exc from excgrp + + raise + + return True + + return None + + +@pytest.fixture(scope="module", params=get_available_backends()) +def anyio_backend(request: Any) -> Any: + return request.param + + +@pytest.fixture +def anyio_backend_name(anyio_backend: Any) -> str: + if isinstance(anyio_backend, str): + return anyio_backend + else: + return anyio_backend[0] + + +@pytest.fixture +def anyio_backend_options(anyio_backend: Any) -> dict[str, Any]: + if isinstance(anyio_backend, str): + return {} + else: + return anyio_backend[1] + + +class FreePortFactory: + """ + Manages port generation based on specified socket kind, ensuring no duplicate + ports are generated. + + This class provides functionality for generating available free ports on the + system. It is initialized with a specific socket kind and can generate ports + for given address families while avoiding reuse of previously generated ports. + + Users should not instantiate this class directly, but use the + ``free_tcp_port_factory`` and ``free_udp_port_factory`` fixtures instead. For simple + uses cases, ``free_tcp_port`` and ``free_udp_port`` can be used instead. + """ + + def __init__(self, kind: socket.SocketKind) -> None: + self._kind = kind + self._generated = set[int]() + + @property + def kind(self) -> socket.SocketKind: + """ + The type of socket connection (e.g., :data:`~socket.SOCK_STREAM` or + :data:`~socket.SOCK_DGRAM`) used to bind for checking port availability + + """ + return self._kind + + def __call__(self, family: socket.AddressFamily | None = None) -> int: + """ + Return an unbound port for the given address family. + + :param family: if omitted, both IPv4 and IPv6 addresses will be tried + :return: a port number + + """ + if family is not None: + families = [family] + else: + families = [socket.AF_INET] + if socket.has_ipv6: + families.append(socket.AF_INET6) + + while True: + port = 0 + with ExitStack() as stack: + for fam in families: + sock = stack.enter_context(socket.socket(fam, self._kind)) + addr = "::1" if fam == socket.AF_INET6 else "127.0.0.1" + try: + sock.bind((addr, port)) + except OSError: + break + + if not port: + port = sock.getsockname()[1] + else: + if port not in self._generated: + self._generated.add(port) + return port + + +@pytest.fixture(scope="session") +def free_tcp_port_factory() -> FreePortFactory: + return FreePortFactory(socket.SOCK_STREAM) + + +@pytest.fixture(scope="session") +def free_udp_port_factory() -> FreePortFactory: + return FreePortFactory(socket.SOCK_DGRAM) + + +@pytest.fixture +def free_tcp_port(free_tcp_port_factory: Callable[[], int]) -> int: + return free_tcp_port_factory() + + +@pytest.fixture +def free_udp_port(free_udp_port_factory: Callable[[], int]) -> int: + return free_udp_port_factory() diff --git a/.venv/lib/python3.14/site-packages/anyio/streams/__init__.py b/.venv/lib/python3.14/site-packages/anyio/streams/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.venv/lib/python3.14/site-packages/anyio/streams/__pycache__/__init__.cpython-314.pyc b/.venv/lib/python3.14/site-packages/anyio/streams/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3a58aa64f455007f84c7ded66c6d65b767f29632 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/anyio/streams/__pycache__/__init__.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/anyio/streams/__pycache__/buffered.cpython-314.pyc b/.venv/lib/python3.14/site-packages/anyio/streams/__pycache__/buffered.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..244b2a2e179bb924418a37b38284734a04eeba9c Binary files /dev/null and b/.venv/lib/python3.14/site-packages/anyio/streams/__pycache__/buffered.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/anyio/streams/__pycache__/file.cpython-314.pyc b/.venv/lib/python3.14/site-packages/anyio/streams/__pycache__/file.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3905753b12b69b859c9f0809f07c55c42a959aef Binary files /dev/null and b/.venv/lib/python3.14/site-packages/anyio/streams/__pycache__/file.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/anyio/streams/__pycache__/memory.cpython-314.pyc b/.venv/lib/python3.14/site-packages/anyio/streams/__pycache__/memory.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0115623790d0ab089c0e8d335b820248989c88c6 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/anyio/streams/__pycache__/memory.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/anyio/streams/__pycache__/stapled.cpython-314.pyc b/.venv/lib/python3.14/site-packages/anyio/streams/__pycache__/stapled.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..901f5a6a0e12d75f15cd72337cfaed4d8cf48c52 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/anyio/streams/__pycache__/stapled.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/anyio/streams/__pycache__/text.cpython-314.pyc b/.venv/lib/python3.14/site-packages/anyio/streams/__pycache__/text.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..518ca43b4fc333be3309962629c1eaa84ea44e7f Binary files /dev/null and b/.venv/lib/python3.14/site-packages/anyio/streams/__pycache__/text.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/anyio/streams/__pycache__/tls.cpython-314.pyc b/.venv/lib/python3.14/site-packages/anyio/streams/__pycache__/tls.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7674beabd5976dde78ddc60a470a105a3cdbb089 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/anyio/streams/__pycache__/tls.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/anyio/streams/buffered.py b/.venv/lib/python3.14/site-packages/anyio/streams/buffered.py new file mode 100644 index 0000000000000000000000000000000000000000..a3b07f73ac44d5876c239fd0317846564ef06f91 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/anyio/streams/buffered.py @@ -0,0 +1,201 @@ +from __future__ import annotations + +__all__ = ( + "BufferedByteReceiveStream", + "BufferedByteStream", + "BufferedConnectable", +) + +import sys +from collections.abc import Callable, Iterable, Mapping +from dataclasses import dataclass, field +from typing import Any, SupportsIndex + +from .. import ClosedResourceError, DelimiterNotFound, EndOfStream, IncompleteRead +from ..abc import ( + AnyByteReceiveStream, + AnyByteStream, + AnyByteStreamConnectable, + ByteReceiveStream, + ByteStream, + ByteStreamConnectable, +) + +if sys.version_info >= (3, 12): + from typing import override +else: + from typing_extensions import override + + +@dataclass(eq=False) +class BufferedByteReceiveStream(ByteReceiveStream): + """ + Wraps any bytes-based receive stream and uses a buffer to provide sophisticated + receiving capabilities in the form of a byte stream. + """ + + receive_stream: AnyByteReceiveStream + _buffer: bytearray = field(init=False, default_factory=bytearray) + _closed: bool = field(init=False, default=False) + + async def aclose(self) -> None: + await self.receive_stream.aclose() + self._closed = True + + @property + def buffer(self) -> bytes: + """The bytes currently in the buffer.""" + return bytes(self._buffer) + + @property + def extra_attributes(self) -> Mapping[Any, Callable[[], Any]]: + return self.receive_stream.extra_attributes + + def feed_data(self, data: Iterable[SupportsIndex], /) -> None: + """ + Append data directly into the buffer. + + Any data in the buffer will be consumed by receive operations before receiving + anything from the wrapped stream. + + :param data: the data to append to the buffer (can be bytes or anything else + that supports ``__index__()``) + + """ + self._buffer.extend(data) + + async def receive(self, max_bytes: int = 65536) -> bytes: + if max_bytes < 1: + raise ValueError("max_bytes must be a positive integer") + + if self._closed: + raise ClosedResourceError + + if self._buffer: + chunk = bytes(self._buffer[:max_bytes]) + del self._buffer[:max_bytes] + return chunk + elif isinstance(self.receive_stream, ByteReceiveStream): + return await self.receive_stream.receive(max_bytes) + else: + # With a bytes-oriented object stream, we need to handle any surplus bytes + # we get from the receive() call + chunk = await self.receive_stream.receive() + if len(chunk) > max_bytes: + # Save the surplus bytes in the buffer + self._buffer.extend(chunk[max_bytes:]) + return chunk[:max_bytes] + else: + return chunk + + async def receive_exactly(self, nbytes: int) -> bytes: + """ + Read exactly the given amount of bytes from the stream. + + :param nbytes: the number of bytes to read + :return: the bytes read + :raises ~anyio.IncompleteRead: if the stream was closed before the requested + amount of bytes could be read from the stream + + """ + while True: + remaining = nbytes - len(self._buffer) + if remaining <= 0: + retval = self._buffer[:nbytes] + del self._buffer[:nbytes] + return bytes(retval) + + try: + if isinstance(self.receive_stream, ByteReceiveStream): + chunk = await self.receive_stream.receive(remaining) + else: + chunk = await self.receive_stream.receive() + except EndOfStream as exc: + raise IncompleteRead from exc + + self._buffer.extend(chunk) + + async def receive_until(self, delimiter: bytes, max_bytes: int) -> bytes: + """ + Read from the stream until the delimiter is found or max_bytes have been read. + + :param delimiter: the marker to look for in the stream + :param max_bytes: maximum number of bytes that will be read before raising + :exc:`~anyio.DelimiterNotFound` + :return: the bytes read (not including the delimiter) + :raises ~anyio.IncompleteRead: if the stream was closed before the delimiter + was found + :raises ~anyio.DelimiterNotFound: if the delimiter is not found within the + bytes read up to the maximum allowed + + """ + delimiter_size = len(delimiter) + offset = 0 + while True: + # Check if the delimiter can be found in the current buffer + index = self._buffer.find(delimiter, offset) + if index >= 0: + found = self._buffer[:index] + del self._buffer[: index + len(delimiter) :] + return bytes(found) + + # Check if the buffer is already at or over the limit + if len(self._buffer) >= max_bytes: + raise DelimiterNotFound(max_bytes) + + # Read more data into the buffer from the socket + try: + data = await self.receive_stream.receive() + except EndOfStream as exc: + raise IncompleteRead from exc + + # Move the offset forward and add the new data to the buffer + offset = max(len(self._buffer) - delimiter_size + 1, 0) + self._buffer.extend(data) + + +class BufferedByteStream(BufferedByteReceiveStream, ByteStream): + """ + A full-duplex variant of :class:`BufferedByteReceiveStream`. All writes are passed + through to the wrapped stream as-is. + """ + + def __init__(self, stream: AnyByteStream): + """ + :param stream: the stream to be wrapped + + """ + super().__init__(stream) + self._stream = stream + + @override + async def send_eof(self) -> None: + await self._stream.send_eof() + + @override + async def send(self, item: bytes) -> None: + await self._stream.send(item) + + +class BufferedConnectable(ByteStreamConnectable): + """ + Wraps a byte stream connectable to produce :class:`BufferedByteStream` connections. + + Use this when you want the streams returned by :meth:`connect` to have the buffered + receive API (e.g. :meth:`~BufferedByteReceiveStream.receive_exactly` and + :meth:`~BufferedByteReceiveStream.receive_until`). + + :param connectable: the byte stream connectable to wrap + """ + + def __init__(self, connectable: AnyByteStreamConnectable): + """ + :param connectable: the connectable to wrap + + """ + self.connectable = connectable + + @override + async def connect(self) -> BufferedByteStream: + stream = await self.connectable.connect() + return BufferedByteStream(stream) diff --git a/.venv/lib/python3.14/site-packages/anyio/streams/file.py b/.venv/lib/python3.14/site-packages/anyio/streams/file.py new file mode 100644 index 0000000000000000000000000000000000000000..b9d7ca4f3341a7ad7e44553e4e15134f223cb750 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/anyio/streams/file.py @@ -0,0 +1,157 @@ +from __future__ import annotations + +__all__ = ( + "FileReadStream", + "FileStreamAttribute", + "FileWriteStream", +) + +from collections.abc import Callable, Mapping +from io import SEEK_SET, UnsupportedOperation +from os import PathLike +from pathlib import Path +from typing import IO, Any + +from .. import ( + BrokenResourceError, + ClosedResourceError, + EndOfStream, + TypedAttributeSet, + to_thread, + typed_attribute, +) +from ..abc import ByteReceiveStream, ByteSendStream + + +class FileStreamAttribute(TypedAttributeSet): + #: the open file descriptor + file: IO[bytes] = typed_attribute() + #: the path of the file on the file system, if available (file must be a real file) + path: Path = typed_attribute() + #: the file number, if available (file must be a real file or a TTY) + fileno: int = typed_attribute() + + +class _BaseFileStream: + def __init__(self, file: IO[bytes]): + self._file = file + + async def aclose(self) -> None: + await to_thread.run_sync(self._file.close) + + @property + def extra_attributes(self) -> Mapping[Any, Callable[[], Any]]: + attributes: dict[Any, Callable[[], Any]] = { + FileStreamAttribute.file: lambda: self._file, + } + + if hasattr(self._file, "name"): + attributes[FileStreamAttribute.path] = lambda: Path(self._file.name) + + try: + self._file.fileno() + except UnsupportedOperation: + pass + else: + attributes[FileStreamAttribute.fileno] = lambda: self._file.fileno() + + return attributes + + +class FileReadStream(_BaseFileStream, ByteReceiveStream): + """ + A byte stream that reads from a file in the file system. + + :param file: a file that has been opened for reading in binary mode + + .. versionadded:: 3.0 + """ + + @classmethod + async def from_path(cls, path: str | PathLike[str]) -> FileReadStream: + """ + Create a file read stream by opening the given file. + + :param path: path of the file to read from + + """ + file = await to_thread.run_sync(Path(path).open, "rb") + return cls(file) + + async def receive(self, max_bytes: int = 65536) -> bytes: + if max_bytes < 1: + raise ValueError("max_bytes must be a positive integer") + + try: + data = await to_thread.run_sync(self._file.read, max_bytes) + except ValueError: + raise ClosedResourceError from None + except OSError as exc: + raise BrokenResourceError from exc + + if data: + return data + else: + raise EndOfStream + + async def seek(self, position: int, whence: int = SEEK_SET) -> int: + """ + Seek the file to the given position. + + .. seealso:: :meth:`io.IOBase.seek` + + .. note:: Not all file descriptors are seekable. + + :param position: position to seek the file to + :param whence: controls how ``position`` is interpreted + :return: the new absolute position + :raises OSError: if the file is not seekable + + """ + return await to_thread.run_sync(self._file.seek, position, whence) + + async def tell(self) -> int: + """ + Return the current stream position. + + .. note:: Not all file descriptors are seekable. + + :return: the current absolute position + :raises OSError: if the file is not seekable + + """ + return await to_thread.run_sync(self._file.tell) + + +class FileWriteStream(_BaseFileStream, ByteSendStream): + """ + A byte stream that writes to a file in the file system. + + :param file: a file that has been opened for writing in binary mode + + .. versionadded:: 3.0 + """ + + @classmethod + async def from_path( + cls, path: str | PathLike[str], append: bool = False + ) -> FileWriteStream: + """ + Create a file write stream by opening the given file for writing. + + :param path: path of the file to write to + :param append: if ``True``, open the file for appending; if ``False``, any + existing file at the given path will be truncated + + """ + mode = "ab" if append else "wb" + file = await to_thread.run_sync(Path(path).open, mode) + return cls(file) + + async def send(self, item: bytes) -> None: + try: + await to_thread.run_sync(self._file.write, item) + except ValueError: + raise ClosedResourceError from None + except OSError as exc: + raise BrokenResourceError from exc diff --git a/.venv/lib/python3.14/site-packages/anyio/streams/memory.py b/.venv/lib/python3.14/site-packages/anyio/streams/memory.py new file mode 100644 index 0000000000000000000000000000000000000000..329a75e44b7af979042c54e65529c7148e67cd8d --- /dev/null +++ b/.venv/lib/python3.14/site-packages/anyio/streams/memory.py @@ -0,0 +1,326 @@ +from __future__ import annotations + +__all__ = ( + "MemoryObjectReceiveStream", + "MemoryObjectSendStream", + "MemoryObjectStreamStatistics", +) + +import warnings +from collections import OrderedDict, deque +from dataclasses import dataclass, field +from types import TracebackType +from typing import Generic, NamedTuple, TypeVar + +from .._core._exceptions import ( + BrokenResourceError, + ClosedResourceError, + EndOfStream, + WouldBlock, +) +from .._core._synchronization import Event +from .._core._testing import TaskInfo, get_current_task +from ..abc import ObjectReceiveStream, ObjectSendStream +from ..lowlevel import checkpoint + +T_Item = TypeVar("T_Item") +T_co = TypeVar("T_co", covariant=True) +T_contra = TypeVar("T_contra", contravariant=True) + + +class MemoryObjectStreamStatistics(NamedTuple): + current_buffer_used: int #: number of items stored in the buffer + #: maximum number of items that can be stored on this stream (or :data:`math.inf`) + max_buffer_size: float + open_send_streams: int #: number of unclosed clones of the send stream + open_receive_streams: int #: number of unclosed clones of the receive stream + #: number of tasks blocked on :meth:`MemoryObjectSendStream.send` + tasks_waiting_send: int + #: number of tasks blocked on :meth:`MemoryObjectReceiveStream.receive` + tasks_waiting_receive: int + + +@dataclass(eq=False) +class _MemoryObjectItemReceiver(Generic[T_Item]): + task_info: TaskInfo = field(init=False, default_factory=get_current_task) + item: T_Item = field(init=False) + + def __repr__(self) -> str: + # When item is not defined, we get following error with default __repr__: + # AttributeError: 'MemoryObjectItemReceiver' object has no attribute 'item' + item = getattr(self, "item", None) + return f"{self.__class__.__name__}(task_info={self.task_info}, item={item!r})" + + +@dataclass(eq=False) +class _MemoryObjectStreamState(Generic[T_Item]): + max_buffer_size: float = field() + buffer: deque[T_Item] = field(init=False, default_factory=deque) + open_send_channels: int = field(init=False, default=0) + open_receive_channels: int = field(init=False, default=0) + waiting_receivers: OrderedDict[Event, _MemoryObjectItemReceiver[T_Item]] = field( + init=False, default_factory=OrderedDict + ) + waiting_senders: OrderedDict[Event, T_Item] = field( + init=False, default_factory=OrderedDict + ) + + def statistics(self) -> MemoryObjectStreamStatistics: + return MemoryObjectStreamStatistics( + len(self.buffer), + self.max_buffer_size, + self.open_send_channels, + self.open_receive_channels, + len(self.waiting_senders), + len(self.waiting_receivers), + ) + + +@dataclass(eq=False) +class MemoryObjectReceiveStream(ObjectReceiveStream[T_co], Generic[T_co]): + _state: _MemoryObjectStreamState[T_co] + _closed: bool = field(init=False, default=False) + + def __post_init__(self) -> None: + self._state.open_receive_channels += 1 + + def receive_nowait(self) -> T_co: + """ + Receive the next item if it can be done without waiting. + + :return: the received item + :raises ~anyio.ClosedResourceError: if this send stream has been closed + :raises ~anyio.EndOfStream: if the buffer is empty and this stream has been + closed from the sending end + :raises ~anyio.WouldBlock: if there are no items in the buffer and no tasks + waiting to send + + """ + if self._closed: + raise ClosedResourceError + + if self._state.waiting_senders: + # Get the item from the next sender + send_event, item = self._state.waiting_senders.popitem(last=False) + self._state.buffer.append(item) + send_event.set() + + if self._state.buffer: + return self._state.buffer.popleft() + elif not self._state.open_send_channels: + raise EndOfStream + + raise WouldBlock + + async def receive(self) -> T_co: + await checkpoint() + try: + return self.receive_nowait() + except WouldBlock: + # Add ourselves in the queue + receive_event = Event() + receiver = _MemoryObjectItemReceiver[T_co]() + self._state.waiting_receivers[receive_event] = receiver + + try: + await receive_event.wait() + finally: + self._state.waiting_receivers.pop(receive_event, None) + + try: + return receiver.item + except AttributeError: + raise EndOfStream from None + + def clone(self) -> MemoryObjectReceiveStream[T_co]: + """ + Create a clone of this receive stream. + + Each clone can be closed separately. Only when all clones have been closed will + the receiving end of the memory stream be considered closed by the sending ends. + + :return: the cloned stream + + """ + if self._closed: + raise ClosedResourceError + + return MemoryObjectReceiveStream(_state=self._state) + + def close(self) -> None: + """ + Close the stream. + + This works the exact same way as :meth:`aclose`, but is provided as a special + case for the benefit of synchronous callbacks. + + """ + if not self._closed: + self._closed = True + self._state.open_receive_channels -= 1 + if self._state.open_receive_channels == 0: + send_events = list(self._state.waiting_senders.keys()) + for event in send_events: + event.set() + + async def aclose(self) -> None: + self.close() + + def statistics(self) -> MemoryObjectStreamStatistics: + """ + Return statistics about the current state of this stream. + + .. versionadded:: 3.0 + """ + return self._state.statistics() + + def __enter__(self) -> MemoryObjectReceiveStream[T_co]: + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> None: + self.close() + + def __del__(self) -> None: + if not self._closed: + warnings.warn( + f"Unclosed <{self.__class__.__name__} at {id(self):x}>", + ResourceWarning, + stacklevel=1, + source=self, + ) + + +@dataclass(eq=False) +class MemoryObjectSendStream(ObjectSendStream[T_contra], Generic[T_contra]): + _state: _MemoryObjectStreamState[T_contra] + _closed: bool = field(init=False, default=False) + + def __post_init__(self) -> None: + self._state.open_send_channels += 1 + + def send_nowait(self, item: T_contra) -> None: + """ + Send an item immediately if it can be done without waiting. + + :param item: the item to send + :raises ~anyio.ClosedResourceError: if this send stream has been closed + :raises ~anyio.BrokenResourceError: if the stream has been closed from the + receiving end + :raises ~anyio.WouldBlock: if the buffer is full and there are no tasks waiting + to receive + + """ + if self._closed: + raise ClosedResourceError + if not self._state.open_receive_channels: + raise BrokenResourceError + + while self._state.waiting_receivers: + receive_event, receiver = self._state.waiting_receivers.popitem(last=False) + if not receiver.task_info.has_pending_cancellation(): + receiver.item = item + receive_event.set() + return + + if len(self._state.buffer) < self._state.max_buffer_size: + self._state.buffer.append(item) + else: + raise WouldBlock + + async def send(self, item: T_contra) -> None: + """ + Send an item to the stream. + + If the buffer is full, this method blocks until there is again room in the + buffer or the item can be sent directly to a receiver. + + :param item: the item to send + :raises ~anyio.ClosedResourceError: if this send stream has been closed + :raises ~anyio.BrokenResourceError: if the stream has been closed from the + receiving end + + """ + await checkpoint() + try: + self.send_nowait(item) + except WouldBlock: + # Wait until there's someone on the receiving end + send_event = Event() + self._state.waiting_senders[send_event] = item + try: + await send_event.wait() + except BaseException: + self._state.waiting_senders.pop(send_event, None) + raise + + if send_event in self._state.waiting_senders: + del self._state.waiting_senders[send_event] + raise BrokenResourceError from None + + def clone(self) -> MemoryObjectSendStream[T_contra]: + """ + Create a clone of this send stream. + + Each clone can be closed separately. Only when all clones have been closed will + the sending end of the memory stream be considered closed by the receiving ends. + + :return: the cloned stream + + """ + if self._closed: + raise ClosedResourceError + + return MemoryObjectSendStream(_state=self._state) + + def close(self) -> None: + """ + Close the stream. + + This works the exact same way as :meth:`aclose`, but is provided as a special + case for the benefit of synchronous callbacks. + + """ + if not self._closed: + self._closed = True + self._state.open_send_channels -= 1 + if self._state.open_send_channels == 0: + receive_events = list(self._state.waiting_receivers.keys()) + self._state.waiting_receivers.clear() + for event in receive_events: + event.set() + + async def aclose(self) -> None: + self.close() + + def statistics(self) -> MemoryObjectStreamStatistics: + """ + Return statistics about the current state of this stream. + + .. versionadded:: 3.0 + """ + return self._state.statistics() + + def __enter__(self) -> MemoryObjectSendStream[T_contra]: + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> None: + self.close() + + def __del__(self) -> None: + if not self._closed: + warnings.warn( + f"Unclosed <{self.__class__.__name__} at {id(self):x}>", + ResourceWarning, + stacklevel=1, + source=self, + ) diff --git a/.venv/lib/python3.14/site-packages/anyio/streams/stapled.py b/.venv/lib/python3.14/site-packages/anyio/streams/stapled.py new file mode 100644 index 0000000000000000000000000000000000000000..9a7534563b1aa9613516cd468daf6e68b4fa8a71 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/anyio/streams/stapled.py @@ -0,0 +1,160 @@ +from __future__ import annotations + +__all__ = ( + "MultiListener", + "StapledByteStream", + "StapledObjectStream", +) + +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from typing import Any, Generic, TypeVar + +from ..abc import ( + ByteReceiveStream, + ByteSendStream, + ByteStream, + Listener, + ObjectReceiveStream, + ObjectSendStream, + ObjectStream, + TaskGroup, +) + +T_Item = TypeVar("T_Item") +T_Stream = TypeVar("T_Stream") + + +@dataclass(eq=False) +class StapledByteStream(ByteStream): + """ + Combines two byte streams into a single, bidirectional byte stream. + + Extra attributes will be provided from both streams, with the receive stream + providing the values in case of a conflict. + + :param ByteSendStream send_stream: the sending byte stream + :param ByteReceiveStream receive_stream: the receiving byte stream + """ + + send_stream: ByteSendStream + receive_stream: ByteReceiveStream + + async def receive(self, max_bytes: int = 65536) -> bytes: + if max_bytes < 1: + raise ValueError("max_bytes must be a positive integer") + + return await self.receive_stream.receive(max_bytes) + + async def send(self, item: bytes) -> None: + await self.send_stream.send(item) + + async def send_eof(self) -> None: + await self.send_stream.aclose() + + async def aclose(self) -> None: + await self.send_stream.aclose() + await self.receive_stream.aclose() + + @property + def extra_attributes(self) -> Mapping[Any, Callable[[], Any]]: + return { + **self.send_stream.extra_attributes, + **self.receive_stream.extra_attributes, + } + + +@dataclass(eq=False) +class StapledObjectStream(ObjectStream[T_Item], Generic[T_Item]): + """ + Combines two object streams into a single, bidirectional object stream. + + Extra attributes will be provided from both streams, with the receive stream + providing the values in case of a conflict. + + :param ObjectSendStream send_stream: the sending object stream + :param ObjectReceiveStream receive_stream: the receiving object stream + """ + + send_stream: ObjectSendStream[T_Item] + receive_stream: ObjectReceiveStream[T_Item] + + async def receive(self) -> T_Item: + return await self.receive_stream.receive() + + async def send(self, item: T_Item) -> None: + await self.send_stream.send(item) + + def send_nowait(self, item: T_Item) -> None: + try: + send_nowait = self.send_stream.send_nowait # type: ignore[attr-defined] + except AttributeError as exc: + raise NotImplementedError( + f"'send_nowait' method not implemented in {type(self.send_stream)}" + ) from exc + + send_nowait(item) + + async def send_eof(self) -> None: + await self.send_stream.aclose() + + async def aclose(self) -> None: + await self.send_stream.aclose() + await self.receive_stream.aclose() + + @property + def extra_attributes(self) -> Mapping[Any, Callable[[], Any]]: + return { + **self.send_stream.extra_attributes, + **self.receive_stream.extra_attributes, + } + + +@dataclass(eq=False) +class MultiListener(Listener[T_Stream], Generic[T_Stream]): + """ + Combines multiple listeners into one, serving connections from all of them at once. + + Any MultiListeners in the given collection of listeners will have their listeners + moved into this one. + + Extra attributes are provided from each listener, with each successive listener + overriding any conflicting attributes from the previous one. + + :param listeners: listeners to serve + :type listeners: Sequence[Listener[T_Stream]] + """ + + listeners: Sequence[Listener[T_Stream]] + + def __post_init__(self) -> None: + listeners: list[Listener[T_Stream]] = [] + for listener in self.listeners: + if isinstance(listener, MultiListener): + listeners.extend(listener.listeners) + del listener.listeners[:] # type: ignore[attr-defined] + else: + listeners.append(listener) + + self.listeners = listeners + + async def serve( + self, handler: Callable[[T_Stream], Any], task_group: TaskGroup | None = None + ) -> None: + from .. import create_task_group + + async with create_task_group() as tg: + for listener in self.listeners: + tg.start_soon(listener.serve, handler, task_group) + + async def aclose(self) -> None: + for listener in self.listeners: + await listener.aclose() + + @property + def extra_attributes(self) -> Mapping[Any, Callable[[], Any]]: + attributes: dict = {} + for listener in self.listeners: + attributes.update(listener.extra_attributes) + + return attributes diff --git a/.venv/lib/python3.14/site-packages/anyio/streams/text.py b/.venv/lib/python3.14/site-packages/anyio/streams/text.py new file mode 100644 index 0000000000000000000000000000000000000000..296cd250459f3848bb333301fff1ac32973f219a --- /dev/null +++ b/.venv/lib/python3.14/site-packages/anyio/streams/text.py @@ -0,0 +1,176 @@ +from __future__ import annotations + +__all__ = ( + "TextConnectable", + "TextReceiveStream", + "TextSendStream", + "TextStream", +) + +import codecs +import sys +from collections.abc import Callable, Mapping +from dataclasses import InitVar, dataclass, field +from typing import Any + +from ..abc import ( + AnyByteReceiveStream, + AnyByteSendStream, + AnyByteStream, + AnyByteStreamConnectable, + ObjectReceiveStream, + ObjectSendStream, + ObjectStream, + ObjectStreamConnectable, +) + +if sys.version_info >= (3, 12): + from typing import override +else: + from typing_extensions import override + + +@dataclass(eq=False) +class TextReceiveStream(ObjectReceiveStream[str]): + """ + Stream wrapper that decodes bytes to strings using the given encoding. + + Decoding is done using :class:`~codecs.IncrementalDecoder` which returns any + completely received unicode characters as soon as they come in. + + :param transport_stream: any bytes-based receive stream + :param encoding: character encoding to use for decoding bytes to strings (defaults + to ``utf-8``) + :param errors: handling scheme for decoding errors (defaults to ``strict``; see the + `codecs module documentation`_ for a comprehensive list of options) + + .. _codecs module documentation: + https://docs.python.org/3/library/codecs.html#codec-objects + """ + + transport_stream: AnyByteReceiveStream + encoding: InitVar[str] = "utf-8" + errors: InitVar[str] = "strict" + _decoder: codecs.IncrementalDecoder = field(init=False) + + def __post_init__(self, encoding: str, errors: str) -> None: + decoder_class = codecs.getincrementaldecoder(encoding) + self._decoder = decoder_class(errors=errors) + + async def receive(self) -> str: + while True: + chunk = await self.transport_stream.receive() + decoded = self._decoder.decode(chunk) + if decoded: + return decoded + + async def aclose(self) -> None: + await self.transport_stream.aclose() + self._decoder.reset() + + @property + def extra_attributes(self) -> Mapping[Any, Callable[[], Any]]: + return self.transport_stream.extra_attributes + + +@dataclass(eq=False) +class TextSendStream(ObjectSendStream[str]): + """ + Sends strings to the wrapped stream as bytes using the given encoding. + + :param AnyByteSendStream transport_stream: any bytes-based send stream + :param str encoding: character encoding to use for encoding strings to bytes + (defaults to ``utf-8``) + :param str errors: handling scheme for encoding errors (defaults to ``strict``; see + the `codecs module documentation`_ for a comprehensive list of options) + + .. _codecs module documentation: + https://docs.python.org/3/library/codecs.html#codec-objects + """ + + transport_stream: AnyByteSendStream + encoding: InitVar[str] = "utf-8" + errors: str = "strict" + _encoder: Callable[..., tuple[bytes, int]] = field(init=False) + + def __post_init__(self, encoding: str) -> None: + self._encoder = codecs.getencoder(encoding) + + async def send(self, item: str) -> None: + encoded = self._encoder(item, self.errors)[0] + await self.transport_stream.send(encoded) + + async def aclose(self) -> None: + await self.transport_stream.aclose() + + @property + def extra_attributes(self) -> Mapping[Any, Callable[[], Any]]: + return self.transport_stream.extra_attributes + + +@dataclass(eq=False) +class TextStream(ObjectStream[str]): + """ + A bidirectional stream that decodes bytes to strings on receive and encodes strings + to bytes on send. + + Extra attributes will be provided from both streams, with the receive stream + providing the values in case of a conflict. + + :param AnyByteStream transport_stream: any bytes-based stream + :param str encoding: character encoding to use for encoding/decoding strings to/from + bytes (defaults to ``utf-8``) + :param str errors: handling scheme for encoding errors (defaults to ``strict``; see + the `codecs module documentation`_ for a comprehensive list of options) + + .. _codecs module documentation: + https://docs.python.org/3/library/codecs.html#codec-objects + """ + + transport_stream: AnyByteStream + encoding: InitVar[str] = "utf-8" + errors: InitVar[str] = "strict" + _receive_stream: TextReceiveStream = field(init=False) + _send_stream: TextSendStream = field(init=False) + + def __post_init__(self, encoding: str, errors: str) -> None: + self._receive_stream = TextReceiveStream( + self.transport_stream, encoding=encoding, errors=errors + ) + self._send_stream = TextSendStream( + self.transport_stream, encoding=encoding, errors=errors + ) + + async def receive(self) -> str: + return await self._receive_stream.receive() + + async def send(self, item: str) -> None: + await self._send_stream.send(item) + + async def send_eof(self) -> None: + await self.transport_stream.send_eof() + + async def aclose(self) -> None: + await self._send_stream.aclose() + await self._receive_stream.aclose() + + @property + def extra_attributes(self) -> Mapping[Any, Callable[[], Any]]: + return { + **self._send_stream.extra_attributes, + **self._receive_stream.extra_attributes, + } + + +class TextConnectable(ObjectStreamConnectable[str]): + def __init__(self, connectable: AnyByteStreamConnectable): + """ + :param connectable: the bytestream endpoint to wrap + + """ + self.connectable = connectable + + @override + async def connect(self) -> TextStream: + stream = await self.connectable.connect() + return TextStream(stream) diff --git a/.venv/lib/python3.14/site-packages/anyio/streams/tls.py b/.venv/lib/python3.14/site-packages/anyio/streams/tls.py new file mode 100644 index 0000000000000000000000000000000000000000..b9ddf1d333143767b737bbd5915bebe1309481b2 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/anyio/streams/tls.py @@ -0,0 +1,436 @@ +from __future__ import annotations + +__all__ = ( + "TLSAttribute", + "TLSConnectable", + "TLSListener", + "TLSStream", +) + +import logging +import re +import ssl +import sys +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from functools import wraps +from ssl import SSLContext +from typing import Any, TypeAlias, TypeVar + +from .. import ( + BrokenResourceError, + EndOfStream, + aclose_forcefully, + get_cancelled_exc_class, + to_thread, +) +from .._core._typedattr import TypedAttributeSet, typed_attribute +from ..abc import ( + AnyByteStream, + AnyByteStreamConnectable, + ByteStream, + ByteStreamConnectable, + Listener, + TaskGroup, +) + +if sys.version_info >= (3, 11): + from typing import TypeVarTuple, Unpack +else: + from typing_extensions import TypeVarTuple, Unpack + +if sys.version_info >= (3, 12): + from typing import override +else: + from typing_extensions import override + +T_Retval = TypeVar("T_Retval") +PosArgsT = TypeVarTuple("PosArgsT") +_PCTRTT: TypeAlias = tuple[tuple[str, str], ...] +_PCTRTTT: TypeAlias = tuple[_PCTRTT, ...] + + +class TLSAttribute(TypedAttributeSet): + """Contains Transport Layer Security related attributes.""" + + #: the selected ALPN protocol + alpn_protocol: str | None = typed_attribute() + #: the channel binding for type ``tls-unique`` + channel_binding_tls_unique: bytes = typed_attribute() + #: the selected cipher + cipher: tuple[str, str, int] = typed_attribute() + #: the peer certificate in dictionary form (see :meth:`ssl.SSLSocket.getpeercert` + # for more information) + peer_certificate: None | (dict[str, str | _PCTRTTT | _PCTRTT]) = typed_attribute() + #: the peer certificate in binary form + peer_certificate_binary: bytes | None = typed_attribute() + #: ``True`` if this is the server side of the connection + server_side: bool = typed_attribute() + #: ciphers shared by the client during the TLS handshake (``None`` if this is the + #: client side) + shared_ciphers: list[tuple[str, str, int]] | None = typed_attribute() + #: the :class:`~ssl.SSLObject` used for encryption + ssl_object: ssl.SSLObject = typed_attribute() + #: ``True`` if this stream does (and expects) a closing TLS handshake when the + #: stream is being closed + standard_compatible: bool = typed_attribute() + #: the TLS protocol version (e.g. ``TLSv1.2``) + tls_version: str = typed_attribute() + + +@dataclass(eq=False) +class TLSStream(ByteStream): + """ + A stream wrapper that encrypts all sent data and decrypts received data. + + This class has no public initializer; use :meth:`wrap` instead. + All extra attributes from :class:`~TLSAttribute` are supported. + + :var AnyByteStream transport_stream: the wrapped stream + + """ + + transport_stream: AnyByteStream + standard_compatible: bool + _ssl_object: ssl.SSLObject + _read_bio: ssl.MemoryBIO + _write_bio: ssl.MemoryBIO + + @classmethod + async def wrap( + cls, + transport_stream: AnyByteStream, + *, + server_side: bool | None = None, + hostname: str | None = None, + ssl_context: ssl.SSLContext | None = None, + standard_compatible: bool = True, + ) -> TLSStream: + """ + Wrap an existing stream with Transport Layer Security. + + This performs a TLS handshake with the peer. + + :param transport_stream: a bytes-transporting stream to wrap + :param server_side: ``True`` if this is the server side of the connection, + ``False`` if this is the client side (if omitted, will be set to ``False`` + if ``hostname`` has been provided, ``False`` otherwise). Used only to create + a default context when an explicit context has not been provided. + :param hostname: host name of the peer (if host name checking is desired) + :param ssl_context: the SSLContext object to use (if not provided, a secure + default will be created) + :param standard_compatible: if ``False``, skip the closing handshake when + closing the connection, and don't raise an exception if the peer does the + same + :raises ~ssl.SSLError: if the TLS handshake fails + + """ + if server_side is None: + server_side = not hostname + + if not ssl_context: + purpose = ( + ssl.Purpose.CLIENT_AUTH if server_side else ssl.Purpose.SERVER_AUTH + ) + ssl_context = ssl.create_default_context(purpose) + + # Re-enable detection of unexpected EOFs if it was disabled by Python + if hasattr(ssl, "OP_IGNORE_UNEXPECTED_EOF"): + ssl_context.options &= ~ssl.OP_IGNORE_UNEXPECTED_EOF + + bio_in = ssl.MemoryBIO() + bio_out = ssl.MemoryBIO() + + # Resolve international host names using IDNA 2008. + # Otherwise wrap_bio() would resolve them with IDNA 2003. + if hostname is not None: + from .._core._sockets import idna2008_resolve + + server_hostname: bytes | None = idna2008_resolve(hostname) + else: + server_hostname = None + + # External SSLContext implementations may do blocking I/O in wrap_bio(), + # but the standard library implementation won't + if type(ssl_context) is ssl.SSLContext: + ssl_object = ssl_context.wrap_bio( + bio_in, + bio_out, + server_side=server_side, + server_hostname=server_hostname, + ) + else: + ssl_object = await to_thread.run_sync( + ssl_context.wrap_bio, + bio_in, + bio_out, + server_side, + server_hostname, + None, + ) + + wrapper = cls( + transport_stream=transport_stream, + standard_compatible=standard_compatible, + _ssl_object=ssl_object, + _read_bio=bio_in, + _write_bio=bio_out, + ) + await wrapper._call_sslobject_method(ssl_object.do_handshake) + return wrapper + + async def _call_sslobject_method( + self, func: Callable[[Unpack[PosArgsT]], T_Retval], *args: Unpack[PosArgsT] + ) -> T_Retval: + while True: + try: + result = func(*args) + except ssl.SSLWantReadError: + try: + # Flush any pending writes first + if self._write_bio.pending: + await self.transport_stream.send(self._write_bio.read()) + + data = await self.transport_stream.receive() + except EndOfStream: + self._read_bio.write_eof() + except OSError as exc: + self._read_bio.write_eof() + self._write_bio.write_eof() + raise BrokenResourceError from exc + else: + self._read_bio.write(data) + except ssl.SSLWantWriteError: + await self.transport_stream.send(self._write_bio.read()) + except ssl.SSLSyscallError as exc: + self._read_bio.write_eof() + self._write_bio.write_eof() + raise BrokenResourceError from exc + except ssl.SSLError as exc: + self._read_bio.write_eof() + self._write_bio.write_eof() + if isinstance(exc, ssl.SSLEOFError) or ( + exc.strerror and "UNEXPECTED_EOF_WHILE_READING" in exc.strerror + ): + if self.standard_compatible: + raise BrokenResourceError from exc + else: + raise EndOfStream from None + + raise + else: + # Flush any pending writes first + if self._write_bio.pending: + await self.transport_stream.send(self._write_bio.read()) + + return result + + async def unwrap(self) -> tuple[AnyByteStream, bytes]: + """ + Does the TLS closing handshake. + + :return: a tuple of (wrapped byte stream, bytes left in the read buffer) + + """ + await self._call_sslobject_method(self._ssl_object.unwrap) + self._read_bio.write_eof() + self._write_bio.write_eof() + return self.transport_stream, self._read_bio.read() + + async def aclose(self) -> None: + if self.standard_compatible: + try: + await self.unwrap() + except BaseException: + await aclose_forcefully(self.transport_stream) + raise + + await self.transport_stream.aclose() + + async def receive(self, max_bytes: int = 65536) -> bytes: + if max_bytes < 1: + raise ValueError("max_bytes must be a positive integer") + + data = await self._call_sslobject_method(self._ssl_object.read, max_bytes) + if not data: + raise EndOfStream + + return data + + async def send(self, item: bytes) -> None: + await self._call_sslobject_method(self._ssl_object.write, item) + + async def send_eof(self) -> None: + tls_version = self.extra(TLSAttribute.tls_version) + match = re.match(r"TLSv(\d+)(?:\.(\d+))?", tls_version) + if match: + major, minor = int(match.group(1)), int(match.group(2) or 0) + if (major, minor) < (1, 3): + raise NotImplementedError( + f"send_eof() requires at least TLSv1.3; current " + f"session uses {tls_version}" + ) + + raise NotImplementedError( + "send_eof() has not yet been implemented for TLS streams" + ) + + @property + def extra_attributes(self) -> Mapping[Any, Callable[[], Any]]: + return { + **self.transport_stream.extra_attributes, + TLSAttribute.alpn_protocol: self._ssl_object.selected_alpn_protocol, + TLSAttribute.channel_binding_tls_unique: ( + self._ssl_object.get_channel_binding + ), + TLSAttribute.cipher: self._ssl_object.cipher, + TLSAttribute.peer_certificate: lambda: self._ssl_object.getpeercert(False), + TLSAttribute.peer_certificate_binary: lambda: self._ssl_object.getpeercert( + True + ), + TLSAttribute.server_side: lambda: self._ssl_object.server_side, + TLSAttribute.shared_ciphers: lambda: ( + self._ssl_object.shared_ciphers() + if self._ssl_object.server_side + else None + ), + TLSAttribute.standard_compatible: lambda: self.standard_compatible, + TLSAttribute.ssl_object: lambda: self._ssl_object, + TLSAttribute.tls_version: self._ssl_object.version, + } + + +@dataclass(eq=False) +class TLSListener(Listener[TLSStream]): + """ + A convenience listener that wraps another listener and auto-negotiates a TLS session + on every accepted connection. + + If the TLS handshake times out or raises an exception, + :meth:`handle_handshake_error` is called to do whatever post-mortem processing is + deemed necessary. + + Supports only the :attr:`~TLSAttribute.standard_compatible` extra attribute. + + :param Listener listener: the listener to wrap + :param ssl_context: the SSL context object + :param standard_compatible: a flag passed through to :meth:`TLSStream.wrap` + :param handshake_timeout: time limit for the TLS handshake + (passed to :func:`~anyio.fail_after`) + """ + + listener: Listener[Any] + ssl_context: ssl.SSLContext + standard_compatible: bool = True + handshake_timeout: float = 30 + + @staticmethod + async def handle_handshake_error(exc: BaseException, stream: AnyByteStream) -> None: + """ + Handle an exception raised during the TLS handshake. + + This method does 3 things: + + #. Forcefully closes the original stream + #. Logs the exception (unless it was a cancellation exception) using the + ``anyio.streams.tls`` logger + #. Reraises the exception if it was a base exception or a cancellation exception + + :param exc: the exception + :param stream: the original stream + + """ + await aclose_forcefully(stream) + + # Log all except cancellation exceptions + if not isinstance(exc, get_cancelled_exc_class()): + # CPython (as of 3.11.5) returns incorrect `sys.exc_info()` here when using + # any asyncio implementation, so we explicitly pass the exception to log + # (https://github.com/python/cpython/issues/108668). Trio does not have this + # issue because it works around the CPython bug. + logging.getLogger(__name__).exception( + "Error during TLS handshake", exc_info=exc + ) + + # Only reraise base exceptions and cancellation exceptions + if not isinstance(exc, Exception) or isinstance(exc, get_cancelled_exc_class()): + raise # noqa: PLE0704 + + async def serve( + self, + handler: Callable[[TLSStream], Any], + task_group: TaskGroup | None = None, + ) -> None: + @wraps(handler) + async def handler_wrapper(stream: AnyByteStream) -> None: + from .. import fail_after + + try: + with fail_after(self.handshake_timeout): + wrapped_stream = await TLSStream.wrap( + stream, + ssl_context=self.ssl_context, + standard_compatible=self.standard_compatible, + ) + except BaseException as exc: + await self.handle_handshake_error(exc, stream) + else: + await handler(wrapped_stream) + + await self.listener.serve(handler_wrapper, task_group) + + async def aclose(self) -> None: + await self.listener.aclose() + + @property + def extra_attributes(self) -> Mapping[Any, Callable[[], Any]]: + return { + TLSAttribute.standard_compatible: lambda: self.standard_compatible, + } + + +class TLSConnectable(ByteStreamConnectable): + """ + Wraps another connectable and does TLS negotiation after a successful connection. + + :param connectable: the connectable to wrap + :param hostname: host name of the server (if host name checking is desired) + :param ssl_context: the SSLContext object to use (if not provided, a secure default + will be created) + :param standard_compatible: if ``False``, skip the closing handshake when closing + the connection, and don't raise an exception if the server does the same + """ + + def __init__( + self, + connectable: AnyByteStreamConnectable, + *, + hostname: str | None = None, + ssl_context: ssl.SSLContext | None = None, + standard_compatible: bool = True, + ) -> None: + self.connectable = connectable + self.ssl_context: SSLContext = ssl_context or ssl.create_default_context( + ssl.Purpose.SERVER_AUTH + ) + if not isinstance(self.ssl_context, ssl.SSLContext): + raise TypeError( + "ssl_context must be an instance of ssl.SSLContext, not " + f"{type(self.ssl_context).__name__}" + ) + self.hostname = hostname + self.standard_compatible = standard_compatible + + @override + async def connect(self) -> TLSStream: + stream = await self.connectable.connect() + try: + return await TLSStream.wrap( + stream, + hostname=self.hostname, + ssl_context=self.ssl_context, + standard_compatible=self.standard_compatible, + ) + except BaseException: + await aclose_forcefully(stream) + raise diff --git a/.venv/lib/python3.14/site-packages/anyio/to_interpreter.py b/.venv/lib/python3.14/site-packages/anyio/to_interpreter.py new file mode 100644 index 0000000000000000000000000000000000000000..54ef4a8b8c58db9ea05e4188cf6d7899c8ff8c72 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/anyio/to_interpreter.py @@ -0,0 +1,246 @@ +from __future__ import annotations + +__all__ = ( + "current_default_interpreter_limiter", + "run_sync", +) + +import atexit +import os +import sys +from collections import deque +from collections.abc import Callable +from typing import Any, Final, TypeVar + +from . import current_time, to_thread +from ._core._exceptions import BrokenWorkerInterpreter +from ._core._synchronization import CapacityLimiter +from .lowlevel import RunVar + +if sys.version_info >= (3, 11): + from typing import TypeVarTuple, Unpack +else: + from typing_extensions import TypeVarTuple, Unpack + +if sys.version_info >= (3, 14): + from concurrent.interpreters import ExecutionFailed, create + + def _interp_call( + func: Callable[..., Any], args: tuple[Any, ...] + ) -> tuple[Any, bool]: + try: + retval = func(*args) + except BaseException as exc: + return exc, True + else: + return retval, False + + class _Worker: + last_used: float = 0 + + def __init__(self) -> None: + self._interpreter = create() + + def destroy(self) -> None: + self._interpreter.close() + + def call( + self, + func: Callable[..., T_Retval], + args: tuple[Any, ...], + ) -> T_Retval: + try: + res, is_exception = self._interpreter.call(_interp_call, func, args) + except ExecutionFailed as exc: + raise BrokenWorkerInterpreter(exc.excinfo) from exc + + if is_exception: + raise res + + return res +elif sys.version_info >= (3, 13): + import _interpqueues + import _interpreters + + UNBOUND: Final = 2 # I have no clue how this works, but it was used in the stdlib + FMT_UNPICKLED: Final = 0 + FMT_PICKLED: Final = 1 + QUEUE_PICKLE_ARGS: Final = (FMT_PICKLED, UNBOUND) + QUEUE_UNPICKLE_ARGS: Final = (FMT_UNPICKLED, UNBOUND) + + _run_func = compile( + """ +import _interpqueues +from _interpreters import NotShareableError +from pickle import loads, dumps, HIGHEST_PROTOCOL + +QUEUE_PICKLE_ARGS = (1, 2) +QUEUE_UNPICKLE_ARGS = (0, 2) + +item = _interpqueues.get(queue_id)[0] +try: + func, args = loads(item) + retval = func(*args) +except BaseException as exc: + is_exception = True + retval = exc +else: + is_exception = False + +try: + _interpqueues.put(queue_id, (retval, is_exception), *QUEUE_UNPICKLE_ARGS) +except NotShareableError: + retval = dumps(retval, HIGHEST_PROTOCOL) + _interpqueues.put(queue_id, (retval, is_exception), *QUEUE_PICKLE_ARGS) + """, + "", + "exec", + ) + + class _Worker: + last_used: float = 0 + + def __init__(self) -> None: + self._interpreter_id = _interpreters.create() + self._queue_id = _interpqueues.create(1, *QUEUE_UNPICKLE_ARGS) + _interpreters.set___main___attrs( + self._interpreter_id, {"queue_id": self._queue_id} + ) + + def destroy(self) -> None: + _interpqueues.destroy(self._queue_id) + _interpreters.destroy(self._interpreter_id) + + def call( + self, + func: Callable[..., T_Retval], + args: tuple[Any, ...], + ) -> T_Retval: + import pickle + + item = pickle.dumps((func, args), pickle.HIGHEST_PROTOCOL) + _interpqueues.put(self._queue_id, item, *QUEUE_PICKLE_ARGS) + exc_info = _interpreters.exec(self._interpreter_id, _run_func) + if exc_info: + raise BrokenWorkerInterpreter(exc_info) + + res = _interpqueues.get(self._queue_id) + (res, is_exception), fmt = res[:2] + if fmt == FMT_PICKLED: + res = pickle.loads(res) + + if is_exception: + raise res + + return res +else: + + class _Worker: + last_used: float = 0 + + def __init__(self) -> None: + raise RuntimeError("subinterpreters require at least Python 3.13") + + def call( + self, + func: Callable[..., T_Retval], + args: tuple[Any, ...], + ) -> T_Retval: + raise NotImplementedError + + def destroy(self) -> None: + pass + + +DEFAULT_CPU_COUNT: Final = 8 # this is just an arbitrarily selected value +MAX_WORKER_IDLE_TIME = ( + 30 # seconds a subinterpreter can be idle before becoming eligible for pruning +) + +T_Retval = TypeVar("T_Retval") +PosArgsT = TypeVarTuple("PosArgsT") + +_idle_workers = RunVar[deque[_Worker]]("_available_workers") +_default_interpreter_limiter = RunVar[CapacityLimiter]("_default_interpreter_limiter") + + +def _stop_workers(workers: deque[_Worker]) -> None: + for worker in workers: + worker.destroy() + + workers.clear() + + +async def run_sync( + func: Callable[[Unpack[PosArgsT]], T_Retval], + *args: Unpack[PosArgsT], + limiter: CapacityLimiter | None = None, +) -> T_Retval: + """ + Call the given function with the given arguments in a subinterpreter. + + .. warning:: On Python 3.13, the :mod:`concurrent.interpreters` module was not yet + available, so the code path for that Python version relies on an undocumented, + private API. As such, it is recommended to not rely on this function for anything + mission-critical on Python 3.13. + + :param func: a callable + :param args: the positional arguments for the callable + :param limiter: capacity limiter to use to limit the total number of subinterpreters + running (if omitted, the default limiter is used) + :return: the result of the call + :raises BrokenWorkerInterpreter: if there's an internal error in a subinterpreter + + """ + if limiter is None: + limiter = current_default_interpreter_limiter() + + try: + idle_workers = _idle_workers.get() + except LookupError: + idle_workers = deque() + _idle_workers.set(idle_workers) + atexit.register(_stop_workers, idle_workers) + + async with limiter: + try: + worker = idle_workers.pop() + except IndexError: + worker = _Worker() + + try: + return await to_thread.run_sync( + worker.call, + func, + args, + limiter=limiter, + ) + finally: + # Prune workers that have been idle for too long + now = current_time() + while idle_workers: + if now - idle_workers[0].last_used <= MAX_WORKER_IDLE_TIME: + break + + await to_thread.run_sync(idle_workers.popleft().destroy, limiter=limiter) + + worker.last_used = current_time() + idle_workers.append(worker) + + +def current_default_interpreter_limiter() -> CapacityLimiter: + """ + Return the capacity limiter used by default to limit the number of concurrently + running subinterpreters. + + Defaults to the number of CPU cores. + + :return: a capacity limiter object + + """ + try: + return _default_interpreter_limiter.get() + except LookupError: + limiter = CapacityLimiter(os.cpu_count() or DEFAULT_CPU_COUNT) + _default_interpreter_limiter.set(limiter) + return limiter diff --git a/.venv/lib/python3.14/site-packages/anyio/to_process.py b/.venv/lib/python3.14/site-packages/anyio/to_process.py new file mode 100644 index 0000000000000000000000000000000000000000..584128ec3b2d4bbd0a9634fc7f2f39caf1fe968d --- /dev/null +++ b/.venv/lib/python3.14/site-packages/anyio/to_process.py @@ -0,0 +1,269 @@ +from __future__ import annotations + +__all__ = ( + "current_default_process_limiter", + "process_worker", + "run_sync", +) + +import os +import pickle +import runpy +import subprocess +import sys +from collections import deque +from collections.abc import Callable +from types import ModuleType +from typing import TypeVar, cast + +from ._core._eventloop import current_time, get_async_backend, get_cancelled_exc_class +from ._core._exceptions import BrokenWorkerProcess +from ._core._subprocesses import open_process +from ._core._synchronization import CapacityLimiter +from ._core._tasks import CancelScope, fail_after +from .abc import ByteReceiveStream, ByteSendStream, Process +from .lowlevel import RunVar, checkpoint_if_cancelled +from .streams.buffered import BufferedByteReceiveStream + +if sys.version_info >= (3, 11): + from typing import TypeVarTuple, Unpack +else: + from typing_extensions import TypeVarTuple, Unpack + +WORKER_MAX_IDLE_TIME = 300 # 5 minutes + +T_Retval = TypeVar("T_Retval") +PosArgsT = TypeVarTuple("PosArgsT") + +_process_pool_workers: RunVar[set[Process]] = RunVar("_process_pool_workers") +_process_pool_idle_workers: RunVar[deque[tuple[Process, float]]] = RunVar( + "_process_pool_idle_workers" +) +_default_process_limiter: RunVar[CapacityLimiter] = RunVar("_default_process_limiter") + + +async def run_sync( # type: ignore[return] + func: Callable[[Unpack[PosArgsT]], T_Retval], + *args: Unpack[PosArgsT], + cancellable: bool = False, + limiter: CapacityLimiter | None = None, +) -> T_Retval: + """ + Call the given function with the given arguments in a worker process. + + If the ``cancellable`` option is enabled and the task waiting for its completion is + cancelled, the worker process running it will be abruptly terminated using SIGKILL + (or ``terminateProcess()`` on Windows). + + :param func: a callable + :param args: positional arguments for the callable + :param cancellable: ``True`` to allow cancellation of the operation while it's + running + :param limiter: capacity limiter to use to limit the total amount of processes + running (if omitted, the default limiter is used) + :raises NoEventLoopError: if no supported asynchronous event loop is running in the + current thread + :return: an awaitable that yields the return value of the function. + + """ + + async def send_raw_command(pickled_cmd: bytes) -> object: + try: + await stdin.send(pickled_cmd) + response = await buffered.receive_until(b"\n", 50) + status, length = response.split(b" ") + if status not in (b"RETURN", b"EXCEPTION"): + raise RuntimeError( + f"Worker process returned unexpected response: {response!r}" + ) + + pickled_response = await buffered.receive_exactly(int(length)) + except BaseException as exc: + workers.discard(process) + try: + process.kill() + with CancelScope(shield=True): + await process.aclose() + except ProcessLookupError: + pass + + if isinstance(exc, get_cancelled_exc_class()): + raise + else: + raise BrokenWorkerProcess from exc + + retval = pickle.loads(pickled_response) + if status == b"EXCEPTION": + assert isinstance(retval, BaseException) + raise retval + else: + return retval + + # First pickle the request before trying to reserve a worker process + await checkpoint_if_cancelled() + request = pickle.dumps(("run", func, args), protocol=pickle.HIGHEST_PROTOCOL) + + # If this is the first run in this event loop thread, set up the necessary variables + try: + workers = _process_pool_workers.get() + idle_workers = _process_pool_idle_workers.get() + except LookupError: + workers = set() + idle_workers = deque() + _process_pool_workers.set(workers) + _process_pool_idle_workers.set(idle_workers) + get_async_backend().setup_process_pool_exit_at_shutdown(workers) + + async with limiter or current_default_process_limiter(): + # Pop processes from the pool (starting from the most recently used) until we + # find one that hasn't exited yet + process: Process + while idle_workers: + process, _idle_since = idle_workers.pop() + if process.returncode is None: + stdin = cast(ByteSendStream, process.stdin) + buffered = BufferedByteReceiveStream( + cast(ByteReceiveStream, process.stdout) + ) + + # Prune any other workers that have been idle for WORKER_MAX_IDLE_TIME + # seconds or longer + now = current_time() + killed_processes: list[Process] = [] + while idle_workers: + if now - idle_workers[0][1] < WORKER_MAX_IDLE_TIME: + break + + process_to_kill, _idle_since = idle_workers.popleft() + process_to_kill.kill() + workers.remove(process_to_kill) + killed_processes.append(process_to_kill) + + with CancelScope(shield=True): + for killed_process in killed_processes: + await killed_process.aclose() + + break + + workers.remove(process) + else: + command = [sys.executable, "-u", "-m", __name__] + process = await open_process( + command, stdin=subprocess.PIPE, stdout=subprocess.PIPE + ) + try: + stdin = cast(ByteSendStream, process.stdin) + buffered = BufferedByteReceiveStream( + cast(ByteReceiveStream, process.stdout) + ) + with fail_after(20): + message = await buffered.receive(6) + + if message != b"READY\n": + raise BrokenWorkerProcess( + f"Worker process returned unexpected response: {message!r}" + ) + + main_module_path = getattr(sys.modules["__main__"], "__file__", None) + pickled = pickle.dumps( + ("init", sys.path, main_module_path), + protocol=pickle.HIGHEST_PROTOCOL, + ) + await send_raw_command(pickled) + except (BrokenWorkerProcess, get_cancelled_exc_class()): + raise + except BaseException as exc: + process.kill() + raise BrokenWorkerProcess( + "Error during worker process initialization" + ) from exc + + workers.add(process) + + with CancelScope(shield=not cancellable): + try: + return cast(T_Retval, await send_raw_command(request)) + finally: + if process in workers: + idle_workers.append((process, current_time())) + + +def current_default_process_limiter() -> CapacityLimiter: + """ + Return the capacity limiter that is used by default to limit the number of worker + processes. + + :return: a capacity limiter object + + """ + try: + return _default_process_limiter.get() + except LookupError: + limiter = CapacityLimiter(os.cpu_count() or 2) + _default_process_limiter.set(limiter) + return limiter + + +def process_worker() -> None: + # Redirect standard streams to os.devnull so that user code won't interfere with the + # parent-worker communication + stdin = sys.stdin + stdout = sys.stdout + sys.stdin = open(os.devnull) + sys.stdout = open(os.devnull, "w") + sys.stderr = open(os.devnull, "w") + + stdout.buffer.write(b"READY\n") + while True: + retval = exception = None + try: + command, *args = pickle.load(stdin.buffer) + except EOFError: + return + except BaseException as exc: + exception = exc + else: + if command == "run": + func, args = args + try: + retval = func(*args) + except BaseException as exc: + exception = exc + elif command == "init": + main_module_path: str | None + sys.path, main_module_path = args + del sys.modules["__main__"] + if main_module_path and os.path.isfile(main_module_path): + # Load the parent's main module but as __mp_main__ instead of + # __main__ (like multiprocessing does) to avoid infinite recursion + try: + main = ModuleType("__mp_main__") + main_content = runpy.run_path( + main_module_path, run_name="__mp_main__" + ) + main.__dict__.update(main_content) + sys.modules["__main__"] = sys.modules["__mp_main__"] = main + except BaseException as exc: + exception = exc + try: + if exception is not None: + status = b"EXCEPTION" + pickled = pickle.dumps(exception, pickle.HIGHEST_PROTOCOL) + else: + status = b"RETURN" + pickled = pickle.dumps(retval, pickle.HIGHEST_PROTOCOL) + except BaseException as exc: + exception = exc + status = b"EXCEPTION" + pickled = pickle.dumps(exc, pickle.HIGHEST_PROTOCOL) + + stdout.buffer.write(b"%s %d\n" % (status, len(pickled))) + stdout.buffer.write(pickled) + + # Respect SIGTERM + if isinstance(exception, SystemExit): + raise exception + + +if __name__ == "__main__": + process_worker() diff --git a/.venv/lib/python3.14/site-packages/anyio/to_thread.py b/.venv/lib/python3.14/site-packages/anyio/to_thread.py new file mode 100644 index 0000000000000000000000000000000000000000..5cdb7253e3c0a4f99ad2f893d31cc32f8fbda298 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/anyio/to_thread.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +__all__ = ( + "current_default_thread_limiter", + "run_sync", +) + +import sys +from collections.abc import Callable +from typing import TYPE_CHECKING, TypeVar +from warnings import warn + +from ._core._eventloop import get_async_backend + +if TYPE_CHECKING: + from ._core._synchronization import CapacityLimiter + +if sys.version_info >= (3, 11): + from typing import TypeVarTuple, Unpack +else: + from typing_extensions import TypeVarTuple, Unpack + +T_Retval = TypeVar("T_Retval") +PosArgsT = TypeVarTuple("PosArgsT") + + +async def run_sync( + func: Callable[[Unpack[PosArgsT]], T_Retval], + *args: Unpack[PosArgsT], + abandon_on_cancel: bool = False, + cancellable: bool | None = None, + limiter: CapacityLimiter | None = None, +) -> T_Retval: + """ + Call the given function with the given arguments in a worker thread. + + If the ``abandon_on_cancel`` option is enabled and the task waiting for its + completion is cancelled, the thread will still run its course but its + return value (or any raised exception) will be ignored. + + :param func: a callable + :param args: positional arguments for the callable + :param abandon_on_cancel: ``True`` to abandon the thread (leaving it to run + unchecked on own) if the host task is cancelled, ``False`` to ignore + cancellations in the host task until the operation has completed in the worker + thread + :param cancellable: deprecated alias of ``abandon_on_cancel``; will override + ``abandon_on_cancel`` if both parameters are passed + :param limiter: capacity limiter to use to limit the total amount of threads running + (if omitted, the default limiter is used) + :raises NoEventLoopError: if no supported asynchronous event loop is running in the + current thread + :return: an awaitable that yields the return value of the function. + + """ + if cancellable is not None: + abandon_on_cancel = cancellable + warn( + "The `cancellable=` keyword argument to `anyio.to_thread.run_sync` is " + "deprecated since AnyIO 4.1.0; use `abandon_on_cancel=` instead", + DeprecationWarning, + stacklevel=2, + ) + + return await get_async_backend().run_sync_in_worker_thread( + func, args, abandon_on_cancel=abandon_on_cancel, limiter=limiter + ) + + +def current_default_thread_limiter() -> CapacityLimiter: + """ + Return the capacity limiter that is used by default to limit the number of + concurrent threads. + + :return: a capacity limiter object + :raises NoEventLoopError: if no supported asynchronous event loop is running in the + current thread + + """ + return get_async_backend().current_default_thread_limiter() diff --git a/.venv/lib/python3.14/site-packages/click-8.5.0.dist-info/INSTALLER b/.venv/lib/python3.14/site-packages/click-8.5.0.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/click-8.5.0.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/.venv/lib/python3.14/site-packages/click-8.5.0.dist-info/METADATA b/.venv/lib/python3.14/site-packages/click-8.5.0.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..7b785308a746c2f777c0d53a6c0f47344d74ecce --- /dev/null +++ b/.venv/lib/python3.14/site-packages/click-8.5.0.dist-info/METADATA @@ -0,0 +1,83 @@ +Metadata-Version: 2.4 +Name: click +Version: 8.5.0 +Summary: Composable command line interface toolkit +Maintainer-email: Pallets +Requires-Python: >=3.10 +Description-Content-Type: text/markdown +License-Expression: BSD-3-Clause +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python +Classifier: Typing :: Typed +License-File: LICENSE.txt +Project-URL: Changes, https://click.palletsprojects.com/page/changes/ +Project-URL: Chat, https://discord.gg/pallets +Project-URL: Documentation, https://click.palletsprojects.com/ +Project-URL: Donate, https://palletsprojects.com/donate +Project-URL: Source, https://github.com/pallets/click/ + +
+ +# Click + +Click is a Python package for creating beautiful command line interfaces +in a composable way with as little code as necessary. It's the "Command +Line Interface Creation Kit". It's highly configurable but comes with +sensible defaults out of the box. + +It aims to make the process of writing command line tools quick and fun +while also preventing any frustration caused by the inability to +implement an intended CLI API. + +Click in three points: + +- Arbitrary nesting of commands +- Automatic help page generation +- Supports lazy loading of subcommands at runtime + + +## A Simple Example + +```python +import click + +@click.command() +@click.option("--count", default=1, help="Number of greetings.") +@click.option("--name", prompt="Your name", help="The person to greet.") +def hello(count, name): + """Simple program that greets NAME for a total of COUNT times.""" + for _ in range(count): + click.echo(f"Hello, {name}!") + +if __name__ == '__main__': + hello() +``` + +``` +$ python hello.py --count=3 +Your name: Click +Hello, Click! +Hello, Click! +Hello, Click! +``` + + +## Donate + +The Pallets organization develops and supports Click and other popular +packages. In order to grow the community of contributors and users, and +allow the maintainers to devote more time to the projects, [please +donate today][]. + +[please donate today]: https://palletsprojects.com/donate + +## Contributing + +See our [detailed contributing documentation][contrib] for many ways to +contribute, including reporting issues, requesting features, asking or answering +questions, and making PRs. + +[contrib]: https://palletsprojects.com/contributing/ + diff --git a/.venv/lib/python3.14/site-packages/click-8.5.0.dist-info/RECORD b/.venv/lib/python3.14/site-packages/click-8.5.0.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..92da42c478f3e8abac3f971f6dda35f0a419f507 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/click-8.5.0.dist-info/RECORD @@ -0,0 +1,40 @@ +click-8.5.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +click-8.5.0.dist-info/METADATA,sha256=6HvOC9GU3nDfuOcI4KawSDAJ8lYRgE6vh7_rY-bCBQE,2567 +click-8.5.0.dist-info/RECORD,, +click-8.5.0.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82 +click-8.5.0.dist-info/licenses/LICENSE.txt,sha256=morRBqOU6FO_4h9C9OctWSgZoigF2ZG18ydQKSkrZY0,1475 +click/__init__.py,sha256=Wr_FTTfUfMeIt-egXpUUeH-MWgt9tCnQ8k10isiZZMo,5145 +click/__pycache__/__init__.cpython-314.pyc,, +click/__pycache__/_compat.cpython-314.pyc,, +click/__pycache__/_termui_impl.cpython-314.pyc,, +click/__pycache__/_textwrap.cpython-314.pyc,, +click/__pycache__/_utils.cpython-314.pyc,, +click/__pycache__/_winconsole.cpython-314.pyc,, +click/__pycache__/core.cpython-314.pyc,, +click/__pycache__/decorators.cpython-314.pyc,, +click/__pycache__/exceptions.cpython-314.pyc,, +click/__pycache__/formatting.cpython-314.pyc,, +click/__pycache__/globals.cpython-314.pyc,, +click/__pycache__/parser.cpython-314.pyc,, +click/__pycache__/shell_completion.cpython-314.pyc,, +click/__pycache__/termui.cpython-314.pyc,, +click/__pycache__/testing.cpython-314.pyc,, +click/__pycache__/types.cpython-314.pyc,, +click/__pycache__/utils.cpython-314.pyc,, +click/_compat.py,sha256=jbHaKWXPPl3GbGxTgzy8YT6hzSRAudDraqg2DrRtSRM,17939 +click/_termui_impl.py,sha256=iLl9m3pblHsQoW2KsmmDe8hMflXCi6cupfL7M5ESM2o,32805 +click/_textwrap.py,sha256=7Z0N7Vmn-66TNSTUwp6OXJbcUXRmYET9h9c2ucD8oQQ,6270 +click/_utils.py,sha256=eCZCtwJtsYD5QYkkNWJ8MY_8ABIjy8MczgMMyVY32rQ,996 +click/_winconsole.py,sha256=KSxfNbMlYRa6GOJuCLgsg2Pb3dVkgJNPqLJPae-Pa10,8543 +click/core.py,sha256=TGWmE8HEB9zpB6ThI7Es7F_g9iCIqLn4b6vUtgxLbXg,147845 +click/decorators.py,sha256=FgaTV2FWkfzfyceUu1FfYAnrNarW96dsAXvc4G2ujVU,21680 +click/exceptions.py,sha256=HvSY34G4auj_bYRR8-T8CU8Jwq_1-OcsRU4ezfozeEk,11862 +click/formatting.py,sha256=8SW2KGkvjfz9Q1NbeojMHuZBN0cfnQJDs4mqDP6oXms,10444 +click/globals.py,sha256=gM-Nh6A4M0HB_SgkaF5M4ncGGMDHc_flHXu9_oh4GEU,1923 +click/parser.py,sha256=oJ-fU_3mvxugIuNtHaCATZ56lgEmHRggjJiSqEgYrjA,19052 +click/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +click/shell_completion.py,sha256=GFIvdd3_5lXpQ5eNGVMlXER4QoZ9P9Z_vcFUPT332gk,26172 +click/termui.py,sha256=jsgCK45FKPhz7EExlKXRYU6W2lC2xKcfYmCzJnYqfpw,35370 +click/testing.py,sha256=dvTmSbxTAA1mfIwN7_FZGXD6A0sPCFogcWWDmyMBAZo,27584 +click/types.py,sha256=HpiJBKen9oq61jDH-5wDikNF-i5nAcl2puUfEL37Ut0,46842 +click/utils.py,sha256=9D1XQ9wHJA5hL-76CmnSsWl7aQAUn4-An_zZfstILb8,21483 diff --git a/.venv/lib/python3.14/site-packages/click-8.5.0.dist-info/WHEEL b/.venv/lib/python3.14/site-packages/click-8.5.0.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..d8b9936dad9ab2513fa6979f411560d3b6b57e37 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/click-8.5.0.dist-info/WHEEL @@ -0,0 +1,4 @@ +Wheel-Version: 1.0 +Generator: flit 3.12.0 +Root-Is-Purelib: true +Tag: py3-none-any diff --git a/.venv/lib/python3.14/site-packages/click-8.5.0.dist-info/licenses/LICENSE.txt b/.venv/lib/python3.14/site-packages/click-8.5.0.dist-info/licenses/LICENSE.txt new file mode 100644 index 0000000000000000000000000000000000000000..d12a849186982399c537c5b9a8fd77bf2edd5eab --- /dev/null +++ b/.venv/lib/python3.14/site-packages/click-8.5.0.dist-info/licenses/LICENSE.txt @@ -0,0 +1,28 @@ +Copyright 2014 Pallets + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/.venv/lib/python3.14/site-packages/click/__init__.py b/.venv/lib/python3.14/site-packages/click/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a34ab023a658c1b38fe75e57781afd3380718638 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/click/__init__.py @@ -0,0 +1,144 @@ +""" +Click is a simple Python module inspired by the stdlib optparse to make +writing command line scripts fun. Unlike other modules, it's based +around a simple API that does not come with too much magic and is +composable. +""" + +from __future__ import annotations + +from .core import Argument as Argument +from .core import Command as Command +from .core import CommandCollection as CommandCollection +from .core import Context as Context +from .core import Group as Group +from .core import Option as Option +from .core import Parameter as Parameter +from .core import ParameterSource as ParameterSource +from .decorators import argument as argument +from .decorators import command as command +from .decorators import confirmation_option as confirmation_option +from .decorators import custom_version_option as custom_version_option +from .decorators import group as group +from .decorators import help_option as help_option +from .decorators import make_pass_decorator as make_pass_decorator +from .decorators import option as option +from .decorators import pass_context as pass_context +from .decorators import pass_obj as pass_obj +from .decorators import password_option as password_option +from .decorators import version_option as version_option +from .exceptions import Abort as Abort +from .exceptions import BadArgumentUsage as BadArgumentUsage +from .exceptions import BadOptionUsage as BadOptionUsage +from .exceptions import BadParameter as BadParameter +from .exceptions import ClickException as ClickException +from .exceptions import FileError as FileError +from .exceptions import MissingParameter as MissingParameter +from .exceptions import NoSuchCommand as NoSuchCommand +from .exceptions import NoSuchOption as NoSuchOption +from .exceptions import UsageError as UsageError +from .formatting import HelpFormatter as HelpFormatter +from .formatting import wrap_text as wrap_text +from .globals import get_current_context as get_current_context +from .termui import clear as clear +from .termui import confirm as confirm +from .termui import echo_via_pager as echo_via_pager +from .termui import edit as edit +from .termui import get_pager_file as get_pager_file +from .termui import getchar as getchar +from .termui import launch as launch +from .termui import pause as pause +from .termui import progressbar as progressbar +from .termui import prompt as prompt +from .termui import secho as secho +from .termui import style as style +from .termui import unstyle as unstyle +from .types import BOOL as BOOL +from .types import Choice as Choice +from .types import DateTime as DateTime +from .types import File as File +from .types import FLOAT as FLOAT +from .types import FloatRange as FloatRange +from .types import INT as INT +from .types import IntRange as IntRange +from .types import ParamType as ParamType +from .types import Path as Path +from .types import STRING as STRING +from .types import Tuple as Tuple +from .types import UNPROCESSED as UNPROCESSED +from .types import UUID as UUID +from .utils import echo as echo +from .utils import format_filename as format_filename +from .utils import get_app_dir as get_app_dir +from .utils import open_file as open_file + + +def __getattr__(name: str) -> object: + import warnings + + if name == "BaseCommand": + from .core import _BaseCommand + + warnings.warn( + "'BaseCommand' is deprecated and will be removed in Click 9.0. Use" + " 'Command' instead.", + DeprecationWarning, + stacklevel=2, + ) + return _BaseCommand + + if name == "MultiCommand": + from .core import _MultiCommand + + warnings.warn( + "'MultiCommand' is deprecated and will be removed in Click 9.0. Use" + " 'Group' instead.", + DeprecationWarning, + stacklevel=2, + ) + return _MultiCommand + + if name == "OptionParser": + from .parser import _OptionParser + + warnings.warn( + "'OptionParser' is deprecated and will be removed in Click 9.0. The" + " old parser is available in 'optparse'.", + DeprecationWarning, + stacklevel=2, + ) + return _OptionParser + + if name == "__version__": + import importlib.metadata + + warnings.warn( + "The '__version__' attribute is deprecated and will be removed in" + " Click 9.1. Use feature detection or" + " 'importlib.metadata.version(\"click\")' instead.", + DeprecationWarning, + stacklevel=2, + ) + return importlib.metadata.version("click") + + if name == "get_binary_stream": + from .utils import _get_binary_stream + + warnings.warn( + "'get_binary_stream' is deprecated and will be removed in Click 9.0.", + DeprecationWarning, + stacklevel=2, + ) + return _get_binary_stream + + if name == "get_text_stream": + from .utils import _get_text_stream + + warnings.warn( + "'get_text_stream' is deprecated and will be removed in Click 9.0.", + DeprecationWarning, + stacklevel=2, + ) + return _get_text_stream + + raise AttributeError(name) diff --git a/.venv/lib/python3.14/site-packages/click/__pycache__/__init__.cpython-314.pyc b/.venv/lib/python3.14/site-packages/click/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e8af2bfeba4e19ca0880ffda284386caa0692afb Binary files /dev/null and b/.venv/lib/python3.14/site-packages/click/__pycache__/__init__.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/click/__pycache__/_compat.cpython-314.pyc b/.venv/lib/python3.14/site-packages/click/__pycache__/_compat.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6fbd887e155f42d650f794d2b44203e7f3c886c5 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/click/__pycache__/_compat.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/click/__pycache__/_termui_impl.cpython-314.pyc b/.venv/lib/python3.14/site-packages/click/__pycache__/_termui_impl.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b725a769d947db1caa8db52539b518ef33cc339b Binary files /dev/null and b/.venv/lib/python3.14/site-packages/click/__pycache__/_termui_impl.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/click/__pycache__/_textwrap.cpython-314.pyc b/.venv/lib/python3.14/site-packages/click/__pycache__/_textwrap.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c8ac8b731c043bbf8e882475aae8a6457d4ba68e Binary files /dev/null and b/.venv/lib/python3.14/site-packages/click/__pycache__/_textwrap.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/click/__pycache__/_utils.cpython-314.pyc b/.venv/lib/python3.14/site-packages/click/__pycache__/_utils.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b5dfd12df4b36e403953ff21a03bc612a68672cd Binary files /dev/null and b/.venv/lib/python3.14/site-packages/click/__pycache__/_utils.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/click/__pycache__/_winconsole.cpython-314.pyc b/.venv/lib/python3.14/site-packages/click/__pycache__/_winconsole.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3807ad27c2f99d82365ff185b6cd007187721268 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/click/__pycache__/_winconsole.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/click/__pycache__/core.cpython-314.pyc b/.venv/lib/python3.14/site-packages/click/__pycache__/core.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..23309a7f319de21f6b47d5fc3e6d7bc73b0a1fc0 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/click/__pycache__/core.cpython-314.pyc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8eefb31cd1a7a35d2ad0bba699301cb946b6ba92c0c830faa6f68341f8fce6de +size 171125 diff --git a/.venv/lib/python3.14/site-packages/click/__pycache__/decorators.cpython-314.pyc b/.venv/lib/python3.14/site-packages/click/__pycache__/decorators.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..37daf461c0c8de6335bc6e64e822b29855f543cd Binary files /dev/null and b/.venv/lib/python3.14/site-packages/click/__pycache__/decorators.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/click/__pycache__/exceptions.cpython-314.pyc b/.venv/lib/python3.14/site-packages/click/__pycache__/exceptions.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..54aeeb8c0125f8db9f43142fcce57b1db456e6f6 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/click/__pycache__/exceptions.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/click/__pycache__/formatting.cpython-314.pyc b/.venv/lib/python3.14/site-packages/click/__pycache__/formatting.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6dda5cde9442078b7d15622baadba74b9408de95 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/click/__pycache__/formatting.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/click/__pycache__/globals.cpython-314.pyc b/.venv/lib/python3.14/site-packages/click/__pycache__/globals.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d47f096b1e3e28343e6c89b295a99d2f0f0284ab Binary files /dev/null and b/.venv/lib/python3.14/site-packages/click/__pycache__/globals.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/click/__pycache__/parser.cpython-314.pyc b/.venv/lib/python3.14/site-packages/click/__pycache__/parser.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..60d3ebec46f13e86475bc18393e664d4e4e37919 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/click/__pycache__/parser.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/click/__pycache__/shell_completion.cpython-314.pyc b/.venv/lib/python3.14/site-packages/click/__pycache__/shell_completion.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e6e57e238bc83178c9703e43987b124bbc5a25d2 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/click/__pycache__/shell_completion.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/click/__pycache__/termui.cpython-314.pyc b/.venv/lib/python3.14/site-packages/click/__pycache__/termui.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..87d64801b3129ee0aec1d155c33f8e7f4ad9e80d Binary files /dev/null and b/.venv/lib/python3.14/site-packages/click/__pycache__/termui.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/click/__pycache__/testing.cpython-314.pyc b/.venv/lib/python3.14/site-packages/click/__pycache__/testing.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1923ef8152bd94477a12ffe12795b932f7681594 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/click/__pycache__/testing.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/click/__pycache__/types.cpython-314.pyc b/.venv/lib/python3.14/site-packages/click/__pycache__/types.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..75658c0efdd005274ec51dc47139e7b7d492e87d Binary files /dev/null and b/.venv/lib/python3.14/site-packages/click/__pycache__/types.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/click/__pycache__/utils.cpython-314.pyc b/.venv/lib/python3.14/site-packages/click/__pycache__/utils.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b13184af599c8c1b9577c6074fe956d8aed7a4f4 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/click/__pycache__/utils.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/click/_compat.py b/.venv/lib/python3.14/site-packages/click/_compat.py new file mode 100644 index 0000000000000000000000000000000000000000..8507a503abf9b5fcaf3285dc3999ae223c7608c8 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/click/_compat.py @@ -0,0 +1,590 @@ +from __future__ import annotations + +import codecs +import collections.abc as cabc +import io +import os +import re +import sys +import typing as t +from types import TracebackType +from weakref import WeakKeyDictionary + +CYGWIN = sys.platform.startswith("cygwin") +WIN = sys.platform.startswith("win") +# One CSI escape sequence per the ECMA-48 grammar: parameter bytes (0x30-0x3F), +# intermediate bytes (0x20-0x2F), then a final byte (0x40-0x7E). Broader than the +# SGR codes Click emits, so foreign sequences (colon-delimited true-color, mouse +# reporting) are stripped too. +_ansi_re = re.compile(r"\033\[[0-?]*[ -/]*[@-~]") + + +def _make_text_stream( + stream: t.BinaryIO, + encoding: str | None, + errors: str | None, + force_readable: bool = False, + force_writable: bool = False, +) -> t.TextIO: + if encoding is None: + encoding = get_best_encoding(stream) + if errors is None: + errors = "replace" + return _NonClosingTextIOWrapper( + stream, + encoding, + errors, + line_buffering=True, + force_readable=force_readable, + force_writable=force_writable, + ) + + +def is_ascii_encoding(encoding: str) -> bool: + """Checks if a given encoding is ascii.""" + try: + return codecs.lookup(encoding).name == "ascii" + except LookupError: + return False + + +def get_best_encoding(stream: t.IO[t.Any]) -> str: + """Returns the default stream encoding if not found.""" + rv = getattr(stream, "encoding", None) or sys.getdefaultencoding() + if is_ascii_encoding(rv): + return "utf-8" + return rv + + +class _NonClosingTextIOWrapper(io.TextIOWrapper): + def __init__( + self, + stream: t.BinaryIO, + encoding: str | None, + errors: str | None, + force_readable: bool = False, + force_writable: bool = False, + **extra: t.Any, + ) -> None: + self._stream = stream = t.cast( + t.BinaryIO, _FixupStream(stream, force_readable, force_writable) + ) + super().__init__(stream, encoding, errors, **extra) + + def __del__(self) -> None: + try: + self.detach() + except Exception: + pass + + def isatty(self) -> bool: + # https://bitbucket.org/pypy/pypy/issue/1803 + return self._stream.isatty() + + +class _FixupStream: + """The new io interface needs more from streams than streams + traditionally implement. As such, this fix-up code is necessary in + some circumstances. + + The forcing of readable and writable flags are there because some tools + put badly patched objects on sys (one such offender are certain version + of jupyter notebook). + """ + + def __init__( + self, + stream: t.BinaryIO, + force_readable: bool = False, + force_writable: bool = False, + ): + self._stream = stream + self._force_readable = force_readable + self._force_writable = force_writable + + def __getattr__(self, name: str) -> t.Any: + return getattr(self._stream, name) + + def read1(self, size: int) -> bytes: + f = getattr(self._stream, "read1", None) + + if f is not None: + return t.cast(bytes, f(size)) + + return self._stream.read(size) + + def readable(self) -> bool: + if self._force_readable: + return True + x = getattr(self._stream, "readable", None) + if x is not None: + return t.cast(bool, x()) + try: + self._stream.read(0) + except Exception: + return False + return True + + def writable(self) -> bool: + if self._force_writable: + return True + x = getattr(self._stream, "writable", None) + if x is not None: + return t.cast(bool, x()) + try: + self._stream.write(b"") + except Exception: + try: + self._stream.write(b"") + except Exception: + return False + return True + + def seekable(self) -> bool: + x = getattr(self._stream, "seekable", None) + if x is not None: + return t.cast(bool, x()) + try: + self._stream.seek(self._stream.tell()) + except Exception: + return False + return True + + +def _is_binary_reader(stream: t.IO[t.Any], default: bool = False) -> bool: + try: + return isinstance(stream.read(0), bytes) + except Exception: + return default + # This happens in some cases where the stream was already + # closed. In this case, we assume the default. + + +def _is_binary_writer(stream: t.IO[t.Any], default: bool = False) -> bool: + try: + stream.write(b"") + except Exception: + try: + stream.write("") + return False + except Exception: + pass + return default + return True + + +def _find_binary_reader(stream: t.IO[t.Any]) -> t.BinaryIO | None: + # We need to figure out if the given stream is already binary. + # This can happen because the official docs recommend detaching + # the streams to get binary streams. Some code might do this, so + # we need to deal with this case explicitly. + if _is_binary_reader(stream, False): + return t.cast(t.BinaryIO, stream) + + buf = getattr(stream, "buffer", None) + + # Same situation here; this time we assume that the buffer is + # actually binary in case it's closed. + if buf is not None and _is_binary_reader(buf, True): + return t.cast(t.BinaryIO, buf) + + return None + + +def _find_binary_writer(stream: t.IO[t.Any]) -> t.BinaryIO | None: + # We need to figure out if the given stream is already binary. + # This can happen because the official docs recommend detaching + # the streams to get binary streams. Some code might do this, so + # we need to deal with this case explicitly. + if _is_binary_writer(stream, False): + return t.cast(t.BinaryIO, stream) + + buf = getattr(stream, "buffer", None) + + # Same situation here; this time we assume that the buffer is + # actually binary in case it's closed. + if buf is not None and _is_binary_writer(buf, True): + return t.cast(t.BinaryIO, buf) + + return None + + +def _stream_is_misconfigured(stream: t.TextIO) -> bool: + """A stream is misconfigured if its encoding is ASCII.""" + # If the stream does not have an encoding set, we assume it's set + # to ASCII. This appears to happen in certain unittest + # environments. It's not quite clear what the correct behavior is + # but this at least will force Click to recover somehow. + return is_ascii_encoding(getattr(stream, "encoding", None) or "ascii") + + +def _is_compat_stream_attr(stream: t.TextIO, attr: str, value: str | None) -> bool: + """A stream attribute is compatible if it is equal to the + desired value or the desired value is unset and the attribute + has a value. + """ + stream_value = getattr(stream, attr, None) + return stream_value == value or (value is None and stream_value is not None) + + +def _is_compatible_text_stream( + stream: t.TextIO, encoding: str | None, errors: str | None +) -> bool: + """Check if a stream's encoding and errors attributes are + compatible with the desired values. + """ + return _is_compat_stream_attr( + stream, "encoding", encoding + ) and _is_compat_stream_attr(stream, "errors", errors) + + +def _force_correct_text_stream( + text_stream: t.IO[t.Any], + encoding: str | None, + errors: str | None, + is_binary: t.Callable[[t.IO[t.Any], bool], bool], + find_binary: t.Callable[[t.IO[t.Any]], t.BinaryIO | None], + force_readable: bool = False, + force_writable: bool = False, +) -> t.TextIO: + if is_binary(text_stream, False): + binary_reader = t.cast(t.BinaryIO, text_stream) + else: + text_stream = t.cast(t.TextIO, text_stream) + # If the stream looks compatible, and won't default to a + # misconfigured ascii encoding, return it as-is. + if _is_compatible_text_stream(text_stream, encoding, errors) and not ( + encoding is None and _stream_is_misconfigured(text_stream) + ): + return text_stream + + # Otherwise, get the underlying binary reader. + possible_binary_reader = find_binary(text_stream) + + # If that's not possible, silently use the original reader + # and get mojibake instead of exceptions. + if possible_binary_reader is None: + return text_stream + + binary_reader = possible_binary_reader + + # Default errors to replace instead of strict in order to get + # something that works. + if errors is None: + errors = "replace" + + # Wrap the binary stream in a text stream with the correct + # encoding parameters. + return _make_text_stream( + binary_reader, + encoding, + errors, + force_readable=force_readable, + force_writable=force_writable, + ) + + +def _force_correct_text_reader( + text_reader: t.IO[t.Any], + encoding: str | None, + errors: str | None, + force_readable: bool = False, +) -> t.TextIO: + return _force_correct_text_stream( + text_reader, + encoding, + errors, + _is_binary_reader, + _find_binary_reader, + force_readable=force_readable, + ) + + +def _force_correct_text_writer( + text_writer: t.IO[t.Any], + encoding: str | None, + errors: str | None, + force_writable: bool = False, +) -> t.TextIO: + return _force_correct_text_stream( + text_writer, + encoding, + errors, + _is_binary_writer, + _find_binary_writer, + force_writable=force_writable, + ) + + +def get_binary_stdin() -> t.BinaryIO: + reader = _find_binary_reader(sys.stdin) + if reader is None: + raise RuntimeError("Was not able to determine binary stream for sys.stdin.") + return reader + + +def get_binary_stdout() -> t.BinaryIO: + writer = _find_binary_writer(sys.stdout) + if writer is None: + raise RuntimeError("Was not able to determine binary stream for sys.stdout.") + return writer + + +def get_binary_stderr() -> t.BinaryIO: + writer = _find_binary_writer(sys.stderr) + if writer is None: + raise RuntimeError("Was not able to determine binary stream for sys.stderr.") + return writer + + +def get_text_stdin(encoding: str | None = None, errors: str | None = None) -> t.TextIO: + rv = _get_windows_console_stream(sys.stdin, encoding, errors) + if rv is not None: + return rv + return _force_correct_text_reader(sys.stdin, encoding, errors, force_readable=True) + + +def get_text_stdout(encoding: str | None = None, errors: str | None = None) -> t.TextIO: + rv = _get_windows_console_stream(sys.stdout, encoding, errors) + if rv is not None: + return rv + return _force_correct_text_writer(sys.stdout, encoding, errors, force_writable=True) + + +def get_text_stderr(encoding: str | None = None, errors: str | None = None) -> t.TextIO: + rv = _get_windows_console_stream(sys.stderr, encoding, errors) + if rv is not None: + return rv + return _force_correct_text_writer(sys.stderr, encoding, errors, force_writable=True) + + +def _wrap_io_open( + file: str | os.PathLike[str] | int, + mode: str, + encoding: str | None, + errors: str | None, +) -> t.IO[t.Any]: + """Handles not passing ``encoding`` and ``errors`` in binary mode.""" + if "b" in mode: + return open(file, mode) + + return open(file, mode, encoding=encoding, errors=errors) + + +def open_stream( + filename: str | os.PathLike[str], + mode: str = "r", + encoding: str | None = None, + errors: str | None = "strict", + atomic: bool = False, +) -> tuple[t.IO[t.Any], bool]: + binary = "b" in mode + filename = os.fspath(filename) + + # Standard streams first. These are simple because they ignore the + # atomic flag. Use fsdecode to handle Path("-"). + if os.fsdecode(filename) == "-": + if any(m in mode for m in ["w", "a", "x"]): + if binary: + return get_binary_stdout(), False + return get_text_stdout(encoding=encoding, errors=errors), False + if binary: + return get_binary_stdin(), False + return get_text_stdin(encoding=encoding, errors=errors), False + + # Non-atomic writes directly go out through the regular open functions. + if not atomic: + return _wrap_io_open(filename, mode, encoding, errors), True + + # Some usability stuff for atomic writes + if "a" in mode: + raise ValueError( + "Appending to an existing file is not supported, because that" + " would involve an expensive `copy`-operation to a temporary" + " file. Open the file in normal `w`-mode and copy explicitly" + " if that's what you're after." + ) + if "x" in mode: + raise ValueError("Use the `overwrite`-parameter instead.") + if "w" not in mode: + raise ValueError("Atomic writes only make sense with `w`-mode.") + + # Atomic writes are more complicated. They work by opening a file + # as a proxy in the same folder and then using the fdopen + # functionality to wrap it in a Python file. Then we wrap it in an + # atomic file that moves the file over on close. + import errno + import random + + try: + perm: int | None = os.stat(filename).st_mode + except OSError: + perm = None + + flags = os.O_RDWR | os.O_CREAT | os.O_EXCL + + if binary: + flags |= getattr(os, "O_BINARY", 0) + + while True: + tmp_filename = os.path.join( + os.path.dirname(filename), + f".__atomic-write{random.randrange(1 << 32):08x}", + ) + try: + fd = os.open(tmp_filename, flags, 0o666 if perm is None else perm) + break + except OSError as e: + if e.errno == errno.EEXIST or ( + os.name == "nt" + and e.errno == errno.EACCES + and os.path.isdir(e.filename) + and os.access(e.filename, os.W_OK) + ): + continue + raise + + if perm is not None: + os.chmod(tmp_filename, perm) # in case perm includes bits in umask + + f = _wrap_io_open(fd, mode, encoding, errors) + af = _AtomicFile(f, tmp_filename, os.path.realpath(filename)) + return t.cast(t.IO[t.Any], af), True + + +class _AtomicFile: + def __init__(self, f: t.IO[t.Any], tmp_filename: str, real_filename: str) -> None: + self._f = f + self._tmp_filename = tmp_filename + self._real_filename = real_filename + self.closed = False + + @property + def name(self) -> str: + return self._real_filename + + def close(self, delete: bool = False) -> None: + if self.closed: + return + self._f.close() + os.replace(self._tmp_filename, self._real_filename) + self.closed = True + + def __getattr__(self, name: str) -> t.Any: + return getattr(self._f, name) + + def __enter__(self) -> _AtomicFile: + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + tb: TracebackType | None, + ) -> None: + self.close(delete=exc_type is not None) + + def __repr__(self) -> str: + return repr(self._f) + + +def strip_ansi(value: str) -> str: + return _ansi_re.sub("", value) + + +def _is_jupyter_kernel_output(stream: t.IO[t.Any]) -> bool: + while isinstance(stream, (_FixupStream, _NonClosingTextIOWrapper)): + stream = stream._stream + + return stream.__class__.__module__.startswith("ipykernel.") + + +def should_strip_ansi( + stream: t.IO[t.Any] | None = None, color: bool | None = None +) -> bool: + if color is None: + if stream is None: + stream = sys.stdin + elif hasattr(stream, "color"): + # ._termui_impl._PagerWriter handles stripping ansi itself, + # so we don't need to strip it here + return False + return not isatty(stream) and not _is_jupyter_kernel_output(stream) + return not color + + +# Double check is needed so mypy does not analyze this on Linux. +if sys.platform.startswith("win") and WIN: + from ._winconsole import _get_windows_console_stream + + def _get_argv_encoding() -> str: + import locale + + return locale.getpreferredencoding() + +else: + + def _get_argv_encoding() -> str: + return getattr(sys.stdin, "encoding", None) or sys.getfilesystemencoding() + + def _get_windows_console_stream( + f: t.TextIO, encoding: str | None, errors: str | None + ) -> t.TextIO | None: + return None + + +def term_len(x: str) -> int: + return len(strip_ansi(x)) + + +def isatty(stream: t.IO[t.Any]) -> bool: + try: + return stream.isatty() + except Exception: + return False + + +def _make_cached_stream_func( + src_func: t.Callable[[], t.TextIO | None], + wrapper_func: t.Callable[[], t.TextIO], +) -> t.Callable[[], t.TextIO | None]: + cache: cabc.MutableMapping[t.TextIO, t.TextIO] = WeakKeyDictionary() + + def func() -> t.TextIO | None: + stream = src_func() + + if stream is None: + return None + + try: + rv = cache.get(stream) + except Exception: + rv = None + if rv is not None: + return rv + rv = wrapper_func() + try: + cache[stream] = rv + except Exception: + pass + return rv + + return func + + +_default_text_stdin = _make_cached_stream_func(lambda: sys.stdin, get_text_stdin) +_default_text_stdout = _make_cached_stream_func(lambda: sys.stdout, get_text_stdout) +_default_text_stderr = _make_cached_stream_func(lambda: sys.stderr, get_text_stderr) + + +binary_streams: cabc.Mapping[str, t.Callable[[], t.BinaryIO]] = { + "stdin": get_binary_stdin, + "stdout": get_binary_stdout, + "stderr": get_binary_stderr, +} + +text_streams: cabc.Mapping[str, t.Callable[[str | None, str | None], t.TextIO]] = { + "stdin": get_text_stdin, + "stdout": get_text_stdout, + "stderr": get_text_stderr, +} diff --git a/.venv/lib/python3.14/site-packages/click/_termui_impl.py b/.venv/lib/python3.14/site-packages/click/_termui_impl.py new file mode 100644 index 0000000000000000000000000000000000000000..70aabf8f17c56ce5ed559ea88ce233a7944dbe3a --- /dev/null +++ b/.venv/lib/python3.14/site-packages/click/_termui_impl.py @@ -0,0 +1,972 @@ +""" +This module contains implementations for the termui module. To keep the +import time of Click down, some infrequently used functionality is +placed in this module and only imported as needed. +""" + +from __future__ import annotations + +import collections.abc as cabc +import contextlib +import math +import os +import shlex +import sys +import time +import typing as t +from gettext import gettext as _ +from io import StringIO +from pathlib import Path +from types import TracebackType + +from ._compat import _default_text_stdout +from ._compat import CYGWIN +from ._compat import get_best_encoding +from ._compat import isatty +from ._compat import strip_ansi +from ._compat import term_len +from ._compat import WIN +from .exceptions import ClickException +from .utils import echo + +V = t.TypeVar("V") + + +if os.name == "nt": + BEFORE_BAR = "\r" + AFTER_BAR = "\n" +else: + BEFORE_BAR = "\r\033[?25l" + AFTER_BAR = "\033[?25h\n" + + +class ProgressBar(t.Generic[V]): + def __init__( + self, + iterable: cabc.Iterable[V] | None, + length: int | None = None, + fill_char: str = "#", + empty_char: str = " ", + bar_template: str = "%(bar)s", + info_sep: str = " ", + hidden: bool = False, + show_eta: bool = True, + show_percent: bool | None = None, + show_pos: bool = False, + item_show_func: t.Callable[[V | None], str | None] | None = None, + label: str | None = None, + file: t.TextIO | None = None, + color: bool | None = None, + update_min_steps: int = 1, + width: int = 30, + ) -> None: + self.fill_char = fill_char + self.empty_char = empty_char + self.bar_template = bar_template + self.info_sep = info_sep + self.hidden = hidden + self.show_eta = show_eta + self.show_percent = show_percent + self.show_pos = show_pos + self.item_show_func = item_show_func + self.label: str = label or "" + + if file is None: + file = _default_text_stdout() + + # There are no standard streams attached to write to. For example, + # pythonw on Windows. + if file is None: + file = StringIO() + + self.file = file + self.color = color + self.update_min_steps = update_min_steps + self._completed_intervals = 0 + self.width: int = width + self.autowidth: bool = width == 0 + + if length is None: + from operator import length_hint + + length = length_hint(iterable, -1) + + if length == -1: + length = None + if iterable is None: + if length is None: + raise TypeError("iterable or length is required") + iterable = t.cast("cabc.Iterable[V]", range(length)) + self.iter: cabc.Iterable[V] = iter(iterable) + self.length = length + self.pos: int = 0 + self.avg: list[float] = [] + self.last_eta: float + self.start: float + self.start = self.last_eta = time.time() + self.eta_known: bool = False + self.finished: bool = False + self.max_width: int | None = None + self.entered: bool = False + self.current_item: V | None = None + self._is_atty = isatty(self.file) + self._last_line: str | None = None + + def __enter__(self) -> ProgressBar[V]: + self.entered = True + self.render_progress() + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + tb: TracebackType | None, + ) -> None: + self.render_finish() + + def __iter__(self) -> cabc.Iterator[V]: + if not self.entered: + raise RuntimeError("You need to use progress bars in a with block.") + self.render_progress() + return self.generator() + + def __next__(self) -> V: + # Iteration is defined in terms of a generator function, + # returned by iter(self); use that to define next(). This works + # because `self.iter` is an iterable consumed by that generator, + # so it is re-entry safe. Calling `next(self.generator())` + # twice works and does "what you want". + return next(iter(self)) + + def render_finish(self) -> None: + # Steps stay pending until they add up to update_min_steps, so a length + # that is not a multiple of it keeps a sub-threshold remainder that + # never reaches pos. Apply it so the bar settles on its true position. + if self._completed_intervals: + self.make_step(self._completed_intervals) + self._completed_intervals = 0 + self.render_progress() + + if self.hidden or not self._is_atty: + return + self.file.write(AFTER_BAR) + self.file.flush() + + @property + def pct(self) -> float: + if self.finished: + return 1.0 + return min(self.pos / (float(self.length or 1) or 1), 1.0) + + @property + def time_per_iteration(self) -> float: + if not self.avg: + return 0.0 + return sum(self.avg) / float(len(self.avg)) + + @property + def eta(self) -> float: + if self.length is not None and not self.finished: + return self.time_per_iteration * (self.length - self.pos) + return 0.0 + + def format_eta(self) -> str: + if self.eta_known: + t = int(self.eta) + seconds = t % 60 + t //= 60 + minutes = t % 60 + t //= 60 + hours = t % 24 + t //= 24 + if t > 0: + return "{d}{day_label} {h:02}:{m:02}:{s:02}".format( + d=t, + day_label=_("d"), + h=hours, + m=minutes, + s=seconds, + ) + else: + return f"{hours:02}:{minutes:02}:{seconds:02}" + return "" + + def format_pos(self) -> str: + pos = str(self.pos) + if self.length is not None: + pos += f"/{self.length}" + return pos + + def format_pct(self) -> str: + return f"{int(self.pct * 100): 4}%"[1:] + + def format_bar(self) -> str: + if self.length is not None: + bar_length = int(self.pct * self.width) + bar = self.fill_char * bar_length + bar += self.empty_char * (self.width - bar_length) + elif self.finished: + bar = self.fill_char * self.width + else: + chars = list(self.empty_char * (self.width or 1)) + if self.time_per_iteration != 0: + chars[ + int( + (math.cos(self.pos * self.time_per_iteration) / 2.0 + 0.5) + * self.width + ) + ] = self.fill_char + bar = "".join(chars) + return bar + + def format_progress_line(self) -> str: + show_percent = self.show_percent + + info_bits = [] + if self.length is not None and show_percent is None: + show_percent = not self.show_pos + + if self.show_pos: + info_bits.append(self.format_pos()) + if show_percent: + info_bits.append(self.format_pct()) + if self.show_eta and self.eta_known and not self.finished: + info_bits.append(self.format_eta()) + if self.item_show_func is not None: + item_info = self.item_show_func(self.current_item) + if item_info is not None: + info_bits.append(item_info) + + return ( + self.bar_template + % { + "label": self.label, + "bar": self.format_bar(), + "info": self.info_sep.join(info_bits), + } + ).rstrip() + + def render_progress(self) -> None: + if self.hidden: + return + + if not self._is_atty: + # Only output the label once if the output is not a TTY. + if self._last_line != self.label: + self._last_line = self.label + echo(self.label, file=self.file, color=self.color) + return + + buf = [] + # Update width in case the terminal has been resized + if self.autowidth: + import shutil + + old_width = self.width + self.width = 0 + clutter_length = term_len(self.format_progress_line()) + new_width = max(0, shutil.get_terminal_size().columns - clutter_length) + if new_width < old_width and self.max_width is not None: + buf.append(BEFORE_BAR) + buf.append(" " * self.max_width) + self.max_width = new_width + self.width = new_width + + clear_width = self.width + if self.max_width is not None: + clear_width = self.max_width + + buf.append(BEFORE_BAR) + line = self.format_progress_line() + line_len = term_len(line) + if self.max_width is None or self.max_width < line_len: + self.max_width = line_len + + buf.append(line) + buf.append(" " * (clear_width - line_len)) + line = "".join(buf) + # Render the line only if it changed. + + if line != self._last_line: + self._last_line = line + echo(line, file=self.file, color=self.color, nl=False) + self.file.flush() + + def make_step(self, n_steps: int) -> None: + self.pos += n_steps + if self.length is not None and self.pos >= self.length: + self.finished = True + + if (time.time() - self.last_eta) < 1.0: + return + + self.last_eta = time.time() + + # self.avg is a rolling list of length <= 7 of steps where steps are + # defined as time elapsed divided by the total progress through + # self.length. + if self.pos: + step = (time.time() - self.start) / self.pos + else: + step = time.time() - self.start + + self.avg = self.avg[-6:] + [step] + + self.eta_known = self.length is not None + + def update(self, n_steps: int, current_item: V | None = None) -> None: + """Update the progress bar by advancing a specified number of + steps, and optionally set the ``current_item`` for this new + position. + + :param n_steps: Number of steps to advance. + :param current_item: Optional item to set as ``current_item`` + for the updated position. + + .. versionchanged:: 8.0 + Added the ``current_item`` optional parameter. + + .. versionchanged:: 8.0 + Only render when the number of steps meets the + ``update_min_steps`` threshold. + """ + if current_item is not None: + self.current_item = current_item + + self._completed_intervals += n_steps + + if self._completed_intervals >= self.update_min_steps: + self.make_step(self._completed_intervals) + self.render_progress() + self._completed_intervals = 0 + + def finish(self) -> None: + self.eta_known = False + self.current_item = None + self.finished = True + + def generator(self) -> cabc.Iterator[V]: + """Return a generator which yields the items added to the bar + during construction, and updates the progress bar *after* the + yielded block returns. + """ + # WARNING: the iterator interface for `ProgressBar` relies on + # this and only works because this is a simple generator which + # doesn't create or manage additional state. If this function + # changes, the impact should be evaluated both against + # `iter(bar)` and `next(bar)`. `next()` in particular may call + # `self.generator()` repeatedly, and this must remain safe in + # order for that interface to work. + if not self.entered: + raise RuntimeError("You need to use progress bars in a with block.") + + if not self._is_atty: + yield from self.iter + else: + for rv in self.iter: + self.current_item = rv + + # This allows show_item_func to be updated before the + # item is processed. Only trigger at the beginning of + # the update interval. + if self._completed_intervals == 0: + self.render_progress() + + yield rv + self.update(1) + + self.finish() + self.render_progress() + + +class _PagerWriter: + """Wrap a pager's output stream to strip ANSI styling when colors are + disabled. + + The wrapped stream is owned by the pager strategy that produced it, so this + wrapper never closes it: ``_pipepager`` closes its pipe to signal EOF to the + pager process, ``_tempfilepager`` closes and removes its temporary file, and + ``_nullpager`` leaves an external stream such as ``sys.stdout`` untouched. + + The ``color`` attribute lets :func:`click.echo` detect that ANSI stripping + is handled here, so it doesn't strip a second time (see + :func:`._compat.should_strip_ansi`). + """ + + def __init__(self, stream: t.TextIO, color: bool) -> None: + self._stream = stream + self.color = color + + def write(self, text: str) -> int: + if not self.color: + text = strip_ansi(text) + + return self._stream.write(text) + + def writelines(self, lines: cabc.Iterable[str]) -> None: + for line in lines: + self.write(line) + + def flush(self) -> None: + self._stream.flush() + + def close(self) -> None: + # The pager strategy owns the stream's lifecycle. Flush pending output, + # but never close the underlying stream from here. + self._stream.flush() + + def __getattr__(self, name: str) -> t.Any: + return getattr(self._stream, name) + + +def _resolve_pager_command(cmd_parts: list[str]) -> tuple[Path, list[str]] | None: + """Resolve a pager ``argv`` to an absolute command path and its parameters. + + The command is looked up with :func:`shutil.which`, which the + :mod:`subprocess` docs recommend on every platform for an unqualified name: + https://docs.python.org/3/library/subprocess.html#popen-constructor + + Returns ``None`` when there is no command to run or it is not on the path, + leaving the caller to pick a fallback. + """ + if not cmd_parts: + return None + + import shutil + + cmd_filepath = shutil.which(cmd_parts[0]) + + if not cmd_filepath: + return None + + # Normalize to an absolute path without resolving symlinks: multi-call + # binaries such as busybox derive their identity from the link they are + # invoked through, so resolving less -> busybox makes them misbehave. + # See https://github.com/pallets/click/issues/2943 and + # https://github.com/pallets/click/pull/2944 + return Path(cmd_filepath).absolute(), cmd_parts[1:] + + +def _pager_contextmanager( + color: bool | None = None, +) -> t.ContextManager[tuple[t.TextIO, bool | None]]: + """Decide what method to use for paging through text.""" + stdout = _default_text_stdout() + + # There are no standard streams attached to write to. For example, + # pythonw on Windows. + if stdout is None: + stdout = StringIO() + + if not isatty(sys.stdin) or not isatty(stdout): + return _nullpager(stdout, color) + + # Split using POSIX mode (the default) so that quote characters are + # stripped from tokens and quoted Windows paths are preserved. + # Non-POSIX mode retains quotes in tokens, and wrapping tokens + # with shlex.quote re-introduces quoting issues on Windows. + pager_cmd_parts = shlex.split(os.environ.get("PAGER", "")) + + if pager_cmd_parts: + # Piping to `more` on Windows adds spurious \r\n, so it gets the temp + # file strategy whatever the user asked for. + use_tempfile = WIN + else: + if os.environ.get("TERM") in ("dumb", "emacs"): + return _nullpager(stdout, color) + + use_tempfile = WIN or sys.platform.startswith("os2") + pager_cmd_parts = ["more"] if use_tempfile else ["less"] + + resolved = _resolve_pager_command(pager_cmd_parts) + + if resolved is None: + # Page through stdout, which _nullpager leaves open for its owner. + return _nullpager(stdout, color) + + cmd_path, cmd_params = resolved + + if use_tempfile: + return _tempfilepager(cmd_path, cmd_params, color) + + return _pipepager(cmd_path, cmd_params, color) + + +@contextlib.contextmanager +def get_pager_file(color: bool | None = None) -> t.Generator[t.TextIO, None, None]: + """Context manager. + + Yields a writable file-like object which can be used as an output pager. + + .. versionadded:: 8.4.0 + + :param color: controls if the pager supports ANSI colors or not. The + default is autodetection. + """ + with _pager_contextmanager(color=color) as (stream, color): + # Every pager strategy yields a text stream, so the only thing left to + # do is strip ANSI styling when colors are disabled. A strategy yields + # None when it has no opinion on colors, which settles as off here. The + # wrapper does not close the stream: each strategy owns its stream's + # lifecycle. + writer = _PagerWriter(stream, color=bool(color)) + try: + yield t.cast(t.TextIO, writer) + finally: + writer.flush() + + +def _less_uses_raw_mode(less_env: str, cmd_params: list[str]) -> bool: + """Detect a raw-control-characters request in a ``less`` invocation. + + Parses the tokens of the ``LESS`` environment variable and the command + line: only short-option clusters (``-R``, ``-rX``, ``-XrY``) and the + ``--raw-control-chars`` long option request raw mode. Filenames and + long-option values may carry the letter ``r`` without being such a + request. + """ + try: + env_tokens = shlex.split(less_env) + except ValueError: + # Unbalanced quotes make ``LESS`` unparsable: fall back to whitespace + # splitting rather than failing the pager over it. + env_tokens = less_env.split() + tokens = env_tokens + cmd_params + + for token in tokens: + if token.startswith("--"): + if token.casefold() == "--raw-control-chars": + return True + elif token.startswith("-") and len(token) > 1: + if "r" in token[1:].casefold(): + return True + + return False + + +@contextlib.contextmanager +def _pipepager( + cmd_path: Path, cmd_params: list[str], color: bool | None = None +) -> t.Iterator[tuple[t.TextIO, bool | None]]: + """Page through text by feeding it to another program. + + Invokes the pager via :class:`subprocess.Popen` with an ``argv`` list. + + Invoking a pager through this might support colors: if piping to + ``less`` and the user hasn't decided on colors, ``LESS=-R`` is set + automatically. + """ + import subprocess + + # Make a local copy of the environment to not affect the global one. + env = dict(os.environ) + + # If we're piping to less and the user hasn't decided on colors, we enable + # them: either the invocation already requests raw control characters, or + # ``LESS=-R`` is injected. Match on the stem so a resolved ``less.exe`` is + # recognized like bare ``less``, case-insensitively on Windows since its + # filesystems are. POSIX keeps an exact match. + cmd_name = cmd_path.stem + if WIN: + # ``_pager_contextmanager`` currently routes every Windows invocation + # to ``_tempfilepager``, so this branch is not reachable on Windows + # today: it keeps the detection correct should that routing ever + # change. + cmd_name = cmd_name.casefold() + + if color is None and cmd_name == "less": + less_env = os.environ.get("LESS", "") + if _less_uses_raw_mode(less_env, cmd_params): + color = True + elif not less_env and not cmd_params: + env["LESS"] = "-R" + color = True + + c = subprocess.Popen( + [str(cmd_path)] + cmd_params, + shell=False, + stdin=subprocess.PIPE, + env=env, + errors="replace", + text=True, + ) + # With ``text=True``, ``c.stdin`` is already a text stream that encodes + # writes for the pager process (honoring ``errors="replace"``). + stdin = t.cast(t.TextIO, c.stdin) + try: + yield stdin, color + except BrokenPipeError: + # In case the pager exited unexpectedly, ignore the broken pipe error. + pass + except Exception as e: + # In case there is an exception we want to close the pager immediately + # and let the caller handle it. + # Otherwise the pager will keep running, and the user may not notice + # the error message, or worse yet it may leave the terminal in a broken state. + c.terminate() + raise e + finally: + # We must close stdin and wait for the pager to exit before we continue + try: + stdin.close() + # Close implies flush, so it might throw a BrokenPipeError if the pager + # process exited already. + except BrokenPipeError: + pass + + # Less doesn't respect ^C, but catches it for its own UI purposes (aborting + # search or other commands inside less). + # + # That means when the user hits ^C, the parent process (click) terminates, + # but less is still alive, paging the output and messing up the terminal. + # + # If the user wants to make the pager exit on ^C, they should set + # `LESS='-K'`. It's not our decision to make. + while True: + try: + c.wait() + except KeyboardInterrupt: + pass + else: + break + + +@contextlib.contextmanager +def _tempfilepager( + cmd_path: Path, cmd_params: list[str], color: bool | None = None +) -> t.Iterator[tuple[t.TextIO, bool | None]]: + """Page through text by invoking a program on a temporary file. + + Used as the primary pager strategy on Windows (where piping to + ``more`` adds spurious ``\\r\\n``), and as a fallback on other + platforms. The command is invoked with any parameters from ``PAGER`` + followed by the temporary file name. + """ + import subprocess + import tempfile + + encoding = get_best_encoding(sys.stdout) + # On Windows, NamedTemporaryFile cannot be opened by another process + # while Python still has it open, so we use delete=False and clean up manually + # rather than using a contextmanager here. + f = tempfile.NamedTemporaryFile( + mode="w", encoding=encoding, errors="replace", delete=False + ) + try: + yield t.cast(t.TextIO, f), color + f.flush() + f.close() + subprocess.call([str(cmd_path), *cmd_params, f.name]) + finally: + # An error raised while paging skips the close() above, and Windows + # refuses to unlink a file the process still holds open. Closing here + # keeps that PermissionError from replacing the original exception. + # close() is idempotent, so this is a no-op on the success path. + f.close() + os.unlink(f.name) + + +@contextlib.contextmanager +def _nullpager( + stream: t.TextIO, color: bool | None = None +) -> t.Iterator[tuple[t.TextIO, bool | None]]: + """Simply print unformatted text. This is the ultimate fallback. + + The stream comes from elsewhere (typically ``sys.stdout``), so its lifecycle + is left untouched: :class:`_PagerWriter` never closes what it wraps, and it + is the only thing :func:`get_pager_file` hands to the caller. + """ + yield stream, color + + +class Editor: + def __init__( + self, + editor: str | None = None, + env: cabc.Mapping[str, str] | None = None, + require_save: bool = True, + extension: str = ".txt", + ) -> None: + self.editor = editor + self.env = env + self.require_save = require_save + self.extension = extension + + def get_editor(self) -> str: + if self.editor is not None: + return self.editor + for key in "VISUAL", "EDITOR": + rv = os.environ.get(key) + if rv: + return rv + if WIN: + return "notepad" + + from shutil import which + + for editor in "sensible-editor", "vim", "nano": + if which(editor) is not None: + return editor + return "vi" + + def edit_files(self, filenames: cabc.Iterable[str | os.PathLike[str]]) -> None: + """Open files in the user's editor.""" + import shlex + import subprocess + + editor = self.get_editor() + environ: dict[str, str] | None = None + + if self.env: + environ = os.environ.copy() + environ.update(self.env) + + try: + # Split in POSIX mode (the default) for the same reasons as + # in pager(): strips quotes from tokens and preserves quoted + # Windows paths. + c = subprocess.Popen( + args=shlex.split(editor) + list(filenames), + env=environ, + ) + exit_code = c.wait() + if exit_code != 0: + raise ClickException( + _("{editor}: Editing failed").format(editor=editor) + ) + except OSError as e: + raise ClickException( + _("{editor}: Editing failed: {e}").format(editor=editor, e=e) + ) from e + + @t.overload + def edit(self, text: bytes | bytearray) -> bytes | None: ... + + # We cannot know whether or not the type expected is str or bytes when None + # is passed, so str is returned as that was what was done before. + @t.overload + def edit(self, text: str | None) -> str | None: ... + + def edit(self, text: str | bytes | bytearray | None) -> str | bytes | None: + import tempfile + + if text is None: + data: bytes | bytearray = b"" + elif isinstance(text, (bytes, bytearray)): + data = text + else: + if text and not text.endswith("\n"): + text += "\n" + + if WIN: + data = text.replace("\n", "\r\n").encode("utf-8-sig") + else: + data = text.encode("utf-8") + + fd, name = tempfile.mkstemp(prefix="editor-", suffix=self.extension) + f: t.BinaryIO + + try: + with os.fdopen(fd, "wb") as f: + f.write(data) + + # If the filesystem resolution is 1 second, like Mac OS + # 10.12 Extended, or 2 seconds, like FAT32, and the editor + # closes very fast, require_save can fail. Set the modified + # time to be 2 seconds in the past to work around this. + os.utime(name, (os.path.getatime(name), os.path.getmtime(name) - 2)) + # Depending on the resolution, the exact value might not be + # recorded, so get the new recorded value. + timestamp = os.path.getmtime(name) + + self.edit_files((name,)) + + if self.require_save and os.path.getmtime(name) == timestamp: + return None + + with open(name, "rb") as f: + rv = f.read() + + if isinstance(text, (bytes, bytearray)): + return rv + + return rv.decode("utf-8-sig").replace("\r\n", "\n") + finally: + os.unlink(name) + + +def open_url(url: str, wait: bool = False, locate: bool = False) -> int: + import subprocess + + def _unquote_file(url: str) -> str: + from urllib.parse import unquote + + if url.startswith("file://"): + url = unquote(url[7:]) + + return url + + if sys.platform == "darwin": + args = ["open"] + if wait: + args.append("-W") + if locate: + args.append("-R") + args.append(_unquote_file(url)) + null = open("/dev/null", "w") + try: + return subprocess.Popen(args, stderr=null).wait() + finally: + null.close() + elif WIN: + if locate: + url = _unquote_file(url) + args = ["explorer", "/select,", url] + try: + return subprocess.call(args) + except OSError: + return 127 + else: + try: + os.startfile(url) # type: ignore[attr-defined] + except OSError: + return 127 + return 0 + elif CYGWIN: + if locate: + url = _unquote_file(url) + args = ["cygstart", os.path.dirname(url)] + else: + args = ["cygstart"] + if wait: + args.append("-w") + args.append(url) + try: + return subprocess.call(args) + except OSError: + # Command not found + return 127 + + try: + if locate: + url = os.path.dirname(_unquote_file(url)) or "." + else: + url = _unquote_file(url) + c = subprocess.Popen(["xdg-open", url]) + if wait: + return c.wait() + return 0 + except OSError: + if url.startswith(("http://", "https://")) and not locate and not wait: + import webbrowser + + webbrowser.open(url) + return 0 + return 1 + + +def _translate_ch_to_exc(ch: str) -> None: + if ch == "\x03": + raise KeyboardInterrupt() + + if ch == "\x04" and not WIN: # Unix-like, Ctrl+D + raise EOFError() + + if ch == "\x1a" and WIN: # Windows, Ctrl+Z + raise EOFError() + + +if sys.platform == "win32": + import msvcrt + + @contextlib.contextmanager + def raw_terminal() -> cabc.Iterator[int]: + yield -1 + + def getchar(echo: bool) -> str: + # The function `getch` will return a bytes object corresponding to + # the pressed character. Since Windows 10 build 1803, it will also + # return \x00 when called a second time after pressing a regular key. + # + # `getwch` does not share this probably-bugged behavior. Moreover, it + # returns a Unicode object by default, which is what we want. + # + # Either of these functions will return \x00 or \xe0 to indicate + # a special key, and you need to call the same function again to get + # the "rest" of the code. The fun part is that \u00e0 is + # "latin small letter a with grave", so if you type that on a French + # keyboard, you _also_ get a \xe0. + # E.g., consider the Up arrow. This returns \xe0 and then \x48. The + # resulting Unicode string reads as "a with grave" + "capital H". + # This is indistinguishable from when the user actually types + # "a with grave" and then "capital H". + # + # When \xe0 is returned, we assume it's part of a special-key sequence + # and call `getwch` again, but that means that when the user types + # the \u00e0 character, `getchar` doesn't return until a second + # character is typed. + # The alternative is returning immediately, but that would mess up + # cross-platform handling of arrow keys and others that start with + # \xe0. Another option is using `getch`, but then we can't reliably + # read non-ASCII characters, because return values of `getch` are + # limited to the current 8-bit codepage. + # + # Anyway, Click doesn't claim to do this Right(tm), and using `getwch` + # is doing the right thing in more situations than with `getch`. + + if echo: + func = t.cast(t.Callable[[], str], msvcrt.getwche) + else: + func = t.cast(t.Callable[[], str], msvcrt.getwch) + + rv = func() + + if rv in ("\x00", "\xe0"): + # \x00 and \xe0 are control characters that indicate special key, + # see above. + rv += func() + + _translate_ch_to_exc(rv) + return rv + +else: + import termios + import tty + + @contextlib.contextmanager + def raw_terminal() -> cabc.Iterator[int]: + f: t.TextIO | None + fd: int + + if not isatty(sys.stdin): + f = open("/dev/tty") + fd = f.fileno() + else: + fd = sys.stdin.fileno() + f = None + + try: + old_settings = termios.tcgetattr(fd) + + try: + tty.setraw(fd) + yield fd + finally: + termios.tcsetattr(fd, termios.TCSADRAIN, old_settings) + sys.stdout.flush() + + if f is not None: + f.close() + except termios.error: + pass + + def getchar(echo: bool) -> str: + with raw_terminal() as fd: + ch = os.read(fd, 32).decode(get_best_encoding(sys.stdin), "replace") + + if echo and isatty(sys.stdout): + sys.stdout.write(ch) + + _translate_ch_to_exc(ch) + return ch diff --git a/.venv/lib/python3.14/site-packages/click/_textwrap.py b/.venv/lib/python3.14/site-packages/click/_textwrap.py new file mode 100644 index 0000000000000000000000000000000000000000..82840f2dff3ce627712c0ece2752382a0f7dab8b --- /dev/null +++ b/.venv/lib/python3.14/site-packages/click/_textwrap.py @@ -0,0 +1,188 @@ +from __future__ import annotations + +import collections.abc as cabc +import textwrap +from contextlib import contextmanager + +from ._compat import _ansi_re +from ._compat import term_len + + +def _truncate_visible(text: str, n: int) -> str: + """Return the longest prefix of ``text`` containing at most ``n`` visible + characters. + + ANSI escape sequences inside the prefix are kept intact and do not count + toward the visible width. A cut is never placed inside an escape sequence. + """ + if n <= 0: + return "" + + visible = 0 + i = 0 + cut = 0 + end = len(text) + while i < end: + m = _ansi_re.match(text, i) + if m is not None: + i = m.end() + continue + visible += 1 + i += 1 + cut = i + if visible >= n: + break + return text[:cut] + + +class TextWrapper(textwrap.TextWrapper): + """``textwrap.TextWrapper`` variant that measures widths by visible + character count. + + ANSI escape sequences embedded in chunks, indents, or the placeholder are + excluded from the width budget. Without this, styled help text (a styled + ``Usage:`` prefix, a colorized option name, ...) would be wrapped earlier + than its visible length warrants and tokens would split mid-word. + """ + + def _handle_long_word( + self, + reversed_chunks: list[str], + cur_line: list[str], + cur_len: int, + width: int, + ) -> None: + space_left = max(width - cur_len, 1) + + if self.break_long_words: + last = reversed_chunks[-1] + cut = _truncate_visible(last, space_left) + res = last[len(cut) :] + cur_line.append(cut) + reversed_chunks[-1] = res + elif not cur_line: + cur_line.append(reversed_chunks.pop()) + + def _wrap_chunks(self, chunks: list[str]) -> list[str]: + """Wrap chunks counting widths in visible characters. + + Mirrors the algorithm of :meth:`textwrap.TextWrapper._wrap_chunks` + with every width measurement routed through + :func:`click._compat.term_len` instead of :func:`len`, so ANSI escape + bytes in chunks, indents, or the placeholder do not inflate the count. + + .. seealso:: + :class:`textwrap.TextWrapper` in the Python standard library documentation: + https://docs.python.org/3/library/textwrap.html#textwrap.TextWrapper + + Reference implementation in CPython: + https://github.com/python/cpython/blob/main/Lib/textwrap.py + """ + lines: list[str] = [] + if self.width <= 0: + raise ValueError(f"invalid width {self.width!r} (must be > 0)") + if self.max_lines is not None: + if self.max_lines > 1: + indent = self.subsequent_indent + else: + indent = self.initial_indent + if term_len(indent) + term_len(self.placeholder.lstrip()) > self.width: + raise ValueError("placeholder too large for max width") + + chunks.reverse() + + while chunks: + cur_line: list[str] = [] + cur_len = 0 + + if lines: + indent = self.subsequent_indent + else: + indent = self.initial_indent + + width = self.width - term_len(indent) + + if self.drop_whitespace and chunks[-1].strip() == "" and lines: + del chunks[-1] + + while chunks: + n = term_len(chunks[-1]) + + if cur_len + n <= width: + cur_line.append(chunks.pop()) + cur_len += n + + else: + break + + if chunks and term_len(chunks[-1]) > width: + self._handle_long_word(chunks, cur_line, cur_len, width) + cur_len = sum(map(term_len, cur_line)) + + if self.drop_whitespace and cur_line and cur_line[-1].strip() == "": + cur_len -= term_len(cur_line[-1]) + del cur_line[-1] + + if cur_line: + if ( + self.max_lines is None + or len(lines) + 1 < self.max_lines + or ( + not chunks + or self.drop_whitespace + and len(chunks) == 1 + and not chunks[0].strip() + ) + and cur_len <= width + ): + lines.append(indent + "".join(cur_line)) + else: + while cur_line: + if ( + cur_line[-1].strip() + and cur_len + term_len(self.placeholder) <= width + ): + cur_line.append(self.placeholder) + lines.append(indent + "".join(cur_line)) + break + cur_len -= term_len(cur_line[-1]) + del cur_line[-1] + else: + if lines: + prev_line = lines[-1].rstrip() + if ( + term_len(prev_line) + term_len(self.placeholder) + <= self.width + ): + lines[-1] = prev_line + self.placeholder + break + lines.append(indent + self.placeholder.lstrip()) + break + + return lines + + @contextmanager + def extra_indent(self, indent: str) -> cabc.Iterator[None]: + old_initial_indent = self.initial_indent + old_subsequent_indent = self.subsequent_indent + self.initial_indent += indent + self.subsequent_indent += indent + + try: + yield + finally: + self.initial_indent = old_initial_indent + self.subsequent_indent = old_subsequent_indent + + def indent_only(self, text: str) -> str: + rv = [] + + for idx, line in enumerate(text.splitlines()): + indent = self.initial_indent + + if idx > 0: + indent = self.subsequent_indent + + rv.append(f"{indent}{line}") + + return "\n".join(rv) diff --git a/.venv/lib/python3.14/site-packages/click/_utils.py b/.venv/lib/python3.14/site-packages/click/_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..05ee2e99757adb19882664e1fc0377ac29c92f80 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/click/_utils.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +import enum +import typing as t + + +class Sentinel(enum.Enum): + """Enum used to define sentinel values. + + .. seealso:: + + `PEP 661 - Sentinel Values `_. + """ + + UNSET = object() + FLAG_NEEDS_VALUE = object() + + def __repr__(self) -> str: + return f"{self.__class__.__name__}.{self.name}" + + +UNSET: t.Literal[Sentinel.UNSET] = Sentinel.UNSET +"""Sentinel used to indicate that a value is not set.""" + +FLAG_NEEDS_VALUE: t.Literal[Sentinel.FLAG_NEEDS_VALUE] = Sentinel.FLAG_NEEDS_VALUE +"""Sentinel used to indicate an option was passed as a flag without a +value but is not a flag option. + +``Option.consume_value`` uses this to prompt or use the ``flag_value``. +""" + +T_UNSET: t.TypeAlias = t.Literal[Sentinel.UNSET] +"""Type hint for the :data:`UNSET` sentinel value.""" + +T_FLAG_NEEDS_VALUE: t.TypeAlias = t.Literal[Sentinel.FLAG_NEEDS_VALUE] +"""Type hint for the :data:`FLAG_NEEDS_VALUE` sentinel value.""" diff --git a/.venv/lib/python3.14/site-packages/click/_winconsole.py b/.venv/lib/python3.14/site-packages/click/_winconsole.py new file mode 100644 index 0000000000000000000000000000000000000000..d25178d66ff7a9c8a1da52e61379da240be625b6 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/click/_winconsole.py @@ -0,0 +1,297 @@ +# This module is based on the excellent work by Adam Bartoš who +# provided a lot of what went into the implementation here in +# the discussion to issue1602 in the Python bug tracker. +# +# There are some general differences in regards to how this works +# compared to the original patches as we do not need to patch +# the entire interpreter but just work in our little world of +# echo and prompt. +from __future__ import annotations + +import collections.abc as cabc +import io +import sys +import time +import typing as t +from ctypes import Array +from ctypes import byref +from ctypes import c_char +from ctypes import c_char_p +from ctypes import c_int +from ctypes import c_ssize_t +from ctypes import c_ulong +from ctypes import c_void_p +from ctypes import POINTER +from ctypes import py_object +from ctypes import Structure +from ctypes.wintypes import DWORD +from ctypes.wintypes import HANDLE +from ctypes.wintypes import LPCWSTR +from ctypes.wintypes import LPWSTR +from gettext import gettext as _ + +from ._compat import _NonClosingTextIOWrapper + +assert sys.platform == "win32" +import msvcrt # noqa: E402 +from ctypes import windll # noqa: E402 +from ctypes import WINFUNCTYPE # noqa: E402 + +c_ssize_p = POINTER(c_ssize_t) + +kernel32 = windll.kernel32 +GetStdHandle = kernel32.GetStdHandle +ReadConsoleW = kernel32.ReadConsoleW +WriteConsoleW = kernel32.WriteConsoleW +GetConsoleMode = kernel32.GetConsoleMode +GetLastError = kernel32.GetLastError +GetCommandLineW = WINFUNCTYPE(LPWSTR)(("GetCommandLineW", windll.kernel32)) +CommandLineToArgvW = WINFUNCTYPE(POINTER(LPWSTR), LPCWSTR, POINTER(c_int))( + ("CommandLineToArgvW", windll.shell32) +) +LocalFree = WINFUNCTYPE(c_void_p, c_void_p)(("LocalFree", windll.kernel32)) + +STDIN_HANDLE = GetStdHandle(-10) +STDOUT_HANDLE = GetStdHandle(-11) +STDERR_HANDLE = GetStdHandle(-12) + +PyBUF_SIMPLE = 0 +PyBUF_WRITABLE = 1 + +ERROR_SUCCESS = 0 +ERROR_NOT_ENOUGH_MEMORY = 8 +ERROR_OPERATION_ABORTED = 995 + +STDIN_FILENO = 0 +STDOUT_FILENO = 1 +STDERR_FILENO = 2 + +EOF = b"\x1a" +MAX_BYTES_WRITTEN = 32767 + +if t.TYPE_CHECKING: + try: + # Using `typing_extensions.Buffer` instead of `collections.abc` + # on Windows for some reason does not have `Sized` implemented. + from collections.abc import Buffer # type: ignore + except ImportError: + from typing_extensions import Buffer + +try: + from ctypes import pythonapi +except ImportError: + # On PyPy we cannot get buffers so our ability to operate here is + # severely limited. + get_buffer = None +else: + + class Py_buffer(Structure): + _fields_ = [ # noqa: RUF012 + ("buf", c_void_p), + ("obj", py_object), + ("len", c_ssize_t), + ("itemsize", c_ssize_t), + ("readonly", c_int), + ("ndim", c_int), + ("format", c_char_p), + ("shape", c_ssize_p), + ("strides", c_ssize_p), + ("suboffsets", c_ssize_p), + ("internal", c_void_p), + ] + + PyObject_GetBuffer = pythonapi.PyObject_GetBuffer + PyBuffer_Release = pythonapi.PyBuffer_Release + + def get_buffer(obj: Buffer, writable: bool = False) -> Array[c_char]: + buf = Py_buffer() + flags: int = PyBUF_WRITABLE if writable else PyBUF_SIMPLE + PyObject_GetBuffer(py_object(obj), byref(buf), flags) + + try: + buffer_type = c_char * buf.len + out: Array[c_char] = buffer_type.from_address(buf.buf) + return out + finally: + PyBuffer_Release(byref(buf)) + + +class _WindowsConsoleRawIOBase(io.RawIOBase): + def __init__(self, handle: int | None) -> None: + self.handle = handle + + def isatty(self) -> t.Literal[True]: + super().isatty() + return True + + +class _WindowsConsoleReader(_WindowsConsoleRawIOBase): + def readable(self) -> t.Literal[True]: + return True + + def readinto(self, b: Buffer) -> int: + bytes_to_be_read = len(b) + if not bytes_to_be_read: + return 0 + elif bytes_to_be_read % 2: + raise ValueError( + "cannot read odd number of bytes from UTF-16-LE encoded console" + ) + + buffer = get_buffer(b, writable=True) + code_units_to_be_read = bytes_to_be_read // 2 + code_units_read = c_ulong() + + rv = ReadConsoleW( + HANDLE(self.handle), + buffer, + code_units_to_be_read, + byref(code_units_read), + None, + ) + if GetLastError() == ERROR_OPERATION_ABORTED: + # wait for KeyboardInterrupt + time.sleep(0.1) + if not rv: + raise OSError(_("Windows error: {error}").format(error=GetLastError())) + + if buffer[0] == EOF: + return 0 + return 2 * code_units_read.value + + +class _WindowsConsoleWriter(_WindowsConsoleRawIOBase): + def writable(self) -> t.Literal[True]: + return True + + @staticmethod + def _get_error_message(errno: int) -> str: + if errno == ERROR_SUCCESS: + return "ERROR_SUCCESS" + elif errno == ERROR_NOT_ENOUGH_MEMORY: + return "ERROR_NOT_ENOUGH_MEMORY" + return _("Windows error: {error}").format(error=errno) + + def write(self, b: Buffer) -> int: + bytes_to_be_written = len(b) + buf = get_buffer(b) + code_units_to_be_written = min(bytes_to_be_written, MAX_BYTES_WRITTEN) // 2 + code_units_written = c_ulong() + + WriteConsoleW( + HANDLE(self.handle), + buf, + code_units_to_be_written, + byref(code_units_written), + None, + ) + bytes_written = 2 * code_units_written.value + + if bytes_written == 0 and bytes_to_be_written > 0: + raise OSError(self._get_error_message(GetLastError())) + return bytes_written + + +class ConsoleStream: + def __init__(self, text_stream: t.TextIO, byte_stream: t.BinaryIO) -> None: + self._text_stream = text_stream + self.buffer = byte_stream + + @property + def name(self) -> str: + return self.buffer.name + + def write(self, x: t.AnyStr) -> int: + if isinstance(x, str): + return self._text_stream.write(x) + try: + self.flush() + except Exception: + pass + return self.buffer.write(x) + + def writelines(self, lines: cabc.Iterable[t.AnyStr]) -> None: + for line in lines: + self.write(line) + + def __getattr__(self, name: str) -> t.Any: + return getattr(self._text_stream, name) + + def isatty(self) -> bool: + return self.buffer.isatty() + + def __repr__(self) -> str: + return f"" + + +def _get_text_stdin(buffer_stream: t.BinaryIO) -> t.TextIO: + text_stream = _NonClosingTextIOWrapper( + io.BufferedReader(_WindowsConsoleReader(STDIN_HANDLE)), + "utf-16-le", + "strict", + line_buffering=True, + ) + return t.cast(t.TextIO, ConsoleStream(text_stream, buffer_stream)) + + +def _get_text_stdout(buffer_stream: t.BinaryIO) -> t.TextIO: + text_stream = _NonClosingTextIOWrapper( + io.BufferedWriter(_WindowsConsoleWriter(STDOUT_HANDLE)), + "utf-16-le", + "strict", + line_buffering=True, + ) + return t.cast(t.TextIO, ConsoleStream(text_stream, buffer_stream)) + + +def _get_text_stderr(buffer_stream: t.BinaryIO) -> t.TextIO: + text_stream = _NonClosingTextIOWrapper( + io.BufferedWriter(_WindowsConsoleWriter(STDERR_HANDLE)), + "utf-16-le", + "strict", + line_buffering=True, + ) + return t.cast(t.TextIO, ConsoleStream(text_stream, buffer_stream)) + + +_stream_factories: cabc.Mapping[int, t.Callable[[t.BinaryIO], t.TextIO]] = { + 0: _get_text_stdin, + 1: _get_text_stdout, + 2: _get_text_stderr, +} + + +def _is_console(f: t.TextIO) -> bool: + if not hasattr(f, "fileno"): + return False + + try: + fileno = f.fileno() + except (OSError, io.UnsupportedOperation): + return False + + handle = msvcrt.get_osfhandle(fileno) + return bool(GetConsoleMode(handle, byref(DWORD()))) + + +def _get_windows_console_stream( + f: t.TextIO, encoding: str | None, errors: str | None +) -> t.TextIO | None: + if ( + get_buffer is None + or encoding not in {"utf-16-le", None} + or errors not in {"strict", None} + or not _is_console(f) + ): + return None + + func = _stream_factories.get(f.fileno()) + if func is None: + return None + + b = getattr(f, "buffer", None) + + if b is None: + return None + + return func(b) diff --git a/.venv/lib/python3.14/site-packages/click/core.py b/.venv/lib/python3.14/site-packages/click/core.py new file mode 100644 index 0000000000000000000000000000000000000000..de129ec2ceaa1e77cab696cad672b0a3ca84413d --- /dev/null +++ b/.venv/lib/python3.14/site-packages/click/core.py @@ -0,0 +1,3799 @@ +from __future__ import annotations + +import collections.abc as cabc +import enum +import errno +import inspect +import os +import sys +import typing as t +from abc import ABC +from abc import abstractmethod +from collections import abc +from collections import Counter +from contextlib import AbstractContextManager +from contextlib import contextmanager +from contextlib import ExitStack +from functools import update_wrapper +from gettext import gettext as _ +from gettext import ngettext +from itertools import repeat +from types import TracebackType + +from . import types +from ._utils import FLAG_NEEDS_VALUE +from ._utils import UNSET +from .exceptions import Abort +from .exceptions import BadParameter +from .exceptions import ClickException +from .exceptions import Exit +from .exceptions import MissingParameter +from .exceptions import NoArgsIsHelpError +from .exceptions import NoSuchCommand +from .exceptions import UsageError +from .formatting import HelpFormatter +from .formatting import join_options +from .globals import pop_context +from .globals import push_context +from .parser import _OptionParser +from .parser import _split_opt +from .termui import confirm +from .termui import prompt +from .termui import style +from .utils import _detect_program_name +from .utils import _expand_args +from .utils import _make_default_short_help +from .utils import _PacifyFlushWrapper +from .utils import echo +from .utils import make_str + +if t.TYPE_CHECKING: + from typing_extensions import Self + + from .shell_completion import CompletionItem + +F = t.TypeVar("F", bound="t.Callable[..., t.Any]") +V = t.TypeVar("V") + +# Reserved storage name of the automatic help option. No user parameter is +# expected to claim it. +_HELP_OPTION_STORAGE_NAME = "_click_default_help" + + +def _complete_visible_commands( + ctx: Context, incomplete: str +) -> cabc.Iterator[tuple[str, Command]]: + """List all the subcommands of a group that start with the + incomplete value and aren't hidden. + + :param ctx: Invocation context for the group. + :param incomplete: Value being completed. May be empty. + """ + multi = t.cast(Group, ctx.command) + + for name in multi.list_commands(ctx): + if name.startswith(incomplete): + command = multi.get_command(ctx, name) + + if command is not None and not command.hidden: + yield name, command + + +def _check_nested_chain( + base_command: Group, cmd_name: str, cmd: Command, register: bool = False +) -> None: + if not base_command.chain or not isinstance(cmd, Group): + return + + if register: + message = ( + f"It is not possible to add the group {cmd_name!r} to another" + f" group {base_command.name!r} that is in chain mode." + ) + else: + message = ( + f"Found the group {cmd_name!r} as subcommand to another group " + f" {base_command.name!r} that is in chain mode. This is not supported." + ) + + raise RuntimeError(message) + + +def _format_deprecated_label(deprecated: bool | str) -> str: + """Return the parenthesized deprecation label shown in help text.""" + label = _("deprecated").upper() + if isinstance(deprecated, str): + return f"({label}: {deprecated})" + return f"({label})" + + +def _format_deprecated_suffix(deprecated: bool | str) -> str: + """Return the trailing reason for a ``DeprecationWarning`` message, + prefixed with a space, or an empty string when no reason was given. + """ + if isinstance(deprecated, str): + return f" {deprecated}" + return "" + + +def batch(iterable: cabc.Iterable[V], batch_size: int) -> list[tuple[V, ...]]: + return list(zip(*repeat(iter(iterable), batch_size), strict=False)) + + +@contextmanager +def augment_usage_errors( + ctx: Context, param: Parameter | None = None +) -> cabc.Generator[None]: + """Context manager that attaches extra information to exceptions.""" + try: + yield + except BadParameter as e: + if e.ctx is None: + e.ctx = ctx + if param is not None and e.param is None: + e.param = param + raise + except UsageError as e: + if e.ctx is None: + e.ctx = ctx + raise + + +def iter_params_for_processing( + invocation_order: cabc.Sequence[Parameter], + declaration_order: cabc.Sequence[Parameter], +) -> list[Parameter]: + """Returns all declared parameters in the order they should be processed. + + The declared parameters are re-shuffled depending on the order in which + they were invoked, as well as the eagerness of each parameters. + + The invocation order takes precedence over the declaration order. I.e. the + order in which the user provided them to the CLI is respected. + + This behavior and its effect on callback evaluation is detailed at: + https://click.palletsprojects.com/en/stable/advanced/#callback-evaluation-order + """ + + def sort_key(item: Parameter) -> tuple[bool, float]: + try: + idx: float = invocation_order.index(item) + except ValueError: + idx = float("inf") + + return not item.is_eager, idx + + return sorted(declaration_order, key=sort_key) + + +class ParameterSource(enum.IntEnum): + """This is an :class:`~enum.IntEnum` that indicates the source of a + parameter's value. + + Use :meth:`click.Context.get_parameter_source` to get the + source for a parameter by name. + + Members are ordered from most explicit to least explicit source. + This allows comparison to check if a value was explicitly provided: + + .. code-block:: python + + source = ctx.get_parameter_source("port") + if source < click.ParameterSource.DEFAULT_MAP: + ... # value was explicitly set + + .. versionchanged:: 8.3.3 + Use :class:`~enum.IntEnum` and reorder members from most to + least explicit. Supports comparison operators. + + .. versionchanged:: 8.0 + Use :class:`~enum.Enum` and drop the ``validate`` method. + + .. versionchanged:: 8.0 + Added the ``PROMPT`` value. + """ + + PROMPT = enum.auto() + """Used a prompt to confirm a default or provide a value.""" + COMMANDLINE = enum.auto() + """The value was provided by the command line args.""" + ENVIRONMENT = enum.auto() + """The value was provided with an environment variable.""" + DEFAULT_MAP = enum.auto() + """Used a default provided by :attr:`Context.default_map`.""" + DEFAULT = enum.auto() + """Used the default specified by the parameter.""" + + +class Context: + """The context is a special internal object that holds state relevant + for the script execution at every single level. It's normally invisible + to commands unless they opt-in to getting access to it. + + The context is useful as it can pass internal objects around and can + control special execution features such as reading data from + environment variables. + + A context can be used as context manager in which case it will call + :meth:`close` on teardown. + + :param command: the command class for this context. + :param parent: the parent context. + :param info_name: the info name for this invocation. Generally this + is the most descriptive name for the script or + command. For the toplevel script it is usually + the name of the script, for commands below that it's + the name of the script. + :param obj: an arbitrary object of user data. + :param auto_envvar_prefix: the prefix to use for automatic environment + variables. If this is `None` then reading + from environment variables is disabled. This + does not affect manually set environment + variables which are always read. + :param default_map: a dictionary (like object) with default values + for parameters. + :param terminal_width: the width of the terminal. The default is + inherit from parent context. If no context + defines the terminal width then auto + detection will be applied. + :param max_content_width: the maximum width for content rendered by + Click (this currently only affects help + pages). This defaults to 80 characters if + not overridden. In other words: even if the + terminal is larger than that, Click will not + format things wider than 80 characters by + default. In addition to that, formatters might + add some safety mapping on the right. + :param resilient_parsing: if this flag is enabled then Click will + parse without any interactivity or callback + invocation. Default values will also be + ignored. This is useful for implementing + things such as completion support. + :param allow_extra_args: if this is set to `True` then extra arguments + at the end will not raise an error and will be + kept on the context. The default is to inherit + from the command. + :param allow_interspersed_args: if this is set to `False` then options + and arguments cannot be mixed. The + default is to inherit from the command. + :param ignore_unknown_options: instructs click to ignore options it does + not know and keeps them for later + processing. + :param help_option_names: optionally a list of strings that define how + the default help parameter is named. The + default is ``['--help']``. + :param token_normalize_func: an optional function that is used to + normalize tokens (options, choices, + etc.). This for instance can be used to + implement case insensitive behavior. + :param color: controls if the terminal supports ANSI colors or not. The + default is autodetection. This is only needed if ANSI + codes are used in texts that Click prints which is by + default not the case. This for instance would affect + help output. + :param show_default: Show the default value for commands. If this + value is not set, it defaults to the value from the parent + context. ``Command.show_default`` overrides this default for the + specific command. + + .. versionchanged:: 8.2 + The ``protected_args`` attribute is deprecated and will be removed in + Click 9.0. ``args`` will contain remaining unparsed tokens. + + .. versionchanged:: 8.1 + The ``show_default`` parameter is overridden by + ``Command.show_default``, instead of the other way around. + + .. versionchanged:: 8.0 + The ``show_default`` parameter defaults to the value from the + parent context. + + .. versionchanged:: 7.1 + Added the ``show_default`` parameter. + + .. versionchanged:: 4.0 + Added the ``color``, ``ignore_unknown_options``, and + ``max_content_width`` parameters. + + .. versionchanged:: 3.0 + Added the ``allow_extra_args`` and ``allow_interspersed_args`` + parameters. + + .. versionchanged:: 2.0 + Added the ``resilient_parsing``, ``help_option_names``, and + ``token_normalize_func`` parameters. + """ + + #: The formatter class to create with :meth:`make_formatter`. + #: + #: .. versionadded:: 8.0 + formatter_class: type[HelpFormatter] = HelpFormatter + + parent: Context | None + command: Command + info_name: str | None + params: dict[str, t.Any] + args: list[str] + _protected_args: list[str] + _opt_prefixes: set[str] + obj: t.Any + _meta: dict[str, t.Any] + default_map: cabc.MutableMapping[str, t.Any] | None + invoked_subcommand: str | None + terminal_width: int | None + max_content_width: int | None + allow_extra_args: bool + allow_interspersed_args: bool + ignore_unknown_options: bool + help_option_names: list[str] + token_normalize_func: t.Callable[[str], str] | None + resilient_parsing: bool + auto_envvar_prefix: str | None + color: bool | None + show_default: bool | None + _close_callbacks: list[t.Callable[[], t.Any]] + _depth: int + _parameter_source: dict[str, ParameterSource] + _param_default_explicit: dict[str, bool] + _exit_stack: ExitStack + + def __init__( + self, + command: Command, + parent: Context | None = None, + info_name: str | None = None, + obj: t.Any | None = None, + auto_envvar_prefix: str | None = None, + default_map: cabc.MutableMapping[str, t.Any] | None = None, + terminal_width: int | None = None, + max_content_width: int | None = None, + resilient_parsing: bool = False, + allow_extra_args: bool | None = None, + allow_interspersed_args: bool | None = None, + ignore_unknown_options: bool | None = None, + help_option_names: list[str] | None = None, + token_normalize_func: t.Callable[[str], str] | None = None, + color: bool | None = None, + show_default: bool | None = None, + ) -> None: + #: the parent context or `None` if none exists. + self.parent = parent + #: the :class:`Command` for this context. + self.command = command + #: the descriptive information name + self.info_name = info_name + #: Map of parameter names to their parsed values. Parameters + #: with ``expose_value=False`` are not stored. + self.params = {} + #: the leftover arguments. + self.args = [] + #: protected arguments. These are arguments that are prepended + #: to `args` when certain parsing scenarios are encountered but + #: must be never propagated to another arguments. This is used + #: to implement nested parsing. + self._protected_args = [] + #: the collected prefixes of the command's options. + self._opt_prefixes = set(parent._opt_prefixes) if parent else set() + + if obj is None and parent is not None: + obj = parent.obj + + #: the user object stored. + self.obj = obj + self._meta = getattr(parent, "meta", {}) + + #: A dictionary (-like object) with defaults for parameters. + if ( + default_map is None + and info_name is not None + and parent is not None + and parent.default_map is not None + ): + default_map = parent.default_map.get(info_name) + + self.default_map = default_map + + #: This flag indicates if a subcommand is going to be executed. A + #: group callback can use this information to figure out if it's + #: being executed directly or because the execution flow passes + #: onwards to a subcommand. By default it's None, but it can be + #: the name of the subcommand to execute. + #: + #: If chaining is enabled this will be set to ``'*'`` in case + #: any commands are executed. It is however not possible to + #: figure out which ones. If you require this knowledge you + #: should use a :func:`result_callback`. + self.invoked_subcommand = None + + if terminal_width is None and parent is not None: + terminal_width = parent.terminal_width + + #: The width of the terminal (None is autodetection). + self.terminal_width = terminal_width + + if max_content_width is None and parent is not None: + max_content_width = parent.max_content_width + + #: The maximum width of formatted content (None implies a sensible + #: default which is 80 for most things). + self.max_content_width = max_content_width + + if allow_extra_args is None: + allow_extra_args = command.allow_extra_args + + #: Indicates if the context allows extra args or if it should + #: fail on parsing. + #: + #: .. versionadded:: 3.0 + self.allow_extra_args = allow_extra_args + + if allow_interspersed_args is None: + allow_interspersed_args = command.allow_interspersed_args + + #: Indicates if the context allows mixing of arguments and + #: options or not. + #: + #: .. versionadded:: 3.0 + self.allow_interspersed_args = allow_interspersed_args + + if ignore_unknown_options is None: + ignore_unknown_options = command.ignore_unknown_options + + #: Instructs click to ignore options that a command does not + #: understand and will store it on the context for later + #: processing. This is primarily useful for situations where you + #: want to call into external programs. Generally this pattern is + #: strongly discouraged because it's not possibly to losslessly + #: forward all arguments. + #: + #: .. versionadded:: 4.0 + self.ignore_unknown_options = ignore_unknown_options + + if help_option_names is None: + if parent is not None: + help_option_names = parent.help_option_names + else: + help_option_names = ["--help"] + + #: The names for the help options. + self.help_option_names = help_option_names + + if token_normalize_func is None and parent is not None: + token_normalize_func = parent.token_normalize_func + + #: An optional normalization function for tokens. This is + #: options, choices, commands etc. + self.token_normalize_func = token_normalize_func + + #: Indicates if resilient parsing is enabled. In that case Click + #: will do its best to not cause any failures and default values + #: will be ignored. Useful for completion. + self.resilient_parsing = resilient_parsing + + # If there is no envvar prefix yet, but the parent has one and + # the command on this level has a name, we can expand the envvar + # prefix automatically. + if auto_envvar_prefix is None: + if ( + parent is not None + and parent.auto_envvar_prefix is not None + and self.info_name is not None + ): + auto_envvar_prefix = ( + f"{parent.auto_envvar_prefix}_{self.info_name.upper()}" + ) + else: + auto_envvar_prefix = auto_envvar_prefix.upper() + + if auto_envvar_prefix is not None: + auto_envvar_prefix = auto_envvar_prefix.replace("-", "_") + + self.auto_envvar_prefix = auto_envvar_prefix + + if color is None and parent is not None: + color = parent.color + + #: Controls if styling output is wanted or not. + self.color = color + + if show_default is None and parent is not None: + show_default = parent.show_default + + #: Show option default values when formatting help text. + self.show_default = show_default + + self._close_callbacks = [] + self._depth = 0 + self._parameter_source = {} + # Tracks whether the option that currently owns each parameter slot in + # :attr:`params` had its ``default`` set explicitly by the user. Used + # to tie-break feature-switch groups where multiple options share a + # parameter name and both fall back to their default value. + # Refs: https://github.com/pallets/click/issues/3403 + self._param_default_explicit = {} + self._exit_stack = ExitStack() + + @property + def protected_args(self) -> list[str]: + import warnings + + warnings.warn( + "'protected_args' is deprecated and will be removed in Click 9.0." + " 'args' will contain remaining unparsed tokens.", + DeprecationWarning, + stacklevel=2, + ) + return self._protected_args + + def to_info_dict(self) -> dict[str, t.Any]: + """Gather information that could be useful for a tool generating + user-facing documentation. This traverses the entire CLI + structure. + + .. code-block:: python + + with Context(cli) as ctx: + info = ctx.to_info_dict() + + .. versionadded:: 8.0 + """ + return { + "command": self.command.to_info_dict(self), + "info_name": self.info_name, + "allow_extra_args": self.allow_extra_args, + "allow_interspersed_args": self.allow_interspersed_args, + "ignore_unknown_options": self.ignore_unknown_options, + "auto_envvar_prefix": self.auto_envvar_prefix, + } + + def __enter__(self) -> Self: + self._depth += 1 + push_context(self) + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + tb: TracebackType | None, + ) -> bool | None: + self._depth -= 1 + exit_result: bool | None = None + if self._depth == 0: + exit_result = self._close_with_exception_info(exc_type, exc_value, tb) + pop_context() + + return exit_result + + @contextmanager + def scope(self, cleanup: bool = True) -> cabc.Generator[Context]: + """This helper method can be used with the context object to promote + it to the current thread local (see :func:`get_current_context`). + The default behavior of this is to invoke the cleanup functions which + can be disabled by setting `cleanup` to `False`. The cleanup + functions are typically used for things such as closing file handles. + + If the cleanup is intended the context object can also be directly + used as a context manager. + + Example usage:: + + with ctx.scope(): + assert get_current_context() is ctx + + This is equivalent:: + + with ctx: + assert get_current_context() is ctx + + .. versionadded:: 5.0 + + :param cleanup: controls if the cleanup functions should be run or + not. The default is to run these functions. In + some situations the context only wants to be + temporarily pushed in which case this can be disabled. + Nested pushes automatically defer the cleanup. + """ + if not cleanup: + self._depth += 1 + try: + with self as rv: + yield rv + finally: + if not cleanup: + self._depth -= 1 + + @property + def meta(self) -> dict[str, t.Any]: + """This is a dictionary which is shared with all the contexts + that are nested. It exists so that click utilities can store some + state here if they need to. It is however the responsibility of + that code to manage this dictionary well. + + The keys are supposed to be unique dotted strings. For instance + module paths are a good choice for it. What is stored in there is + irrelevant for the operation of click. However what is important is + that code that places data here adheres to the general semantics of + the system. + + Example usage:: + + LANG_KEY = f'{__name__}.lang' + + def set_language(value): + ctx = get_current_context() + ctx.meta[LANG_KEY] = value + + def get_language(): + return get_current_context().meta.get(LANG_KEY, 'en_US') + + .. versionadded:: 5.0 + """ + return self._meta + + def make_formatter(self) -> HelpFormatter: + """Creates the :class:`~click.HelpFormatter` for the help and + usage output. + + To quickly customize the formatter class used without overriding + this method, set the :attr:`formatter_class` attribute. + + .. versionchanged:: 8.0 + Added the :attr:`formatter_class` attribute. + """ + return self.formatter_class( + width=self.terminal_width, max_width=self.max_content_width + ) + + def with_resource(self, context_manager: AbstractContextManager[V]) -> V: + """Register a resource as if it were used in a ``with`` + statement. The resource will be cleaned up when the context is + popped. + + Uses :meth:`contextlib.ExitStack.enter_context`. It calls the + resource's ``__enter__()`` method and returns the result. When + the context is popped, it closes the stack, which calls the + resource's ``__exit__()`` method. + + To register a cleanup function for something that isn't a + context manager, use :meth:`call_on_close`. Or use something + from :mod:`contextlib` to turn it into a context manager first. + + .. code-block:: python + + @click.group() + @click.option("--name") + @click.pass_context + def cli(ctx): + ctx.obj = ctx.with_resource(connect_db(name)) + + :param context_manager: The context manager to enter. + :return: Whatever ``context_manager.__enter__()`` returns. + + .. versionadded:: 8.0 + """ + return self._exit_stack.enter_context(context_manager) + + def call_on_close(self, f: t.Callable[..., t.Any]) -> t.Callable[..., t.Any]: + """Register a function to be called when the context tears down. + + This can be used to close resources opened during the script + execution. Resources that support Python's context manager + protocol which would be used in a ``with`` statement should be + registered with :meth:`with_resource` instead. + + :param f: The function to execute on teardown. + """ + return self._exit_stack.callback(f) + + def close(self) -> None: + """Invoke all close callbacks registered with + :meth:`call_on_close`, and exit all context managers entered + with :meth:`with_resource`. + """ + self._close_with_exception_info(None, None, None) + + def _close_with_exception_info( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + tb: TracebackType | None, + ) -> bool | None: + """Unwind the exit stack by calling its :meth:`__exit__` providing the exception + information to allow for exception handling by the various resources registered + using :meth;`with_resource` + + :return: Whatever ``exit_stack.__exit__()`` returns. + """ + exit_result = self._exit_stack.__exit__(exc_type, exc_value, tb) + # In case the context is reused, create a new exit stack. + self._exit_stack = ExitStack() + + return exit_result + + @property + def command_path(self) -> str: + """The computed command path. This is used for the ``usage`` + information on the help page. It's automatically created by + combining the info names of the chain of contexts to the root. + """ + rv = "" + if self.info_name is not None: + rv = self.info_name + if self.parent is not None: + parent_command_path = [self.parent.command_path] + + if isinstance(self.parent.command, Command): + for param in self.parent.command.get_params(self): + parent_command_path.extend(param.get_usage_pieces(self)) + + rv = f"{' '.join(parent_command_path)} {rv}" + return rv.lstrip() + + def find_root(self) -> Context: + """Finds the outermost context.""" + node = self + while node.parent is not None: + node = node.parent + return node + + def find_object(self, object_type: type[V]) -> V | None: + """Finds the closest object of a given type.""" + node: Context | None = self + + while node is not None: + if isinstance(node.obj, object_type): + return node.obj + + node = node.parent + + return None + + def ensure_object(self, object_type: type[V]) -> V: + """Like :meth:`find_object` but sets the innermost object to a + new instance of `object_type` if it does not exist. + """ + rv = self.find_object(object_type) + if rv is None: + self.obj = rv = object_type() + return rv + + def _default_map_has(self, name: str | None) -> bool: + """Check if :attr:`default_map` contains a real value for ``name``. + + Returns ``False`` when the key is absent, the map is ``None``, + ``name`` is ``None``, or the stored value is the internal + :data:`UNSET` sentinel. + """ + return ( + name is not None + and self.default_map is not None + and name in self.default_map + and self.default_map[name] is not UNSET + ) + + @t.overload + def lookup_default( + self, name: str, call: t.Literal[True] = True + ) -> t.Any | None: ... + + @t.overload + def lookup_default( + self, name: str, call: t.Literal[False] = ... + ) -> t.Any | t.Callable[[], t.Any] | None: ... + + def lookup_default(self, name: str, call: bool = True) -> t.Any | None: + """Get the default for a parameter from :attr:`default_map`. + + :param name: Name of the parameter. + :param call: If the default is a callable, call it. Disable to + return the callable instead. + + .. versionchanged:: 8.0 + Added the ``call`` parameter. + """ + if not self._default_map_has(name): + return None + + # Assert to make the type checker happy. + assert self.default_map is not None + value = self.default_map[name] + + if call and callable(value): + return value() + + return value + + def fail(self, message: str) -> t.NoReturn: + """Aborts the execution of the program with a specific error + message. + + :param message: the error message to fail with. + """ + raise UsageError(message, self) + + def abort(self) -> t.NoReturn: + """Aborts the script.""" + raise Abort() + + def exit(self, code: int = 0) -> t.NoReturn: + """Exits the application with a given exit code. + + .. versionchanged:: 8.2 + Callbacks and context managers registered with :meth:`call_on_close` + and :meth:`with_resource` are closed before exiting. + """ + self.close() + raise Exit(code) + + def get_usage(self) -> str: + """Helper method to get formatted usage string for the current + context and command. + """ + return self.command.get_usage(self) + + def get_help(self) -> str: + """Helper method to get formatted help page for the current + context and command. + """ + return self.command.get_help(self) + + def _make_sub_context(self, command: Command) -> Context: + """Create a new context of the same type as this context, but + for a new command. + + :meta private: + """ + return type(self)(command, info_name=command.name, parent=self) + + @t.overload + def invoke( + self, callback: t.Callable[..., V], /, *args: t.Any, **kwargs: t.Any + ) -> V: ... + + @t.overload + def invoke(self, callback: Command, /, *args: t.Any, **kwargs: t.Any) -> t.Any: ... + + def invoke( + self, callback: Command | t.Callable[..., V], /, *args: t.Any, **kwargs: t.Any + ) -> t.Any | V: + """Invokes a command callback in exactly the way it expects. There + are two ways to invoke this method: + + 1. the first argument can be a callback and all other arguments and + keyword arguments are forwarded directly to the function. + 2. the first argument is a click command object. In that case all + arguments are forwarded as well but proper click parameters + (options and click arguments) must be keyword arguments and Click + will fill in defaults. + + .. versionchanged:: 8.0 + All ``kwargs`` are tracked in :attr:`params` so they will be + passed if :meth:`forward` is called at multiple levels. + + .. versionchanged:: 3.2 + A new context is created, and missing arguments use default values. + """ + if isinstance(callback, Command): + other_cmd = callback + + if other_cmd.callback is None: + raise TypeError( + "The given command does not have a callback that can be invoked." + ) + else: + callback = t.cast("t.Callable[..., V]", other_cmd.callback) + + ctx = self._make_sub_context(other_cmd) + + for param in other_cmd.params: + if param.name not in kwargs and param.expose_value: + default_value = param.get_default(ctx) + # We explicitly hide the :attr:`UNSET` value to the user, as we + # choose to make it an implementation detail. And because ``invoke`` + # has been designed as part of Click public API, we return ``None`` + # instead. Refs: + # https://github.com/pallets/click/issues/3066 + # https://github.com/pallets/click/issues/3065 + # https://github.com/pallets/click/pull/3068 + if default_value is UNSET: + default_value = None + kwargs[param.name] = param.type_cast_value(ctx, default_value) + + # Track all kwargs as params, so that forward() will pass + # them on in subsequent calls. + ctx.params.update(kwargs) + else: + ctx = self + + with augment_usage_errors(self), ctx: + return callback(*args, **kwargs) + + def forward(self, cmd: Command, /, *args: t.Any, **kwargs: t.Any) -> t.Any: + """Similar to :meth:`invoke` but fills in default keyword + arguments from the current context if the other command expects + it. This cannot invoke callbacks directly, only other commands. + + .. versionchanged:: 8.0 + All ``kwargs`` are tracked in :attr:`params` so they will be + passed if ``forward`` is called at multiple levels. + """ + # Can only forward to other commands, not direct callbacks. + if not isinstance(cmd, Command): + raise TypeError("Callback is not a command.") + + for param in self.params: + if param not in kwargs: + kwargs[param] = self.params[param] + + return self.invoke(cmd, *args, **kwargs) + + def set_parameter_source(self, name: str, source: ParameterSource) -> None: + """Set the source of a parameter. This indicates the location + from which the value of the parameter was obtained. + + :param name: The name of the parameter. + :param source: A member of :class:`~click.core.ParameterSource`. + """ + self._parameter_source[name] = source + + def get_parameter_source(self, name: str) -> ParameterSource | None: + """Get the source of a parameter. This indicates the location + from which the value of the parameter was obtained. + + This can be useful for determining when a user specified a value + on the command line that is the same as the default value. It + will be :attr:`~click.core.ParameterSource.DEFAULT` only if the + value was actually taken from the default. + + :param name: The name of the parameter. + :rtype: ParameterSource + + .. versionchanged:: 8.0 + Returns ``None`` if the parameter was not provided from any + source. + """ + return self._parameter_source.get(name) + + +class Command: + """Commands are the basic building block of command line interfaces in + Click. A basic command handles command line parsing and might dispatch + more parsing to commands nested below it. + + :param name: the name of the command to use unless a group overrides it. + :param context_settings: an optional dictionary with defaults that are + passed to the context object. + :param callback: the callback to invoke. This is optional. + :param params: the parameters to register with this command. This can + be either :class:`Option` or :class:`Argument` objects. + :param help: the help string to use for this command. + :param epilog: like the help string but it's printed at the end of the + help page after everything else. + :param short_help: the short help to use for this command. This is + shown on the command listing of the parent command. + :param add_help_option: by default each command registers a ``--help`` + option. This can be disabled by this parameter. + :param no_args_is_help: this controls what happens if no arguments are + provided. This option is disabled by default. + If enabled this will add ``--help`` as argument + if no arguments are passed + :param hidden: hide this command from help outputs. + :param deprecated: If ``True`` or non-empty string, issues a message + indicating that the command is deprecated and highlights + its deprecation in --help. The message can be customized + by using a string as the value. + + .. versionchanged:: 8.2 + This is the base class for all commands, not ``BaseCommand``. + ``deprecated`` can be set to a string as well to customize the + deprecation message. + + .. versionchanged:: 8.1 + ``help``, ``epilog``, and ``short_help`` are stored unprocessed, + all formatting is done when outputting help text, not at init, + and is done even if not using the ``@command`` decorator. + + .. versionchanged:: 8.0 + Added a ``repr`` showing the command name. + + .. versionchanged:: 7.1 + Added the ``no_args_is_help`` parameter. + + .. versionchanged:: 2.0 + Added the ``context_settings`` parameter. + """ + + #: The context class to create with :meth:`make_context`. + #: + #: .. versionadded:: 8.0 + context_class: type[Context] = Context + + #: the default for the :attr:`Context.allow_extra_args` flag. + allow_extra_args = False + + #: the default for the :attr:`Context.allow_interspersed_args` flag. + allow_interspersed_args = True + + #: the default for the :attr:`Context.ignore_unknown_options` flag. + ignore_unknown_options = False + + name: str | None + context_settings: cabc.MutableMapping[str, t.Any] + callback: t.Callable[..., t.Any] | None + params: list[Parameter] + help: str | None + epilog: str | None + options_metavar: str | None + short_help: str | None + add_help_option: bool + _help_option: Option | None + no_args_is_help: bool + hidden: bool + deprecated: bool | str + + def __init__( + self, + name: str | None, + context_settings: cabc.MutableMapping[str, t.Any] | None = None, + callback: t.Callable[..., t.Any] | None = None, + params: list[Parameter] | None = None, + help: str | None = None, + epilog: str | None = None, + short_help: str | None = None, + options_metavar: str | None = "[OPTIONS]", + add_help_option: bool = True, + no_args_is_help: bool = False, + hidden: bool = False, + deprecated: bool | str = False, + ) -> None: + #: the name the command thinks it has. Upon registering a command + #: on a :class:`Group` the group will default the command name + #: with this information. You should instead use the + #: :class:`Context`\'s :attr:`~Context.info_name` attribute. + self.name = name + + if context_settings is None: + context_settings = {} + + #: an optional dictionary with defaults passed to the context. + self.context_settings = context_settings + + #: the callback to execute when the command fires. This might be + #: `None` in which case nothing happens. + self.callback = callback + #: the list of parameters for this command in the order they + #: should show up in the help page and execute. Eager parameters + #: will automatically be handled before non eager ones. + self.params = params or [] + self.help = help + self.epilog = epilog + self.options_metavar = options_metavar + self.short_help = short_help + self.add_help_option = add_help_option + self._help_option = None + self.no_args_is_help = no_args_is_help + self.hidden = hidden + self.deprecated = deprecated + + def to_info_dict(self, ctx: Context) -> dict[str, t.Any]: + return { + "name": self.name, + "params": [param.to_info_dict() for param in self.get_params(ctx)], + "help": self.help, + "epilog": self.epilog, + "short_help": self.short_help, + "hidden": self.hidden, + "deprecated": self.deprecated, + } + + def __repr__(self) -> str: + return f"<{self.__class__.__name__} {self.name}>" + + def get_usage(self, ctx: Context) -> str: + """Formats the usage line into a string and returns it. + + Calls :meth:`format_usage` internally. + """ + formatter = ctx.make_formatter() + self.format_usage(ctx, formatter) + return formatter.getvalue().rstrip("\n") + + def get_params(self, ctx: Context) -> list[Parameter]: + params = self.params + help_option = self.get_help_option(ctx) + + if help_option is not None: + params = [*params, help_option] + + if __debug__: + import warnings + + opts = [opt for param in params for opt in param.opts] + opts_counter = Counter(opts) + duplicate_opts = (opt for opt, count in opts_counter.items() if count > 1) + + for duplicate_opt in duplicate_opts: + warnings.warn( + ( + f"The parameter {duplicate_opt} is used more than once. " + "Remove its duplicate as parameters should be unique." + ), + stacklevel=3, + ) + + # Options may deliberately share a storage name to compete for + # the same value (feature switches), but an argument sharing a + # name silently overwrites the other parameter's value. + names_counter = Counter(param.name for param in params) + duplicate_names = ( + name for name, count in names_counter.items() if count > 1 + ) + + for duplicate_name in duplicate_names: + sharers = [param for param in params if param.name == duplicate_name] + + if help_option in sharers: + warnings.warn( + ( + f"The name {duplicate_name!r} is reserved for the " + "automatic help option. Give the parameter a " + "different name." + ), + stacklevel=3, + ) + elif any(isinstance(param, Argument) for param in sharers): + warnings.warn( + ( + f"The name {duplicate_name!r} is used by an argument " + "and another parameter. They will overwrite each " + "other's value during parsing. Give each parameter " + "a unique name." + ), + stacklevel=3, + ) + + return params + + def format_usage(self, ctx: Context, formatter: HelpFormatter) -> None: + """Writes the usage line into the formatter. + + This is a low-level method called by :meth:`get_usage`. + """ + pieces = self.collect_usage_pieces(ctx) + formatter.write_usage(ctx.command_path, " ".join(pieces)) + + def collect_usage_pieces(self, ctx: Context) -> list[str]: + """Returns all the pieces that go into the usage line and returns + it as a list of strings. + """ + rv = [self.options_metavar] if self.options_metavar else [] + + for param in self.get_params(ctx): + rv.extend(param.get_usage_pieces(ctx)) + + return rv + + def get_help_option_names(self, ctx: Context) -> list[str]: + """Returns the names for the help option. + + Drops duplicates and names already reserved by another parameter. Order of + :attr:`Context.help_option_names` is preserved, so the result is stable. + + .. versionchanged:: 8.5.0 + Names keep their declaration order. + """ + all_names = dict.fromkeys(ctx.help_option_names) + for param in self.params: + for name in (*param.opts, *param.secondary_opts): + all_names.pop(name, None) + return list(all_names) + + def get_help_option(self, ctx: Context) -> Option | None: + """Returns the help option object. + + Skipped if :attr:`add_help_option` is ``False``. + + .. versionchanged:: 8.5.0 + The help option stores its value under the reserved name + ``_click_default_help``, so a parameter named ``help`` no + longer breaks parsing. + + .. versionchanged:: 8.1.8 + The help option is now cached to avoid creating it multiple times. + """ + help_option_names = self.get_help_option_names(ctx) + + if not help_option_names or not self.add_help_option: + return None + + # Cache the help option object in private _help_option attribute to + # avoid creating it multiple times. Not doing this will break the + # callback ordering by iter_params_for_processing(), which relies on + # object comparison. + if self._help_option is None: + # Avoid circular import. + from .decorators import help_option + + # The help option never exposes its value, so it uses a reserved + # storage name, keeping it clear of user parameters (like an + # argument named "help") that would otherwise clobber its value. + help_option(*help_option_names, _HELP_OPTION_STORAGE_NAME)(self) + self._help_option = self.params.pop() # type: ignore[assignment] + + return self._help_option + + def make_parser(self, ctx: Context) -> _OptionParser: + """Creates the underlying option parser for this command.""" + parser = _OptionParser(ctx) + for param in self.get_params(ctx): + param.add_to_parser(parser, ctx) + return parser + + def get_help(self, ctx: Context) -> str: + """Formats the help into a string and returns it. + + Calls :meth:`format_help` internally. + """ + formatter = ctx.make_formatter() + self.format_help(ctx, formatter) + return formatter.getvalue().rstrip("\n") + + def get_short_help_str(self, limit: int = 45) -> str: + """Gets short help for the command or makes it by shortening the + long help string. + """ + if self.short_help: + text = inspect.cleandoc(self.short_help) + elif self.help: + text = _make_default_short_help(self.help, limit) + else: + text = "" + + if self.deprecated: + text = f"{_(text)} {_format_deprecated_label(self.deprecated)}" + + return text.strip() + + def format_help(self, ctx: Context, formatter: HelpFormatter) -> None: + """Writes the help into the formatter if it exists. + + This is a low-level method called by :meth:`get_help`. + + This calls the following methods: + + - :meth:`format_usage` + - :meth:`format_help_text` + - :meth:`format_arguments` + - :meth:`format_options` + - :meth:`format_epilog` + """ + self.format_usage(ctx, formatter) + self.format_help_text(ctx, formatter) + self.format_arguments(ctx, formatter) + self.format_options(ctx, formatter) + self.format_epilog(ctx, formatter) + + def format_help_text(self, ctx: Context, formatter: HelpFormatter) -> None: + """Writes the help text to the formatter if it exists.""" + if self.help is not None: + # truncate the help text to the first form feed + text = inspect.cleandoc(self.help).partition("\f")[0] + else: + text = "" + + if self.deprecated: + label = _format_deprecated_label(self.deprecated) + text = f"{_(text)} {label}" if text else label + + if text: + formatter.write_paragraph() + + with formatter.indentation(): + formatter.write_text(text) + + def format_options(self, ctx: Context, formatter: HelpFormatter) -> None: + """Writes all the options into the formatter if they exist.""" + opts = [] + for param in self.get_params(ctx): + rv = param.get_help_record(ctx) + if rv is not None and not isinstance(param, Argument): + opts.append(rv) + + if opts: + with formatter.section(_("Options")): + formatter.write_dl(opts) + + def format_arguments(self, ctx: Context, formatter: HelpFormatter) -> None: + """Writes the arguments that have a help record into the formatter.""" + args = [] + for param in self.get_params(ctx): + rv = param.get_help_record(ctx) + if rv is not None and isinstance(param, Argument): + args.append(rv) + + if args: + with formatter.section(_("Positional arguments")): + formatter.write_dl(args) + + def format_epilog(self, ctx: Context, formatter: HelpFormatter) -> None: + """Writes the epilog into the formatter if it exists.""" + if self.epilog: + epilog = inspect.cleandoc(self.epilog) + formatter.write_paragraph() + + with formatter.indentation(): + formatter.write_text(epilog) + + def make_context( + self, + info_name: str | None, + args: list[str], + parent: Context | None = None, + **extra: t.Any, + ) -> Context: + """This function when given an info name and arguments will kick + off the parsing and create a new :class:`Context`. It does not + invoke the actual command callback though. + + To quickly customize the context class used without overriding + this method, set the :attr:`context_class` attribute. + + :param info_name: the info name for this invocation. Generally this + is the most descriptive name for the script or + command. For the toplevel script it's usually + the name of the script, for commands below it's + the name of the command. + :param args: the arguments to parse as list of strings. + :param parent: the parent context if available. + :param extra: extra keyword arguments forwarded to the context + constructor. + + .. versionchanged:: 8.0 + Added the :attr:`context_class` attribute. + """ + for key, value in self.context_settings.items(): + if key not in extra: + extra[key] = value + + ctx = self.context_class(self, info_name=info_name, parent=parent, **extra) + + with ctx.scope(cleanup=False): + self.parse_args(ctx, args) + return ctx + + def parse_args(self, ctx: Context, args: list[str]) -> list[str]: + if not args and self.no_args_is_help and not ctx.resilient_parsing: + raise NoArgsIsHelpError(ctx) + + parser = self.make_parser(ctx) + opts, args, param_order = parser.parse_args(args=args) + + for param in iter_params_for_processing(param_order, self.get_params(ctx)): + _, args = param.handle_parse_result(ctx, opts, args) + + # We now have all parameters' values into `ctx.params`, but the data may contain + # the `UNSET` sentinel. + # Convert `UNSET` to `None` to ensure that the user doesn't see `UNSET`. + # + # Waiting until after the initial parse to convert allows us to treat `UNSET` + # more like a missing value when multiple params use the same name. + # Refs: + # https://github.com/pallets/click/issues/3071 + # https://github.com/pallets/click/pull/3079 + for name, value in ctx.params.items(): + if value is UNSET: + ctx.params[name] = None + + if args and not ctx.allow_extra_args and not ctx.resilient_parsing: + ctx.fail( + ngettext( + "Got unexpected extra argument ({args})", + "Got unexpected extra arguments ({args})", + len(args), + ).format(args=" ".join(map(str, args))) + ) + + ctx.args = args + ctx._opt_prefixes.update(parser._opt_prefixes) + return args + + def invoke(self, ctx: Context) -> t.Any: + """Given a context, this invokes the attached callback (if it exists) + in the right way. + """ + if self.deprecated: + message = _( + "DeprecationWarning: The command {name!r} is deprecated.{extra_message}" + ).format( + name=self.name, + extra_message=_format_deprecated_suffix(self.deprecated), + ) + echo(style(message, fg="red"), err=True) + + if self.callback is not None: + return ctx.invoke(self.callback, **ctx.params) + + def shell_complete(self, ctx: Context, incomplete: str) -> list[CompletionItem]: + """Return a list of completions for the incomplete value. Looks + at the names of options and chained multi-commands. + + Any command could be part of a chained multi-command, so sibling + commands are valid at any point during command completion. + + :param ctx: Invocation context for this command. + :param incomplete: Value being completed. May be empty. + + .. versionadded:: 8.0 + """ + from click.shell_completion import CompletionItem + + results: list[CompletionItem] = [] + + if incomplete and not incomplete[0].isalnum(): + for param in self.get_params(ctx): + if ( + not isinstance(param, Option) + or param.hidden + or ( + not param.multiple + and ctx.get_parameter_source(param.name) + is ParameterSource.COMMANDLINE + ) + ): + continue + + results.extend( + CompletionItem(name, help=param.help) + for name in [*param.opts, *param.secondary_opts] + if name.startswith(incomplete) + ) + + while ctx.parent is not None: + ctx = ctx.parent + + if isinstance(ctx.command, Group) and ctx.command.chain: + results.extend( + CompletionItem(name, help=command.get_short_help_str()) + for name, command in _complete_visible_commands(ctx, incomplete) + if name not in ctx._protected_args + ) + + return results + + @t.overload + def main( + self, + args: cabc.Sequence[str] | None = None, + prog_name: str | None = None, + complete_var: str | None = None, + standalone_mode: t.Literal[True] = True, + **extra: t.Any, + ) -> t.NoReturn: ... + + @t.overload + def main( + self, + args: cabc.Sequence[str] | None = None, + prog_name: str | None = None, + complete_var: str | None = None, + standalone_mode: bool = ..., + **extra: t.Any, + ) -> t.Any: ... + + def main( + self, + args: cabc.Sequence[str] | None = None, + prog_name: str | None = None, + complete_var: str | None = None, + standalone_mode: bool = True, + windows_expand_args: bool = True, + **extra: t.Any, + ) -> t.Any: + """This is the way to invoke a script with all the bells and + whistles as a command line application. This will always terminate + the application after a call. If this is not wanted, ``SystemExit`` + needs to be caught. + + This method is also available by directly calling the instance of + a :class:`Command`. + + :param args: the arguments that should be used for parsing. If not + provided, ``sys.argv[1:]`` is used. + :param prog_name: the program name that should be used. By default + the program name is constructed by taking the file + name from ``sys.argv[0]``. + :param complete_var: the environment variable that controls the + bash completion support. The default is + ``"__COMPLETE"`` with prog_name in + uppercase. + :param standalone_mode: the default behavior is to invoke the script + in standalone mode. Click will then + handle exceptions and convert them into + error messages and the function will never + return but shut down the interpreter. If + this is set to `False` they will be + propagated to the caller and the return + value of this function is the return value + of :meth:`invoke`. + :param windows_expand_args: Expand glob patterns, user dir, and + env vars in command line args on Windows. + :param extra: extra keyword arguments are forwarded to the context + constructor. See :class:`Context` for more information. + + .. versionchanged:: 8.0.1 + Added the ``windows_expand_args`` parameter to allow + disabling command line arg expansion on Windows. + + .. versionchanged:: 8.0 + When taking arguments from ``sys.argv`` on Windows, glob + patterns, user dir, and env vars are expanded. + + .. versionchanged:: 3.0 + Added the ``standalone_mode`` parameter. + """ + if args is None: + args = sys.argv[1:] + + if os.name == "nt" and windows_expand_args: + args = _expand_args(args) + else: + args = list(args) + + if prog_name is None: + prog_name = _detect_program_name() + + # Process shell completion requests and exit early. + self._main_shell_completion(extra, prog_name, complete_var) + + try: + try: + with self.make_context(prog_name, args, **extra) as ctx: + rv = self.invoke(ctx) + if not standalone_mode: + return rv + # it's not safe to `ctx.exit(rv)` here! + # note that `rv` may actually contain data like "1" which + # has obvious effects + # more subtle case: `rv=[None, None]` can come out of + # chained commands which all returned `None` -- so it's not + # even always obvious that `rv` indicates success/failure + # by its truthiness/falsiness + ctx.exit() + except (EOFError, KeyboardInterrupt) as e: + echo(file=sys.stderr) + raise Abort() from e + except ClickException as e: + if not standalone_mode: + raise + e.show() + sys.exit(e.exit_code) + except OSError as e: + if e.errno == errno.EPIPE: + sys.stdout = t.cast(t.TextIO, _PacifyFlushWrapper(sys.stdout)) + sys.stderr = t.cast(t.TextIO, _PacifyFlushWrapper(sys.stderr)) + sys.exit(1) + else: + raise + except Exit as e: + if standalone_mode: + sys.exit(e.exit_code) + else: + # in non-standalone mode, return the exit code + # note that this is only reached if `self.invoke` above raises + # an Exit explicitly -- thus bypassing the check there which + # would return its result + # the results of non-standalone execution may therefore be + # somewhat ambiguous: if there are codepaths which lead to + # `ctx.exit(1)` and to `return 1`, the caller won't be able to + # tell the difference between the two + return e.exit_code + except Abort: + if not standalone_mode: + raise + echo(_("Aborted!"), file=sys.stderr) + sys.exit(1) + + def _main_shell_completion( + self, + ctx_args: cabc.MutableMapping[str, t.Any], + prog_name: str, + complete_var: str | None = None, + ) -> None: + """Check if the shell is asking for tab completion, process + that, then exit early. Called from :meth:`main` before the + program is invoked. + + :param prog_name: Name of the executable in the shell. + :param complete_var: Name of the environment variable that holds + the completion instruction. Defaults to + ``_{PROG_NAME}_COMPLETE``. + + .. versionchanged:: 8.2.0 + Dots (``.``) in ``prog_name`` are replaced with underscores (``_``). + """ + if complete_var is None: + complete_name = prog_name.replace("-", "_").replace(".", "_") + complete_var = f"_{complete_name}_COMPLETE".upper() + + instruction = os.environ.get(complete_var) + + if not instruction: + return + + from .shell_completion import shell_complete + + rv = shell_complete(self, ctx_args, prog_name, complete_var, instruction) + sys.exit(rv) + + def __call__(self, *args: t.Any, **kwargs: t.Any) -> t.Any: + """Alias for :meth:`main`.""" + return self.main(*args, **kwargs) + + +class _FakeSubclassCheck(type): + def __subclasscheck__(cls, subclass: type) -> bool: + return issubclass(subclass, cls.__bases__[0]) + + def __instancecheck__(cls, instance: t.Any) -> bool: + return isinstance(instance, cls.__bases__[0]) + + +class _BaseCommand(Command, metaclass=_FakeSubclassCheck): + """ + .. deprecated:: 8.2 + Will be removed in Click 9.0. Use ``Command`` instead. + """ + + +class Group(Command): + """A group is a command that nests other commands (or more groups). + + :param name: The name of the group command. + :param commands: Map names to :class:`Command` objects. Can be a list, which + will use :attr:`Command.name` as the keys. + :param invoke_without_command: Invoke the group's callback even if a + subcommand is not given. + :param no_args_is_help: If no arguments are given, show the group's help and + exit. Defaults to the opposite of ``invoke_without_command``. + :param subcommand_metavar: How to represent the subcommand argument in help. + The default will represent whether ``chain`` is set or not. + :param chain: Allow passing more than one subcommand argument. After parsing + a command's arguments, if any arguments remain another command will be + matched, and so on. + :param result_callback: A function to call after the group's and + subcommand's callbacks. The value returned by the subcommand is passed. + If ``chain`` is enabled, the value will be a list of values returned by + all the commands. If ``invoke_without_command`` is enabled, the value + will be the value returned by the group's callback, or an empty list if + ``chain`` is enabled. + :param kwargs: Other arguments passed to :class:`Command`. + + .. versionchanged:: 8.0 + The ``commands`` argument can be a list of command objects. + + .. versionchanged:: 8.2 + Merged with and replaces the ``MultiCommand`` base class. + """ + + allow_extra_args = True + allow_interspersed_args = False + + #: If set, this is used by the group's :meth:`command` decorator + #: as the default :class:`Command` class. This is useful to make all + #: subcommands use a custom command class. + #: + #: .. versionadded:: 8.0 + command_class: type[Command] | None = None + + #: If set, this is used by the group's :meth:`group` decorator + #: as the default :class:`Group` class. This is useful to make all + #: subgroups use a custom group class. + #: + #: If set to the special value :class:`type` (literally + #: ``group_class = type``), this group's class will be used as the + #: default class. This makes a custom group class continue to make + #: custom groups. + #: + #: .. versionadded:: 8.0 + group_class: type[Group | type] | None = None + # Literal[type] isn't valid, so use Type[type] + + commands: cabc.MutableMapping[str, Command] + invoke_without_command: bool + subcommand_metavar: str + chain: bool + _result_callback: t.Callable[..., t.Any] | None + + def __init__( + self, + name: str | None = None, + commands: cabc.MutableMapping[str, Command] + | cabc.Sequence[Command] + | None = None, + invoke_without_command: bool = False, + no_args_is_help: bool | None = None, + subcommand_metavar: str | None = None, + chain: bool = False, + result_callback: t.Callable[..., t.Any] | None = None, + **kwargs: t.Any, + ) -> None: + super().__init__(name, **kwargs) + + if commands is None: + commands = {} + elif isinstance(commands, abc.Sequence): + commands = {c.name: c for c in commands if c.name is not None} + + #: The registered subcommands by their exported names. + self.commands = commands + + if no_args_is_help is None: + no_args_is_help = not invoke_without_command + + self.no_args_is_help = no_args_is_help + self.invoke_without_command = invoke_without_command + + if subcommand_metavar is None: + # When the group can run without a subcommand, the leading command + # token is optional, so wrap it in brackets to reflect that. + if chain: + if invoke_without_command: + subcommand_metavar = "[COMMAND1] [ARGS]... [COMMAND2 [ARGS]...]..." + else: + subcommand_metavar = "COMMAND1 [ARGS]... [COMMAND2 [ARGS]...]..." + elif invoke_without_command: + subcommand_metavar = "[COMMAND] [ARGS]..." + else: + subcommand_metavar = "COMMAND [ARGS]..." + + self.subcommand_metavar = subcommand_metavar + self.chain = chain + # The result callback that is stored. This can be set or + # overridden with the :func:`result_callback` decorator. + self._result_callback = result_callback + + if self.chain: + for param in self.params: + if isinstance(param, Argument) and not param.required: + raise RuntimeError( + "A group in chain mode cannot have optional arguments." + ) + + def to_info_dict(self, ctx: Context) -> dict[str, t.Any]: + info_dict = super().to_info_dict(ctx) + commands = {} + + for name in self.list_commands(ctx): + command = self.get_command(ctx, name) + + if command is None: + continue + + sub_ctx = ctx._make_sub_context(command) + + with sub_ctx.scope(cleanup=False): + commands[name] = command.to_info_dict(sub_ctx) + + info_dict.update(commands=commands, chain=self.chain) + return info_dict + + def add_command(self, cmd: Command, name: str | None = None) -> None: + """Registers another :class:`Command` with this group. If the name + is not provided, the name of the command is used. + """ + name = name or cmd.name + if name is None: + raise TypeError("Command has no name.") + _check_nested_chain(self, name, cmd, register=True) + self.commands[name] = cmd + + @t.overload + def command(self, __func: t.Callable[..., t.Any]) -> Command: ... + + @t.overload + def command( + self, *args: t.Any, **kwargs: t.Any + ) -> t.Callable[[t.Callable[..., t.Any]], Command]: ... + + def command( + self, *args: t.Any, **kwargs: t.Any + ) -> t.Callable[[t.Callable[..., t.Any]], Command] | Command: + """A shortcut decorator for declaring and attaching a command to + the group. This takes the same arguments as :func:`command` and + immediately registers the created command with this group by + calling :meth:`add_command`. + + To customize the command class used, set the + :attr:`command_class` attribute. + + .. versionchanged:: 8.1 + This decorator can be applied without parentheses. + + .. versionchanged:: 8.0 + Added the :attr:`command_class` attribute. + """ + from .decorators import command + + func: t.Callable[..., t.Any] | None = None + + if args and callable(args[0]): + assert len(args) == 1 and not kwargs, ( + "Use 'command(**kwargs)(callable)' to provide arguments." + ) + (func,) = args + args = () + + if self.command_class and kwargs.get("cls") is None: + kwargs["cls"] = self.command_class + + def decorator(f: t.Callable[..., t.Any]) -> Command: + cmd: Command = command(*args, **kwargs)(f) + self.add_command(cmd) + return cmd + + if func is not None: + return decorator(func) + + return decorator + + @t.overload + def group(self, __func: t.Callable[..., t.Any]) -> Group: ... + + @t.overload + def group( + self, *args: t.Any, **kwargs: t.Any + ) -> t.Callable[[t.Callable[..., t.Any]], Group]: ... + + def group( + self, *args: t.Any, **kwargs: t.Any + ) -> t.Callable[[t.Callable[..., t.Any]], Group] | Group: + """A shortcut decorator for declaring and attaching a group to + the group. This takes the same arguments as :func:`group` and + immediately registers the created group with this group by + calling :meth:`add_command`. + + To customize the group class used, set the :attr:`group_class` + attribute. + + .. versionchanged:: 8.1 + This decorator can be applied without parentheses. + + .. versionchanged:: 8.0 + Added the :attr:`group_class` attribute. + """ + from .decorators import group + + func: t.Callable[..., t.Any] | None = None + + if args and callable(args[0]): + assert len(args) == 1 and not kwargs, ( + "Use 'group(**kwargs)(callable)' to provide arguments." + ) + (func,) = args + args = () + + if self.group_class is not None and kwargs.get("cls") is None: + if self.group_class is type: + kwargs["cls"] = type(self) + else: + kwargs["cls"] = self.group_class + + def decorator(f: t.Callable[..., t.Any]) -> Group: + cmd: Group = group(*args, **kwargs)(f) + self.add_command(cmd) + return cmd + + if func is not None: + return decorator(func) + + return decorator + + def result_callback(self, replace: bool = False) -> t.Callable[[F], F]: + """Adds a result callback to the command. By default if a + result callback is already registered this will chain them but + this can be disabled with the `replace` parameter. The result + callback is invoked with the return value of the subcommand + (or the list of return values from all subcommands if chaining + is enabled) as well as the parameters as they would be passed + to the main callback. + + Example:: + + @click.group() + @click.option('-i', '--input', default=23) + def cli(input): + return 42 + + @cli.result_callback() + def process_result(result, input): + return result + input + + :param replace: if set to `True` an already existing result + callback will be removed. + + .. versionchanged:: 8.0 + Renamed from ``resultcallback``. + + .. versionadded:: 3.0 + """ + + def decorator(f: F) -> F: + old_callback = self._result_callback + + if old_callback is None or replace: + self._result_callback = f + return f + + def function(value: t.Any, /, *args: t.Any, **kwargs: t.Any) -> t.Any: + inner = old_callback(value, *args, **kwargs) + return f(inner, *args, **kwargs) + + self._result_callback = rv = update_wrapper(t.cast(F, function), f) + return rv # type: ignore[return-value] + + return decorator + + def get_command(self, ctx: Context, cmd_name: str) -> Command | None: + """Given a context and a command name, this returns a :class:`Command` + object if it exists or returns ``None``. + """ + return self.commands.get(cmd_name) + + def list_commands(self, ctx: Context) -> list[str]: + """Returns a list of subcommand names in the order they should appear.""" + return sorted(self.commands) + + def collect_usage_pieces(self, ctx: Context) -> list[str]: + rv = super().collect_usage_pieces(ctx) + rv.append(self.subcommand_metavar) + return rv + + def format_options(self, ctx: Context, formatter: HelpFormatter) -> None: + super().format_options(ctx, formatter) + self.format_commands(ctx, formatter) + + def format_commands(self, ctx: Context, formatter: HelpFormatter) -> None: + """Extra format methods for multi methods that adds all the commands + after the options. + """ + commands = [] + for subcommand in self.list_commands(ctx): + cmd = self.get_command(ctx, subcommand) + # What is this, the tool lied about a command. Ignore it + if cmd is None: + continue + if cmd.hidden: + continue + + commands.append((subcommand, cmd)) + + # allow for 3 times the default spacing + if len(commands): + limit = formatter.width - 6 - max(len(cmd[0]) for cmd in commands) + + rows = [] + for subcommand, cmd in commands: + help = cmd.get_short_help_str(limit) + rows.append((subcommand, help)) + + if rows: + with formatter.section(_("Commands")): + formatter.write_dl(rows) + + def parse_args(self, ctx: Context, args: list[str]) -> list[str]: + if not args and self.no_args_is_help and not ctx.resilient_parsing: + raise NoArgsIsHelpError(ctx) + + rest = super().parse_args(ctx, args) + + if self.chain: + ctx._protected_args = rest + ctx.args = [] + elif rest: + ctx._protected_args, ctx.args = rest[:1], rest[1:] + + return ctx.args + + def invoke(self, ctx: Context) -> t.Any: + def _process_result(value: t.Any) -> t.Any: + if self._result_callback is not None: + value = ctx.invoke(self._result_callback, value, **ctx.params) + return value + + if not ctx._protected_args: + if self.invoke_without_command: + # No subcommand was invoked, so the result callback is + # invoked with the group return value for regular + # groups, or an empty list for chained groups. + with ctx: + rv = super().invoke(ctx) + return _process_result([] if self.chain else rv) + ctx.fail(_("Missing command.")) + + # Fetch args back out + args = [*ctx._protected_args, *ctx.args] + ctx.args = [] + ctx._protected_args = [] + + # If we're not in chain mode, we only allow the invocation of a + # single command but we also inform the current context about the + # name of the command to invoke. + if not self.chain: + # Make sure the context is entered so we do not clean up + # resources until the result processor has worked. + with ctx: + cmd_name, cmd, args = self.resolve_command(ctx, args) + assert cmd is not None + ctx.invoked_subcommand = cmd_name + super().invoke(ctx) + sub_ctx = cmd.make_context(cmd_name, args, parent=ctx) + with sub_ctx: + return _process_result(sub_ctx.command.invoke(sub_ctx)) + + # In chain mode we create the contexts step by step, but after the + # base command has been invoked. Because at that point we do not + # know the subcommands yet, the invoked subcommand attribute is + # set to ``*`` to inform the command that subcommands are executed + # but nothing else. + with ctx: + ctx.invoked_subcommand = "*" if args else None + super().invoke(ctx) + + # Otherwise we make every single context and invoke them in a + # chain. In that case the return value to the result processor + # is the list of all invoked subcommand's results. + contexts = [] + while args: + cmd_name, cmd, args = self.resolve_command(ctx, args) + assert cmd is not None + sub_ctx = cmd.make_context( + cmd_name, + args, + parent=ctx, + allow_extra_args=True, + allow_interspersed_args=False, + ) + contexts.append(sub_ctx) + args, sub_ctx.args = sub_ctx.args, [] + + rv = [] + for sub_ctx in contexts: + with sub_ctx: + rv.append(sub_ctx.command.invoke(sub_ctx)) + return _process_result(rv) + + def resolve_command( + self, ctx: Context, args: list[str] + ) -> tuple[str | None, Command | None, list[str]]: + cmd_name = make_str(args[0]) + + # Get the command + cmd = self.get_command(ctx, cmd_name) + + # If we can't find the command but there is a normalization + # function available, we try with that one. + if cmd is None and ctx.token_normalize_func is not None: + cmd_name = ctx.token_normalize_func(cmd_name) + cmd = self.get_command(ctx, cmd_name) + + # If we don't find the command we want to show an error message + # to the user that it was not provided. However, there is + # something else we should do: if the first argument looks like + # an option we want to kick off parsing again for arguments to + # resolve things like --help which now should go to the main + # place. + if cmd is None and not ctx.resilient_parsing: + if _split_opt(cmd_name)[0]: + self.parse_args(ctx, args) + raise NoSuchCommand(cmd_name, possibilities=self.commands, ctx=ctx) + return cmd_name if cmd else None, cmd, args[1:] + + def shell_complete(self, ctx: Context, incomplete: str) -> list[CompletionItem]: + """Return a list of completions for the incomplete value. Looks + at the names of options, subcommands, and chained + multi-commands. + + :param ctx: Invocation context for this command. + :param incomplete: Value being completed. May be empty. + + .. versionadded:: 8.0 + """ + from click.shell_completion import CompletionItem + + results = [ + CompletionItem(name, help=command.get_short_help_str()) + for name, command in _complete_visible_commands(ctx, incomplete) + ] + results.extend(super().shell_complete(ctx, incomplete)) + return results + + +class _MultiCommand(Group, metaclass=_FakeSubclassCheck): + """ + .. deprecated:: 8.2 + Will be removed in Click 9.0. Use ``Group`` instead. + """ + + +class CommandCollection(Group): + """A :class:`Group` that looks up subcommands on other groups. If a command + is not found on this group, each registered source is checked in order. + Parameters on a source are not added to this group, and a source's callback + is not invoked when invoking its commands. In other words, this "flattens" + commands in many groups into this one group. + + :param name: The name of the group command. + :param sources: A list of :class:`Group` objects to look up commands from. + :param kwargs: Other arguments passed to :class:`Group`. + + .. versionchanged:: 8.2 + This is a subclass of ``Group``. Commands are looked up first on this + group, then each of its sources. + """ + + sources: list[Group] + + def __init__( + self, + name: str | None = None, + sources: list[Group] | None = None, + **kwargs: t.Any, + ) -> None: + super().__init__(name, **kwargs) + #: The list of registered groups. + self.sources = sources or [] + + def add_source(self, group: Group) -> None: + """Add a group as a source of commands.""" + self.sources.append(group) + + def get_command(self, ctx: Context, cmd_name: str) -> Command | None: + rv = super().get_command(ctx, cmd_name) + + if rv is not None: + return rv + + for source in self.sources: + rv = source.get_command(ctx, cmd_name) + + if rv is not None: + if self.chain: + _check_nested_chain(self, cmd_name, rv) + + return rv + + return None + + def list_commands(self, ctx: Context) -> list[str]: + rv: set[str] = set(super().list_commands(ctx)) + + for source in self.sources: + rv.update(source.list_commands(ctx)) + + return sorted(rv) + + +def _check_iter(value: cabc.Iterable[V]) -> cabc.Iterator[V]: + """Check if the value is iterable but not a string. Raises a type + error, or return an iterator over the value. + """ + if isinstance(value, str): + raise TypeError + + return iter(value) + + +class Parameter(ABC): + r"""A parameter to a command comes in two versions: they are either + :class:`Option`\s or :class:`Argument`\s. Other subclasses are currently + not supported by design as some of the internals for parsing are + intentionally not finalized. + + Some settings are supported by both options and arguments. + + :param param_decls: the parameter declarations for this option or + argument. This is a list of flags or argument + names. + :param type: the type that should be used. Either a :class:`ParamType` + or a Python type. The latter is converted into the former + automatically if supported. + :param required: controls if this is optional or not. + :param default: the default value if omitted. This can also be a callable, + in which case it's invoked when the default is needed + without any arguments. + :param callback: A function to further process or validate the value + after type conversion. It is called as ``f(ctx, param, value)`` + and must return the value. It is called for all sources, + including prompts. + :param nargs: the number of arguments to match. If not ``1`` the return + value is a tuple instead of single value. The default for + nargs is ``1`` (except if the type is a tuple, then it's + the arity of the tuple). If ``nargs=-1``, all remaining + parameters are collected. + :param metavar: how the value is represented in the help page. + :param expose_value: if this is `True` then the value is passed onwards + to the command callback and stored on the context, + otherwise it's skipped. + :param is_eager: eager values are processed before non eager ones. This + should not be set for arguments or it will inverse the + order of processing. + :param envvar: environment variable(s) that are used to provide a default value for + this parameter. This can be a string or a sequence of strings. If a sequence is + given, only the first non-empty environment variable is used for the parameter. + :param shell_complete: A function that returns custom shell + completions. Used instead of the param's type completion if + given. Takes ``ctx, param, incomplete`` and must return a list + of :class:`~click.shell_completion.CompletionItem` or a list of + strings. + :param deprecated: If ``True`` or non-empty string, issues a message + indicating that the argument is deprecated and highlights + its deprecation in --help. The message can be customized + by using a string as the value. A deprecated parameter + cannot be required, a ValueError will be raised otherwise. + + .. versionchanged:: 8.2.0 + Introduction of ``deprecated``. + + .. versionchanged:: 8.2 + Adding duplicate parameter names to a :class:`~click.core.Command` will + result in a ``UserWarning`` being shown. + + .. versionchanged:: 8.2 + Adding duplicate parameter names to a :class:`~click.core.Command` will + result in a ``UserWarning`` being shown. + + .. versionchanged:: 8.0 + ``process_value`` validates required parameters and bounded + ``nargs``, and invokes the parameter callback before returning + the value. This allows the callback to validate prompts. + ``full_process_value`` is removed. + + .. versionchanged:: 8.0 + ``autocompletion`` is renamed to ``shell_complete`` and has new + semantics described above. The old name is deprecated and will + be removed in 8.1, until then it will be wrapped to match the + new requirements. + + .. versionchanged:: 8.0 + For ``multiple=True, nargs>1``, the default must be a list of + tuples. + + .. versionchanged:: 8.0 + Setting a default is no longer required for ``nargs>1``, it will + default to ``None``. ``multiple=True`` or ``nargs=-1`` will + default to ``()``. + + .. versionchanged:: 7.1 + Empty environment variables are ignored rather than taking the + empty string value. This makes it possible for scripts to clear + variables if they can't unset them. + + .. versionchanged:: 2.0 + Changed signature for parameter callback to also be passed the + parameter. The old callback format will still work, but it will + raise a warning to give you a chance to migrate the code easier. + """ + + param_type_name = "parameter" + + name: str + opts: list[str] + secondary_opts: list[str] + # `Parameter.type` is annotated in `__init__` to avoid confusing mypy + required: bool + callback: t.Callable[[Context, Parameter, t.Any], t.Any] | None + nargs: int + multiple: bool + expose_value: bool + default: t.Any | t.Callable[[], t.Any] | None + _default_explicit: bool + is_eager: bool + metavar: str | None + envvar: str | cabc.Sequence[str] | None + _custom_shell_complete: ( + t.Callable[[Context, Parameter, str], list[CompletionItem] | list[str]] | None + ) + deprecated: bool | str + + def __init__( + self, + param_decls: cabc.Sequence[str] | None = None, + type: types.ParamType[t.Any] | t.Any | None = None, + required: bool = False, + # XXX The default historically embed two concepts: + # - the declaration of a Parameter object carrying the default (handy to + # arbitrage the default value of coupled Parameters sharing the same + # self.name, like flag options), + # - and the actual value of the default. + # It is confusing and is the source of many issues discussed in: + # https://github.com/pallets/click/pull/3030 + # In the future, we might think of splitting it in two, not unlike + # Option.is_flag and Option.flag_value: we could have something like + # Parameter.is_default and Parameter.default_value. + default: t.Any | t.Callable[[], t.Any] | None = UNSET, + callback: t.Callable[[Context, Parameter, t.Any], t.Any] | None = None, + nargs: int | None = None, + multiple: bool = False, + metavar: str | None = None, + expose_value: bool = True, + is_eager: bool = False, + envvar: str | cabc.Sequence[str] | None = None, + shell_complete: t.Callable[ + [Context, Parameter, str], list[CompletionItem] | list[str] + ] + | None = None, + deprecated: bool | str = False, + ) -> None: + self.name, self.opts, self.secondary_opts = self._parse_decls( + param_decls or (), expose_value + ) + self.type: types.ParamType[t.Any] = types.convert_type(type, default) + + # Default nargs to what the type tells us if we have that + # information available. + if nargs is None: + if self.type.is_composite: + nargs = self.type.arity + else: + nargs = 1 + + self.required = required + self.callback = callback + self.nargs = nargs + self.multiple = multiple + self.expose_value = expose_value + self.default = default + # Whether the user passed ``default`` explicitly to the constructor. + # Captured before any auto-derived default (like ``False`` for boolean + # flags in :class:`Option`) replaces the :data:`UNSET` sentinel, so it + # remains ``False`` when the default was inferred rather than chosen. + # Refs: https://github.com/pallets/click/issues/3403 + self._default_explicit = default is not UNSET + self.is_eager = is_eager + self.metavar = metavar + self.envvar = envvar + self._custom_shell_complete = shell_complete + self.deprecated = deprecated + + if __debug__: + if self.type.is_composite and nargs != self.type.arity: + raise ValueError( + f"'nargs' must be {self.type.arity} (or None) for" + f" type {self.type!r}, but it was {nargs}." + ) + + if required and deprecated: + raise ValueError( + f"The {self.param_type_name} '{self.human_readable_name}' " + "is deprecated and still required. A deprecated " + f"{self.param_type_name} cannot be required." + ) + + @staticmethod + def _hide_unset(value: t.Any) -> t.Any: + """Present the internal :data:`UNSET` sentinel as ``None`` at a boundary that + exposes a parameter's value (introspection, prompts), keeping the sentinel an + implementation detail. + """ + return None if value is UNSET else value + + def to_info_dict(self) -> dict[str, t.Any]: + """Gather information that could be useful for a tool generating + user-facing documentation. + + Use :meth:`click.Context.to_info_dict` to traverse the entire + CLI structure. + + .. versionchanged:: 8.3.0 + Returns ``None`` for the :attr:`default` if it was not set. + + .. versionadded:: 8.0 + """ + return { + "name": self.name, + "param_type_name": self.param_type_name, + "opts": self.opts, + "secondary_opts": self.secondary_opts, + "type": self.type.to_info_dict(), + "required": self.required, + "nargs": self.nargs, + "multiple": self.multiple, + "default": self._hide_unset(self.default), + "envvar": self.envvar, + } + + def __repr__(self) -> str: + return f"<{self.__class__.__name__} {self.name}>" + + @abstractmethod + def _parse_decls( + self, decls: cabc.Sequence[str], expose_value: bool + ) -> tuple[str, list[str], list[str]]: ... + + @property + def human_readable_name(self) -> str: + """Returns the human readable name of this parameter. This is the + same as the name for options, but the metavar for arguments. + """ + return self.name + + def make_metavar(self, ctx: Context) -> str: + if self.metavar is not None: + return self.metavar + + metavar = self.type.get_metavar(param=self, ctx=ctx) + + if metavar is None: + metavar = self.type.name.upper() + + if self.nargs != 1: + metavar += "..." + + return metavar + + @t.overload + def get_default( + self, ctx: Context, call: t.Literal[True] = True + ) -> t.Any | None: ... + + @t.overload + def get_default( + self, ctx: Context, call: bool = ... + ) -> t.Any | t.Callable[[], t.Any] | None: ... + + def get_default( + self, ctx: Context, call: bool = True + ) -> t.Any | t.Callable[[], t.Any] | None: + """Get the default for the parameter. Tries + :meth:`Context.lookup_default` first, then the local default. + + :param ctx: Current context. + :param call: If the default is a callable, call it. Disable to + return the callable instead. + + .. versionchanged:: 8.0.2 + Type casting is no longer performed when getting a default. + + .. versionchanged:: 8.0.1 + Type casting can fail in resilient parsing mode. Invalid + defaults will not prevent showing help text. + + .. versionchanged:: 8.0 + Looks at ``ctx.default_map`` first. + + .. versionchanged:: 8.0 + Added the ``call`` parameter. + """ + value = ctx.lookup_default(self.name, call=False) + + if value is None and not ctx._default_map_has(self.name): + value = self.default + + if call and callable(value): + value = value() + + return value + + @abstractmethod + def add_to_parser(self, parser: _OptionParser, ctx: Context) -> None: ... + + def consume_value( + self, ctx: Context, opts: cabc.Mapping[str, t.Any] + ) -> tuple[t.Any, ParameterSource]: + """Returns the parameter value produced by the parser. + + If the parser did not produce a value from user input, the value is either + sourced from the environment variable, the default map, or the parameter's + default value. In that order of precedence. + + If no value is found, an internal sentinel value is returned. + + :meta private: + """ + # Collect from the parse the value passed by the user to the CLI. + value = opts.get(self.name, UNSET) + # If the value is set, it means it was sourced from the command line by the + # parser, otherwise it left unset by default. + source = ( + ParameterSource.COMMANDLINE + if value is not UNSET + else ParameterSource.DEFAULT + ) + + if value is UNSET: + envvar_value = self.value_from_envvar(ctx) + if envvar_value is not None: + value = envvar_value + source = ParameterSource.ENVIRONMENT + + if value is UNSET: + default_map_value = ctx.lookup_default(self.name) + if default_map_value is not None or ctx._default_map_has(self.name): + value = default_map_value + source = ParameterSource.DEFAULT_MAP + + # A string from default_map must be split for multi-value + # parameters, matching value_from_envvar behavior. + if isinstance(value, str) and self.nargs != 1: + value = self.type.split_envvar_value(value) + + if value is UNSET: + default_value = self.get_default(ctx) + if default_value is not UNSET: + value = default_value + source = ParameterSource.DEFAULT + + return value, source + + def type_cast_value(self, ctx: Context, value: t.Any) -> t.Any: + """Convert and validate a value against the parameter's + :attr:`type`, :attr:`multiple`, and :attr:`nargs`. + """ + if value is None: + if self.multiple or self.nargs == -1: + return () + else: + return value + + def check_iter(value: t.Any) -> cabc.Iterator[t.Any]: + try: + return _check_iter(value) + except TypeError: + # This should only happen when passing in args manually, + # the parser should construct an iterable when parsing + # the command line. + raise BadParameter( + _("Value must be an iterable."), ctx=ctx, param=self + ) from None + + # Define the conversion function based on nargs and type. + + if self.nargs == 1 or self.type.is_composite: + + def convert(value: t.Any) -> t.Any: + return self.type(value, param=self, ctx=ctx) + + elif self.nargs == -1: + + def convert(value: t.Any) -> t.Any: # tuple[t.Any, ...] + return tuple(self.type(x, self, ctx) for x in check_iter(value)) + + else: # nargs > 1 + + def convert(value: t.Any) -> t.Any: # tuple[t.Any, ...] + value = tuple(check_iter(value)) + + if len(value) != self.nargs: + raise BadParameter( + ngettext( + "Takes {nargs} values but 1 was given.", + "Takes {nargs} values but {len} were given.", + len(value), + ).format(nargs=self.nargs, len=len(value)), + ctx=ctx, + param=self, + ) + + return tuple(self.type(x, self, ctx) for x in value) + + if self.multiple: + return tuple(convert(x) for x in check_iter(value)) + + return convert(value) + + def value_is_missing(self, value: t.Any) -> bool: + """A value is considered missing if: + + - it is :attr:`UNSET`, + - or if it is an empty sequence while the parameter is suppose to have + non-single value (i.e. :attr:`nargs` is not ``1`` or :attr:`multiple` is + set). + + :meta private: + """ + if value is UNSET: + return True + + if (self.nargs != 1 or self.multiple) and value == (): + return True + + return False + + def process_value(self, ctx: Context, value: t.Any) -> t.Any: + """Process the value of this parameter: + + 1. Type cast the value using :meth:`type_cast_value`. + 2. Check if the value is missing (see: :meth:`value_is_missing`), and raise + :exc:`MissingParameter` if it is required. + 3. If a :attr:`callback` is set, call it to have the value replaced by the + result of the callback. If the value was not set, the callback receive + ``None``. This keep the legacy behavior as it was before the introduction of + the :attr:`UNSET` sentinel. + + :meta private: + """ + # shelter `type_cast_value` from ever seeing an `UNSET` value by handling the + # cases in which `UNSET` gets special treatment explicitly at this layer + # + # Refs: + # https://github.com/pallets/click/issues/3069 + if value is UNSET: + if self.multiple or self.nargs == -1: + value = () + else: + value = self.type_cast_value(ctx, value) + + if self.required and self.value_is_missing(value): + raise MissingParameter(ctx=ctx, param=self) + + if self.callback is not None: + # Legacy case: UNSET is not exposed directly to the callback, but converted + # to None. + if value is UNSET: + value = None + + # Search for parameters with UNSET values in the context. + unset_keys = {k: None for k, v in ctx.params.items() if v is UNSET} + # No UNSET values, call the callback as usual. + if not unset_keys: + value = self.callback(ctx, self, value) + + # Legacy case: provide a temporarily manipulated context to the callback + # to hide UNSET values as None. + # + # Refs: + # https://github.com/pallets/click/issues/3136 + # https://github.com/pallets/click/pull/3137 + else: + # Add another layer to the context stack to clearly hint that the + # context is temporarily modified. + with ctx: + # Update the context parameters to replace UNSET with None. + ctx.params.update(unset_keys) + # Feed these fake context parameters to the callback. + value = self.callback(ctx, self, value) + # Restore the UNSET values in the context parameters. + ctx.params.update( + { + k: UNSET + for k in unset_keys + # Only restore keys that are present and still None, in case + # the callback modified other parameters. + if k in ctx.params and ctx.params[k] is None + } + ) + + return value + + def resolve_envvar_value(self, ctx: Context) -> str | None: + """Returns the value found in the environment variable(s) attached to this + parameter. + + Environment variables values are `always returned as strings + `_. + + This method returns ``None`` if: + + - the :attr:`envvar` property is not set on the :class:`Parameter`, + - the environment variable is not found in the environment, + - the variable is found in the environment but its value is empty (i.e. the + environment variable is present but has an empty string). + + If :attr:`envvar` is setup with multiple environment variables, + then only the first non-empty value is returned. + + .. caution:: + + The raw value extracted from the environment is not normalized and is + returned as-is. Any normalization or reconciliation is performed later by + the :class:`Parameter`'s :attr:`type`. + + :meta private: + """ + if not self.envvar: + return None + + if isinstance(self.envvar, str): + rv = os.environ.get(self.envvar) + + if rv: + return rv + else: + for envvar in self.envvar: + rv = os.environ.get(envvar) + + # Return the first non-empty value of the list of environment variables. + if rv: + return rv + # Else, absence of value is interpreted as an environment variable that + # is not set, so proceed to the next one. + + return None + + def value_from_envvar(self, ctx: Context) -> str | cabc.Sequence[str] | None: + """Process the raw environment variable string for this parameter. + + Returns the string as-is or splits it into a sequence of strings if the + parameter is expecting multiple values (i.e. its :attr:`nargs` property is set + to a value other than ``1``). + + :meta private: + """ + rv = self.resolve_envvar_value(ctx) + + if rv is not None and self.nargs != 1: + return self.type.split_envvar_value(rv) + + return rv + + def handle_parse_result( + self, ctx: Context, opts: cabc.Mapping[str, t.Any], args: list[str] + ) -> tuple[t.Any, list[str]]: + """Process the value produced by the parser from user input. + + Always process the value through the Parameter's :attr:`type`, wherever it + comes from. + + If the parameter is deprecated, this method warn the user about it. But only if + the value has been explicitly set by the user (and as such, is not coming from + a default). + + :meta private: + """ + # Capture the slot's existing state before we mutate + # ``_parameter_source`` so the write decision below can compare our + # incoming source against the source of the option that already wrote + # the slot (if any). + existing_value = ctx.params.get(self.name, UNSET) + existing_source = ctx.get_parameter_source(self.name) + existing_default_explicit = ctx._param_default_explicit.get(self.name, False) + + with augment_usage_errors(ctx, param=self): + value, source = self.consume_value(ctx, opts) + + # Record the source before processing so eager callbacks and type + # conversion can inspect it. Restored after arbitration if this + # option loses a feature-switch group. + ctx.set_parameter_source(self.name, source) + + # Display a deprecation warning if necessary. + if ( + self.deprecated + and value is not UNSET + and source < ParameterSource.DEFAULT_MAP + ): + message = _( + "DeprecationWarning: The {param_type} {name!r} is deprecated." + "{extra_message}" + ).format( + param_type=self.param_type_name, + name=self.human_readable_name, + extra_message=_format_deprecated_suffix(self.deprecated), + ) + echo(style(message, fg="red"), err=True) + + # Process the value through the parameter's type. + try: + value = self.process_value(ctx, value) + except Exception: + if not ctx.resilient_parsing: + raise + # In resilient parsing mode, we do not want to fail the command if the + # value is incompatible with the parameter type, so we reset the value + # to UNSET, which will be interpreted as a missing value. + value = UNSET + + # Arbitrate the slot when several parameters target the same variable + # name (feature-switch groups). See: https://github.com/pallets/click/issues/3403 + slot_empty = existing_value is UNSET + more_explicit = existing_source is not None and source < existing_source + same_source = existing_source is not None and source == existing_source + auto_would_downgrade_explicit = ( + same_source + and source == ParameterSource.DEFAULT + and existing_default_explicit + and not self._default_explicit + ) + is_winner = ( + slot_empty + or more_explicit + or (same_source and not auto_would_downgrade_explicit) + ) + + if is_winner: + if self.expose_value: + ctx.params[self.name] = value + ctx._param_default_explicit[self.name] = self._default_explicit + elif existing_source is not None: + # Lost arbitration; restore the winning option's source. + ctx.set_parameter_source(self.name, existing_source) + # else: ctx.params[self.name] was populated by code that bypassed + # handle_parse_result (from another option's callback for example). Keep + # the provisional source recorded before process_value so downstream + # lookups don't return ``None``. + + return value, args + + def get_help_record(self, ctx: Context) -> tuple[str, str] | None: + return None + + def get_usage_pieces(self, ctx: Context) -> list[str]: + return [] + + def get_error_hint(self, ctx: Context | None) -> str: + """Get a stringified version of the param for use in error messages to + indicate which param caused the error. + + .. versionchanged:: 8.4.0 + ``ctx`` can be ``None``. + """ + hint_list = self.opts or [self.human_readable_name] + return " / ".join(f"'{x}'" for x in hint_list) + + def shell_complete(self, ctx: Context, incomplete: str) -> list[CompletionItem]: + """Return a list of completions for the incomplete value. If a + ``shell_complete`` function was given during init, it is used. + Otherwise, the :attr:`type` + :meth:`~click.types.ParamType[t.Any].shell_complete` function is used. + + :param ctx: Invocation context for this command. + :param incomplete: Value being completed. May be empty. + + .. versionadded:: 8.0 + """ + if self._custom_shell_complete is not None: + results = self._custom_shell_complete(ctx, self, incomplete) + + if results and isinstance(results[0], str): + from click.shell_completion import CompletionItem + + results = [CompletionItem(c) for c in results] + + return t.cast("list[CompletionItem]", results) + + return self.type.shell_complete(ctx, self, incomplete) + + +class Option(Parameter): + """Options are usually optional values on the command line and + have some extra features that arguments don't have. + + All other parameters are passed onwards to the parameter constructor. + + :param show_default: Show the default value for this option in its + help text. Values are not shown by default, unless + :attr:`Context.show_default` is ``True``. If this value is a + string, it shows that string in parentheses instead of the + actual value. This is particularly useful for dynamic options. + For single option boolean flags, the default remains hidden if + its value is ``False``. + :param show_envvar: Controls if an environment variable should be + shown on the help page and error messages. + Normally, environment variables are not shown. + :param prompt: If set to ``True`` or a non empty string then the + user will be prompted for input. If set to ``True`` the prompt + will be the option name capitalized. A deprecated option cannot be + prompted. + :param confirmation_prompt: Prompt a second time to confirm the + value if it was prompted for. Can be set to a string instead of + ``True`` to customize the message. + :param prompt_required: If set to ``False``, the user will be + prompted for input only when the option was specified as a flag + without a value. + :param hide_input: If this is ``True`` then the input on the prompt + will be hidden from the user. This is useful for password input. + :param is_flag: forces this option to act as a flag. The default is + auto detection. + :param flag_value: which value should be used for this flag if it's + enabled. This is set to a boolean automatically if + the option string contains a slash to mark two options. + :param multiple: if this is set to `True` then the argument is accepted + multiple times and recorded. This is similar to ``nargs`` + in how it works but supports arbitrary number of + arguments. + :param count: this flag makes an option increment an integer. + :param allow_from_autoenv: if this is enabled then the value of this + parameter will be pulled from an environment + variable in case a prefix is defined on the + context. + :param help: the help string. + :param hidden: hide this option from help outputs. + :param attrs: Other command arguments described in :class:`Parameter`. + + .. versionchanged:: 8.4.0 + Non-basic ``flag_value`` types (not ``str``, ``int``, ``float``, or + ``bool``) are passed through unchanged instead of being stringified. + Previously, ``type=click.UNPROCESSED`` was required to preserve them. + + .. versionchanged:: 8.2 + ``envvar`` used with ``flag_value`` will always use the ``flag_value``, + previously it would use the value of the environment variable. + + .. versionchanged:: 8.1 + Help text indentation is cleaned here instead of only in the + ``@option`` decorator. + + .. versionchanged:: 8.1 + The ``show_default`` parameter overrides + ``Context.show_default``. + + .. versionchanged:: 8.1 + The default of a single option boolean flag is not shown if the + default value is ``False``. + + .. versionchanged:: 8.0.1 + ``type`` is detected from ``flag_value`` if given, for basic Python + types (``str``, ``int``, ``float``, ``bool``). + """ + + param_type_name = "option" + + prompt: str | None + confirmation_prompt: bool | str + prompt_required: bool + hide_input: bool + hidden: bool + + _flag_needs_value: bool + is_flag: bool + flag_value: t.Any + type: types.ParamType[t.Any] + default: t.Any | t.Callable[[], t.Any] | None + + count: bool + allow_from_autoenv: bool + help: str | None + show_default: bool | str | None + show_choices: bool + show_envvar: bool + + def __init__( + self, + param_decls: cabc.Sequence[str] | None = None, + show_default: bool | str | None = None, + prompt: bool | str = False, + confirmation_prompt: bool | str = False, + prompt_required: bool = True, + hide_input: bool = False, + is_flag: bool | None = None, + flag_value: t.Any = UNSET, + multiple: bool = False, + count: bool = False, + allow_from_autoenv: bool = True, + type: types.ParamType[t.Any] | t.Any | None = None, + help: str | None = None, + hidden: bool = False, + show_choices: bool = True, + show_envvar: bool = False, + deprecated: bool | str = False, + **attrs: t.Any, + ) -> None: + if help: + help = inspect.cleandoc(help) + + super().__init__( + param_decls, type=type, multiple=multiple, deprecated=deprecated, **attrs + ) + + # Phase 1: prompt-related attributes. ``_infer_flag_kind`` reads ``self.prompt`` + # and ``self.prompt_required`` so this must run first. + if prompt is True: + if not self.name: + raise TypeError("'name' is required with 'prompt=True'.") + + prompt_text = self.name.replace("_", " ").capitalize() + elif prompt is False: + prompt_text = None + else: + prompt_text = prompt + + if deprecated: + label = _format_deprecated_label(deprecated) + help = f"{help} {label}" if help else label + + self.prompt = prompt_text + self.confirmation_prompt = confirmation_prompt + self.prompt_required = prompt_required + self.hide_input = hide_input + self.hidden = hidden + + # Phase 2: flag-kind inference. + self.is_flag, self._flag_needs_value = self._infer_flag_kind( + is_flag, flag_value + ) + + # Phase 3: type inference. Override the type set by :meth:`Parameter.__init__` + # when this option is a flag or a count. + self.type = self._pick_type(type, flag_value, count, self.is_flag) + + # Phase 4: store the raw ``flag_value`` and ``count`` settings. + # ``self.flag_value`` and ``self.default`` deliberately keep the :data:`UNSET` + # sentinel when the user didn't pass them. Auto-derived values are resolved + # lazily in :meth:`_resolve_lazy_default` and :attr:`flag_activation_value`. + # Keeping the raw values means ``is UNSET`` reliably answers "did the user pass + # this?": #3403 needs it for arbitration, and any future feature needing the + # same distinction can reuse it without reintroducing parallel "was it + # explicit?" tracking. + self.flag_value = flag_value + self.count = count + if count and self.default is UNSET: + self.default = 0 + + self.allow_from_autoenv = allow_from_autoenv + self.help = help + self.show_default = show_default + self.show_choices = show_choices + self.show_envvar = show_envvar + + # Phase 5: validate. Raises on illegal kwarg combinations. + self._validate(prompt, deprecated) + + @property + def is_bool_flag(self) -> bool: + """``True`` when this option is a flag with a boolean type. + + Derived from :attr:`is_flag` and :attr:`type`; computed on access so it cannot + drift if a subclass replaces :attr:`type` after construction. + """ + return self.is_flag and isinstance(self.type, types.BoolParamType) + + @property + def flag_activation_value(self) -> t.Any: + """Value the function receives when this flag is activated on the command line. + + Resolves a missing :attr:`flag_value` to ``True`` for actual flag options and + ``None`` otherwise. Used by the parser bridge (:meth:`add_to_parser`) and the + runtime (:meth:`consume_value`) so :attr:`flag_value` can keep the :data:`UNSET` + sentinel for "did the user pass one?" introspection. + """ + if self.flag_value is UNSET: + return True if self.is_flag else None + return self.flag_value + + def _infer_flag_kind( + self, is_flag: bool | None, flag_value: t.Any + ) -> tuple[bool, bool]: + """Resolve ``is_flag`` and the parser hint ``_flag_needs_value``. + + Returns ``(is_flag, flag_needs_value)``, where ``_flag_needs_value`` tells the + parser this option is a flag that cannot be used standalone and needs a value. + The parser uses it to decide whether to treat the next CLI token as the flag's + value or as a new option. If ``prompt`` is enabled with + ``prompt_required=False``, it opens the door for an interactive value, hence the + initial condition. Ref: https://github.com/pallets/click/issues/3084 + """ + needs_value = self.prompt is not None and not self.prompt_required + + if is_flag is None: + # Implicitly a flag because flag_value was set. + if flag_value is not UNSET: + return True, needs_value + # Not a flag, but when used as a flag it shows a prompt. + if needs_value: + return False, needs_value + # Implicitly a flag because secondary options names were given. + if self.secondary_opts: + return True, needs_value + return False, needs_value + + if is_flag is False and not needs_value: + # Explicit ``is_flag=False`` with a flag-like value/default still makes the + # option flag-shaped to the parser. + needs_value = flag_value is not UNSET or self.default is UNSET + + return bool(is_flag), needs_value + + def _pick_type( + self, + type_arg: t.Any, + flag_value: t.Any, + count: bool, + is_flag: bool, + ) -> types.ParamType[t.Any]: + """Pick the final :class:`ParamType` for this option. + + :meth:`Parameter.__init__` already stored ``self.type`` from the explicit + ``type`` argument or from ``default``. This method either returns that as-is, or + overrides it for flag and count options (which are inferred from ``flag_value`` + rather than ``default``). + """ + if type_arg is not None: + return self.type + + if count: + return types.IntRange(min=0) + + if not is_flag: + return self.type + + # A flag without a flag_value is a boolean flag. + if flag_value is UNSET or isinstance(flag_value, bool): + return types.BoolParamType() + + guessed: types.ParamType[t.Any] = types.convert_type(None, flag_value) + if ( + isinstance(guessed, types.StringParamType) + and not isinstance(flag_value, str) + and flag_value is not None + ): + # The flag_value type couldn't be auto-detected (not str, int, float, or + # bool). Since flag_value is a programmer-provided Python object, not CLI + # input, pass it through unchanged instead of stringifying it. + return types.UNPROCESSED + return guessed + + def _resolve_lazy_default(self, value: t.Any) -> t.Any: + """Apply lazy auto-derivations to a default-style value. + + Shared between :meth:`get_default` (the runtime path) and :meth:`to_info_dict` + (the introspection path) so the two views cannot drift apart. Callables are + *not* invoked here: that is :meth:`get_default`'s responsibility. Rules: + + * ``UNSET`` resolves to ``False`` for a non-required boolean flag, and to ``()`` + for a non-required, non-prompted multi flag. + * ``True`` resolves to :attr:`flag_value` for a non-boolean flag (the "activate + this flag by default" shorthand). Boolean flags keep ``True`` as a literal. + """ + if value is UNSET and self.is_flag: + if self.multiple and not self.required and not self.prompt: + return () + if self.is_bool_flag and not self.required: + return False + if value is True and self.is_flag and not self.is_bool_flag: + # Use ``flag_activation_value`` so an unset ``flag_value`` resolves to + # ``True`` (the bool-flag activation value) rather than leaking the + # :data:`UNSET` sentinel. + return self.flag_activation_value + return value + + def _validate(self, prompt: bool | str, deprecated: bool | str) -> None: + """Raise :class:`TypeError` / :class:`ValueError` on illegal kwarg combinations. + + Called once, after every other attribute has been assigned, so each check can + read the final state. + """ + if not __debug__: + return + if deprecated and prompt: + raise ValueError("`deprecated` options cannot use `prompt`.") + if self.nargs == -1: + raise TypeError("nargs=-1 is not supported for options.") + if not self.is_bool_flag and self.secondary_opts: + raise TypeError("Secondary flag is not valid for non-boolean flag.") + if self.is_bool_flag and self.hide_input and self.prompt is not None: + raise TypeError("'prompt' with 'hide_input' is not valid for boolean flag.") + if self.count: + if self.multiple: + raise TypeError("'count' is not valid with 'multiple'.") + if self.is_flag: + raise TypeError("'count' is not valid with 'is_flag'.") + + def to_info_dict(self) -> dict[str, t.Any]: + """ + .. versionchanged:: 8.5.0 + ``default`` and ``flag_value`` reflect the auto-derived values (``False`` + for unset boolean-flag defaults, ``True`` for unset boolean-flag activation + values, etc.) when no explicit value was passed, matching what the function + would receive at call time. + + .. versionchanged:: 8.3.0 + Returns ``None`` for the :attr:`flag_value` if it was not set. + """ + info_dict = super().to_info_dict() + info_dict.update( + default=self._hide_unset(self._resolve_lazy_default(self.default)), + help=self.help, + prompt=self.prompt, + is_flag=self.is_flag, + flag_value=self.flag_activation_value, + count=self.count, + hidden=self.hidden, + ) + return info_dict + + def get_default( + self, ctx: Context, call: bool = True + ) -> t.Any | t.Callable[[], t.Any] | None: + """Return the default value for this option. + + Several auto-derived defaults are resolved lazily here rather than eagerly in + :meth:`__init__`. This keeps :attr:`default` raw at construction time, so + ``self.default is UNSET`` reliably answers "did the user pass a default?" for + feature-switch-group arbitration and for anything else that needs to distinguish + "absent" from a chosen value. The resolution rules live in + :meth:`_resolve_lazy_default`. + + .. versionchanged:: 8.5.0 + ``UNSET`` defaults for boolean and multi flags are now resolved here instead + of being coerced in :meth:`__init__`. Reading :attr:`default` directly + returns the user-supplied value (or ``UNSET`` if none was passed) rather + than the auto-derived one. + + .. versionchanged:: 8.3.3 + ``default=True`` is no longer substituted with ``flag_value`` for boolean + flags, fixing negative boolean flags like + ``flag_value=False, default=True``. + """ + raw = super().get_default(ctx, call=False) + value = self._resolve_lazy_default(raw) + # Only invoke the value as a callable when the lazy resolver passed it through + # unchanged. If the resolver substituted ``True`` -> :attr:`flag_value`, the + # result is the programmer-supplied ``flag_value`` (often a class or enum), + # which must NOT be instantiated here. See + # https://github.com/pallets/click/issues/3121. + if value is raw and call and callable(value): + value = value() + return value + + def get_error_hint(self, ctx: Context | None) -> str: + result = super().get_error_hint(ctx) + if self.show_envvar and self.envvar is not None: + result += f" (env var: '{self.envvar}')" + return result + + def _parse_decls( + self, decls: cabc.Sequence[str], expose_value: bool + ) -> tuple[str, list[str], list[str]]: + opts = [] + secondary_opts = [] + name = None + possible_names = [] + + for decl in decls: + if decl.isidentifier(): + if name is not None: + raise TypeError(_("Name '{name}' defined twice").format(name=name)) + name = decl + else: + split_char = ";" if decl[:1] == "/" else "/" + if split_char in decl: + first, second = decl.split(split_char, 1) + first = first.rstrip() + if first: + possible_names.append(_split_opt(first)) + opts.append(first) + second = second.lstrip() + if second: + secondary_opts.append(second.lstrip()) + if first == second: + raise ValueError( + _( + "Boolean option {decl!r} cannot use the" + " same flag for true/false." + ).format(decl=decl) + ) + else: + possible_names.append(_split_opt(decl)) + opts.append(decl) + + if name is None and possible_names: + possible_names.sort(key=lambda x: -len(x[0])) # group long options first + name = possible_names[0][1].replace("-", "_").lower() + if not name.isidentifier(): + name = None + + if name is None: + if not expose_value: + return "", opts, secondary_opts + raise TypeError( + _( + "Could not determine name for option with declarations {decls!r}" + ).format(decls=decls) + ) + + if not opts and not secondary_opts: + raise TypeError( + _( + "No options defined but a name was passed ({name})." + " Did you mean to declare an argument instead? Did" + " you mean to pass '--{name}'?" + ).format(name=name) + ) + + return name, opts, secondary_opts + + def add_to_parser(self, parser: _OptionParser, ctx: Context) -> None: + if self.multiple: + action = "append" + elif self.count: + action = "count" + else: + action = "store" + + if self.is_flag: + action = f"{action}_const" + + if self.is_bool_flag and self.secondary_opts: + parser.add_option( + obj=self, opts=self.opts, dest=self.name, action=action, const=True + ) + parser.add_option( + obj=self, + opts=self.secondary_opts, + dest=self.name, + action=action, + const=False, + ) + else: + parser.add_option( + obj=self, + opts=self.opts, + dest=self.name, + action=action, + # ``flag_activation_value`` resolves UNSET to the right parser-store + # constant (``True`` for bool flags) so the raw :attr:`flag_value` + # can keep the sentinel. + const=self.flag_activation_value, + ) + else: + parser.add_option( + obj=self, + opts=self.opts, + dest=self.name, + action=action, + nargs=self.nargs, + ) + + def get_help_record(self, ctx: Context) -> tuple[str, str] | None: + if self.hidden: + return None + + any_prefix_is_slash = False + + def _write_opts(opts: cabc.Sequence[str]) -> str: + nonlocal any_prefix_is_slash + + rv, any_slashes = join_options(opts) + + if any_slashes: + any_prefix_is_slash = True + + if not self.is_flag and not self.count: + rv += f" {self.make_metavar(ctx=ctx)}" + + return rv + + rv = [_write_opts(self.opts)] + + if self.secondary_opts: + rv.append(_write_opts(self.secondary_opts)) + + help = self.help or "" + + extra = self.get_help_extra(ctx) + extra_items = [] + if "envvars" in extra: + extra_items.append( + _("env var: {var}").format(var=", ".join(extra["envvars"])) + ) + if "default" in extra: + extra_items.append(_("default: {default}").format(default=extra["default"])) + if "range" in extra: + extra_items.append(extra["range"]) + if "required" in extra: + extra_items.append(_(extra["required"])) + + if extra_items: + extra_str = "; ".join(extra_items) + help = f"{help} [{extra_str}]" if help else f"[{extra_str}]" + + return ("; " if any_prefix_is_slash else " / ").join(rv), help + + def get_help_extra(self, ctx: Context) -> types.OptionHelpExtra: + extra: types.OptionHelpExtra = {} + + if self.show_envvar: + envvar = self.envvar + + if envvar is None: + if ( + self.allow_from_autoenv + and ctx.auto_envvar_prefix is not None + and self.name + ): + envvar = f"{ctx.auto_envvar_prefix}_{self.name.upper()}" + + if envvar is not None: + if isinstance(envvar, str): + extra["envvars"] = (envvar,) + else: + extra["envvars"] = tuple(str(d) for d in envvar) + + # Temporarily enable resilient parsing to avoid type casting + # failing for the default. Might be possible to extend this to + # help formatting in general. + resilient = ctx.resilient_parsing + ctx.resilient_parsing = True + + try: + default_value = self.get_default(ctx, call=False) + finally: + ctx.resilient_parsing = resilient + + show_default = False + show_default_is_str = False + + if self.show_default is not None: + if isinstance(self.show_default, str): + show_default_is_str = show_default = True + else: + show_default = self.show_default + elif ctx.show_default is not None: + show_default = ctx.show_default + + if show_default_is_str or ( + show_default and (default_value not in (None, UNSET)) + ): + if show_default_is_str: + default_string = f"({self.show_default})" + elif isinstance(default_value, (list, tuple)): + default_string = ", ".join(str(d) for d in default_value) + elif isinstance(default_value, enum.Enum): + default_string = default_value.name + elif inspect.isfunction(default_value): + default_string = _("(dynamic)") + elif self.is_bool_flag and self.secondary_opts: + # For boolean flags that have distinct True/False opts, + # use the opt without prefix instead of the value. + default_string = _split_opt( + (self.opts if default_value else self.secondary_opts)[0] + )[1] + elif self.is_bool_flag and not self.secondary_opts and not default_value: + default_string = "" + elif isinstance(default_value, str) and default_value == "": + default_string = '""' + else: + default_string = str(default_value) + + if default_string: + extra["default"] = default_string + + if ( + isinstance(self.type, types._NumberRangeBase) + # skip count with default range type + and not (self.count and self.type.min == 0 and self.type.max is None) + ): + range_str = self.type._describe_range() + + if range_str: + extra["range"] = range_str + + if self.required: + extra["required"] = "required" + + return extra + + def prompt_for_value(self, ctx: Context) -> t.Any: + """This is an alternative flow that can be activated in the full + value processing if a value does not exist. It will prompt the + user until a valid value exists and then returns the processed + value as result. + """ + assert self.prompt is not None + + # Calculate the default before prompting anything to lock in the value before + # attempting any user interaction. + default = self.get_default(ctx) + + # A boolean flag can use a simplified [y/n] confirmation prompt. + if self.is_bool_flag: + # If we have no boolean default, we force the user to explicitly provide + # one. + if default in (UNSET, None): + default = None + # Nothing prevent you to declare an option that is simultaneously: + # 1) auto-detected as a boolean flag, + # 2) allowed to prompt, and + # 3) still declare a non-boolean default. + # This forced casting into a boolean is necessary to align any non-boolean + # default to the prompt, which is going to be a [y/n]-style confirmation + # because the option is still a boolean flag. That way, instead of [y/n], + # we get [Y/n] or [y/N] depending on the truthy value of the default. + # Refs: https://github.com/pallets/click/pull/3030#discussion_r2289180249 + else: + default = bool(default) + return confirm(self.prompt, default) + + # If show_default is given, provide this to `prompt` as well, + # otherwise we use `prompt`'s default behavior + prompt_kwargs: t.Any = {} + if self.show_default is not None: + prompt_kwargs["show_default"] = self.show_default + + return prompt( + self.prompt, + # Use ``None`` to inform the prompt() function to reiterate until a valid + # value is provided by the user if we have no default. + default=self._hide_unset(default), + type=self.type, + hide_input=self.hide_input, + show_choices=self.show_choices, + confirmation_prompt=self.confirmation_prompt, + value_proc=lambda x: self.process_value(ctx, x), + **prompt_kwargs, + ) + + def resolve_envvar_value(self, ctx: Context) -> str | None: + """:class:`Option` resolves its environment variable the same way as + :func:`Parameter.resolve_envvar_value`, but it also supports + :attr:`Context.auto_envvar_prefix`. If we could not find an environment from + the :attr:`envvar` property, we fallback on :attr:`Context.auto_envvar_prefix` + to build dynamiccaly the environment variable name using the + :python:`{ctx.auto_envvar_prefix}_{self.name.upper()}` template. + + :meta private: + """ + rv = super().resolve_envvar_value(ctx) + + if rv is not None: + return rv + + if self.allow_from_autoenv and ctx.auto_envvar_prefix is not None and self.name: + envvar = f"{ctx.auto_envvar_prefix}_{self.name.upper()}" + rv = os.environ.get(envvar) + + if rv: + return rv + + return None + + def value_from_envvar(self, ctx: Context) -> t.Any: + """For :class:`Option`, this method processes the raw environment variable + string the same way as :func:`Parameter.value_from_envvar` does. + + But in the case of non-boolean flags, the value is analyzed to determine if the + flag is activated or not, and returns a boolean of its activation, or the + :attr:`flag_value` if the latter is set. + + This method also takes care of repeated options (i.e. options with + :attr:`multiple` set to ``True``). + + :meta private: + """ + rv = self.resolve_envvar_value(ctx) + + # Absent environment variable or an empty string is interpreted as unset. + if rv is None: + return None + + # Non-boolean flags are more liberal in what they accept. But a flag being a + # flag, its envvar value still needs to be analyzed to determine if the flag is + # activated or not. + if self.is_flag and not self.is_bool_flag: + # An exact match against ``flag_value`` (a non-bool flag always has one + # explicitly set) returns it directly. Otherwise the value is analyzed as a + # boolean: a truthy reading activates the flag and the function receives + # ``flag_value``; a falsy reading produces ``False``; an unrecognized + # reading falls through as ``None``. Folding the substitution here means + # :meth:`consume_value` no longer has to repeat it in a source-checked + # branch. + if rv == self.flag_value: + return self.flag_value + parsed = types.BoolParamType.str_to_bool(rv) + if parsed is None: + return None + return self.flag_value if parsed else False + + # Split the envvar value if it is allowed to be repeated. + value_depth = (self.nargs != 1) + bool(self.multiple) + if value_depth > 0: + multi_rv = self.type.split_envvar_value(rv) + if self.multiple and self.nargs != 1: + multi_rv = batch(multi_rv, self.nargs) # type: ignore[assignment] + + return multi_rv + + return rv + + def consume_value( + self, ctx: Context, opts: cabc.Mapping[str, Parameter] + ) -> tuple[t.Any, ParameterSource]: + """For :class:`Option`, the value can be collected from an interactive prompt + if the option is a flag that needs a value (and the :attr:`prompt` property is + set). + + Additionally, this method handles flag option that are activated without a + value, in which case the :attr:`flag_value` is returned. + + :meta private: + """ + value, source = super().consume_value(ctx, opts) + + # The parser emits a sentinel when a flag is allowed to be used without a value. + # Resolve it to a prompt or to the activation value depending on the option's + # configuration. + if value is FLAG_NEEDS_VALUE: + # If the option allows for a prompt, start an interaction with the user. + if self.prompt is not None and not ctx.resilient_parsing: + value = self.prompt_for_value(ctx) + source = ParameterSource.PROMPT + # Else the flag takes its activation value (resolves UNSET). + else: + value = self.flag_activation_value + source = ParameterSource.COMMANDLINE + + # Re-interpret a multiple option that the parser sent through as a list still + # containing the FLAG_NEEDS_VALUE sentinel, replacing each occurrence with the + # activation value. + elif ( + self.multiple + and value is not UNSET + and isinstance(value, cabc.Iterable) + and source < ParameterSource.DEFAULT_MAP + and any(v is FLAG_NEEDS_VALUE for v in value) + ): + value = [ + self.flag_activation_value if v is FLAG_NEEDS_VALUE else v + for v in value + ] + source = ParameterSource.COMMANDLINE + + # The value wasn't set, or used the param's default, prompt for one to the user + # if prompting is enabled. + elif ( + (value is UNSET or source >= ParameterSource.DEFAULT_MAP) + and self.prompt is not None + and (self.required or self.prompt_required) + and not ctx.resilient_parsing + ): + value = self.prompt_for_value(ctx) + source = ParameterSource.PROMPT + + return value, source + + def process_value(self, ctx: Context, value: t.Any) -> t.Any: + # process_value has to be overridden on Options in order to capture + # `value == UNSET` cases before `type_cast_value()` gets called. + # + # Refs: + # https://github.com/pallets/click/issues/3069 + if self.is_flag and not self.required and self.is_bool_flag and value is UNSET: + value = False + + if self.callback is not None: + value = self.callback(ctx, self, value) + + return value + + # in the normal case, rely on Parameter.process_value + return super().process_value(ctx, value) + + +class Argument(Parameter): + """Arguments are positional parameters to a command. They generally + provide fewer features than options but can have infinite ``nargs`` + and are required by default. + + All parameters are passed onwards to the constructor of :class:`Parameter`. + + :param help: the help string. + + .. versionchanged:: 8.5.0 + Added the ``help`` parameter. + """ + + param_type_name = "argument" + + def __init__( + self, + param_decls: cabc.Sequence[str], + required: bool | None = None, + help: str | None = None, + **attrs: t.Any, + ) -> None: + # Auto-detect the requirement status of the argument if not explicitly set. + if required is None: + # The argument gets automatically required if it has no explicit default + # value set and is setup to match at least one value. + if attrs.get("default", UNSET) is UNSET: + required = attrs.get("nargs", 1) > 0 + # If the argument has a default value, it is not required. + else: + required = False + + if "multiple" in attrs: + raise TypeError("__init__() got an unexpected keyword argument 'multiple'.") + + deprecated = attrs.get("deprecated", False) + + if help: + help = inspect.cleandoc(help) + + if deprecated: + label = _format_deprecated_label(deprecated) + help = f"{help} {label}" if help else label + + self.help = help + + super().__init__(param_decls, required=required, **attrs) + + def to_info_dict(self) -> dict[str, t.Any]: + info_dict = super().to_info_dict() + info_dict.update(help=self.help) + return info_dict + + @property + def human_readable_name(self) -> str: + if self.metavar is not None: + return self.metavar + return self.name.upper() + + def make_metavar(self, ctx: Context) -> str: + if self.metavar is not None: + return self.metavar + var = self.type.get_metavar(param=self, ctx=ctx) + if not var: + var = self.name.upper() + # Types like ``Choice`` and ``DateTime`` already surround their metavar + # with square brackets to enumerate the allowed values. Reuse those + # outer brackets as the optional-argument indicator instead of wrapping + # the metavar in a second pair, which would produce ``[[a|b|c]]``. + already_bracketed = var.startswith("[") and var.endswith("]") + if self.deprecated: + var += "!" + if not self.required and not already_bracketed: + var = f"[{var}]" + if self.nargs != 1: + var += "..." + return var + + def _parse_decls( + self, decls: cabc.Sequence[str], expose_value: bool + ) -> tuple[str, list[str], list[str]]: + if not decls: + if not expose_value: + return "", [], [] + raise TypeError("Argument is marked as exposed, but does not have a name.") + if len(decls) == 1: + name = arg = decls[0] + name = name.replace("-", "_").lower() + else: + raise TypeError( + _( + "Arguments take exactly one parameter declaration, got" + " {length}: {decls}." + ).format(length=len(decls), decls=decls) + ) + return name, [arg], [] + + def get_usage_pieces(self, ctx: Context) -> list[str]: + return [self.make_metavar(ctx)] + + def get_help_record(self, ctx: Context) -> tuple[str, str] | None: + if self.help is None: + return None + + return self.make_metavar(ctx), self.help + + def get_error_hint(self, ctx: Context | None) -> str: + if ctx is not None: + return f"'{self.make_metavar(ctx)}'" + return f"'{self.human_readable_name}'" + + def add_to_parser(self, parser: _OptionParser, ctx: Context) -> None: + parser.add_argument(dest=self.name, nargs=self.nargs, obj=self) + + +def __getattr__(name: str) -> object: + import warnings + + if name == "BaseCommand": + warnings.warn( + "'BaseCommand' is deprecated and will be removed in Click 9.0. Use" + " 'Command' instead.", + DeprecationWarning, + stacklevel=2, + ) + return _BaseCommand + + if name == "MultiCommand": + warnings.warn( + "'MultiCommand' is deprecated and will be removed in Click 9.0. Use" + " 'Group' instead.", + DeprecationWarning, + stacklevel=2, + ) + return _MultiCommand + + raise AttributeError(name) diff --git a/.venv/lib/python3.14/site-packages/click/decorators.py b/.venv/lib/python3.14/site-packages/click/decorators.py new file mode 100644 index 0000000000000000000000000000000000000000..cc42e5d42c48e7bddca9346bc4dd4f240a30c2e3 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/click/decorators.py @@ -0,0 +1,627 @@ +from __future__ import annotations + +import inspect +import typing as t +from functools import update_wrapper +from gettext import gettext as _ + +from .core import Argument +from .core import Command +from .core import Context +from .core import Group +from .core import Option +from .core import Parameter +from .globals import get_current_context +from .utils import echo + +if t.TYPE_CHECKING: + import typing_extensions as te + + P = te.ParamSpec("P") + +R = t.TypeVar("R") +T = t.TypeVar("T") +_AnyCallable = t.Callable[..., t.Any] +FC = t.TypeVar("FC", bound="_AnyCallable | Command") + + +def pass_context(f: t.Callable[te.Concatenate[Context, P], R]) -> t.Callable[P, R]: + """Marks a callback as wanting to receive the current context + object as first argument. + """ + + def new_func(*args: P.args, **kwargs: P.kwargs) -> R: + return f(get_current_context(), *args, **kwargs) + + return update_wrapper(new_func, f) + + +def pass_obj(f: t.Callable[te.Concatenate[T, P], R]) -> t.Callable[P, R]: + """Similar to :func:`pass_context`, but only pass the object on the + context onwards (:attr:`Context.obj`). This is useful if that object + represents the state of a nested system. + """ + + def new_func(*args: P.args, **kwargs: P.kwargs) -> R: + return f(get_current_context().obj, *args, **kwargs) + + return update_wrapper(new_func, f) + + +def make_pass_decorator( + object_type: type[T], ensure: bool = False +) -> t.Callable[[t.Callable[te.Concatenate[T, P], R]], t.Callable[P, R]]: + """Given an object type this creates a decorator that will work + similar to :func:`pass_obj` but instead of passing the object of the + current context, it will find the innermost context of type + :func:`object_type`. + + This generates a decorator that works roughly like this:: + + from functools import update_wrapper + + def decorator(f): + @pass_context + def new_func(ctx, *args, **kwargs): + obj = ctx.find_object(object_type) + return ctx.invoke(f, obj, *args, **kwargs) + return update_wrapper(new_func, f) + return decorator + + :param object_type: the type of the object to pass. + :param ensure: if set to `True`, a new object will be created and + remembered on the context if it's not there yet. + """ + + def decorator(f: t.Callable[te.Concatenate[T, P], R]) -> t.Callable[P, R]: + def new_func(*args: P.args, **kwargs: P.kwargs) -> R: + ctx = get_current_context() + + obj: T | None + if ensure: + obj = ctx.ensure_object(object_type) + else: + obj = ctx.find_object(object_type) + + if obj is None: + raise RuntimeError( + "Managed to invoke callback without a context" + f" object of type {object_type.__name__!r}" + " existing." + ) + + return ctx.invoke(f, obj, *args, **kwargs) + + return update_wrapper(new_func, f) + + return decorator + + +def pass_meta_key( + key: str, *, doc_description: str | None = None +) -> t.Callable[[t.Callable[te.Concatenate[T, P], R]], t.Callable[P, R]]: + """Create a decorator that passes a key from + :attr:`click.Context.meta` as the first argument to the decorated + function. + + :param key: Key in ``Context.meta`` to pass. + :param doc_description: Description of the object being passed, + inserted into the decorator's docstring. Defaults to "the 'key' + key from Context.meta". + + .. versionadded:: 8.0 + """ + + def decorator(f: t.Callable[te.Concatenate[T, P], R]) -> t.Callable[P, R]: + def new_func(*args: P.args, **kwargs: P.kwargs) -> R: + ctx = get_current_context() + obj = ctx.meta[key] + return ctx.invoke(f, obj, *args, **kwargs) + + return update_wrapper(new_func, f) + + if doc_description is None: + doc_description = f"the {key!r} key from :attr:`click.Context.meta`" + + decorator.__doc__ = ( + f"Decorator that passes {doc_description} as the first argument" + " to the decorated function." + ) + return decorator + + +CmdType = t.TypeVar("CmdType", bound=Command) + + +# variant: no call, directly as decorator for a function. +@t.overload +def command(name: _AnyCallable) -> Command: ... + + +# variant: with positional name and with positional or keyword cls argument: +# @command(namearg, CommandCls, ...) or @command(namearg, cls=CommandCls, ...) +@t.overload +def command( + name: str | None, + cls: type[CmdType], + **attrs: t.Any, +) -> t.Callable[[_AnyCallable], CmdType]: ... + + +# variant: name omitted, cls _must_ be a keyword argument, @command(cls=CommandCls, ...) +@t.overload +def command( + name: None = None, + *, + cls: type[CmdType], + **attrs: t.Any, +) -> t.Callable[[_AnyCallable], CmdType]: ... + + +# variant: with optional string name, no cls argument provided. +@t.overload +def command( + name: str | None = ..., cls: None = None, **attrs: t.Any +) -> t.Callable[[_AnyCallable], Command]: ... + + +def command( + name: str | _AnyCallable | None = None, + cls: type[CmdType] | None = None, + **attrs: t.Any, +) -> Command | t.Callable[[_AnyCallable], Command | CmdType]: + r"""Creates a new :class:`Command` and uses the decorated function as + callback. This will also automatically attach all decorated + :func:`option`\s and :func:`argument`\s as parameters to the command. + + The name of the command defaults to the name of the function, converted to + lowercase, with underscores ``_`` replaced by dashes ``-``, and the suffixes + ``_command``, ``_cmd``, ``_group``, and ``_grp`` are removed. For example, + ``init_data_command`` becomes ``init-data``. + + All keyword arguments are forwarded to the underlying command class. + For the ``params`` argument, any decorated params are appended to + the end of the list. + + Once decorated the function turns into a :class:`Command` instance + that can be invoked as a command line utility or be attached to a + command :class:`Group`. + + :param name: The name of the command. Defaults to modifying the function's + name as described above. + :param cls: The command class to create. Defaults to :class:`Command`. + + .. versionchanged:: 8.2 + The suffixes ``_command``, ``_cmd``, ``_group``, and ``_grp`` are + removed when generating the name. + + .. versionchanged:: 8.1 + This decorator can be applied without parentheses. + + .. versionchanged:: 8.1 + The ``params`` argument can be used. Decorated params are + appended to the end of the list. + """ + + func: t.Callable[[_AnyCallable], t.Any] | None = None + + if callable(name): + func = name + name = None + assert cls is None, "Use 'command(cls=cls)(callable)' to specify a class." + assert not attrs, "Use 'command(**kwargs)(callable)' to provide arguments." + + if cls is None: + cls = t.cast("type[CmdType]", Command) + + def decorator(f: _AnyCallable) -> CmdType: + if isinstance(f, Command): + raise TypeError("Attempted to convert a callback into a command twice.") + + attr_params = attrs.pop("params", None) + params = attr_params if attr_params is not None else [] + + try: + decorator_params = f.__click_params__ # type: ignore + except AttributeError: + pass + else: + del f.__click_params__ # type: ignore + params.extend(reversed(decorator_params)) + + if attrs.get("help") is None: + attrs["help"] = f.__doc__ + + if t.TYPE_CHECKING: + assert cls is not None + assert not callable(name) + + if name is not None: + cmd_name = name + else: + cmd_name = f.__name__.lower().replace("_", "-") + cmd_left, sep, suffix = cmd_name.rpartition("-") + + if sep and suffix in {"command", "cmd", "group", "grp"}: + cmd_name = cmd_left + + cmd = cls(name=cmd_name, callback=f, params=params, **attrs) + cmd.__doc__ = f.__doc__ + return cmd + + if func is not None: + return decorator(func) + + return decorator + + +GrpType = t.TypeVar("GrpType", bound=Group) + + +# variant: no call, directly as decorator for a function. +@t.overload +def group(name: _AnyCallable) -> Group: ... + + +# variant: with positional name and with positional or keyword cls argument: +# @group(namearg, GroupCls, ...) or @group(namearg, cls=GroupCls, ...) +@t.overload +def group( + name: str | None, + cls: type[GrpType], + **attrs: t.Any, +) -> t.Callable[[_AnyCallable], GrpType]: ... + + +# variant: name omitted, cls _must_ be a keyword argument, @group(cmd=GroupCls, ...) +@t.overload +def group( + name: None = None, + *, + cls: type[GrpType], + **attrs: t.Any, +) -> t.Callable[[_AnyCallable], GrpType]: ... + + +# variant: with optional string name, no cls argument provided. +@t.overload +def group( + name: str | None = ..., cls: None = None, **attrs: t.Any +) -> t.Callable[[_AnyCallable], Group]: ... + + +def group( + name: str | _AnyCallable | None = None, + cls: type[GrpType] | None = None, + **attrs: t.Any, +) -> Group | t.Callable[[_AnyCallable], Group | GrpType]: + """Creates a new :class:`Group` with a function as callback. This + works otherwise the same as :func:`command` just that the `cls` + parameter is set to :class:`Group`. + + .. versionchanged:: 8.1 + This decorator can be applied without parentheses. + """ + if cls is None: + cls = t.cast("type[GrpType]", Group) + + if callable(name): + return command(cls=cls, **attrs)(name) + + return command(name, cls, **attrs) + + +def _param_memo(f: t.Callable[..., t.Any], param: Parameter) -> None: + if isinstance(f, Command): + f.params.append(param) + else: + if not hasattr(f, "__click_params__"): + f.__click_params__ = [] # type: ignore + + f.__click_params__.append(param) # type: ignore + + +def argument( + *param_decls: str, cls: type[Argument] | None = None, **attrs: t.Any +) -> t.Callable[[FC], FC]: + """Attaches an argument to the command. All positional arguments are + passed as parameter declarations to :class:`Argument`; all keyword + arguments are forwarded unchanged (except ``cls``). + This is equivalent to creating an :class:`Argument` instance manually + and attaching it to the :attr:`Command.params` list. + + For the default argument class, refer to :class:`Argument` and + :class:`Parameter` for descriptions of parameters. + + :param cls: the argument class to instantiate. This defaults to + :class:`Argument`. + :param param_decls: Passed as positional arguments to the constructor of + ``cls``. + :param attrs: Passed as keyword arguments to the constructor of ``cls``. + """ + if cls is None: + cls = Argument + + def decorator(f: FC) -> FC: + _param_memo(f, cls(param_decls, **attrs)) + return f + + return decorator + + +def option( + *param_decls: str, cls: type[Option] | None = None, **attrs: t.Any +) -> t.Callable[[FC], FC]: + """Attaches an option to the command. All positional arguments are + passed as parameter declarations to :class:`Option`; all keyword + arguments are forwarded unchanged (except ``cls``). + This is equivalent to creating an :class:`Option` instance manually + and attaching it to the :attr:`Command.params` list. + + For the default option class, refer to :class:`Option` and + :class:`Parameter` for descriptions of parameters. + + :param cls: the option class to instantiate. This defaults to + :class:`Option`. + :param param_decls: Passed as positional arguments to the constructor of + ``cls``. + :param attrs: Passed as keyword arguments to the constructor of ``cls``. + """ + if cls is None: + cls = Option + + def decorator(f: FC) -> FC: + _param_memo(f, cls(param_decls, **attrs)) + return f + + return decorator + + +def confirmation_option(*param_decls: str, **kwargs: t.Any) -> t.Callable[[FC], FC]: + """Add a ``--yes`` option which shows a prompt before continuing if + not passed. If the prompt is declined, the program will exit. + + :param param_decls: One or more option names. Defaults to the single + value ``"--yes"``. + :param kwargs: Extra arguments are passed to :func:`option`. + """ + + def callback(ctx: Context, param: Parameter, value: bool) -> None: + if not value: + ctx.abort() + + if not param_decls: + param_decls = ("--yes",) + + kwargs.setdefault("is_flag", True) + kwargs.setdefault("callback", callback) + kwargs.setdefault("expose_value", False) + kwargs.setdefault("prompt", _("Do you want to continue?")) + kwargs.setdefault("help", _("Confirm the action without prompting.")) + return option(*param_decls, **kwargs) + + +def password_option(*param_decls: str, **kwargs: t.Any) -> t.Callable[[FC], FC]: + """Add a ``--password`` option which prompts for a password, hiding + input and asking to enter the value again for confirmation. + + :param param_decls: One or more option names. Defaults to the single + value ``"--password"``. + :param kwargs: Extra arguments are passed to :func:`option`. + """ + if not param_decls: + param_decls = ("--password",) + + kwargs.setdefault("prompt", True) + kwargs.setdefault("confirmation_prompt", True) + kwargs.setdefault("hide_input", True) + return option(*param_decls, **kwargs) + + +def version_option( + version: str | None = None, + *param_decls: str, + package_name: str | None = None, + prog_name: str | None = None, + message: str | None = None, + **kwargs: t.Any, +) -> t.Callable[[FC], FC]: + """Add a ``--version`` option which immediately prints the version + number and exits the program. + + If ``version`` is not provided, Click will try to detect it using + :func:`importlib.metadata.version` to get the version for the + ``package_name``. + + If ``package_name`` is not provided, Click will try to detect it by + inspecting the stack frames. If the detected (or given) name does + not match an installed distribution, Click resolves it as an import + (top-level module) name via + :func:`importlib.metadata.packages_distributions`, so e.g. ``PIL`` + resolves to the ``Pillow`` distribution. + + .. note:: + The parameters and message variables accepted by this option are + frozen: no new slots will be added, to keep the common case simple + and predictable. If you need values it does not expose, such as a + file path, the Python version, or git metadata, use + :func:`custom_version_option` to render the output yourself. + + Rationale: `discussion #3527 + `_. + + :param version: The version number to show. If not provided, Click + will try to detect it. + :param param_decls: One or more option names. Defaults to the single + value ``"--version"``. + :param package_name: The package name to detect the version from. If + not provided, Click will try to detect it. + :param prog_name: The name of the CLI to show in the message. If not + provided, it will be detected from the command. + :param message: The message to show. The values ``%(prog)s``, + ``%(package)s``, and ``%(version)s`` are available. Defaults to + ``"%(prog)s, version %(version)s"``. + :param kwargs: Extra arguments are passed to :func:`option`. + :raise RuntimeError: ``version`` could not be detected. + + .. versionchanged:: 8.0 + Add the ``package_name`` parameter, and the ``%(package)s`` + value for messages. + + .. versionchanged:: 8.0 + Use :mod:`importlib.metadata` instead of ``pkg_resources``. The + version is detected based on the package name, not the entry + point name. The Python package name must match the installed + package name, or be passed with ``package_name=``. + + .. versionchanged:: 8.4.2 + When ``package_name`` does not match an installed distribution, + Click now resolves it as an import (top-level module). + """ + if message is None: + message = _("%(prog)s, version %(version)s") + + if version is None and package_name is None: + frame = inspect.currentframe() + f_back = frame.f_back if frame is not None else None + f_globals = f_back.f_globals if f_back is not None else None + # break reference cycle + # https://docs.python.org/3/library/inspect.html#the-interpreter-stack + del frame + + if f_globals is not None: + package_name = f_globals.get("__name__") + + if package_name == "__main__": + package_name = f_globals.get("__package__") + + if package_name: + package_name = package_name.partition(".")[0] + + def callback(ctx: Context, param: Parameter, value: bool) -> None: + if not value or ctx.resilient_parsing: + return + + nonlocal prog_name + nonlocal version + nonlocal package_name + + if prog_name is None: + prog_name = ctx.find_root().info_name + + if version is None and package_name is not None: + import importlib.metadata + + try: + version = importlib.metadata.version(package_name) + except importlib.metadata.PackageNotFoundError: + # The given name didn't match an installed distribution. + # Try resolving it as an import (top-level module) name, + # e.g. ``PIL`` is provided by the ``Pillow`` distribution. + distributions = importlib.metadata.packages_distributions().get( + package_name, [] + ) + if len(distributions) == 1: + package_name = distributions[0] + version = importlib.metadata.version(package_name) + elif len(distributions) > 1: + raise RuntimeError( + f"{package_name!r} maps to multiple installed" + f" distributions ({', '.join(distributions)})." + " Pass 'package_name' to disambiguate." + ) from None + else: + raise RuntimeError( + f"{package_name!r} is not installed. Try passing" + " 'package_name' instead." + ) from None + + if version is None: + raise RuntimeError( + f"Could not determine the version for {package_name!r} automatically." + ) + + echo( + message % {"prog": prog_name, "package": package_name, "version": version}, + color=ctx.color, + ) + ctx.exit() + + if not param_decls: + param_decls = ("--version",) + + kwargs.setdefault("is_flag", True) + kwargs.setdefault("expose_value", False) + kwargs.setdefault("is_eager", True) + kwargs.setdefault("help", _("Show the version and exit.")) + kwargs["callback"] = callback + return option(*param_decls, **kwargs) + + +def custom_version_option( + callback: t.Callable[[Context], str], + *param_decls: str, + **kwargs: t.Any, +) -> t.Callable[[FC], FC]: + """Add a ``--version`` option whose output is produced by ``callback``. + + This is the customizable companion to :func:`version_option`. Where + :func:`version_option` is intentionally limited to a fixed message and + a small set of values, this option calls ``callback`` to build the + whole string to print. Use it when you need values that + :func:`version_option` does not expose, such as a file path, the + Python version, or git metadata. + + :param callback: Called with the current :class:`Context` when the + option is invoked. Its return value is printed, then the program + exits. + :param param_decls: One or more option names. Defaults to the single + value ``--version``. + :param kwargs: Extra arguments are passed to :func:`option`. + + .. versionadded:: 8.5.0 + """ + + def show_version(ctx: Context, param: Parameter, value: bool) -> None: + if not value or ctx.resilient_parsing: + return + + echo(callback(ctx), color=ctx.color) + ctx.exit() + + if not param_decls: + param_decls = ("--version",) + + kwargs.setdefault("is_flag", True) + kwargs.setdefault("expose_value", False) + kwargs.setdefault("is_eager", True) + kwargs.setdefault("help", _("Show the version and exit.")) + kwargs["callback"] = show_version + return option(*param_decls, **kwargs) + + +def help_option(*param_decls: str, **kwargs: t.Any) -> t.Callable[[FC], FC]: + """Pre-configured ``--help`` option which immediately prints the help page + and exits the program. + + :param param_decls: One or more option names. Defaults to the single + value ``"--help"``. + :param kwargs: Extra arguments are passed to :func:`option`. + """ + + def show_help(ctx: Context, param: Parameter, value: bool) -> None: + """Callback that print the help page on ```` and exits.""" + if value and not ctx.resilient_parsing: + echo(ctx.get_help(), color=ctx.color) + ctx.exit() + + if not param_decls: + param_decls = ("--help",) + + kwargs.setdefault("is_flag", True) + kwargs.setdefault("expose_value", False) + kwargs.setdefault("is_eager", True) + kwargs.setdefault("help", _("Show this message and exit.")) + kwargs.setdefault("callback", show_help) + + return option(*param_decls, **kwargs) diff --git a/.venv/lib/python3.14/site-packages/click/exceptions.py b/.venv/lib/python3.14/site-packages/click/exceptions.py new file mode 100644 index 0000000000000000000000000000000000000000..6272c38a448d57241d9a6bff4191897f4a9bb7d6 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/click/exceptions.py @@ -0,0 +1,378 @@ +from __future__ import annotations + +import collections.abc as cabc +import typing as t +from gettext import gettext as _ +from gettext import ngettext + +from ._compat import get_text_stderr +from .globals import resolve_color_default +from .utils import echo +from .utils import format_filename + +if t.TYPE_CHECKING: + from .core import Command + from .core import Context + from .core import Parameter + + +def _join_param_hints(param_hint: cabc.Sequence[str] | str | None) -> str | None: + if param_hint is not None and not isinstance(param_hint, str): + return " / ".join(repr(x) for x in param_hint) + + return param_hint + + +def _format_possibilities(possibilities: list[str]) -> str: + possibility_str = ", ".join(repr(p) for p in sorted(possibilities)) + return ngettext( + "Did you mean {possibility}?", + "(Did you mean one of: {possibilities}?)", + len(possibilities), + ).format(possibility=possibility_str, possibilities=possibility_str) + + +class ClickException(Exception): + """An exception that Click can handle and show to the user.""" + + #: The exit code for this exception. + exit_code: t.ClassVar[int] = 1 + + show_color: t.Final[bool | None] + message: t.Final[str] + + def __init__(self, message: str) -> None: + super().__init__(message) + # The context will be removed by the time we print the message, so cache + # the color settings here to be used later on (in `show`) + self.show_color = resolve_color_default() + self.message = message + + def format_message(self) -> str: + return self.message + + def __str__(self) -> str: + return self.message + + def show(self, file: t.IO[t.Any] | None = None) -> None: + if file is None: + file = get_text_stderr() + + echo( + _("Error: {message}").format(message=self.format_message()), + file=file, + color=self.show_color, + ) + + +class UsageError(ClickException): + """An internal exception that signals a usage error. This typically + aborts any further handling. + + :param message: the error message to display. + :param ctx: optionally the context that caused this error. Click will + fill in the context automatically in some situations. + """ + + exit_code: t.ClassVar[int] = 2 + + ctx: Context | None + cmd: t.Final[Command | None] + + def __init__(self, message: str, ctx: Context | None = None) -> None: + super().__init__(message) + self.ctx = ctx + self.cmd = self.ctx.command if self.ctx else None + + def show(self, file: t.IO[t.Any] | None = None) -> None: + if file is None: + file = get_text_stderr() + color = None + hint = "" + if ( + self.ctx is not None + and self.ctx.command.get_help_option(self.ctx) is not None + ): + help_names = self.ctx.command.get_help_option_names(self.ctx) + # Pick the longest name (like ``--help`` over ``-h``) for + # readability in error messages. + hint = _("Try '{command} {option}' for help.").format( + command=self.ctx.command_path, + option=max(help_names, key=len), + ) + hint = f"{hint}\n" + if self.ctx is not None: + color = self.ctx.color + echo(f"{self.ctx.get_usage()}\n{hint}", file=file, color=color) + echo( + _("Error: {message}").format(message=self.format_message()), + file=file, + color=color, + ) + + +class BadParameter(UsageError): + """An exception that formats out a standardized error message for a + bad parameter. This is useful when thrown from a callback or type as + Click will attach contextual information to it (for instance, which + parameter it is). + + .. versionadded:: 2.0 + + :param param: the parameter object that caused this error. This can + be left out, and Click will attach this info itself + if possible. + :param param_hint: a string that shows up as parameter name. This + can be used as alternative to `param` in cases + where custom validation should happen. If it is + a string it's used as such, if it's a list then + each item is quoted and separated. + """ + + param: Parameter | None + param_hint: cabc.Sequence[str] | str | None + + def __init__( + self, + message: str, + ctx: Context | None = None, + param: Parameter | None = None, + param_hint: cabc.Sequence[str] | str | None = None, + ) -> None: + super().__init__(message, ctx) + self.param = param + self.param_hint = param_hint + + def format_message(self) -> str: + if self.param_hint is not None: + param_hint = self.param_hint + elif self.param is not None: + param_hint = self.param.get_error_hint(self.ctx) + else: + return _("Invalid value: {message}").format(message=self.message) + + return _("Invalid value for {param_hint}: {message}").format( + param_hint=_join_param_hints(param_hint), message=self.message + ) + + +class MissingParameter(BadParameter): + """Raised if click required an option or argument but it was not + provided when invoking the script. + + .. versionadded:: 4.0 + + :param param_type: a string that indicates the type of the parameter. + The default is to inherit the parameter type from + the given `param`. Valid values are ``'parameter'``, + ``'option'`` or ``'argument'``. + """ + + param_type: t.Final[str | None] + + def __init__( + self, + message: str | None = None, + ctx: Context | None = None, + param: Parameter | None = None, + param_hint: cabc.Sequence[str] | str | None = None, + param_type: str | None = None, + ) -> None: + super().__init__(message or "", ctx, param, param_hint) + self.param_type = param_type + + def format_message(self) -> str: + if self.param_hint is not None: + param_hint: cabc.Sequence[str] | str | None = self.param_hint + elif self.param is not None: + param_hint = self.param.get_error_hint(self.ctx) + else: + param_hint = None + + param_hint = _join_param_hints(param_hint) + param_hint = f" {param_hint}" if param_hint else "" + + param_type = self.param_type + if param_type is None and self.param is not None: + param_type = self.param.param_type_name + + msg = self.message + if self.param is not None: + msg_extra = self.param.type.get_missing_message( + param=self.param, ctx=self.ctx + ) + if msg_extra: + if msg: + msg += f". {msg_extra}" + else: + msg = msg_extra + + msg = f" {msg}" if msg else "" + + # Translate param_type for known types. + if param_type == "argument": + missing = _("Missing argument") + elif param_type == "option": + missing = _("Missing option") + elif param_type == "parameter": + missing = _("Missing parameter") + else: + missing = _("Missing {param_type}").format(param_type=param_type) + + return f"{missing}{param_hint}.{msg}" + + def __str__(self) -> str: + if not self.message: + param_name = self.param.name if self.param else None + return _("Missing parameter: {param_name}").format(param_name=param_name) + else: + return self.message + + +class NoSuchOption(UsageError): + """Raised if Click attempted to handle an option that does not exist. + + .. versionadded:: 4.0 + """ + + option_name: t.Final[str] + possibilities: t.Final[list[str] | None] + + def __init__( + self, + option_name: str, + message: str | None = None, + possibilities: cabc.Iterable[str] | None = None, + ctx: Context | None = None, + ) -> None: + if message is None: + message = _("No such option {name!r}.").format(name=option_name) + + super().__init__(message, ctx) + self.option_name = option_name + + if possibilities: + from difflib import get_close_matches + + possibilities_ = get_close_matches(option_name, possibilities) + else: + possibilities_ = None + self.possibilities = possibilities_ + + def format_message(self) -> str: + if not self.possibilities: + return self.message + return f"{self.message} {_format_possibilities(self.possibilities)}" + + +class NoSuchCommand(UsageError): + """Raised if Click attempted to handle a command that does not exist. + + .. versionadded:: 8.4.0 + """ + + command_name: t.Final[str] + possibilities: t.Final[list[str] | None] + + def __init__( + self, + command_name: str, + message: str | None = None, + possibilities: cabc.Iterable[str] | None = None, + ctx: Context | None = None, + ) -> None: + if message is None: + message = _("No such command {name!r}.").format(name=command_name) + + super().__init__(message, ctx) + self.command_name = command_name + + if possibilities: + from difflib import get_close_matches + + possibilities_ = get_close_matches(command_name, possibilities) + else: + possibilities_ = None + self.possibilities = possibilities_ + + def format_message(self) -> str: + if not self.possibilities: + return self.message + return f"{self.message} {_format_possibilities(self.possibilities)}" + + +class BadOptionUsage(UsageError): + """Raised if an option is generally supplied but the use of the option + was incorrect. This is for instance raised if the number of arguments + for an option is not correct. + + .. versionadded:: 4.0 + + :param option_name: the name of the option being used incorrectly. + """ + + option_name: t.Final[str] + + def __init__( + self, option_name: str, message: str, ctx: Context | None = None + ) -> None: + super().__init__(message, ctx) + self.option_name = option_name + + +class BadArgumentUsage(UsageError): + """Raised if an argument is generally supplied but the use of the argument + was incorrect. This is for instance raised if the number of values + for an argument is not correct. + + .. versionadded:: 6.0 + """ + + +class NoArgsIsHelpError(UsageError): + ctx: Context + + def __init__(self, ctx: Context) -> None: + super().__init__(ctx.get_help(), ctx=ctx) + + def show(self, file: t.IO[t.Any] | None = None) -> None: + echo(self.format_message(), file=file, err=True, color=self.ctx.color) + + +class FileError(ClickException): + """Raised if a file cannot be opened.""" + + ui_filename: t.Final[str] + filename: t.Final[str] + + def __init__(self, filename: str, hint: str | None = None) -> None: + if hint is None: + hint = _("unknown error") + + super().__init__(hint) + self.ui_filename = format_filename(filename) + self.filename = filename + + def format_message(self) -> str: + return _("Could not open file {filename!r}: {message}").format( + filename=self.ui_filename, message=self.message + ) + + +class Abort(RuntimeError): + """An internal signalling exception that signals Click to abort.""" + + +class Exit(RuntimeError): + """An exception that indicates that the application should exit with some + status code. + + :param code: the status code to exit with. + """ + + __slots__ = ("exit_code",) + + exit_code: t.Final[int] + + def __init__(self, code: int = 0) -> None: + self.exit_code = code diff --git a/.venv/lib/python3.14/site-packages/click/formatting.py b/.venv/lib/python3.14/site-packages/click/formatting.py new file mode 100644 index 0000000000000000000000000000000000000000..c4aa2de571a1eb17bfeb1853e315d28bc968e74c --- /dev/null +++ b/.venv/lib/python3.14/site-packages/click/formatting.py @@ -0,0 +1,320 @@ +from __future__ import annotations + +import collections.abc as cabc +from contextlib import contextmanager +from gettext import gettext as _ + +from ._compat import term_len +from .parser import _split_opt + +# Can force a width. This is used by the test system +FORCED_WIDTH: int | None = None + + +def measure_table(rows: cabc.Iterable[tuple[str, str]]) -> tuple[int, ...]: + widths: dict[int, int] = {} + + for row in rows: + for idx, col in enumerate(row): + widths[idx] = max(widths.get(idx, 0), term_len(col)) + + return tuple(y for x, y in sorted(widths.items())) + + +def iter_rows( + rows: cabc.Iterable[tuple[str, str]], col_count: int +) -> cabc.Iterator[tuple[str, ...]]: + for row in rows: + yield row + ("",) * (col_count - len(row)) + + +def wrap_text( + text: str, + width: int = 78, + initial_indent: str = "", + subsequent_indent: str = "", + preserve_paragraphs: bool = False, +) -> str: + """A helper function that intelligently wraps text. By default, it + assumes that it operates on a single paragraph of text but if the + `preserve_paragraphs` parameter is provided it will intelligently + handle paragraphs (defined by two empty lines). + + If paragraphs are handled, a paragraph can be prefixed with an empty + line containing the ``\\b`` character (``\\x08``) to indicate that + no rewrapping should happen in that block. + + :param text: the text that should be rewrapped. + :param width: the maximum width for the text. + :param initial_indent: the initial indent that should be placed on the + first line as a string. + :param subsequent_indent: the indent string that should be placed on + each consecutive line. + :param preserve_paragraphs: if this flag is set then the wrapping will + intelligently handle paragraphs. + + .. versionchanged:: 8.4.0 + Width is measured in visible characters. ANSI escape sequences in + ``text``, ``initial_indent``, or ``subsequent_indent`` no longer + count toward the width budget, so styled input wraps based on what + the user sees instead of raw byte length. + """ + from ._textwrap import TextWrapper + + text = text.expandtabs() + wrapper = TextWrapper( + width, + initial_indent=initial_indent, + subsequent_indent=subsequent_indent, + replace_whitespace=False, + ) + if not preserve_paragraphs: + return wrapper.fill(text) + + p: list[tuple[int, bool, str]] = [] + buf: list[str] = [] + indent = None + + def _flush_par() -> None: + if not buf: + return + if buf[0].strip() == "\b": + p.append((indent or 0, True, "\n".join(buf[1:]))) + else: + p.append((indent or 0, False, " ".join(buf))) + del buf[:] + + for line in text.splitlines(): + if not line: + _flush_par() + indent = None + else: + if indent is None: + orig_len = term_len(line) + line = line.lstrip() + indent = orig_len - term_len(line) + buf.append(line) + _flush_par() + + rv = [] + for indent, raw, text in p: + with wrapper.extra_indent(" " * indent): + if raw: + rv.append(wrapper.indent_only(text)) + else: + rv.append(wrapper.fill(text)) + + return "\n\n".join(rv) + + +class HelpFormatter: + """This class helps with formatting text-based help pages. It's + usually just needed for very special internal cases, but it's also + exposed so that developers can write their own fancy outputs. + + At present, it always writes into memory. + + :param indent_increment: the additional increment for each level. + :param width: the width for the text. This defaults to the terminal + width clamped to a maximum of 78. + """ + + indent_increment: int + width: int + current_indent: int + buffer: list[str] + + def __init__( + self, + indent_increment: int = 2, + width: int | None = None, + max_width: int | None = None, + ) -> None: + self.indent_increment = indent_increment + if max_width is None: + max_width = 80 + if width is None: + import shutil + + width = FORCED_WIDTH + if width is None: + width = max(min(shutil.get_terminal_size().columns, max_width) - 2, 50) + self.width = width + self.current_indent = 0 + self.buffer = [] + + def write(self, string: str) -> None: + """Writes a unicode string into the internal buffer.""" + self.buffer.append(string) + + def indent(self) -> None: + """Increases the indentation.""" + self.current_indent += self.indent_increment + + def dedent(self) -> None: + """Decreases the indentation.""" + self.current_indent -= self.indent_increment + + def write_usage(self, prog: str, args: str = "", prefix: str | None = None) -> None: + """Writes a usage line into the buffer. + + :param prog: the program name. + :param args: whitespace separated list of arguments. + :param prefix: The prefix for the first line. Defaults to + ``"Usage: "``. + """ + if prefix is None: + prefix = "{usage} ".format(usage=_("Usage:")) + + usage_prefix = f"{prefix:>{self.current_indent}}{prog} " + text_width = self.width - self.current_indent + + if not args: + # Without args, the prefix's trailing space and the wrap_text + # call that would normally place args on the line are both + # unnecessary. Emit just the prefix line. + self.write(usage_prefix.rstrip(" ")) + self.write("\n") + return + + if text_width >= (term_len(usage_prefix) + 20): + # The arguments will fit to the right of the prefix. + indent = " " * term_len(usage_prefix) + self.write( + wrap_text( + args, + text_width, + initial_indent=usage_prefix, + subsequent_indent=indent, + ) + ) + else: + # The prefix is too long, put the arguments on the next line. + self.write(usage_prefix) + self.write("\n") + indent = " " * (max(self.current_indent, term_len(prefix)) + 4) + self.write( + wrap_text( + args, text_width, initial_indent=indent, subsequent_indent=indent + ) + ) + + self.write("\n") + + def write_heading(self, heading: str) -> None: + """Writes a heading into the buffer.""" + self.write(f"{'':>{self.current_indent}}{heading}:\n") + + def write_paragraph(self) -> None: + """Writes a paragraph into the buffer.""" + if self.buffer: + self.write("\n") + + def write_text(self, text: str) -> None: + """Writes re-indented text into the buffer. This rewraps and + preserves paragraphs. + """ + indent = " " * self.current_indent + self.write( + wrap_text( + text, + self.width, + initial_indent=indent, + subsequent_indent=indent, + preserve_paragraphs=True, + ) + ) + self.write("\n") + + def write_dl( + self, + rows: cabc.Iterable[tuple[str, str]], + col_max: int = 30, + col_spacing: int = 2, + ) -> None: + """Writes a definition list into the buffer. This is how options + and commands are usually formatted. + + :param rows: a list of two item tuples for the terms and values. + :param col_max: the maximum width of the first column. + :param col_spacing: the number of spaces between the first and + second column. + """ + rows = list(rows) + widths = measure_table(rows) + if len(widths) != 2: + raise TypeError("Expected two columns for definition list") + + first_col = min(widths[0], col_max) + col_spacing + + for first, second in iter_rows(rows, len(widths)): + self.write(f"{'':>{self.current_indent}}{first}") + if not second: + self.write("\n") + continue + if term_len(first) <= first_col - col_spacing: + self.write(" " * (first_col - term_len(first))) + else: + self.write("\n") + self.write(" " * (first_col + self.current_indent)) + + text_width = max(self.width - first_col - 2, 10) + wrapped_text = wrap_text(second, text_width, preserve_paragraphs=True) + lines = wrapped_text.splitlines() + + if lines: + self.write(f"{lines[0]}\n") + + for line in lines[1:]: + self.write(f"{'':>{first_col + self.current_indent}}{line}\n") + else: + self.write("\n") + + @contextmanager + def section(self, name: str) -> cabc.Generator[None]: + """Helpful context manager that writes a paragraph, a heading, + and the indents. + + :param name: the section name that is written as heading. + """ + self.write_paragraph() + self.write_heading(name) + self.indent() + try: + yield + finally: + self.dedent() + + @contextmanager + def indentation(self) -> cabc.Generator[None]: + """A context manager that increases the indentation.""" + self.indent() + try: + yield + finally: + self.dedent() + + def getvalue(self) -> str: + """Returns the buffer contents.""" + return "".join(self.buffer) + + +def join_options(options: cabc.Iterable[str]) -> tuple[str, bool]: + """Given a list of option strings this joins them in the most appropriate + way and returns them in the form ``(formatted_string, + any_prefix_is_slash)`` where the second item in the tuple is a flag that + indicates if any of the option prefixes was a slash. + """ + rv = [] + any_prefix_is_slash = False + + for opt in options: + prefix = _split_opt(opt)[0] + + if prefix == "/": + any_prefix_is_slash = True + + rv.append((len(prefix), opt)) + + rv.sort(key=lambda x: x[0]) + return ", ".join(x[1] for x in rv), any_prefix_is_slash diff --git a/.venv/lib/python3.14/site-packages/click/globals.py b/.venv/lib/python3.14/site-packages/click/globals.py new file mode 100644 index 0000000000000000000000000000000000000000..a2f91723d21cefdf658be327174e1f8ccdc20fcd --- /dev/null +++ b/.venv/lib/python3.14/site-packages/click/globals.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +import typing as t +from threading import local + +if t.TYPE_CHECKING: + from .core import Context + +_local = local() + + +@t.overload +def get_current_context(silent: t.Literal[False] = False) -> Context: ... + + +@t.overload +def get_current_context(silent: bool = ...) -> Context | None: ... + + +def get_current_context(silent: bool = False) -> Context | None: + """Returns the current click context. This can be used as a way to + access the current context object from anywhere. This is a more implicit + alternative to the :func:`pass_context` decorator. This function is + primarily useful for helpers such as :func:`echo` which might be + interested in changing its behavior based on the current context. + + To push the current context, :meth:`Context.scope` can be used. + + .. versionadded:: 5.0 + + :param silent: if set to `True` the return value is `None` if no context + is available. The default behavior is to raise a + :exc:`RuntimeError`. + """ + try: + return t.cast("Context", _local.stack[-1]) + except (AttributeError, IndexError) as e: + if not silent: + raise RuntimeError("There is no active click context.") from e + + return None + + +def push_context(ctx: Context) -> None: + """Pushes a new context to the current stack.""" + _local.__dict__.setdefault("stack", []).append(ctx) + + +def pop_context() -> None: + """Removes the top level from the stack.""" + _local.stack.pop() + + +def resolve_color_default(color: bool | None = None) -> bool | None: + """Internal helper to get the default value of the color flag. If a + value is passed it's returned unchanged, otherwise it's looked up from + the current context. + """ + if color is not None: + return color + + ctx = get_current_context(silent=True) + + if ctx is not None: + return ctx.color + + return None diff --git a/.venv/lib/python3.14/site-packages/click/parser.py b/.venv/lib/python3.14/site-packages/click/parser.py new file mode 100644 index 0000000000000000000000000000000000000000..4fcbf7caa83a474cee2d3ea25da56b0399dd893a --- /dev/null +++ b/.venv/lib/python3.14/site-packages/click/parser.py @@ -0,0 +1,533 @@ +""" +This module started out as largely a copy paste from the stdlib's +optparse module with the features removed that we do not need from +optparse because we implement them in Click on a higher level (for +instance type handling, help formatting and a lot more). + +The plan is to remove more and more from here over time. + +The reason this is a different module and not optparse from the stdlib +is that there are differences in 2.x and 3.x about the error messages +generated and optparse in the stdlib uses gettext for no good reason +and might cause us issues. + +Click uses parts of optparse written by Gregory P. Ward and maintained +by the Python Software Foundation. This is limited to code in parser.py. + +Copyright 2001-2006 Gregory P. Ward. All rights reserved. +Copyright 2002-2006 Python Software Foundation. All rights reserved. +""" + +# This code uses parts of optparse written by Gregory P. Ward and +# maintained by the Python Software Foundation. +# Copyright 2001-2006 Gregory P. Ward +# Copyright 2002-2006 Python Software Foundation +from __future__ import annotations + +import collections.abc as cabc +import typing as t +from collections import deque +from gettext import gettext as _ +from gettext import ngettext + +from ._utils import FLAG_NEEDS_VALUE +from ._utils import UNSET +from .exceptions import BadArgumentUsage +from .exceptions import BadOptionUsage +from .exceptions import NoSuchOption +from .exceptions import UsageError + +if t.TYPE_CHECKING: + from ._utils import T_FLAG_NEEDS_VALUE + from ._utils import T_UNSET + from .core import Argument as CoreArgument + from .core import Context + from .core import Option as CoreOption + from .core import Parameter as CoreParameter + +V = t.TypeVar("V") + + +def _unpack_args( + args: cabc.Sequence[str], nargs_spec: cabc.Sequence[int] +) -> tuple[cabc.Sequence[str | cabc.Sequence[str | T_UNSET] | T_UNSET], list[str]]: + """Given an iterable of arguments and an iterable of nargs specifications, + it returns a tuple with all the unpacked arguments at the first index + and all remaining arguments as the second. + + The nargs specification is the number of arguments that should be consumed + or `-1` to indicate that this position should eat up all the remainders. + + Missing items are filled with ``UNSET``. + """ + args = deque(args) + nargs_spec = deque(nargs_spec) + rv: list[str | tuple[str | T_UNSET, ...] | T_UNSET] = [] + spos: int | None = None + + def _fetch(c: deque[str]) -> str | T_UNSET: + try: + if spos is None: + return c.popleft() + else: + return c.pop() + except IndexError: + return UNSET + + while nargs_spec: + if spos is None: + nargs = nargs_spec.popleft() + else: + nargs = nargs_spec.pop() + + if nargs == 1: + rv.append(_fetch(args)) + elif nargs > 1: + x: list[str | T_UNSET] = [_fetch(args) for _ in range(nargs)] + + # If we're reversed, we're pulling in the arguments in reverse, + # so we need to turn them around. + if spos is not None: + x.reverse() + + rv.append(tuple(x)) + elif nargs < 0: + if spos is not None: + raise TypeError("Cannot have two nargs < 0") + + spos = len(rv) + rv.append(UNSET) + + # spos is the position of the wildcard (star). If it's not `None`, + # we fill it with the remainder. + if spos is not None: + rv[spos] = tuple(args) + args = [] + rv[spos + 1 :] = reversed(rv[spos + 1 :]) + + return tuple(rv), list(args) + + +def _split_opt(opt: str) -> tuple[str, str]: + first = opt[:1] + if first.isalnum(): + return "", opt + if opt[1:2] == first: + return opt[:2], opt[2:] + return first, opt[1:] + + +def _normalize_opt(opt: str, ctx: Context | None) -> str: + if ctx is None or ctx.token_normalize_func is None: + return opt + prefix, opt = _split_opt(opt) + return f"{prefix}{ctx.token_normalize_func(opt)}" + + +class _Option: + def __init__( + self, + obj: CoreOption, + opts: cabc.Sequence[str], + dest: str | None, + action: str | None = None, + nargs: int = 1, + const: t.Any | None = None, + ): + self._short_opts = [] + self._long_opts = [] + self.prefixes: set[str] = set() + + for opt in opts: + prefix, value = _split_opt(opt) + if not prefix: + raise ValueError( + _("Invalid start character for option ({option})").format( + option=opt + ) + ) + self.prefixes.add(prefix[0]) + if len(prefix) == 1 and len(value) == 1: + self._short_opts.append(opt) + else: + self._long_opts.append(opt) + self.prefixes.add(prefix) + + if action is None: + action = "store" + + self.dest = dest + self.action = action + self.nargs = nargs + self.const = const + self.obj = obj + + @property + def takes_value(self) -> bool: + return self.action in ("store", "append") + + def process(self, value: t.Any, state: _ParsingState) -> None: + if self.action == "store": + state.opts[self.dest] = value # type: ignore + elif self.action == "store_const": + state.opts[self.dest] = self.const # type: ignore + elif self.action == "append": + state.opts.setdefault(self.dest, []).append(value) # type: ignore + elif self.action == "append_const": + state.opts.setdefault(self.dest, []).append(self.const) # type: ignore + elif self.action == "count": + state.opts[self.dest] = state.opts.get(self.dest, 0) + 1 # type: ignore + else: + raise ValueError(f"unknown action '{self.action}'") + state.order.append(self.obj) + + +class _Argument: + def __init__(self, obj: CoreArgument, dest: str | None, nargs: int = 1): + self.dest = dest + self.nargs = nargs + self.obj = obj + + def process( + self, + value: str | cabc.Sequence[str | T_UNSET] | T_UNSET, + state: _ParsingState, + ) -> None: + if self.nargs > 1: + assert isinstance(value, cabc.Sequence) + holes = sum(x is UNSET for x in value) + if holes == len(value): + value = UNSET + elif holes != 0: + raise BadArgumentUsage( + _("Argument {name!r} takes {nargs} values.").format( + name=self.dest, nargs=self.nargs + ) + ) + + # We failed to collect any argument value so we consider the argument as unset. + if value == (): + value = UNSET + + state.opts[self.dest] = value # type: ignore + state.order.append(self.obj) + + +class _ParsingState: + def __init__(self, rargs: list[str]) -> None: + self.opts: dict[str, t.Any] = {} + self.largs: list[str] = [] + self.rargs = rargs + self.order: list[CoreParameter] = [] + + +class _OptionParser: + """The option parser is an internal class that is ultimately used to + parse options and arguments. It's modelled after optparse and brings + a similar but vastly simplified API. It should generally not be used + directly as the high level Click classes wrap it for you. + + It's not nearly as extensible as optparse or argparse as it does not + implement features that are implemented on a higher level (such as + types or defaults). + + :param ctx: optionally the :class:`~click.Context` where this parser + should go with. + + .. deprecated:: 8.2 + Will be removed in Click 9.0. + """ + + def __init__(self, ctx: Context | None = None) -> None: + #: The :class:`~click.Context` for this parser. This might be + #: `None` for some advanced use cases. + self.ctx = ctx + #: This controls how the parser deals with interspersed arguments. + #: If this is set to `False`, the parser will stop on the first + #: non-option. Click uses this to implement nested subcommands + #: safely. + self.allow_interspersed_args: bool = True + #: This tells the parser how to deal with unknown options. By + #: default it will error out (which is sensible), but there is a + #: second mode where it will ignore it and continue processing + #: after shifting all the unknown options into the resulting args. + self.ignore_unknown_options: bool = False + + if ctx is not None: + self.allow_interspersed_args = ctx.allow_interspersed_args + self.ignore_unknown_options = ctx.ignore_unknown_options + + self._short_opt: dict[str, _Option] = {} + self._long_opt: dict[str, _Option] = {} + self._opt_prefixes = {"-", "--"} + self._args: list[_Argument] = [] + + def add_option( + self, + obj: CoreOption, + opts: cabc.Sequence[str], + dest: str | None, + action: str | None = None, + nargs: int = 1, + const: t.Any | None = None, + ) -> None: + """Adds a new option named `dest` to the parser. The destination + is not inferred (unlike with optparse) and needs to be explicitly + provided. Action can be any of ``store``, ``store_const``, + ``append``, ``append_const`` or ``count``. + + The `obj` can be used to identify the option in the order list + that is returned from the parser. + """ + opts = [_normalize_opt(opt, self.ctx) for opt in opts] + option = _Option(obj, opts, dest, action=action, nargs=nargs, const=const) + self._opt_prefixes.update(option.prefixes) + for opt in option._short_opts: + self._short_opt[opt] = option + for opt in option._long_opts: + self._long_opt[opt] = option + + def add_argument(self, obj: CoreArgument, dest: str | None, nargs: int = 1) -> None: + """Adds a positional argument named `dest` to the parser. + + The `obj` can be used to identify the option in the order list + that is returned from the parser. + """ + self._args.append(_Argument(obj, dest=dest, nargs=nargs)) + + def parse_args( + self, args: list[str] + ) -> tuple[dict[str, t.Any], list[str], list[CoreParameter]]: + """Parses positional arguments and returns ``(values, args, order)`` + for the parsed options and arguments as well as the leftover + arguments if there are any. The order is a list of objects as they + appear on the command line. If arguments appear multiple times they + will be memorized multiple times as well. + """ + state = _ParsingState(args) + try: + self._process_args_for_options(state) + self._process_args_for_args(state) + except UsageError: + if self.ctx is None or not self.ctx.resilient_parsing: + raise + return state.opts, state.largs, state.order + + def _process_args_for_args(self, state: _ParsingState) -> None: + pargs, args = _unpack_args( + state.largs + state.rargs, [x.nargs for x in self._args] + ) + + for idx, arg in enumerate(self._args): + arg.process(pargs[idx], state) + + state.largs = args + state.rargs = [] + + def _process_args_for_options(self, state: _ParsingState) -> None: + while state.rargs: + arg = state.rargs.pop(0) + arglen = len(arg) + # Double dashes always handled explicitly regardless of what + # prefixes are valid. + if arg == "--": + return + elif arg[:1] in self._opt_prefixes and arglen > 1: + self._process_opts(arg, state) + elif self.allow_interspersed_args: + state.largs.append(arg) + else: + state.rargs.insert(0, arg) + return + + # Say this is the original argument list: + # [arg0, arg1, ..., arg(i-1), arg(i), arg(i+1), ..., arg(N-1)] + # ^ + # (we are about to process arg(i)). + # + # Then rargs is [arg(i), ..., arg(N-1)] and largs is a *subset* of + # [arg0, ..., arg(i-1)] (any options and their arguments will have + # been removed from largs). + # + # The while loop will usually consume 1 or more arguments per pass. + # If it consumes 1 (eg. arg is an option that takes no arguments), + # then after _process_arg() is done the situation is: + # + # largs = subset of [arg0, ..., arg(i)] + # rargs = [arg(i+1), ..., arg(N-1)] + # + # If allow_interspersed_args is false, largs will always be + # *empty* -- still a subset of [arg0, ..., arg(i-1)], but + # not a very interesting subset! + + def _match_long_opt( + self, opt: str, explicit_value: str | None, state: _ParsingState + ) -> None: + if opt not in self._long_opt: + raise NoSuchOption(opt, possibilities=self._long_opt, ctx=self.ctx) + + option = self._long_opt[opt] + if option.takes_value: + # At this point it's safe to modify rargs by injecting the + # explicit value, because no exception is raised in this + # branch. This means that the inserted value will be fully + # consumed. + if explicit_value is not None: + state.rargs.insert(0, explicit_value) + + value = self._get_value_from_state(opt, option, state) + + elif explicit_value is not None: + raise BadOptionUsage( + opt, _("Option {name!r} does not take a value.").format(name=opt) + ) + + else: + value = UNSET + + option.process(value, state) + + def _match_short_opt(self, arg: str, state: _ParsingState) -> None: + stop = False + i = 1 + prefix = arg[0] + unknown_options = [] + + for ch in arg[1:]: + opt = _normalize_opt(f"{prefix}{ch}", self.ctx) + option = self._short_opt.get(opt) + i += 1 + + if not option: + if self.ignore_unknown_options: + unknown_options.append(ch) + continue + raise NoSuchOption(opt, ctx=self.ctx) + if option.takes_value: + # Any characters left in arg? Pretend they're the + # next arg, and stop consuming characters of arg. + if i < len(arg): + state.rargs.insert(0, arg[i:]) + stop = True + + value = self._get_value_from_state(opt, option, state) + + else: + value = UNSET + + option.process(value, state) + + if stop: + break + + # If we got any unknown options we recombine the string of the + # remaining options and re-attach the prefix, then report that + # to the state as new large. This way there is basic combinatorics + # that can be achieved while still ignoring unknown arguments. + if self.ignore_unknown_options and unknown_options: + state.largs.append(f"{prefix}{''.join(unknown_options)}") + + def _get_value_from_state( + self, option_name: str, option: _Option, state: _ParsingState + ) -> str | cabc.Sequence[str] | T_UNSET | T_FLAG_NEEDS_VALUE: + nargs = option.nargs + + value: str | cabc.Sequence[str] | T_UNSET | T_FLAG_NEEDS_VALUE + + if len(state.rargs) < nargs: + if option.obj._flag_needs_value: + # Option allows omitting the value. + value = FLAG_NEEDS_VALUE + else: + raise BadOptionUsage( + option_name, + ngettext( + "Option {name!r} requires an argument.", + "Option {name!r} requires {nargs} arguments.", + nargs, + ).format(name=option_name, nargs=nargs), + ) + elif nargs == 1: + next_rarg = state.rargs[0] + + if ( + option.obj._flag_needs_value + and isinstance(next_rarg, str) + and next_rarg[:1] in self._opt_prefixes + and len(next_rarg) > 1 + ): + # The next arg looks like the start of an option, don't + # use it as the value if omitting the value is allowed. + value = FLAG_NEEDS_VALUE + else: + value = state.rargs.pop(0) + else: + value = tuple(state.rargs[:nargs]) + del state.rargs[:nargs] + + return value + + def _process_opts(self, arg: str, state: _ParsingState) -> None: + explicit_value = None + # Long option handling happens in two parts. The first part is + # supporting explicitly attached values. In any case, we will try + # to long match the option first. + if "=" in arg: + long_opt, explicit_value = arg.split("=", 1) + else: + long_opt = arg + norm_long_opt = _normalize_opt(long_opt, self.ctx) + + # At this point we will match the (assumed) long option through + # the long option matching code. Note that this allows options + # like "-foo" to be matched as long options. + try: + self._match_long_opt(norm_long_opt, explicit_value, state) + except NoSuchOption: + # At this point the long option matching failed, and we need + # to try with short options. However there is a special rule + # which says, that if we have a two character options prefix + # (applies to "--foo" for instance), we do not dispatch to the + # short option code and will instead raise the no option + # error. + if arg[:2] not in self._opt_prefixes: + self._match_short_opt(arg, state) + return + + if not self.ignore_unknown_options: + raise + + state.largs.append(arg) + + +def __getattr__(name: str) -> object: + import warnings + + if name in { + "OptionParser", + "Argument", + "Option", + "split_opt", + "normalize_opt", + "ParsingState", + }: + warnings.warn( + f"'parser.{name}' is deprecated and will be removed in Click 9.0." + " The old parser is available in 'optparse'.", + DeprecationWarning, + stacklevel=2, + ) + return globals()[f"_{name}"] + + if name == "split_arg_string": + from .shell_completion import split_arg_string + + warnings.warn( + "Importing 'parser.split_arg_string' is deprecated, it will only be" + " available in 'shell_completion' in Click 9.0.", + DeprecationWarning, + stacklevel=2, + ) + return split_arg_string + + raise AttributeError(name) diff --git a/.venv/lib/python3.14/site-packages/click/py.typed b/.venv/lib/python3.14/site-packages/click/py.typed new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.venv/lib/python3.14/site-packages/click/shell_completion.py b/.venv/lib/python3.14/site-packages/click/shell_completion.py new file mode 100644 index 0000000000000000000000000000000000000000..8a2668b09f1525ae95e814fa92a23bfdbee2c21d --- /dev/null +++ b/.venv/lib/python3.14/site-packages/click/shell_completion.py @@ -0,0 +1,801 @@ +from __future__ import annotations + +import collections.abc as cabc +import os +import re +import typing as t +from gettext import gettext as _ + +from .core import Argument +from .core import Command +from .core import Context +from .core import Group +from .core import Option +from .core import Parameter +from .core import ParameterSource +from .utils import echo + + +def shell_complete( + cli: Command, + ctx_args: cabc.MutableMapping[str, t.Any], + prog_name: str, + complete_var: str, + instruction: str, +) -> t.Literal[0, 1]: + """Perform shell completion for the given CLI program. + + :param cli: Command being called. + :param ctx_args: Extra arguments to pass to + ``cli.make_context``. + :param prog_name: Name of the executable in the shell. + :param complete_var: Name of the environment variable that holds + the completion instruction. + :param instruction: Value of ``complete_var`` with the completion + instruction and shell, in the form ``instruction_shell``. + :return: Status code to exit with. + """ + shell, _, instruction = instruction.partition("_") + comp_cls = get_completion_class(shell) + + if comp_cls is None: + return 1 + + comp = comp_cls(cli, ctx_args, prog_name, complete_var) + + # Write bytes, otherwise Windows text stdout translates LF to CRLF and breaks. + if instruction == "source": + echo(comp.source().encode(), nl=False) + return 0 + + if instruction == "complete": + echo(comp.complete().encode()) + return 0 + + return 1 + + +if t.TYPE_CHECKING: + from typing_extensions import TypeVar + + # `Any` is used as default for backwards compatibility (instead of e.g. `str`) + _ValueT_co = TypeVar("_ValueT_co", covariant=True, default=t.Any) +else: + _ValueT_co = t.TypeVar("_ValueT_co", covariant=True) + + +class CompletionItem(t.Generic[_ValueT_co]): + """Represents a completion value and metadata about the value. The + default metadata is ``type`` to indicate special shell handling, + and ``help`` if a shell supports showing a help string next to the + value. + + Arbitrary parameters can be passed when creating the object, and + accessed using ``item.attr``. If an attribute wasn't passed, + accessing it returns ``None``. + + :param value: The completion suggestion. + :param type: Tells the shell script to provide special completion + support for the type. Click uses ``"dir"`` and ``"file"``. + :param help: String shown next to the value if supported. + :param kwargs: Arbitrary metadata. The built-in implementations + don't use this, but custom type completions paired with custom + shell support could use it. + """ + + __slots__ = ("value", "type", "help", "_info") + + def __init__( + self, + value: _ValueT_co, + type: str = "plain", + help: str | None = None, + **kwargs: t.Any, + ) -> None: + self.value: _ValueT_co = value + self.type: str = type + self.help: str | None = help + self._info = kwargs + + def __getattr__(self, name: str) -> t.Any: + return self._info.get(name) + + +# Only Bash >= 4.4 has the nosort option. +_SOURCE_BASH = """\ +%(complete_func)s() { + local IFS=$'\\n' + local response + + response=$(env COMP_WORDS="${COMP_WORDS[*]}" COMP_CWORD=$COMP_CWORD \ +%(complete_var)s=bash_complete $1) + + for completion in $response; do + IFS=',' read type value <<< "$completion" + + if [[ $type == 'dir' ]]; then + COMPREPLY=() + compopt -o dirnames + elif [[ $type == 'file' ]]; then + COMPREPLY=() + compopt -o default + elif [[ $type == 'plain' ]]; then + COMPREPLY+=($value) + fi + done + + return 0 +} + +%(complete_func)s_setup() { + complete -o nosort -F %(complete_func)s %(prog_name)s +} + +%(complete_func)s_setup; +""" + +# See ZshComplete.format_completion below, and issue #2703, before +# changing this script. +# +# (TL;DR: _describe is picky about the format, but this Zsh script snippet +# is already widely deployed. So freeze this script, and use clever-ish +# handling of colons in ZshComplet.format_completion.) +_SOURCE_ZSH = """\ +#compdef %(prog_name)s + +%(complete_func)s() { + local -a completions + local -a completions_with_descriptions + local -a response + (( ! $+commands[%(prog_name)s] )) && return 1 + + response=("${(@f)$(env COMP_WORDS="${words[*]}" COMP_CWORD=$((CURRENT-1)) \ +%(complete_var)s=zsh_complete %(prog_name)s)}") + + for type key descr in ${response}; do + if [[ "$type" == "plain" ]]; then + if [[ "$descr" == "_" ]]; then + completions+=("$key") + else + completions_with_descriptions+=("$key":"$descr") + fi + elif [[ "$type" == "dir" ]]; then + _path_files -/ + elif [[ "$type" == "file" ]]; then + _path_files -f + fi + done + + if [ -n "$completions_with_descriptions" ]; then + _describe -V unsorted completions_with_descriptions -U + fi + + if [ -n "$completions" ]; then + compadd -U -V unsorted -a completions + fi +} + +if [[ $zsh_eval_context[-1] == loadautofunc ]]; then + # autoload from fpath, call function directly + %(complete_func)s "$@" +else + # eval/source/. command, register function for later + compdef %(complete_func)s %(prog_name)s +fi +""" + +_SOURCE_FISH = """\ +function %(complete_func)s; + set -l response (env %(complete_var)s=fish_complete COMP_WORDS=(commandline -cp) \ +COMP_CWORD=(commandline -t) %(prog_name)s); + + for completion in $response; + set -l metadata (string split "," $completion); + + if test $metadata[1] = "dir"; + __fish_complete_directories $metadata[2]; + else if test $metadata[1] = "file"; + __fish_complete_path $metadata[2]; + else if test $metadata[1] = "plain"; + echo $metadata[2]; + end; + end; +end; + +complete --no-files --command %(prog_name)s --arguments \ +"(%(complete_func)s)"; +""" + +# Compatible with Windows PowerShell 5.1+ and PowerShell (pwsh) 7+. +# Uses Register-ArgumentCompleter -Native, which receives the command AST +# as parsed by PowerShell. The command text is forwarded verbatim through +# COMP_WORDS so the Python side can reuse the same shlex-based splitting +# used by the bash/zsh completers. +_SOURCE_POWERSHELL = """\ +Register-ArgumentCompleter -Native -CommandName %(prog_name)s -ScriptBlock { + param($wordToComplete, $commandAst, $cursorPosition) + + $prev_complete = $env:%(complete_var)s + $prev_words = $env:COMP_WORDS + $prev_cword = $env:COMP_CWORD + + $env:%(complete_var)s = 'powershell_complete' + $env:COMP_WORDS = $commandAst.ToString() + if ($wordToComplete) { + $env:COMP_CWORD = $commandAst.CommandElements.Count - 1 + } else { + $env:COMP_CWORD = $commandAst.CommandElements.Count + } + + try { + $response = & %(prog_name)s 2>$null + } finally { + $env:%(complete_var)s = $prev_complete + $env:COMP_WORDS = $prev_words + $env:COMP_CWORD = $prev_cword + } + + if (-not $response) { return } + + $prefix = "$wordToComplete*" + $lines = $response -split "`n" + for ($i = 0; $i + 2 -lt $lines.Count; $i += 3) { + $type = $lines[$i] + $value = $lines[$i + 1] + $descr = $lines[$i + 2] + if (-not $type) { continue } + + if ($type -eq 'plain') { + $tip = if ($descr -and $descr -ne '_') { $descr } else { $value } + [System.Management.Automation.CompletionResult]::new( + $value, $value, 'ParameterValue', $tip) + } elseif ($type -eq 'dir') { + Get-ChildItem -Directory -Path $prefix -ErrorAction SilentlyContinue | + ForEach-Object { + [System.Management.Automation.CompletionResult]::new( + $_.FullName, $_.Name, 'ProviderContainer', $_.FullName) + } + } elseif ($type -eq 'file') { + Get-ChildItem -Path $prefix -ErrorAction SilentlyContinue | + ForEach-Object { + $kind = 'ProviderItem' + if ($_.PSIsContainer) { $kind = 'ProviderContainer' } + [System.Management.Automation.CompletionResult]::new( + $_.FullName, $_.Name, $kind, $_.FullName) + } + } + } +} +""" + + +class _SourceVarsDict(t.TypedDict): + complete_func: str + complete_var: str + prog_name: str + + +class ShellComplete: + """Base class for providing shell completion support. A subclass for + a given shell will override attributes and methods to implement the + completion instructions (``source`` and ``complete``). + + :param cli: Command being called. + :param prog_name: Name of the executable in the shell. + :param complete_var: Name of the environment variable that holds + the completion instruction. + + .. versionadded:: 8.0 + """ + + name: t.ClassVar[str] + """Name to register the shell as with :func:`add_completion_class`. + This is used in completion instructions (``{name}_source`` and + ``{name}_complete``). + """ + + source_template: t.ClassVar[str] + """Completion script template formatted by :meth:`source`. This must + be provided by subclasses. + """ + + cli: Command + ctx_args: cabc.MutableMapping[str, t.Any] + prog_name: str + complete_var: str + + def __init__( + self, + cli: Command, + ctx_args: cabc.MutableMapping[str, t.Any], + prog_name: str, + complete_var: str, + ) -> None: + self.cli = cli + self.ctx_args = ctx_args + self.prog_name = prog_name + self.complete_var = complete_var + + @property + def func_name(self) -> str: + """The name of the shell function defined by the completion + script. + """ + safe_name = re.sub(r"\W*", "", self.prog_name.replace("-", "_"), flags=re.ASCII) + return f"_{safe_name}_completion" + + def source_vars(self) -> _SourceVarsDict: + """Vars for formatting :attr:`source_template`. + + By default this provides ``complete_func``, ``complete_var``, + and ``prog_name``. + """ + return { + "complete_func": self.func_name, + "complete_var": self.complete_var, + "prog_name": self.prog_name, + } + + def source(self) -> str: + """Produce the shell script that defines the completion + function. By default this ``%``-style formats + :attr:`source_template` with the dict returned by + :meth:`source_vars`. + """ + return self.source_template % self.source_vars() + + def get_completion_args(self) -> tuple[list[str], str]: + """Use the env vars defined by the shell script to return a + tuple of ``args, incomplete``. This must be implemented by + subclasses. + """ + raise NotImplementedError + + def get_completions( + self, args: list[str], incomplete: str + ) -> list[CompletionItem[str]]: + """Determine the context and last complete command or parameter + from the complete args. Call that object's ``shell_complete`` + method to get the completions for the incomplete value. + + :param args: List of complete args before the incomplete value. + :param incomplete: Value being completed. May be empty. + """ + ctx = _resolve_context(self.cli, self.ctx_args, self.prog_name, args) + obj, incomplete = _resolve_incomplete(ctx, args, incomplete) + return obj.shell_complete(ctx, incomplete) + + def format_completion(self, item: CompletionItem[str]) -> str: + """Format a completion item into the form recognized by the + shell script. This must be implemented by subclasses. + + :param item: Completion item to format. + """ + raise NotImplementedError + + def complete(self) -> str: + """Produce the completion data to send back to the shell. + + By default this calls :meth:`get_completion_args`, gets the + completions, then calls :meth:`format_completion` for each + completion. + """ + args, incomplete = self.get_completion_args() + completions = self.get_completions(args, incomplete) + out = [self.format_completion(item) for item in completions] + return "\n".join(out) + + +class BashComplete(ShellComplete): + """Shell completion for Bash.""" + + name: t.ClassVar[str] = "bash" + source_template: t.ClassVar[str] = _SOURCE_BASH + + @staticmethod + def _check_version() -> None: + import shutil + import subprocess + + bash_exe = shutil.which("bash") + + if bash_exe is None: + match = None + else: + output = subprocess.run( + [bash_exe, "--norc", "-c", 'echo "${BASH_VERSION}"'], + stdout=subprocess.PIPE, + ) + match = re.search(r"^(\d+)\.(\d+)\.\d+", output.stdout.decode()) + + if match is not None: + major, minor = match.groups() + + if major < "4" or major == "4" and minor < "4": + echo( + _( + "Shell completion is not supported for Bash" + " versions older than 4.4." + ), + err=True, + ) + else: + echo( + _("Couldn't detect Bash version, shell completion is not supported."), + err=True, + ) + + def source(self) -> str: + self._check_version() + return super().source() + + def get_completion_args(self) -> tuple[list[str], str]: + cwords = split_arg_string(os.environ["COMP_WORDS"]) + cword = int(os.environ["COMP_CWORD"]) + args = cwords[1:cword] + + try: + incomplete = cwords[cword] + except IndexError: + incomplete = "" + + return args, incomplete + + def format_completion(self, item: CompletionItem[t.Any]) -> str: + return f"{item.type},{item.value}" + + +class ZshComplete(ShellComplete): + """Shell completion for Zsh.""" + + name: t.ClassVar[str] = "zsh" + source_template: t.ClassVar[str] = _SOURCE_ZSH + + def get_completion_args(self) -> tuple[list[str], str]: + cwords = split_arg_string(os.environ["COMP_WORDS"]) + cword = int(os.environ["COMP_CWORD"]) + args = cwords[1:cword] + + try: + incomplete = cwords[cword] + except IndexError: + incomplete = "" + + return args, incomplete + + def format_completion(self, item: CompletionItem[str]) -> str: + help_ = item.help or "_" + # The zsh completion script uses `_describe` on items with help + # texts (which splits the item help from the item value at the + # first unescaped colon) and `compadd` on items without help + # text (which uses the item value as-is and does not support + # colon escaping). So escape colons in the item value if and + # only if the item help is not the sentinel "_" value, as used + # by the completion script. + # + # (The zsh completion script is potentially widely deployed, and + # thus harder to fix than this method.) + # + # See issue #1812 and issue #2703 for further context. + value = item.value.replace(":", r"\:") if help_ != "_" else item.value + return f"{item.type}\n{value}\n{help_}" + + +class FishComplete(ShellComplete): + """Shell completion for Fish.""" + + name: t.ClassVar[str] = "fish" + source_template: t.ClassVar[str] = _SOURCE_FISH + + def get_completion_args(self) -> tuple[list[str], str]: + cwords = split_arg_string(os.environ["COMP_WORDS"]) + incomplete = os.environ["COMP_CWORD"] + if incomplete: + incomplete = split_arg_string(incomplete)[0] + args = cwords[1:] + + # Fish stores the partial word in both COMP_WORDS and + # COMP_CWORD, remove it from complete args. + if incomplete and args and args[-1] == incomplete: + args.pop() + + return args, incomplete + + def format_completion(self, item: CompletionItem[str]) -> str: + """ + .. versionchanged:: 8.4.2 + Escape newlines and replace tabs with spaces in the help text to + fix completion errors with multi-line help strings. + """ + # According to https://fishshell.com/docs/current/cmds/complete.html + # Command substitutions found in ARGUMENTS should return a newline- + # separated list of arguments, and each argument may optionally have a tab + # character followed by the argument description. + if item.help: + help_ = item.help.replace("\n", "\\n").replace("\t", " ") + return f"{item.type},{item.value}\t{help_}" + + return f"{item.type},{item.value}" + + +class PowerShellComplete(ShellComplete): + """Shell completion for PowerShell (Windows PowerShell 5.1+ and pwsh 7+). + + .. versionadded:: 8.5.0 + """ + + name: t.ClassVar[str] = "powershell" + source_template: t.ClassVar[str] = _SOURCE_POWERSHELL + + def get_completion_args(self) -> tuple[list[str], str]: + cwords = split_arg_string(os.environ["COMP_WORDS"]) + cword = int(os.environ["COMP_CWORD"]) + args = cwords[1:cword] + + try: + incomplete = cwords[cword] + except IndexError: + incomplete = "" + + return args, incomplete + + def format_completion(self, item: CompletionItem[str]) -> str: + # PowerShell parses the response by splitting on newlines and + # consuming three lines per completion (type, value, help). + # Multi-line help text would break that framing, so collapse + # newlines and carriage returns to spaces. The literal "_" + # sentinel matches what ZshComplete uses for "no help text". + if item.help: + help_ = item.help.replace("\r", " ").replace("\n", " ") + else: + help_ = "_" + return f"{item.type}\n{item.value}\n{help_}" + + +_available_shells: t.Final[dict[str, type[ShellComplete]]] = { + "bash": BashComplete, + "fish": FishComplete, + "powershell": PowerShellComplete, + "zsh": ZshComplete, +} + +_ShellCompleteT = t.TypeVar("_ShellCompleteT", bound="ShellComplete") + + +def add_completion_class( + cls: type[_ShellCompleteT], name: str | None = None +) -> type[_ShellCompleteT]: + """Register a :class:`ShellComplete` subclass under the given name. + The name will be provided by the completion instruction environment + variable during completion. + + :param cls: The completion class that will handle completion for the + shell. + :param name: Name to register the class under. Defaults to the + class's ``name`` attribute. + """ + if name is None: + name = cls.name + + _available_shells[name] = cls + + return cls + + +@t.overload +def get_completion_class(shell: t.Literal["bash"]) -> type[BashComplete]: ... +@t.overload +def get_completion_class(shell: t.Literal["fish"]) -> type[FishComplete]: ... +@t.overload +def get_completion_class(shell: t.Literal["zsh"]) -> type[ZshComplete]: ... +@t.overload +def get_completion_class(shell: str) -> type[ShellComplete] | None: ... +def get_completion_class(shell: str) -> type[ShellComplete] | None: + """Look up a registered :class:`ShellComplete` subclass by the name + provided by the completion instruction environment variable. If the + name isn't registered, returns ``None``. + + :param shell: Name the class is registered under. + """ + return _available_shells.get(shell) + + +def split_arg_string(string: str) -> list[str]: + """Split an argument string as with :func:`shlex.split`, but don't + fail if the string is incomplete. Ignores a missing closing quote or + incomplete escape sequence and uses the partial token as-is. + + .. code-block:: python + + split_arg_string("example 'my file") + ["example", "my file"] + + split_arg_string("example my\\") + ["example", "my"] + + :param string: String to split. + + .. versionchanged:: 8.2 + Moved to ``shell_completion`` from ``parser``. + """ + import shlex + + lex = shlex.shlex(string, posix=True) + lex.whitespace_split = True + lex.commenters = "" + out: list[str] = [] + + try: + out.extend(lex) + except ValueError: + # Raised when end-of-string is reached in an invalid state. Use + # the partial token as-is. The quote or escape character is in + # lex.state, not lex.token. + out.append(lex.token) + + return out + + +def _is_incomplete_argument(ctx: Context, param: Parameter) -> bool: + """Determine if the given parameter is an argument that can still + accept values. + + :param ctx: Invocation context for the command represented by the + parsed complete args. + :param param: Argument object being checked. + """ + if not isinstance(param, Argument): + return False + + value = ctx.params.get(param.name) + return ( + param.nargs == -1 + or ctx.get_parameter_source(param.name) is not ParameterSource.COMMANDLINE + or ( + param.nargs > 1 + and isinstance(value, (tuple, list)) + and len(value) < param.nargs + ) + ) + + +def _start_of_option(ctx: Context, value: str) -> bool: + """Check if the value looks like the start of an option.""" + if not value: + return False + + c = value[0] + return c in ctx._opt_prefixes + + +def _is_incomplete_option(ctx: Context, args: list[str], param: Parameter) -> bool: + """Determine if the given parameter is an option that needs a value. + + :param args: List of complete args before the incomplete value. + :param param: Option object being checked. + """ + if not isinstance(param, Option): + return False + + if param.is_flag or param.count: + return False + + last_option = None + + for index, arg in enumerate(reversed(args)): + if index + 1 > param.nargs: + break + + if _start_of_option(ctx, arg): + last_option = arg + break + + return last_option is not None and last_option in param.opts + + +def _resolve_context( + cli: Command, + ctx_args: cabc.MutableMapping[str, t.Any], + prog_name: str, + args: list[str], +) -> Context: + """Produce the context hierarchy starting with the command and + traversing the complete arguments. This only follows the commands, + it doesn't trigger input prompts or callbacks. + + :param cli: Command being called. + :param prog_name: Name of the executable in the shell. + :param args: List of complete args before the incomplete value. + """ + ctx_args["resilient_parsing"] = True + with cli.make_context(prog_name, args.copy(), **ctx_args) as ctx: + args = ctx._protected_args + ctx.args + + while args: + command = ctx.command + + if isinstance(command, Group): + if not command.chain: + name, cmd, args = command.resolve_command(ctx, args) + + if cmd is None: + return ctx + + with cmd.make_context( + name, args, parent=ctx, resilient_parsing=True + ) as sub_ctx: + ctx = sub_ctx + args = ctx._protected_args + ctx.args + else: + sub_ctx = ctx + + while args: + name, cmd, args = command.resolve_command(ctx, args) + + if cmd is None: + return ctx + + with cmd.make_context( + name, + args, + parent=ctx, + allow_extra_args=True, + allow_interspersed_args=False, + resilient_parsing=True, + ) as sub_sub_ctx: + sub_ctx = sub_sub_ctx + args = sub_ctx.args + + ctx = sub_ctx + args = [*sub_ctx._protected_args, *sub_ctx.args] + else: + break + + return ctx + + +def _resolve_incomplete( + ctx: Context, args: list[str], incomplete: str +) -> tuple[Command | Parameter, str]: + """Find the Click object that will handle the completion of the + incomplete value. Return the object and the incomplete value. + + :param ctx: Invocation context for the command represented by + the parsed complete args. + :param args: List of complete args before the incomplete value. + :param incomplete: Value being completed. May be empty. + """ + # Different shells treat an "=" between a long option name and + # value differently. Might keep the value joined, return the "=" + # as a separate item, or return the split name and value. Always + # split and discard the "=" to make completion easier. + if incomplete == "=": + incomplete = "" + elif "=" in incomplete and _start_of_option(ctx, incomplete): + name, _, incomplete = incomplete.partition("=") + args.append(name) + + # The "--" marker tells Click to stop treating values as options + # even if they start with the option character. If it hasn't been + # given and the incomplete arg looks like an option, the current + # command will provide option name completions. + if "--" not in args and _start_of_option(ctx, incomplete): + return ctx.command, incomplete + + params = ctx.command.get_params(ctx) + + # If the last complete arg is an option name with an incomplete + # value, the option will provide value completions. + for param in params: + if _is_incomplete_option(ctx, args, param): + return param, incomplete + + # It's not an option name or value. The first argument without a + # parsed value will provide value completions. + for param in params: + if _is_incomplete_argument(ctx, param): + return param, incomplete + + # There were no unparsed arguments, the command may be a group that + # will provide command name completions. + return ctx.command, incomplete diff --git a/.venv/lib/python3.14/site-packages/click/termui.py b/.venv/lib/python3.14/site-packages/click/termui.py new file mode 100644 index 0000000000000000000000000000000000000000..e1315f23744ee9055126971c1cf350428aeab06b --- /dev/null +++ b/.venv/lib/python3.14/site-packages/click/termui.py @@ -0,0 +1,1014 @@ +from __future__ import annotations + +import collections.abc as cabc +import inspect +import io +import itertools +import os +import re +import sys +import typing as t +from contextlib import AbstractContextManager +from contextlib import redirect_stdout +from gettext import gettext as _ + +from . import _compat +from ._compat import isatty +from ._compat import strip_ansi +from .exceptions import Abort +from .exceptions import UsageError +from .globals import resolve_color_default +from .types import Choice +from .types import convert_type +from .types import ParamType +from .utils import _LazyFile +from .utils import echo + +if t.TYPE_CHECKING: + from ._termui_impl import ProgressBar + +V = t.TypeVar("V") + +# The prompt functions to use. The doc tools currently override these +# functions to customize how they work. +visible_prompt_func: t.Callable[[str], str] = input + +_ansi_colors = { + "black": 30, + "red": 31, + "green": 32, + "yellow": 33, + "blue": 34, + "magenta": 35, + "cyan": 36, + "white": 37, + "reset": 39, + "bright_black": 90, + "bright_red": 91, + "bright_green": 92, + "bright_yellow": 93, + "bright_blue": 94, + "bright_magenta": 95, + "bright_cyan": 96, + "bright_white": 97, +} +_ansi_reset_all = "\033[0m" + + +_HIDDEN_INPUT_MASK = "'***'" + + +def _mask_hidden_input(message: str, value: str) -> str: + """Replace occurrences of ``value`` in ``message`` with a fixed mask. + + Both ``repr(value)`` (the form built-in :class:`ParamType` errors use + via ``{value!r}``) and the raw value are masked. The raw-value pass + uses word-boundary lookarounds so a substring like ``"1"`` does not + match inside ``"10"``, and ``"ent"`` does not match inside + ``"Authentication"``. The empty string is skipped to avoid matching + at every boundary. + """ + message = message.replace(repr(value), _HIDDEN_INPUT_MASK) + if value: + message = re.sub( + rf"(? str: + import getpass + + return getpass.getpass(prompt) + + +def _readline_prompt(func: t.Callable[[str], str], text: str, err: bool) -> str: + """Call a prompt function, passing the full prompt so readline can + handle line editing and cursor positioning correctly. + + The prompt is handed to *func* (such as :func:`input`) rather than + written through :func:`echo`, so it has to strip ANSI color and style + codes itself when the destination stream does not support them. Without + this the prompt would keep codes that :func:`echo` removes from the + rest of the output. + """ + stream = sys.stderr if err else sys.stdout + + # Look up ``should_strip_ansi`` on the module so that ``CliRunner``, + # which patches it there during test isolation, is honored. + if _compat.should_strip_ansi(stream, resolve_color_default()): + text = strip_ansi(text) + + if err: + with redirect_stdout(sys.stderr): + return func(text) + return func(text) + + +def _build_prompt( + text: str, + suffix: str, + show_default: bool | str = False, + default: object | None = None, + show_choices: bool = True, + type: object | None = None, +) -> str: + prompt = text + if type is not None and show_choices and isinstance(type, Choice): + prompt += f" ({', '.join(map(str, type.choices))})" + default_preview = "" + if show_default: + if isinstance(show_default, str): + default_preview = f" [({show_default})]" + elif default is not None: + default_preview = f" [{_format_default(default)}]" + return f"{prompt}{default_preview}{suffix}" + + +def _format_default(default: V) -> V | str: + if isinstance(default, (io.IOBase, _LazyFile)): + name = getattr(default, "name", None) + + if name is not None: + return str(name) + + return default + + +@t.overload +def prompt( + text: str, + default: str | None = None, + hide_input: bool = False, + confirmation_prompt: bool | str = False, + type: None = None, + value_proc: None = None, + prompt_suffix: str = ": ", + show_default: bool | str = True, + err: bool = False, + show_choices: bool = True, +) -> str: ... + + +@t.overload +def prompt( + text: str, + default: V | str | None = None, + hide_input: bool = False, + confirmation_prompt: bool | str = False, + type: ParamType[V, str] | type[V] | None = None, + value_proc: t.Callable[[str], V] | None = None, + prompt_suffix: str = ": ", + show_default: bool | str = True, + err: bool = False, + show_choices: bool = True, +) -> V: ... + + +def prompt( + text: str, + default: V | str | None = None, + hide_input: bool = False, + confirmation_prompt: bool | str = False, + type: ParamType[V, str] | type[V] | None = None, + value_proc: t.Callable[[str], V] | None = None, + prompt_suffix: str = ": ", + show_default: bool | str = True, + err: bool = False, + show_choices: bool = True, +) -> V: + """Prompts a user for input. This is a convenience function that can + be used to prompt a user for input later. + + If the user aborts the input by sending an interrupt signal, this + function will catch it and raise a :exc:`Abort` exception. + + :param text: the text to show for the prompt. + :param default: the default value to use if no input happens. If this + is not given it will prompt until it's aborted. + :param hide_input: if this is set to true then the input value will + be hidden. + :param confirmation_prompt: Prompt a second time to confirm the + value. Can be set to a string instead of ``True`` to customize + the message. + :param type: the type to use to check the value against. + :param value_proc: if this parameter is provided it's a function that + is invoked instead of the type conversion to + convert a value. + :param prompt_suffix: a suffix that should be added to the prompt. + :param show_default: shows or hides the default value in the prompt. + If this value is a string, it shows that string + in parentheses instead of the actual value. + :param err: if set to true the file defaults to ``stderr`` instead of + ``stdout``, the same as with echo. + :param show_choices: Show or hide choices if the passed type is a Choice. + For example if type is a Choice of either day or week, + show_choices is true and text is "Group by" then the + prompt will be "Group by (day, week): ". + + .. versionchanged:: 8.5.0 + Generically typed: the return type is narrowed by ``type``, + ``value_proc``, or ``default`` instead of being ``Any``. Runtime + behavior is unchanged. + + .. versionchanged:: 8.3.3 + ``show_default`` can be a string to show a custom value instead + of the actual default, matching the help text behavior. + + .. versionchanged:: 8.3.1 + A space is no longer appended to the prompt. + + .. versionadded:: 8.0 + ``confirmation_prompt`` can be a custom string. + + .. versionadded:: 7.0 + Added the ``show_choices`` parameter. + + .. versionadded:: 6.0 + Added unicode support for cmd.exe on Windows. + + .. versionadded:: 4.0 + Added the `err` parameter. + + """ + + def prompt_func(text: str) -> str: + f = hidden_prompt_func if hide_input else visible_prompt_func + try: + return _readline_prompt(f, text, err) + except (KeyboardInterrupt, EOFError): + # getpass doesn't print a newline if the user aborts input with ^C. + # Allegedly this behavior is inherited from getpass(3). + # A doc bug has been filed at https://bugs.python.org/issue24711 + if hide_input: + echo(None, err=err) + raise Abort() from None + + if value_proc is None: + value_proc = convert_type(type, default) + + prompt = _build_prompt( + text, prompt_suffix, show_default, default, show_choices, type + ) + + if confirmation_prompt: + if confirmation_prompt is True: + confirmation_prompt = _("Repeat for confirmation") + + confirmation_prompt = _build_prompt(confirmation_prompt, prompt_suffix) + + while True: + while True: + value = prompt_func(prompt) + if value: + break + elif default is not None: + # Defaults of any type are accepted and round trip through + # value_proc like typed input, so the annotation is only + # accurate for typed input. + value = t.cast("str", default) + break + try: + result = value_proc(value) + except UsageError as e: + message = _mask_hidden_input(e.message, value) if hide_input else e.message + echo(_("Error: {message}").format(message=message), err=err) + continue + if not confirmation_prompt: + return result + while True: + value2 = prompt_func(confirmation_prompt) + is_empty = not value and not value2 + if value2 or is_empty: + break + if value == value2: + return result + echo(_("Error: The two entered values do not match."), err=err) + + +def confirm( + text: str, + default: bool | None = False, + abort: bool = False, + prompt_suffix: str = ": ", + show_default: bool = True, + err: bool = False, +) -> bool: + """Prompts for confirmation (yes/no question). + + If the user aborts the input by sending a interrupt signal this + function will catch it and raise a :exc:`Abort` exception. + + :param text: the question to ask. + :param default: The default value to use when no input is given. If + ``None``, repeat until input is given. + :param abort: if this is set to `True` a negative answer aborts the + exception by raising :exc:`Abort`. + :param prompt_suffix: a suffix that should be added to the prompt. + :param show_default: shows or hides the default value in the prompt. + :param err: if set to true the file defaults to ``stderr`` instead of + ``stdout``, the same as with echo. + + .. versionchanged:: 8.3.1 + A space is no longer appended to the prompt. + + .. versionchanged:: 8.0 + Repeat until input is given if ``default`` is ``None``. + + .. versionadded:: 4.0 + Added the ``err`` parameter. + """ + prompt = _build_prompt( + text, + prompt_suffix, + show_default, + "y/n" if default is None else ("Y/n" if default else "y/N"), + ) + + while True: + try: + value = _readline_prompt(visible_prompt_func, prompt, err).lower().strip() + except (KeyboardInterrupt, EOFError): + raise Abort() from None + if value in ("y", "yes"): + rv = True + elif value in ("n", "no"): + rv = False + elif default is not None and value == "": + rv = default + else: + echo(_("Error: invalid input"), err=err) + continue + break + if abort and not rv: + raise Abort() + return rv + + +def get_pager_file( + color: bool | None = None, +) -> t.ContextManager[t.TextIO]: + """Context manager. + + Yields a writable file-like object which can be used as an output pager. + + .. versionadded:: 8.4.0 + + :param color: controls if the pager supports ANSI colors or not. The + default is autodetection. + """ + from ._termui_impl import get_pager_file + + color = resolve_color_default(color) + + return get_pager_file(color=color) + + +def echo_via_pager( + text_or_generator: cabc.Iterable[str] | t.Callable[[], cabc.Iterable[str]] | str, + color: bool | None = None, +) -> None: + """This function takes a text and shows it via an environment specific + pager on stdout. + + .. versionchanged:: 3.0 + Added the `color` flag. + + :param text_or_generator: the text to page, or alternatively, a + generator emitting the text to page. + :param color: controls if the pager supports ANSI colors or not. The + default is autodetection. + """ + + if inspect.isgeneratorfunction(text_or_generator): + i = t.cast("t.Callable[[], cabc.Iterable[str]]", text_or_generator)() + elif isinstance(text_or_generator, str): + i = [text_or_generator] + else: + i = iter(t.cast("cabc.Iterable[str]", text_or_generator)) + + # convert every element of i to a text type if necessary + text_generator = (el if isinstance(el, str) else str(el) for el in i) + + with get_pager_file(color=color) as pager: + for text in itertools.chain(text_generator, "\n"): + pager.write(text) + # Flush after each write so a slow generator streams to the pager + # incrementally rather than staying invisible until the pipe buffer + # fills (~8 KB). + pager.flush() + + +@t.overload +def progressbar( + *, + length: int, + label: str | None = None, + hidden: bool = False, + show_eta: bool = True, + show_percent: bool | None = None, + show_pos: bool = False, + fill_char: str = "#", + empty_char: str = "-", + bar_template: str = "%(label)s [%(bar)s] %(info)s", + info_sep: str = " ", + width: int = 36, + file: t.TextIO | None = None, + color: bool | None = None, + update_min_steps: int = 1, +) -> ProgressBar[int]: ... + + +@t.overload +def progressbar( + iterable: cabc.Iterable[V] | None = None, + length: int | None = None, + label: str | None = None, + hidden: bool = False, + show_eta: bool = True, + show_percent: bool | None = None, + show_pos: bool = False, + item_show_func: t.Callable[[V | None], str | None] | None = None, + fill_char: str = "#", + empty_char: str = "-", + bar_template: str = "%(label)s [%(bar)s] %(info)s", + info_sep: str = " ", + width: int = 36, + file: t.TextIO | None = None, + color: bool | None = None, + update_min_steps: int = 1, +) -> ProgressBar[V]: ... + + +def progressbar( + iterable: cabc.Iterable[V] | None = None, + length: int | None = None, + label: str | None = None, + hidden: bool = False, + show_eta: bool = True, + show_percent: bool | None = None, + show_pos: bool = False, + item_show_func: t.Callable[[V | None], str | None] | None = None, + fill_char: str = "#", + empty_char: str = "-", + bar_template: str = "%(label)s [%(bar)s] %(info)s", + info_sep: str = " ", + width: int = 36, + file: t.TextIO | None = None, + color: bool | None = None, + update_min_steps: int = 1, +) -> ProgressBar[V]: + """This function creates an iterable context manager that can be used + to iterate over something while showing a progress bar. It will + either iterate over the `iterable` or `length` items (that are counted + up). While iteration happens, this function will print a rendered + progress bar to the given `file` (defaults to stdout) and will attempt + to calculate remaining time and more. By default, this progress bar + will not be rendered if the file is not a terminal. + + The context manager creates the progress bar. When the context + manager is entered the progress bar is already created. With every + iteration over the progress bar, the iterable passed to the bar is + advanced and the bar is updated. When the context manager exits, + a newline is printed and the progress bar is finalized on screen. + + Note: The progress bar is currently designed for use cases where the + total progress can be expected to take at least several seconds. + Because of this, the ProgressBar class object won't display + progress that is considered too fast, and progress where the time + between steps is less than a second. + + No printing must happen or the progress bar will be unintentionally + destroyed. + + Example usage:: + + with progressbar(items) as bar: + for item in bar: + do_something_with(item) + + Alternatively, if no iterable is specified, one can manually update the + progress bar through the `update()` method instead of directly + iterating over the progress bar. The update method accepts the number + of steps to increment the bar with:: + + with progressbar(length=chunks.total_bytes) as bar: + for chunk in chunks: + process_chunk(chunk) + bar.update(chunks.bytes) + + The ``update()`` method also takes an optional value specifying the + ``current_item`` at the new position. This is useful when used + together with ``item_show_func`` to customize the output for each + manual step:: + + with click.progressbar( + length=total_size, + label='Unzipping archive', + item_show_func=lambda a: a.filename + ) as bar: + for archive in zip_file: + archive.extract() + bar.update(archive.size, archive) + + :param iterable: an iterable to iterate over. If not provided the length + is required. + :param length: the number of items to iterate over. By default the + progressbar will attempt to ask the iterator about its + length, which might or might not work. If an iterable is + also provided this parameter can be used to override the + length. If an iterable is not provided the progress bar + will iterate over a range of that length. + :param label: the label to show next to the progress bar. + :param hidden: hide the progressbar. Defaults to ``False``. When no tty is + detected, it will only print the progressbar label. Setting this to + ``False`` also disables that. + :param show_eta: enables or disables the estimated time display. This is + automatically disabled if the length cannot be + determined. + :param show_percent: enables or disables the percentage display. The + default is `True` if the iterable has a length or + `False` if not. + :param show_pos: enables or disables the absolute position display. The + default is `False`. + :param item_show_func: A function called with the current item which + can return a string to show next to the progress bar. If the + function returns ``None`` nothing is shown. The current item can + be ``None``, such as when entering and exiting the bar. + :param fill_char: the character to use to show the filled part of the + progress bar. + :param empty_char: the character to use to show the non-filled part of + the progress bar. + :param bar_template: the format string to use as template for the bar. + The parameters in it are ``label`` for the label, + ``bar`` for the progress bar and ``info`` for the + info section. + :param info_sep: the separator between multiple info items (eta etc.) + :param width: the width of the progress bar in characters, 0 means full + terminal width + :param file: The file to write to. If this is not a terminal then + only the label is printed. + :param color: controls if the terminal supports ANSI colors or not. The + default is autodetection. This is only needed if ANSI + codes are included anywhere in the progress bar output + which is not the case by default. + :param update_min_steps: Render only when this many updates have + completed. This allows tuning for very fast iterators. + + .. versionadded:: 8.2 + The ``hidden`` argument. + + .. versionchanged:: 8.0 + Output is shown even if execution time is less than 0.5 seconds. + + .. versionchanged:: 8.0 + ``item_show_func`` shows the current item, not the previous one. + + .. versionchanged:: 8.0 + Labels are echoed if the output is not a TTY. Reverts a change + in 7.0 that removed all output. + + .. versionadded:: 8.0 + The ``update_min_steps`` parameter. + + .. versionadded:: 4.0 + The ``color`` parameter and ``update`` method. + + .. versionadded:: 2.0 + """ + from ._termui_impl import ProgressBar + + color = resolve_color_default(color) + return ProgressBar( + iterable=iterable, + length=length, + hidden=hidden, + show_eta=show_eta, + show_percent=show_percent, + show_pos=show_pos, + item_show_func=item_show_func, + fill_char=fill_char, + empty_char=empty_char, + bar_template=bar_template, + info_sep=info_sep, + file=file, + label=label, + width=width, + color=color, + update_min_steps=update_min_steps, + ) + + +def clear() -> None: + """Clears the terminal screen. This will have the effect of clearing + the whole visible space of the terminal and moving the cursor to the + top left. This does not do anything if not connected to a terminal. + + .. versionadded:: 2.0 + """ + if not isatty(sys.stdout): + return + + # ANSI escape \033[2J clears the screen, \033[1;1H moves the cursor + echo("\033[2J\033[1;1H", nl=False) + + +def _interpret_color(color: int | tuple[int, int, int] | str, offset: int = 0) -> str: + """Interprets a color value and returns the corresponding ANSI code.""" + if isinstance(color, str) and color in _ansi_colors: + return str(_ansi_colors[color] + offset) + + # bool is an int subclass: without the exclusion, True and False would + # silently render as the palette indices 1 and 0. + elif isinstance(color, int) and not isinstance(color, bool): + if 0 <= color <= 255: + return f"{38 + offset};5;{color:d}" + + elif ( + isinstance(color, (tuple, list)) + and len(color) == 3 + and all( + isinstance(c, int) and not isinstance(c, bool) and 0 <= c <= 255 + for c in color + ) + ): + r, g, b = color + return f"{38 + offset};2;{r:d};{g:d};{b:d}" + + raise ValueError(_("Unknown color {colour!r}").format(colour=color)) + + +def style( + text: t.Any, + fg: int | tuple[int, int, int] | str | None = None, + bg: int | tuple[int, int, int] | str | None = None, + bold: bool | None = None, + dim: bool | None = None, + underline: bool | None = None, + overline: bool | None = None, + italic: bool | None = None, + blink: bool | None = None, + reverse: bool | None = None, + strikethrough: bool | None = None, + reset: bool = True, +) -> str: + """Styles a text with ANSI styles and returns the new string. By + default the styling is self contained which means that at the end + of the string a reset code is issued. This can be prevented by + passing ``reset=False``. + + Examples:: + + click.echo(click.style('Hello World!', fg='green')) + click.echo(click.style('ATTENTION!', blink=True)) + click.echo(click.style('Some things', reverse=True, fg='cyan')) + click.echo(click.style('More colors', fg=(255, 12, 128), bg=117)) + + Supported color names: + + * ``black`` (might be a gray) + * ``red`` + * ``green`` + * ``yellow`` (might be an orange) + * ``blue`` + * ``magenta`` + * ``cyan`` + * ``white`` (might be light gray) + * ``bright_black`` + * ``bright_red`` + * ``bright_green`` + * ``bright_yellow`` + * ``bright_blue`` + * ``bright_magenta`` + * ``bright_cyan`` + * ``bright_white`` + * ``reset`` (reset the color code only) + + If the terminal supports it, color may also be specified as: + + - An integer in the interval [0, 255]. The terminal must support + 8-bit/256-color mode. + - An RGB tuple of three integers in [0, 255]. The terminal must + support 24-bit/true-color mode. + + See https://en.wikipedia.org/wiki/ANSI_color and + https://gist.github.com/XVilka/8346728 for more information. + + :param text: the string to style with ansi codes. + :param fg: if provided this will become the foreground color. + :param bg: if provided this will become the background color. + :param bold: if provided this will enable or disable bold mode. + :param dim: if provided this will enable or disable dim mode. This is + badly supported. + :param underline: if provided this will enable or disable underline. + :param overline: if provided this will enable or disable overline. + :param italic: if provided this will enable or disable italic. + :param blink: if provided this will enable or disable blinking. + :param reverse: if provided this will enable or disable inverse + rendering (foreground becomes background and the + other way round). + :param strikethrough: if provided this will enable or disable + striking through text. + :param reset: by default a reset-all code is added at the end of the + string which means that styles do not carry over. This + can be disabled to compose styles. + + .. versionchanged:: 8.5.0 + All invalid color values raise :exc:`ValueError`. 256-color index + ``0`` is no longer ignored. + + .. versionchanged:: 8.0 + A non-string ``message`` is converted to a string. + + .. versionchanged:: 8.0 + Added support for 256 and RGB color codes. + + .. versionchanged:: 8.0 + Added the ``strikethrough``, ``italic``, and ``overline`` + parameters. + + .. versionchanged:: 7.0 + Added support for bright colors. + + .. versionadded:: 2.0 + """ + if not isinstance(text, str): + text = str(text) + + bits = [] + + if fg is not None: + bits.append(f"\033[{_interpret_color(fg)}m") + + if bg is not None: + bits.append(f"\033[{_interpret_color(bg, 10)}m") + + if bold is not None: + bits.append(f"\033[{1 if bold else 22}m") + if dim is not None: + bits.append(f"\033[{2 if dim else 22}m") + if underline is not None: + bits.append(f"\033[{4 if underline else 24}m") + if overline is not None: + bits.append(f"\033[{53 if overline else 55}m") + if italic is not None: + bits.append(f"\033[{3 if italic else 23}m") + if blink is not None: + bits.append(f"\033[{5 if blink else 25}m") + if reverse is not None: + bits.append(f"\033[{7 if reverse else 27}m") + if strikethrough is not None: + bits.append(f"\033[{9 if strikethrough else 29}m") + bits.append(text) + if reset: + bits.append(_ansi_reset_all) + return "".join(bits) + + +def unstyle(text: str) -> str: + """Removes ANSI styling information from a string. Usually it's not + necessary to use this function as Click's echo function will + automatically remove styling if necessary. + + .. versionadded:: 2.0 + + :param text: the text to remove style information from. + """ + return strip_ansi(text) + + +def secho( + message: t.Any | None = None, + file: t.IO[t.AnyStr] | None = None, + nl: bool = True, + err: bool = False, + color: bool | None = None, + **styles: t.Any, +) -> None: + """This function combines :func:`echo` and :func:`style` into one + call. As such the following two calls are the same:: + + click.secho('Hello World!', fg='green') + click.echo(click.style('Hello World!', fg='green')) + + All keyword arguments are forwarded to the underlying functions + depending on which one they go with. + + Non-string types will be converted to :class:`str`. However, + :class:`bytes` are passed directly to :meth:`echo` without applying + style. If you want to style bytes that represent text, call + :meth:`bytes.decode` first. + + .. versionchanged:: 8.0 + A non-string ``message`` is converted to a string. Bytes are + passed through without style applied. + + .. versionadded:: 2.0 + """ + if message is not None and not isinstance(message, (bytes, bytearray)): + message = style(message, **styles) + + return echo(message, file=file, nl=nl, err=err, color=color) + + +@t.overload +def edit( + text: bytes | bytearray, + editor: str | None = None, + env: cabc.Mapping[str, str] | None = None, + require_save: bool = False, + extension: str = ".txt", +) -> bytes | None: ... + + +@t.overload +def edit( + text: str, + editor: str | None = None, + env: cabc.Mapping[str, str] | None = None, + require_save: bool = True, + extension: str = ".txt", +) -> str | None: ... + + +@t.overload +def edit( + text: None = None, + editor: str | None = None, + env: cabc.Mapping[str, str] | None = None, + require_save: bool = True, + extension: str = ".txt", + filename: str + | os.PathLike[str] + | cabc.Iterable[str | os.PathLike[str]] + | None = None, +) -> None: ... + + +def edit( + text: str | bytes | bytearray | None = None, + editor: str | None = None, + env: cabc.Mapping[str, str] | None = None, + require_save: bool = True, + extension: str = ".txt", + filename: str + | os.PathLike[str] + | cabc.Iterable[str | os.PathLike[str]] + | None = None, +) -> str | bytes | bytearray | None: + r"""Edits the given text in the defined editor. If an editor is given + (should be the full path to the executable but the regular operating + system search path is used for finding the executable) it overrides + the detected editor. Optionally, some environment variables can be + used. If the editor is closed without changes, `None` is returned. In + case a file is edited directly the return value is always `None` and + `require_save` and `extension` are ignored. + + If the editor cannot be opened a :exc:`UsageError` is raised. + + Note for Windows: to simplify cross-platform usage, the newlines are + automatically converted from POSIX to Windows and vice versa. As such, + the message here will have ``\n`` as newline markers. + + :param text: the text to edit. + :param editor: optionally the editor to use. Defaults to automatic + detection. + :param env: environment variables to forward to the editor. + :param require_save: if this is true, then not saving in the editor + will make the return value become `None`. + :param extension: the extension to tell the editor about. This defaults + to `.txt` but changing this might change syntax + highlighting. + :param filename: if provided it will edit this file instead of the + provided text contents. It will not use a temporary + file as an indirection in that case. It accepts a path + or any iterable of paths. If the editor supports + editing multiple files at once, a sequence of files may + be passed as well. Invoke `click.file` once per file + instead if multiple files cannot be managed at once or + editing the files serially is desired. + + .. versionchanged:: 8.2.0 + ``filename`` now accepts any ``Iterable[str]`` in addition to a ``str`` + if the ``editor`` supports editing multiple files at once. + + .. versionchanged:: 8.5.0 + ``filename`` accepts ``os.PathLike`` values in addition to strings. + + """ + from ._termui_impl import Editor + + ed = Editor(editor=editor, env=env, require_save=require_save, extension=extension) + + if filename is None: + return ed.edit(text) + + if isinstance(filename, (str, os.PathLike)): + filename = (filename,) + + ed.edit_files(filenames=filename) + return None + + +def launch(url: str, wait: bool = False, locate: bool = False) -> int: + """This function launches the given URL (or filename) in the default + viewer application for this file type. If this is an executable, it + might launch the executable in a new session. The return value is + the exit code of the launched application. Usually, ``0`` indicates + success. + + Examples:: + + click.launch('https://click.palletsprojects.com/') + click.launch('/my/downloaded/file', locate=True) + + .. versionadded:: 2.0 + + :param url: URL or filename of the thing to launch. + :param wait: Wait for the program to exit before returning. This + only works if the launched program blocks. In particular, + ``xdg-open`` on Linux does not block. + :param locate: if this is set to `True` then instead of launching the + application associated with the URL it will attempt to + launch a file manager with the file located. This + might have weird effects if the URL does not point to + the filesystem. + """ + from ._termui_impl import open_url + + return open_url(url, wait=wait, locate=locate) + + +# If this is provided, getchar() calls into this instead. This is used +# for unittesting purposes. +_getchar: t.Callable[[bool], str] | None = None + + +def getchar(echo: bool = False) -> str: + """Fetches a single character from the terminal and returns it. This + will always return a unicode character and under certain rare + circumstances this might return more than one character. The + situations which more than one character is returned is when for + whatever reason multiple characters end up in the terminal buffer or + standard input was not actually a terminal. + + Note that this will always read from the terminal, even if something + is piped into the standard input. + + Note for Windows: in rare cases when typing non-ASCII characters, this + function might wait for a second character and then return both at once. + This is because certain Unicode characters look like special-key markers. + + .. versionadded:: 2.0 + + :param echo: if set to `True`, the character read will also show up on + the terminal. The default is to not show it. + """ + global _getchar + + if _getchar is None: + from ._termui_impl import getchar as f + + _getchar = f + + return _getchar(echo) + + +def raw_terminal() -> AbstractContextManager[int]: + from ._termui_impl import raw_terminal as f + + return f() + + +def pause(info: str | None = None, err: bool = False) -> None: + """This command stops execution and waits for the user to press any + key to continue. This is similar to the Windows batch "pause" + command. If the program is not run through a terminal, this command + will instead do nothing. + + .. versionadded:: 2.0 + + .. versionadded:: 4.0 + Added the `err` parameter. + + :param info: The message to print before pausing. Defaults to + ``"Press any key to continue..."``. + :param err: if set to message goes to ``stderr`` instead of + ``stdout``, the same as with echo. + """ + if not isatty(sys.stdin) or not isatty(sys.stdout): + return + + if info is None: + info = _("Press any key to continue...") + + try: + if info: + echo(info, nl=False, err=err) + try: + getchar() + except (KeyboardInterrupt, EOFError): + pass + finally: + if info: + echo(err=err) diff --git a/.venv/lib/python3.14/site-packages/click/testing.py b/.venv/lib/python3.14/site-packages/click/testing.py new file mode 100644 index 0000000000000000000000000000000000000000..cc123d4d15fda16262db16cefc467b1fffd9ac5d --- /dev/null +++ b/.venv/lib/python3.14/site-packages/click/testing.py @@ -0,0 +1,798 @@ +from __future__ import annotations + +import collections.abc as cabc +import contextlib +import io +import os +import pdb +import shlex +import sys +import tempfile +import typing as t +from types import TracebackType + +from . import _compat +from . import formatting +from . import termui +from . import utils +from ._compat import _find_binary_reader + +if t.TYPE_CHECKING: + from _typeshed import ReadableBuffer + + from .core import Command + +if sys.platform == "win32": + CaptureMode: t.TypeAlias = t.Literal["sys"] # pyright: ignore[reportRedeclaration] +else: + CaptureMode: t.TypeAlias = t.Literal["sys", "fd"] # pyright: ignore[reportRedeclaration] +ExceptionInfo: t.TypeAlias = tuple[type[BaseException], BaseException, TracebackType] + + +class EchoingStdin: + _input: t.BinaryIO + _output: t.BinaryIO + _paused: bool + + def __init__(self, input: t.BinaryIO, output: t.BinaryIO) -> None: + self._input = input + self._output = output + self._paused = False + + def __getattr__(self, x: str) -> t.Any: + return getattr(self._input, x) + + def _echo(self, rv: bytes) -> bytes: + if not self._paused: + self._output.write(rv) + + return rv + + def read(self, n: int = -1) -> bytes: + return self._echo(self._input.read(n)) + + def read1(self, n: int = -1) -> bytes: + return self._echo(self._input.read1(n)) # type: ignore + + def readline(self, n: int = -1) -> bytes: + return self._echo(self._input.readline(n)) + + def readlines(self) -> list[bytes]: + return [self._echo(x) for x in self._input.readlines()] + + def __iter__(self) -> cabc.Iterator[bytes]: + return iter(self._echo(x) for x in self._input) + + def __repr__(self) -> str: + return repr(self._input) + + +@contextlib.contextmanager +def _pause_echo(stream: EchoingStdin | None) -> cabc.Generator[None]: + if stream is None: + yield + else: + stream._paused = True + yield + stream._paused = False + + +class _FDCapture: + """Redirect a file descriptor to a temporary file for capture. + + Saves the current target of *targetfd* via :func:`os.dup`, then + redirects it to a temporary file via :func:`os.dup2`. On + :meth:`stop`, restores the original ``fd`` and returns the captured + bytes. Inspired by Pytest's ``FDCapture``. + + .. versionadded:: 8.4.0 + """ + + _targetfd: int + saved_fd: int + _tmpfile: t.BinaryIO | None + + def __init__(self, targetfd: int) -> None: + self._targetfd = targetfd + self.saved_fd = -1 + self._tmpfile = None + + def start(self) -> None: + self.saved_fd = os.dup(self._targetfd) + self._tmpfile = tempfile.TemporaryFile(buffering=0) + os.dup2(self._tmpfile.fileno(), self._targetfd) + + def stop(self) -> bytes: + assert self._tmpfile is not None, "_FDCapture.start() was not called" + os.dup2(self.saved_fd, self._targetfd) + os.close(self.saved_fd) + self.saved_fd = -1 + self._tmpfile.seek(0) + data = self._tmpfile.read() + self._tmpfile.close() + self._tmpfile = None + return data + + +class BytesIOCopy(io.BytesIO): + """Patch ``io.BytesIO`` to let the written stream be copied to another. + + .. versionadded:: 8.2 + """ + + copy_to: io.BytesIO + + def __init__(self, copy_to: io.BytesIO) -> None: + super().__init__() + self.copy_to = copy_to + + def flush(self) -> None: + super().flush() + self.copy_to.flush() + + def write(self, b: ReadableBuffer) -> int: + self.copy_to.write(b) + return super().write(b) + + +class StreamMixer: + """Mixes `` and `` streams. + + The result is available in the ``output`` attribute. + + .. versionadded:: 8.2 + """ + + output: io.BytesIO + stdout: BytesIOCopy + stderr: BytesIOCopy + + def __init__(self) -> None: + self.output = io.BytesIO() + self.stdout = BytesIOCopy(copy_to=self.output) + self.stderr = BytesIOCopy(copy_to=self.output) + + +class _NamedTextIOWrapper(io.TextIOWrapper): + """A :class:`~io.TextIOWrapper` with custom ``name`` and ``mode`` + that does not close its underlying buffer. + + When ``CliRunner`` runs in ``fd`` mode, ``_original_fd`` is patched to + point at the saved (pre-redirection) ``fd``, so C-level consumers that call + :meth:`fileno` (like ``faulthandler`` or ``subprocess``) keep working. In + the default ``sys`` mode ``_original_fd`` stays at ``-1`` and + :meth:`fileno` raises :exc:`io.UnsupportedOperation`, matching the + pre-``8.3.3`` behavior. + """ + + _name: str + _mode: str + _original_fd: int + + def __init__( + self, + buffer: t.BinaryIO, + name: str, + mode: str, + **kwargs: t.Any, + ) -> None: + super().__init__(buffer, **kwargs) + self._name = name + self._mode = mode + self._original_fd = -1 + + def close(self) -> None: + """The buffer this object contains belongs to some other object, + so prevent the default ``__del__`` implementation from closing + that buffer. + + .. versionadded:: 8.3.2 + """ + + def fileno(self) -> int: + """Return the file descriptor of the saved original stream when + ``CliRunner`` runs in ``fd`` mode. Otherwise delegate to + :class:`~io.TextIOWrapper`, which raises + :exc:`io.UnsupportedOperation` for a ``BytesIO``-backed buffer. + """ + if self._original_fd >= 0: + return self._original_fd + return super().fileno() + + @property + def name(self) -> str: + return self._name + + @property + def mode(self) -> str: + return self._mode + + +def make_input_stream( + input: str | bytes | t.IO[t.Any] | None, charset: str +) -> t.BinaryIO: + # Is already an input stream. + if hasattr(input, "read"): + rv = _find_binary_reader(t.cast("t.IO[t.Any]", input)) + + if rv is not None: + return rv + + raise TypeError("Could not find binary reader for input stream.") + + if input is None: + input = b"" + elif isinstance(input, str): + input = input.encode(charset) + + return io.BytesIO(input) + + +class Result: + """Holds the captured result of an invoked CLI script. + + :param runner: The runner that created the result + :param stdout_bytes: The standard output as bytes. + :param stderr_bytes: The standard error as bytes. + :param output_bytes: A mix of ``stdout_bytes`` and ``stderr_bytes``, as the + user would see it in its terminal. + :param return_value: The value returned from the invoked command. + :param exit_code: The exit code as integer. + :param exception: The exception that happened if one did. + :param exc_info: Exception information (exception type, exception instance, + traceback type). + + .. versionchanged:: 8.2 + ``stderr_bytes`` no longer optional, ``output_bytes`` introduced and + ``mix_stderr`` has been removed. + + .. versionadded:: 8.0 + Added ``return_value``. + """ + + runner: CliRunner + stdout_bytes: bytes + stderr_bytes: bytes + output_bytes: bytes + return_value: t.Any + exit_code: int + exception: BaseException | None + exc_info: ExceptionInfo | None + + def __init__( + self, + runner: CliRunner, + stdout_bytes: bytes, + stderr_bytes: bytes, + output_bytes: bytes, + return_value: t.Any, + exit_code: int, + exception: BaseException | None, + exc_info: ExceptionInfo | None = None, + ) -> None: + self.runner = runner + self.stdout_bytes = stdout_bytes + self.stderr_bytes = stderr_bytes + self.output_bytes = output_bytes + self.return_value = return_value + self.exit_code = exit_code + self.exception = exception + self.exc_info = exc_info + + @property + def output(self) -> str: + """The terminal output as unicode string, as the user would see it. + + .. versionchanged:: 8.2 + No longer a proxy for ``self.stdout``. Now has its own independent stream + that is mixing `` and ``, in the order they were written. + """ + return self.output_bytes.decode(self.runner.charset, "replace").replace( + "\r\n", "\n" + ) + + @property + def stdout(self) -> str: + """The standard output as unicode string.""" + return self.stdout_bytes.decode(self.runner.charset, "replace").replace( + "\r\n", "\n" + ) + + @property + def stderr(self) -> str: + """The standard error as unicode string. + + .. versionchanged:: 8.2 + No longer raise an exception, always returns the `` string. + """ + return self.stderr_bytes.decode(self.runner.charset, "replace").replace( + "\r\n", "\n" + ) + + def __repr__(self) -> str: + exc_str = repr(self.exception) if self.exception else "okay" + return f"<{type(self).__name__} {exc_str}>" + + +class CliRunner: + """The CLI runner provides functionality to invoke a Click command line + script for unittesting purposes in a isolated environment. This only + works in single-threaded systems without any concurrency as it changes the + global interpreter state. + + :param charset: the character set for the input and output data. + :param env: a dictionary with environment variables for overriding. + :param echo_stdin: if this is set to `True`, then reading from `` writes + to ``. This is useful for showing examples in + some circumstances. Note that regular prompts + will automatically echo the input. + :param catch_exceptions: Whether to catch any exceptions other than + ``SystemExit`` when running :meth:`~CliRunner.invoke`. + :param capture: Selects the output capture strategy. ``sys`` (default) + captures Python-level writes only and leaves + :meth:`sys.stdout.fileno` raising :exc:`io.UnsupportedOperation`, so + user code that calls :func:`os.dup2` on ``sys.stdout.fileno()`` cannot + clobber the host runner's stdout. ``fd`` redirects file descriptors + ``1`` and ``2`` via :func:`os.dup2` to a temporary file, also catching + output from stale stream references, C extensions, and subprocesses. + ``fd`` is not supported on Windows. + + .. versionchanged:: 8.4.0 + Added the ``capture`` parameter. The default ``sys`` mode no longer + exposes the original fd through :meth:`fileno`, reverting the change + introduced in ``8.3.3`` that broke Pytest's ``fd``-level capture + teardown. Use ``capture="fd"`` to restore that behavior with proper + isolation. :issue:`3384` + + .. versionchanged:: 8.2 + Added the ``catch_exceptions`` parameter. + + .. versionchanged:: 8.2 + ``mix_stderr`` parameter has been removed. + """ + + charset: str + env: cabc.Mapping[str, str | None] + echo_stdin: bool + catch_exceptions: bool + capture: CaptureMode + + def __init__( + self, + charset: str = "utf-8", + env: cabc.Mapping[str, str | None] | None = None, + echo_stdin: bool = False, + catch_exceptions: bool = True, + capture: CaptureMode = "sys", + ) -> None: + if capture not in {"sys", "fd"}: + raise ValueError( + f"capture={capture!r} is not valid. Choose from 'sys' or 'fd'." + ) + if capture == "fd" and sys.platform == "win32": + raise ValueError( + f"capture={capture!r} is not supported on Windows. Use 'sys'." + ) + self.charset = charset + self.env = env or {} + self.echo_stdin = echo_stdin + self.catch_exceptions = catch_exceptions + self.capture = capture + + def get_default_prog_name(self, cli: Command) -> str: + """Given a command object it will return the default program name + for it. The default is the `name` attribute or ``"root"`` if not + set. + """ + return cli.name or "root" + + def make_env( + self, overrides: cabc.Mapping[str, str | None] | None = None + ) -> cabc.Mapping[str, str | None]: + """Returns the environment overrides for invoking a script.""" + rv = dict(self.env) + if overrides: + rv.update(overrides) + return rv + + @contextlib.contextmanager + def isolation( + self, + input: str | bytes | t.IO[t.Any] | None = None, + env: cabc.Mapping[str, str | None] | None = None, + color: bool = False, + ) -> cabc.Generator[tuple[io.BytesIO, io.BytesIO, io.BytesIO]]: + """A context manager that sets up the isolation for invoking of a + command line tool. This sets up `` with the given input data + and `os.environ` with the overrides from the given dictionary. + This also rebinds some internals in Click to be mocked (like the + prompt functionality). + + This is automatically done in the :meth:`invoke` method. + + :param input: the input stream to put into `sys.stdin`. + :param env: the environment overrides as dictionary. + :param color: whether the output should contain color codes. The + application can still override this explicitly. + + .. versionadded:: 8.2 + An additional output stream is returned, which is a mix of + `` and `` streams. + + .. versionchanged:: 8.2 + Always returns the `` stream. + + .. versionchanged:: 8.0 + `` is opened with ``errors="backslashreplace"`` + instead of the default ``"strict"``. + + .. versionchanged:: 4.0 + Added the ``color`` parameter. + """ + bytes_input = make_input_stream(input, self.charset) + echo_input = None + + old_stdin = sys.stdin + old_stdout = sys.stdout + old_stderr = sys.stderr + old_forced_width = formatting.FORCED_WIDTH + formatting.FORCED_WIDTH = 80 + + env = self.make_env(env) + + stream_mixer = StreamMixer() + + if self.echo_stdin: + bytes_input = echo_input = t.cast( + t.BinaryIO, EchoingStdin(bytes_input, stream_mixer.stdout) + ) + + sys.stdin = text_input = _NamedTextIOWrapper( + bytes_input, encoding=self.charset, name="", mode="r" + ) + + if self.echo_stdin: + # Force unbuffered reads, otherwise TextIOWrapper reads a + # large chunk which is echoed early. + text_input._CHUNK_SIZE = 1 # type: ignore + + sys.stdout = _NamedTextIOWrapper( + stream_mixer.stdout, + encoding=self.charset, + name="", + mode="w", + ) + + sys.stderr = _NamedTextIOWrapper( + stream_mixer.stderr, + encoding=self.charset, + name="", + mode="w", + errors="backslashreplace", + ) + + @_pause_echo(echo_input) # type: ignore + def visible_input(prompt: str | None = None) -> str: + sys.stdout.write(prompt or "") + try: + val = next(text_input).rstrip("\r\n") + except StopIteration as e: + raise EOFError() from e + sys.stdout.write(f"{val}\n") + sys.stdout.flush() + return val + + @_pause_echo(echo_input) # type: ignore + def hidden_input(prompt: str | None = None) -> str: + sys.stdout.write(f"{prompt or ''}\n") + sys.stdout.flush() + try: + return next(text_input).rstrip("\r\n") + except StopIteration as e: + raise EOFError() from e + + @_pause_echo(echo_input) # type: ignore + def _getchar(echo: bool) -> str: + char = sys.stdin.read(1) + + if echo: + sys.stdout.write(char) + + sys.stdout.flush() + return char + + default_color = color + + def should_strip_ansi( + stream: t.IO[t.Any] | None = None, color: bool | None = None + ) -> bool: + if color is None: + return not default_color + return not color + + old_visible_prompt_func = termui.visible_prompt_func + old_hidden_prompt_func = termui.hidden_prompt_func + old__getchar_func = termui._getchar + old_should_strip_ansi = utils.should_strip_ansi # type: ignore + old__compat_should_strip_ansi = _compat.should_strip_ansi + old_pdb_init = pdb.Pdb.__init__ + termui.visible_prompt_func = visible_input + termui.hidden_prompt_func = hidden_input + termui._getchar = _getchar + utils.should_strip_ansi = should_strip_ansi # type: ignore + _compat.should_strip_ansi = should_strip_ansi + + def _patched_pdb_init( + self: pdb.Pdb, + completekey: str = "tab", + stdin: t.IO[str] | None = None, + stdout: t.IO[str] | None = None, + **kwargs: t.Any, + ) -> None: + """Default ``pdb.Pdb`` to real terminal streams during + ``CliRunner`` isolation. + + Without this patch, ``pdb.Pdb.__init__`` inherits from + ``cmd.Cmd`` which falls back to ``sys.stdin``/``sys.stdout`` + when no explicit streams are provided. During isolation + those are ``BytesIO``-backed wrappers, so the debugger + reads from an empty buffer and writes to captured output, + making interactive debugging impossible. + + By defaulting to ``sys.__stdin__``/``sys.__stdout__`` (the + original terminal streams Python preserves regardless of + redirection), debuggers can interact with the user while + ``click.echo`` output is still captured normally. + + This covers ``pdb.set_trace()``, ``breakpoint()``, + ``pdb.post_mortem()``, and debuggers that subclass + ``pdb.Pdb`` (ipdb, pdbpp). Explicit ``stdin``/``stdout`` + arguments are honored and not overridden. Debuggers that + do not subclass ``pdb.Pdb`` (pudb, debugpy) are not + covered. + """ + if stdin is None: + stdin = sys.__stdin__ + if stdout is None: + stdout = sys.__stdout__ + old_pdb_init( + self, completekey=completekey, stdin=stdin, stdout=stdout, **kwargs + ) + + pdb.Pdb.__init__ = _patched_pdb_init # type: ignore[assignment] + + old_env = {} + try: + for key, value in env.items(): + old_env[key] = os.environ.get(key) + if value is None: + try: + del os.environ[key] + except Exception: + pass + else: + os.environ[key] = value + yield (stream_mixer.stdout, stream_mixer.stderr, stream_mixer.output) + finally: + for key, value in old_env.items(): + if value is None: + try: + del os.environ[key] + except Exception: + pass + else: + os.environ[key] = value + sys.stdout = old_stdout + sys.stderr = old_stderr + sys.stdin = old_stdin + termui.visible_prompt_func = old_visible_prompt_func + termui.hidden_prompt_func = old_hidden_prompt_func + termui._getchar = old__getchar_func + utils.should_strip_ansi = old_should_strip_ansi # type: ignore + _compat.should_strip_ansi = old__compat_should_strip_ansi + formatting.FORCED_WIDTH = old_forced_width + pdb.Pdb.__init__ = old_pdb_init # type: ignore[method-assign] + + def invoke( + self, + cli: Command, + args: str | cabc.Sequence[str] | None = None, + input: str | bytes | t.IO[t.Any] | None = None, + env: cabc.Mapping[str, str | None] | None = None, + catch_exceptions: bool | None = None, + color: bool = False, + **extra: t.Any, + ) -> Result: + """Invokes a command in an isolated environment. The arguments are + forwarded directly to the command line script, the `extra` keyword + arguments are passed to the :meth:`~clickpkg.Command.main` function of + the command. + + This returns a :class:`Result` object. + + :param cli: the command to invoke + :param args: the arguments to invoke. It may be given as an iterable + or a string. When given as string it will be interpreted + as a Unix shell command. More details at + :func:`shlex.split`. + :param input: the input data for `sys.stdin`. + :param env: the environment overrides. + :param catch_exceptions: Whether to catch any other exceptions than + ``SystemExit``. If :data:`None`, the value + from :class:`CliRunner` is used. + :param extra: the keyword arguments to pass to :meth:`main`. + :param color: whether the output should contain color codes. The + application can still override this explicitly. + + .. versionadded:: 8.2 + The result object has the ``output_bytes`` attribute with + the mix of ``stdout_bytes`` and ``stderr_bytes``, as the user would + see it in its terminal. + + .. versionchanged:: 8.2 + The result object always returns the ``stderr_bytes`` stream. + + .. versionchanged:: 8.0 + The result object has the ``return_value`` attribute with + the value returned from the invoked command. + + .. versionchanged:: 4.0 + Added the ``color`` parameter. + + .. versionchanged:: 3.0 + Added the ``catch_exceptions`` parameter. + + .. versionchanged:: 3.0 + The result object has the ``exc_info`` attribute with the + traceback if available. + """ + exc_info = None + if catch_exceptions is None: + catch_exceptions = self.catch_exceptions + + # Set up fd capture before isolation replaces sys.stdout and sys.stderr. + cap_out: _FDCapture | None = None + cap_err: _FDCapture | None = None + + if self.capture == "fd": + cap_out = _FDCapture(1) + cap_err = _FDCapture(2) + try: + cap_out.start() + cap_err.start() + except OSError: + cap_out = cap_err = None + + with self.isolation(input=input, env=env, color=color) as outstreams: + # Point the captured streams' fileno() at the saved (original) + # fd so that C-level consumers like faulthandler keep working + # while fd 1/2 are redirected to the capture tmpfile. + if cap_out is not None and cap_err is not None: + sys.stdout._original_fd = cap_out.saved_fd # type: ignore[union-attr] + sys.stderr._original_fd = cap_err.saved_fd # type: ignore[union-attr] + + return_value = None + exception: BaseException | None = None + exit_code = 0 + + if isinstance(args, str): + args = shlex.split(args) + + try: + prog_name = extra.pop("prog_name") + except KeyError: + prog_name = self.get_default_prog_name(cli) + + try: + return_value = cli.main(args=args or (), prog_name=prog_name, **extra) + except SystemExit as e: + exc_info = sys.exc_info() + e_code = t.cast("int | t.Any | None", e.code) + + if e_code is None: + e_code = 0 + + if e_code != 0: + exception = e + + if not isinstance(e_code, int): + sys.stdout.write(str(e_code)) + sys.stdout.write("\n") + e_code = 1 + + exit_code = e_code + + except Exception as e: + if not catch_exceptions: + raise + exception = e + exit_code = 1 + exc_info = sys.exc_info() + finally: + sys.stdout.flush() + sys.stderr.flush() + + # Stop fd capture and merge the captured bytes into + # the stdout/stderr BytesIO streams. BytesIOCopy mirrors + # those writes into outstreams[2] automatically. + if cap_out is not None and cap_err is not None: + fd_out = cap_out.stop() + fd_err = cap_err.stop() + if fd_out: + outstreams[0].write(fd_out) + if fd_err: + outstreams[1].write(fd_err) + + stdout = outstreams[0].getvalue() + stderr = outstreams[1].getvalue() + output = outstreams[2].getvalue() + + return Result( + runner=self, + stdout_bytes=stdout, + stderr_bytes=stderr, + output_bytes=output, + return_value=return_value, + exit_code=exit_code, + exception=exception, + exc_info=exc_info, # type: ignore + ) + + @contextlib.contextmanager + def isolated_filesystem( + self, temp_dir: str | os.PathLike[str] | None = None + ) -> cabc.Generator[str]: + """A context manager that creates a temporary directory and + changes the current working directory to it. This isolates tests + that affect the contents of the CWD to prevent them from + interfering with each other. + + .. warning:: + This helper predates Python 3 and modern pytest, and is not + thread-safe: it relies on :func:`os.chdir`, which mutates + process-global state, and :meth:`invoke` swaps the + process-global standard streams too. Parallelize tests with + processes (``pytest-xdist``), not threads. Locking the + runner (:pr:`3511`, :pr:`3520`, :pr:`3530`) and a + ``set_filesystem()`` API (:issue:`3123`) were declined: + neither removes the global-state mutation. See :issue:`3700` + and :issue:`3501`. + + :param temp_dir: Create the temporary directory under this + directory. If given, the created directory is not removed + when exiting. + + .. deprecated:: 8.5.0 + Will be removed in Click 9.0. Use + :class:`tempfile.TemporaryDirectory` or pytest's + ``tmp_path`` fixture with absolute paths instead. + + .. versionchanged:: 8.0 + Added the ``temp_dir`` parameter. + """ + import warnings + + warnings.warn( + "'isolated_filesystem' is deprecated and will be removed in Click" + " 9.0. Use 'tempfile.TemporaryDirectory' or pytest's 'tmp_path'" + " fixture with absolute paths instead.", + DeprecationWarning, + stacklevel=3, + ) + + cwd = os.getcwd() + dt = tempfile.mkdtemp(dir=temp_dir) + os.chdir(dt) + + try: + yield dt + finally: + os.chdir(cwd) + + if temp_dir is None: + import shutil + + try: + shutil.rmtree(dt) + except OSError: + pass diff --git a/.venv/lib/python3.14/site-packages/click/types.py b/.venv/lib/python3.14/site-packages/click/types.py new file mode 100644 index 0000000000000000000000000000000000000000..0eb84621d3ceb92e240f1523e2e58d5951335633 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/click/types.py @@ -0,0 +1,1422 @@ +from __future__ import annotations + +import abc +import collections.abc as cabc +import enum +import os +import stat +import sys +import typing as t +import uuid +from datetime import datetime +from gettext import gettext as _ +from gettext import ngettext + +from ._compat import _get_argv_encoding +from ._compat import open_stream +from .exceptions import BadParameter +from .utils import _LazyFile +from .utils import _safecall +from .utils import format_filename + +if t.TYPE_CHECKING: + import typing_extensions as te + + from .core import Context + from .core import Parameter + from .shell_completion import CompletionItem + +_ValueT = t.TypeVar("_ValueT") +_ValueT_contra = t.TypeVar("_ValueT_contra", contravariant=True) +_ValueT_co = t.TypeVar("_ValueT_co", covariant=True) + +# The input type parameter of ParamType defaults to Any. TypeVar defaults +# (PEP 696) landed in typing on Python 3.13, so type checkers get the +# default from typing_extensions, which is never imported at runtime. On +# older Pythons the runtime TypeVar carries no default; +# ParamType.__class_getitem__ fills in the omitted parameter instead. +if t.TYPE_CHECKING: + _InputT_contra = te.TypeVar("_InputT_contra", contravariant=True, default=t.Any) +elif sys.version_info >= (3, 13): + _InputT_contra = t.TypeVar("_InputT_contra", contravariant=True, default=t.Any) +else: + _InputT_contra = t.TypeVar("_InputT_contra", contravariant=True) + +_FloatValueT = t.TypeVar("_FloatValueT", bound=float) +_FloatValueT_co = t.TypeVar("_FloatValueT_co", bound=float, covariant=True) + + +class ParamTypeInfoDict(t.TypedDict): + param_type: str + name: str + + +class ParamType(t.Generic[_ValueT_co, _InputT_contra], abc.ABC): + """Represents the type of a parameter. Validates and converts values + from the command line or Python into the correct type. + + To implement a custom type, subclass and implement at least the + following: + + - The :attr:`name` class attribute must be set. + - Calling an instance of the type with ``None`` must return + ``None``. This is already implemented by default. + - :meth:`convert` must convert string values to the correct type. + - :meth:`convert` must accept values that are already the correct + type. + - It must be able to convert a value if the ``ctx`` and ``param`` + arguments are ``None``. This can occur when converting prompt + input. + + .. versionchanged:: 8.4.0 + Now a generic abstract base class. Parameterize with the + converted value type (``ParamType[int]`` for an integer-returning + type) so that :meth:`convert` and downstream consumers carry the + narrowed return type. + + .. versionchanged:: 8.5.0 + Accepts a second optional type parameter for the input value type + that :meth:`convert` accepts (``ParamType[int, str]`` for a type + converting strings to integers), defaulting to ``Any``. + """ + + is_composite: t.ClassVar[bool] = False + arity: int = 1 # read-only + + #: the descriptive name of this type + name: str + + #: if a list of this type is expected and the value is pulled from a + #: string environment variable, this is what splits it up. `None` + #: means any whitespace. For all parameters the general rule is that + #: whitespace splits them up. The exception are paths and files which + #: are split by ``os.path.pathsep`` by default (":" on Unix and ";" on + #: Windows). + envvar_list_splitter: t.ClassVar[str | None] = None + + if sys.version_info < (3, 13): + # ``_InputT_contra`` carries its ``Any`` default only for type + # checkers: ``TypeVar(default=...)`` (PEP 696) is unavailable at + # runtime before Python 3.13. Fill in the omitted input type + # parameter by hand so ``ParamType[int]`` keeps working. + def __class_getitem__(cls, params: t.Any) -> t.Any: + if cls is ParamType: + if not isinstance(params, tuple): + params = (params,) + if len(params) == 1: + params = (*params, t.Any) + # Checkers cannot see Generic.__class_getitem__ through super(). + return super().__class_getitem__(params) # type: ignore[misc] + + def to_info_dict(self) -> ParamTypeInfoDict: + """Gather information that could be useful for a tool generating + user-facing documentation. + + Use :meth:`click.Context.to_info_dict` to traverse the entire + CLI structure. + + .. versionadded:: 8.0 + """ + # The class name without the "ParamType" suffix. + param_type = type(self).__name__.partition("ParamType")[0] + param_type = param_type.partition("ParameterType")[0] + + # Custom subclasses might not remember to set a name. + if hasattr(self, "name"): + name = self.name + else: + name = param_type + + return {"param_type": param_type, "name": name} + + @t.overload + def __call__( + self, + value: None, + param: Parameter | None = None, + ctx: Context | None = None, + ) -> None: ... + + @t.overload + def __call__( + self, + value: _InputT_contra, + param: Parameter | None = None, + ctx: Context | None = None, + ) -> _ValueT_co: ... + + def __call__( + self, + value: _InputT_contra | None, + param: Parameter | None = None, + ctx: Context | None = None, + ) -> _ValueT_co | None: + if value is not None: + return self.convert(value, param, ctx) + return None + + def get_metavar(self, param: Parameter, ctx: Context) -> str | None: + """Returns the metavar default for this param if it provides one.""" + + def get_missing_message(self, param: Parameter, ctx: Context | None) -> str | None: + """Optionally might return extra information about a missing + parameter. + + .. versionadded:: 2.0 + """ + + def convert( + self, value: _InputT_contra, param: Parameter | None, ctx: Context | None + ) -> _ValueT_co: + """Convert the value to the correct type. This is not called if + the value is ``None`` (the missing value). + + This must accept string values from the command line, as well as + values that are already the correct type. It may also convert + other compatible types. + + The ``param`` and ``ctx`` arguments may be ``None`` in certain + situations, such as when converting prompt input. + + If the value cannot be converted, call :meth:`fail` with a + descriptive message. + + :param value: The value to convert. + :param param: The parameter that is using this type to convert + its value. May be ``None``. + :param ctx: The current context that arrived at this value. May + be ``None``. + """ + # The default returns the value as-is so subclasses that only customize + # metadata are not forced to redeclare ``convert``. + return t.cast("_ValueT_co", value) + + def split_envvar_value(self, rv: str) -> cabc.Sequence[str]: + """Given a value from an environment variable this splits it up + into small chunks depending on the defined envvar list splitter. + + If the splitter is set to `None`, which means that whitespace splits, + then leading and trailing whitespace is ignored. Otherwise, leading + and trailing splitters usually lead to empty items being included. + """ + return (rv or "").split(self.envvar_list_splitter) + + def fail( + self, + message: str, + param: Parameter | None = None, + ctx: Context | None = None, + ) -> t.NoReturn: + """Helper method to fail with an invalid value message.""" + raise BadParameter(message, ctx=ctx, param=param) + + def shell_complete( + self, ctx: Context, param: Parameter, incomplete: str + ) -> list[CompletionItem]: + """Return a list of + :class:`~click.shell_completion.CompletionItem` objects for the + incomplete value. Most types do not provide completions, but + some do, and this allows custom types to provide custom + completions as well. + + :param ctx: Invocation context for this command. + :param param: The parameter that is requesting completion. + :param incomplete: Value being completed. May be empty. + + .. versionadded:: 8.0 + """ + return [] + + +class CompositeParamType(ParamType[_ValueT_co]): + is_composite: t.ClassVar[bool] = True + + @property + @abc.abstractmethod + def arity(self) -> int: ... # type: ignore[override] + + +if t.TYPE_CHECKING: + # on Python 3.10 this will raise a TypeError + + class FuncParamTypeInfoDict( + ParamTypeInfoDict, + t.Generic[_ValueT_contra, _ValueT_co], + ): + func: t.Callable[[_ValueT_contra], _ValueT_co] +else: + + class FuncParamTypeInfoDict(ParamTypeInfoDict): + func: t.Callable[[t.Any], t.Any] + + +class FuncParamType(ParamType[_ValueT_co], t.Generic[_ValueT_contra, _ValueT_co]): + name: str + func: t.Callable[[_ValueT_contra], _ValueT_co] + + def __init__(self, func: t.Callable[[_ValueT_contra], _ValueT_co]) -> None: + self.name = func.__name__ + self.func = func + + def to_info_dict(self) -> FuncParamTypeInfoDict[_ValueT_contra, _ValueT_co]: + return {"func": self.func, **super().to_info_dict()} + + def convert( + self, value: _ValueT_contra, param: Parameter | None, ctx: Context | None + ) -> _ValueT_co: + try: + return self.func(value) + except ValueError as exc: + message = str(exc) + + if not message: + try: + message = str(value) + except UnicodeError: + message = t.cast("bytes", value).decode("utf-8", "replace") + + self.fail(message, param, ctx) + + +class UnprocessedParamType(ParamType[t.Any]): + name = "text" + + def convert( + self, value: _ValueT, param: Parameter | None, ctx: Context | None + ) -> _ValueT: + return value + + def __repr__(self) -> str: + return "UNPROCESSED" + + +class StringParamType(ParamType[str]): + name = "text" + + def convert( + self, value: t.Any, param: Parameter | None, ctx: Context | None + ) -> str: + if isinstance(value, bytes): + enc = _get_argv_encoding() + try: + return value.decode(enc) + except UnicodeError: + fs_enc = sys.getfilesystemencoding() + if fs_enc != enc: + try: + return value.decode(fs_enc) + except UnicodeError: + return value.decode("utf-8", "replace") + else: + return value.decode("utf-8", "replace") + return str(value) + + def __repr__(self) -> str: + return "STRING" + + +if t.TYPE_CHECKING: + # on Python 3.10 this will raise a TypeError + + class ChoiceInfoDict(ParamTypeInfoDict, t.Generic[_ValueT_co]): + choices: tuple[_ValueT_co, ...] + case_sensitive: bool +else: + + class ChoiceInfoDict(ParamTypeInfoDict): + choices: tuple[t.Any, ...] + case_sensitive: bool + + +class Choice(ParamType[_ValueT_co], t.Generic[_ValueT_co]): + """The choice type allows a value to be checked against a fixed set + of supported values. + + You may pass any iterable value which will be converted to a tuple + and thus will only be iterated once. + + The resulting value will always be one of the originally passed choices. + See :meth:`normalize_choice` for more info on the mapping of strings + to choices. See :ref:`choice-opts` for an example. + + :param case_sensitive: Set to false to make choices case + insensitive. Defaults to true. + + .. versionchanged:: 8.4.0 + Now generic in the choice value type. Parameterize with the type of + the choice values (``Choice[HashType]`` for an enum, ``Choice[str]`` + for plain strings) to enable type-checked consumers. + + .. versionchanged:: 8.2.0 + Non-``str`` ``choices`` are now supported. It can additionally be any + iterable. Before you were not recommended to pass anything but a list or + tuple. + + .. versionadded:: 8.2.0 + Choice normalization can be overridden via :meth:`normalize_choice`. + """ + + name: str = "choice" + + choices: tuple[_ValueT_co, ...] + case_sensitive: bool + + def __init__( + self, choices: cabc.Iterable[_ValueT_co], case_sensitive: bool = True + ) -> None: + self.choices = tuple(choices) + self.case_sensitive = case_sensitive + + def to_info_dict(self) -> ChoiceInfoDict[_ValueT_co]: + return { + "choices": self.choices, + "case_sensitive": self.case_sensitive, + **super().to_info_dict(), + } + + def _normalized_mapping( + self, ctx: Context | None = None + ) -> cabc.Mapping[_ValueT_co, str]: + """ + Returns mapping where keys are the original choices and the values are + the normalized values that are accepted via the command line. + + This is a simple wrapper around :meth:`normalize_choice`, use that + instead which is supported. + """ + return { + choice: self.normalize_choice( + choice=choice, + ctx=ctx, + ) + for choice in self.choices + } + + def normalize_choice(self, choice: object, ctx: Context | None) -> str: + """ + Normalize a choice value, used to map a passed string to a choice. + Each choice must have a unique normalized value. + + By default uses :meth:`Context.token_normalize_func` and if not case + sensitive, convert it to a casefolded value. + + .. versionadded:: 8.2.0 + """ + normed_value = choice.name if isinstance(choice, enum.Enum) else str(choice) + + if ctx is not None and ctx.token_normalize_func is not None: + normed_value = ctx.token_normalize_func(normed_value) + + if not self.case_sensitive: + normed_value = normed_value.casefold() + + return normed_value + + def get_metavar(self, param: Parameter, ctx: Context) -> str | None: + if param.param_type_name == "option" and not param.show_choices: # type: ignore[attr-defined] + choice_metavars = [ + convert_type(type(choice)).name.upper() for choice in self.choices + ] + choices_str = "|".join([*dict.fromkeys(choice_metavars)]) + else: + choices_str = "|".join( + [str(i) for i in self._normalized_mapping(ctx=ctx).values()] + ) + + # Use curly braces to indicate a required argument. + if param.required and param.param_type_name == "argument": + return f"{{{choices_str}}}" + + # Use square braces to indicate an option or optional argument. + return f"[{choices_str}]" + + def get_missing_message(self, param: Parameter, ctx: Context | None) -> str: + """ + Message shown when no choice is passed. + + .. versionchanged:: 8.2.0 Added ``ctx`` argument. + """ + return _("Choose from:\n\t{choices}").format( + choices=",\n\t".join(self._normalized_mapping(ctx=ctx).values()) + ) + + def convert( + self, value: t.Any, param: Parameter | None, ctx: Context | None + ) -> _ValueT_co: + """ + For a given value from the parser, normalize it and find its + matching normalized value in the list of choices. Then return the + matched "original" choice. + """ + normed_value = self.normalize_choice(choice=value, ctx=ctx) + normalized_mapping = self._normalized_mapping(ctx=ctx) + + try: + return next( + original + for original, normalized in normalized_mapping.items() + if normalized == normed_value + ) + except StopIteration: + self.fail( + self.get_invalid_choice_message(value=value, ctx=ctx), + param=param, + ctx=ctx, + ) + + def get_invalid_choice_message(self, value: t.Any, ctx: Context | None) -> str: + """Get the error message when the given choice is invalid. + + :param value: The invalid value. + + .. versionadded:: 8.2 + """ + choices_str = ", ".join(map(repr, self._normalized_mapping(ctx=ctx).values())) + return ngettext( + "{value!r} is not {choice}.", + "{value!r} is not one of {choices}.", + len(self.choices), + ).format(value=value, choice=choices_str, choices=choices_str) + + def __repr__(self) -> str: + return _("Choice({choices})").format(choices=list(self.choices)) + + def shell_complete( + self, ctx: Context, param: Parameter, incomplete: str + ) -> list[CompletionItem]: + """Complete choices that start with the incomplete value. + + :param ctx: Invocation context for this command. + :param param: The parameter that is requesting completion. + :param incomplete: Value being completed. May be empty. + + .. versionadded:: 8.0 + """ + from click.shell_completion import CompletionItem + + str_choices = [self.normalize_choice(choice, ctx) for choice in self.choices] + if self.case_sensitive: + matched = (c for c in str_choices if c.startswith(incomplete)) + else: + incomplete = incomplete.lower() + matched = (c for c in str_choices if c.lower().startswith(incomplete)) + + return [CompletionItem(c) for c in matched] + + +class DateTimeInfoDict(ParamTypeInfoDict): + formats: cabc.Sequence[str] + + +class DateTime(ParamType[datetime]): + """The DateTime type converts date strings into `datetime` objects. + + The format strings which are checked are configurable, but default to some + common (non-timezone aware) ISO 8601 formats. + + When specifying *DateTime* formats, you should only pass a list or a tuple. + Other iterables, like generators, may lead to surprising results. + + The format strings are processed using ``datetime.strptime``, and this + consequently defines the format strings which are allowed. + + Parsing is tried using each format, in order, and the first format which + parses successfully is used. + + :param formats: A list or tuple of date format strings, in the order in + which they should be tried. Defaults to + ``'%Y-%m-%d'``, ``'%Y-%m-%dT%H:%M:%S'``, + ``'%Y-%m-%d %H:%M:%S'``. + """ + + name = "datetime" + + formats: cabc.Sequence[str] + + def __init__(self, formats: cabc.Sequence[str] | None = None): + self.formats = formats or [ + "%Y-%m-%d", + "%Y-%m-%dT%H:%M:%S", + "%Y-%m-%d %H:%M:%S", + ] + + def to_info_dict(self) -> DateTimeInfoDict: + return {"formats": self.formats, **super().to_info_dict()} + + def get_metavar(self, param: Parameter, ctx: Context) -> str: + return f"[{'|'.join(self.formats)}]" + + def _try_to_convert_date(self, value: t.Any, format: str) -> datetime | None: + try: + return datetime.strptime(value, format) + except ValueError: + return None + + def convert( + self, value: t.Any, param: Parameter | None, ctx: Context | None + ) -> datetime: + if isinstance(value, datetime): + return value + + for format in self.formats: + converted = self._try_to_convert_date(value, format) + + if converted is not None: + return converted + + formats_str = ", ".join(map(repr, self.formats)) + self.fail( + ngettext( + "{value!r} does not match the format {format}.", + "{value!r} does not match the formats {formats}.", + len(self.formats), + ).format(value=value, format=formats_str, formats=formats_str), + param, + ctx, + ) + + def __repr__(self) -> str: + return "DateTime" + + +class _NumberParamTypeBase( + ParamType[_ValueT_co], t.Generic[_ValueT_contra, _ValueT_co] +): + _number_class: t.Callable[[_ValueT_contra], _ValueT_co] + + def convert( + self, value: _ValueT_contra, param: Parameter | None, ctx: Context | None + ) -> _ValueT_co: + try: + return self._number_class(value) + except ValueError: + self.fail( + _("{value!r} is not a valid {number_type}.").format( + value=value, number_type=self.name + ), + param, + ctx, + ) + + +if t.TYPE_CHECKING: + # on Python 3.10 this will raise a TypeError + + class NumberRangeInfoDict(ParamTypeInfoDict, t.Generic[_FloatValueT_co]): + min: _FloatValueT_co | None + max: _FloatValueT_co | None + min_open: bool + max_open: bool + clamp: bool +else: + + class NumberRangeInfoDict(ParamTypeInfoDict): + min: t.Any | None + max: t.Any | None + min_open: bool + max_open: bool + clamp: bool + + +class _NumberRangeBase( + _NumberParamTypeBase[_ValueT_contra, _FloatValueT_co], + t.Generic[_ValueT_contra, _FloatValueT_co], +): + min: _FloatValueT_co | None + max: _FloatValueT_co | None + min_open: bool + max_open: bool + clamp: bool + + def __init__( + self, + min: _FloatValueT_co | None = None, + max: _FloatValueT_co | None = None, + min_open: bool = False, + max_open: bool = False, + clamp: bool = False, + ) -> None: + self.min = min + self.max = max + self.min_open = min_open + self.max_open = max_open + self.clamp = clamp + + def to_info_dict(self) -> NumberRangeInfoDict[_FloatValueT_co]: + return { + "min": self.min, + "max": self.max, + "min_open": self.min_open, + "max_open": self.max_open, + "clamp": self.clamp, + **super().to_info_dict(), + } + + def convert( + self, value: _ValueT_contra, param: Parameter | None, ctx: Context | None + ) -> _FloatValueT_co: + import operator + + rv = super().convert(value, param, ctx) + min = self.min + max = self.max + lt_min: bool = min is not None and ( + operator.le if self.min_open else operator.lt + )(rv, min) + gt_max: bool = max is not None and ( + operator.ge if self.max_open else operator.gt + )(rv, max) + + if self.clamp: + if min is not None and lt_min: + return self._clamp(min, 1, self.min_open) + + if max is not None and gt_max: + return self._clamp(max, -1, self.max_open) + + if lt_min or gt_max: + self.fail( + _("{value} is not in the range {range}.").format( + value=rv, range=self._describe_range() + ), + param, + ctx, + ) + + return rv + + @abc.abstractmethod + def _clamp( + # Covariant type variables cannot be used in input positions, so we use a + # separate method-scoped type variable instead. + self: _NumberRangeBase[t.Any, _FloatValueT], + bound: _FloatValueT, + dir: t.Literal[1, -1], + open: bool, + ) -> _FloatValueT: + """Find the valid value to clamp to bound in the given + direction. + + :param bound: The boundary value. + :param dir: 1 or -1 indicating the direction to move. + :param open: If true, the range does not include the bound. + """ + ... + + def _describe_range(self) -> str: + """Describe the range for use in help text.""" + if self.min is None: + op = "<" if self.max_open else "<=" + return f"x{op}{self.max}" + + if self.max is None: + op = ">" if self.min_open else ">=" + return f"x{op}{self.min}" + + lop = "<" if self.min_open else "<=" + rop = "<" if self.max_open else "<=" + return f"{self.min}{lop}x{rop}{self.max}" + + def __repr__(self) -> str: + clamp = " clamped" if self.clamp else "" + return f"<{type(self).__name__} {self._describe_range()}{clamp}>" + + +class IntParamType(_NumberParamTypeBase[t.SupportsInt | t.SupportsIndex, int]): + name = "integer" + _number_class = int + + def __repr__(self) -> str: + return "INT" + + +class IntRange(_NumberRangeBase[int, int], IntParamType): + """Restrict an :data:`click.INT` value to a range of accepted + values. See :ref:`ranges`. + + If ``min`` or ``max`` are not passed, any value is accepted in that + direction. If ``min_open`` or ``max_open`` are enabled, the + corresponding boundary is not included in the range. + + If ``clamp`` is enabled, a value outside the range is clamped to the + boundary instead of failing. + + .. versionchanged:: 8.0 + Added the ``min_open`` and ``max_open`` parameters. + """ + + name = "integer range" + + def _clamp(self, bound: int, dir: t.Literal[1, -1], open: bool) -> int: + if not open: + return bound + + return bound + dir + + +class FloatParamType(_NumberParamTypeBase[t.SupportsFloat | t.SupportsIndex, float]): + name = "float" + _number_class = float + + def __repr__(self) -> str: + return "FLOAT" + + +class FloatRange(_NumberRangeBase[float, float], FloatParamType): + """Restrict a :data:`click.FLOAT` value to a range of accepted + values. See :ref:`ranges`. + + If ``min`` or ``max`` are not passed, any value is accepted in that + direction. If ``min_open`` or ``max_open`` are enabled, the + corresponding boundary is not included in the range. + + If ``clamp`` is enabled, a value outside the range is clamped to the + boundary instead of failing. This is not supported if either + boundary is marked ``open``. + + .. versionchanged:: 8.0 + Added the ``min_open`` and ``max_open`` parameters. + """ + + name = "float range" + + def __init__( + self, + min: float | None = None, + max: float | None = None, + min_open: bool = False, + max_open: bool = False, + clamp: bool = False, + ) -> None: + super().__init__( + min=min, max=max, min_open=min_open, max_open=max_open, clamp=clamp + ) + + if (min_open or max_open) and clamp: + raise TypeError("Clamping is not supported for open bounds.") + + def _clamp(self, bound: float, dir: t.Literal[1, -1], open: bool) -> float: + if not open: + return bound + + # Could use math.nextafter here, but clamping an + # open float range doesn't seem to be particularly useful. It's + # left up to the user to write a callback to do it if needed. + raise RuntimeError("Clamping is not supported for open bounds.") + + +class BoolParamType(ParamType[bool]): + name = "boolean" + + bool_states: dict[str, bool] = { + "1": True, + "0": False, + "yes": True, + "no": False, + "true": True, + "false": False, + "on": True, + "off": False, + "t": True, + "f": False, + "y": True, + "n": False, + # Absence of value is considered False. + "": False, + } + """A mapping of string values to boolean states. + + Mapping is inspired by :py:attr:`configparser.ConfigParser.BOOLEAN_STATES` + and extends it. + + .. caution:: + String values are lower-cased, as the ``str_to_bool`` comparison function + below is case-insensitive. + + .. warning:: + The mapping is not exhaustive, and does not cover all possible boolean strings + representations. It will remains as it is to avoid endless bikeshedding. + + Future work my be considered to make this mapping user-configurable from public + API. + """ + + @staticmethod + def str_to_bool(value: str | bool) -> bool | None: + """Convert a string to a boolean value. + + If the value is already a boolean, it is returned as-is. If the value is a + string, it is stripped of whitespaces and lower-cased, then checked against + the known boolean states pre-defined in the `BoolParamType.bool_states` mapping + above. + + Returns `None` if the value does not match any known boolean state. + """ + if isinstance(value, bool): + return value + return BoolParamType.bool_states.get(value.strip().lower()) + + def convert( + self, value: t.Any, param: Parameter | None, ctx: Context | None + ) -> bool: + normalized = self.str_to_bool(value) + if normalized is None: + self.fail( + _( + "{value!r} is not a valid boolean. Recognized values: {states}" + ).format(value=value, states=", ".join(sorted(self.bool_states))), + param, + ctx, + ) + return normalized + + def __repr__(self) -> str: + return "BOOL" + + +class UUIDParameterType(ParamType[uuid.UUID]): + name = "uuid" + + def convert( + self, value: uuid.UUID | str, param: Parameter | None, ctx: Context | None + ) -> uuid.UUID: + if isinstance(value, uuid.UUID): + return value + + value = value.strip() + + try: + return uuid.UUID(value) + except ValueError: + self.fail( + _("{value!r} is not a valid UUID.").format(value=value), param, ctx + ) + + def __repr__(self) -> str: + return "UUID" + + +class FileInfoDict(ParamTypeInfoDict): + mode: str + encoding: str | None + + +class File(ParamType[t.IO[t.Any]]): + """Declares a parameter to be a file for reading or writing. The file + is automatically closed once the context tears down (after the command + finished working). + + Files can be opened for reading or writing. The special value ``-`` + indicates stdin or stdout depending on the mode. + + By default, the file is opened for reading text data, but it can also be + opened in binary mode or for writing. The encoding parameter can be used + to force a specific encoding. + + The `lazy` flag controls if the file should be opened immediately or upon + first IO. The default is to be non-lazy for standard input and output + streams as well as files opened for reading, `lazy` otherwise. When opening a + file lazily for reading, it is still opened temporarily for validation, but + will not be held open until first IO. lazy is mainly useful when opening + for writing to avoid creating the file until it is needed. + + Files can also be opened atomically in which case all writes go into a + separate file in the same folder and upon completion the file will + be moved over to the original location. This is useful if a file + regularly read by other users is modified. + + See :ref:`file-args` for more information. + + .. versionchanged:: 2.0 + Added the ``atomic`` parameter. + """ + + name = "filename" + envvar_list_splitter: t.ClassVar[str] = os.path.pathsep + + mode: str + encoding: str | None + errors: str | None + lazy: bool | None + atomic: bool + + def __init__( + self, + mode: str = "r", + encoding: str | None = None, + errors: str | None = "strict", + lazy: bool | None = None, + atomic: bool = False, + ) -> None: + self.mode = mode + self.encoding = encoding + self.errors = errors + self.lazy = lazy + self.atomic = atomic + + def to_info_dict(self) -> FileInfoDict: + return { + "mode": self.mode, + "encoding": self.encoding, + **super().to_info_dict(), + } + + def resolve_lazy_flag(self, value: str | os.PathLike[str]) -> bool: + if self.lazy is not None: + return self.lazy + if os.fspath(value) == "-": + return False + elif "w" in self.mode: + return True + return False + + def convert( + self, + value: str | os.PathLike[str] | t.IO[t.Any], + param: Parameter | None, + ctx: Context | None, + ) -> t.IO[t.Any]: + if _is_file_like(value): + return value + + try: + lazy = self.resolve_lazy_flag(value) + + if lazy: + lf = _LazyFile( + value, self.mode, self.encoding, self.errors, atomic=self.atomic + ) + + if ctx is not None: + ctx.call_on_close(lf.close_intelligently) + + return t.cast("t.IO[t.Any]", lf) + + f, should_close = open_stream( + value, self.mode, self.encoding, self.errors, atomic=self.atomic + ) + + # If a context is provided, we automatically close the file + # at the end of the context execution (or flush out). If a + # context does not exist, it's the caller's responsibility to + # properly close the file. This for instance happens when the + # type is used with prompts. + if ctx is not None: + if should_close: + ctx.call_on_close(_safecall(f.close)) + else: + ctx.call_on_close(_safecall(f.flush)) + + return f + except OSError as e: + self.fail( + f"'{format_filename(value)}': {e.strerror}", + param, + ctx, + ) + + def shell_complete( + self, ctx: Context, param: Parameter, incomplete: str + ) -> list[CompletionItem]: + """Return a special completion marker that tells the completion + system to use the shell to provide file path completions. + + :param ctx: Invocation context for this command. + :param param: The parameter that is requesting completion. + :param incomplete: Value being completed. May be empty. + + .. versionadded:: 8.0 + """ + from click.shell_completion import CompletionItem + + return [CompletionItem(incomplete, type="file")] + + +def _is_file_like(value: t.Any) -> te.TypeIs[t.IO[t.Any]]: + return hasattr(value, "read") or hasattr(value, "write") + + +class PathInfoDict(ParamTypeInfoDict): + exists: bool + file_okay: bool + dir_okay: bool + writable: bool + readable: bool + allow_dash: bool + + +class Path(ParamType[str | bytes | os.PathLike[str]]): + """The ``Path`` type is similar to the :class:`File` type, but + returns the filename instead of an open file. Various checks can be + enabled to validate the type of file and permissions. + + :param exists: The file or directory needs to exist for the value to + be valid. If this is not set to ``True``, and the file does not + exist, then all further checks are silently skipped. + :param file_okay: Allow a file as a value. + :param dir_okay: Allow a directory as a value. + :param readable: if true, a readable check is performed. + :param writable: if true, a writable check is performed. + :param executable: if true, an executable check is performed. + :param resolve_path: Make the value absolute and resolve any + symlinks. A ``~`` is not expanded, as this is supposed to be + done by the shell only. + :param allow_dash: Allow a single dash as a value, which indicates + a standard stream (but does not open it). Use + :func:`~click.open_file` to handle opening this value. + :param path_type: Convert the incoming path value to this type. If + ``None``, keep Python's default, which is ``str``. Useful to + convert to :class:`pathlib.Path`. + + .. versionchanged:: 8.1 + Added the ``executable`` parameter. + + .. versionchanged:: 8.0 + Allow passing ``path_type=pathlib.Path``. + + .. versionchanged:: 6.0 + Added the ``allow_dash`` parameter. + """ + + envvar_list_splitter: t.ClassVar[str] = os.path.pathsep + + exists: bool + file_okay: bool + dir_okay: bool + readable: bool + writable: bool + executable: bool + resolve_path: bool + allow_dash: bool + name: str + + def __init__( + self, + exists: bool = False, + file_okay: bool = True, + dir_okay: bool = True, + writable: bool = False, + readable: bool = True, + resolve_path: bool = False, + allow_dash: bool = False, + path_type: type | None = None, + executable: bool = False, + ) -> None: + self.exists = exists + self.file_okay = file_okay + self.dir_okay = dir_okay + self.readable = readable + self.writable = writable + self.executable = executable + self.resolve_path = resolve_path + self.allow_dash = allow_dash + self.type: type | None = path_type + + if self.file_okay and not self.dir_okay: + self.name = _("file") + elif self.dir_okay and not self.file_okay: + self.name = _("directory") + else: + self.name = _("path") + + def to_info_dict(self) -> PathInfoDict: + return { + "exists": self.exists, + "file_okay": self.file_okay, + "dir_okay": self.dir_okay, + "writable": self.writable, + "readable": self.readable, + "allow_dash": self.allow_dash, + **super().to_info_dict(), + } + + def coerce_path_result( + self, value: str | os.PathLike[str] + ) -> str | bytes | os.PathLike[str]: + if self.type is not None and not isinstance(value, self.type): + if self.type is str: + return os.fsdecode(value) + elif self.type is bytes: + return os.fsencode(value) + else: + return t.cast("os.PathLike[str]", self.type(value)) + + return value + + def convert( + self, + value: str | os.PathLike[str], + param: Parameter | None, + ctx: Context | None, + ) -> str | bytes | os.PathLike[str]: + rv = value + + dash = b"-" if isinstance(rv, bytes) else "-" + is_dash = self.file_okay and self.allow_dash and rv == dash + + if not is_dash: + if self.resolve_path: + rv = os.path.realpath(rv) + + try: + st = os.stat(rv) + except OSError: + if not self.exists: + return self.coerce_path_result(rv) + self.fail( + _("{name} {filename!r} does not exist.").format( + name=self.name.title(), filename=format_filename(value) + ), + param, + ctx, + ) + + if not self.file_okay and stat.S_ISREG(st.st_mode): + self.fail( + _("{name} {filename!r} is a file.").format( + name=self.name.title(), filename=format_filename(value) + ), + param, + ctx, + ) + if not self.dir_okay and stat.S_ISDIR(st.st_mode): + self.fail( + _("{name} {filename!r} is a directory.").format( + name=self.name.title(), filename=format_filename(value) + ), + param, + ctx, + ) + + if self.readable and not os.access(rv, os.R_OK): + self.fail( + _("{name} {filename!r} is not readable.").format( + name=self.name.title(), filename=format_filename(value) + ), + param, + ctx, + ) + + if self.writable and not os.access(rv, os.W_OK): + self.fail( + _("{name} {filename!r} is not writable.").format( + name=self.name.title(), filename=format_filename(value) + ), + param, + ctx, + ) + + if self.executable and not os.access(value, os.X_OK): + self.fail( + _("{name} {filename!r} is not executable.").format( + name=self.name.title(), filename=format_filename(value) + ), + param, + ctx, + ) + + return self.coerce_path_result(rv) + + def shell_complete( + self, ctx: Context, param: Parameter, incomplete: str + ) -> list[CompletionItem]: + """Return a special completion marker that tells the completion + system to use the shell to provide path completions for only + directories or any paths. + + :param ctx: Invocation context for this command. + :param param: The parameter that is requesting completion. + :param incomplete: Value being completed. May be empty. + + .. versionadded:: 8.0 + """ + from click.shell_completion import CompletionItem + + type = "dir" if self.dir_okay and not self.file_okay else "file" + return [CompletionItem(incomplete, type=type)] + + +class TupleInfoDict(ParamTypeInfoDict): + types: cabc.Sequence[ParamTypeInfoDict] + + +class Tuple(CompositeParamType[tuple[t.Any, ...]]): + """The default behavior of Click is to apply a type on a value directly. + This works well in most cases, except for when `nargs` is set to a fixed + count and different types should be used for different items. In this + case the :class:`Tuple` type can be used. This type can only be used + if `nargs` is set to a fixed number. + + For more information see :ref:`tuple-type`. + + This can be selected by using a Python tuple literal as a type. + + :param types: a list of types that should be used for the tuple items. + """ + + def __init__(self, types: cabc.Sequence[type[t.Any] | ParamType[t.Any]]) -> None: + self.types: cabc.Sequence[ParamType[t.Any]] = [convert_type(ty) for ty in types] + + def to_info_dict(self) -> TupleInfoDict: + return { + "types": [ty.to_info_dict() for ty in self.types], + **super().to_info_dict(), + } + + @property + def name(self) -> str: # type: ignore[override] + return f"<{' '.join(ty.name for ty in self.types)}>" + + @property + def arity(self) -> int: # type: ignore[override] + return len(self.types) + + def convert( + self, value: t.Any, param: Parameter | None, ctx: Context | None + ) -> tuple[t.Any, ...]: + len_type = len(self.types) + len_value = len(value) + + if len_value != len_type: + self.fail( + ngettext( + "{len_type} values are required, but {len_value} was given.", + "{len_type} values are required, but {len_value} were given.", + len_value, + ).format(len_type=len_type, len_value=len_value), + param=param, + ctx=ctx, + ) + + return tuple( + ty(x, param, ctx) for ty, x in zip(self.types, value, strict=False) + ) + + +def _guess_type( + ty: type[t.Any] | ParamType[t.Any] | None, + default: t.Any | None, +) -> type[t.Any] | tuple[type[t.Any], ...] | ParamType[t.Any] | None: + """Infer a type from *ty* or *default*. + + Returns *ty* unchanged when it is not ``None``. Otherwise inspects + *default* to produce a ``type``, a ``tuple`` of types (for tuple + defaults), or ``None``. + """ + if ty is not None: + return ty + + if default is None: + return None + + if not isinstance(default, (tuple, list)): + return type(default) + + # If the default is empty, return None so convert_type falls + # through to STRING. + if not default: + return None + + item = default[0] + + # A sequence of iterables needs to detect the inner types. + # Can't call convert_type recursively because that would + # incorrectly unwind the tuple to a single type. + if isinstance(item, (tuple, list)): + return tuple(map(type, item)) + + return type(item) + + +@t.overload +def convert_type(ty: None, default: None = None) -> StringParamType: ... +@t.overload +def convert_type( + ty: type | ParamType[t.Any], default: t.Any | None = None +) -> ParamType[t.Any]: ... +@t.overload +def convert_type( + ty: t.Any | None, default: t.Any | None = None +) -> ParamType[t.Any]: ... +def convert_type( + ty: t.Any | None = None, default: t.Any | None = None +) -> ParamType[t.Any]: + """Find the most appropriate :class:`ParamType` for the given Python + type. If the type isn't provided, it can be inferred from a default + value. + """ + guessed = _guess_type(ty, default) + is_guessed = guessed is not ty + + if isinstance(guessed, tuple): + return Tuple(guessed) + + if isinstance(guessed, ParamType): + return guessed + + if guessed is str or guessed is None: + return STRING + + if guessed is int: + return INT + + if guessed is float: + return FLOAT + + if guessed is bool: + return BOOL + + if is_guessed: + return STRING + + if __debug__: + try: + if issubclass(guessed, ParamType): + raise AssertionError( + f"Attempted to use an uninstantiated parameter type ({guessed})." + ) + except TypeError: + # guessed is an instance (correct), so issubclass fails. + pass + + return FuncParamType(guessed) + + +#: A dummy parameter type that just does nothing. From a user's +#: perspective this appears to just be the same as `STRING` but +#: internally no string conversion takes place if the input was bytes. +#: This is usually useful when working with file paths as they can +#: appear in bytes and unicode. +#: +#: For path related uses the :class:`Path` type is a better choice but +#: there are situations where an unprocessed type is useful which is why +#: it is provided. +#: +#: .. versionadded:: 4.0 +UNPROCESSED: t.Final[UnprocessedParamType] = UnprocessedParamType() + +#: A unicode string parameter type which is the implicit default. This +#: can also be selected by using ``str`` as type. +STRING: t.Final[StringParamType] = StringParamType() + +#: An integer parameter. This can also be selected by using ``int`` as +#: type. +INT: t.Final[IntParamType] = IntParamType() + +#: A floating point value parameter. This can also be selected by using +#: ``float`` as type. +FLOAT: t.Final[FloatParamType] = FloatParamType() + +#: A boolean parameter. This is the default for boolean flags. This can +#: also be selected by using ``bool`` as a type. +BOOL: t.Final[BoolParamType] = BoolParamType() + +#: A UUID parameter. +UUID: t.Final[UUIDParameterType] = UUIDParameterType() + + +class OptionHelpExtra(t.TypedDict, total=False): + envvars: tuple[str, ...] + default: str + range: str + required: str diff --git a/.venv/lib/python3.14/site-packages/click/utils.py b/.venv/lib/python3.14/site-packages/click/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..b529eb04b34189f08e1211c95c7c8a30ca778490 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/click/utils.py @@ -0,0 +1,688 @@ +from __future__ import annotations + +import collections.abc as cabc +import os +import re +import sys +import typing as t +from functools import update_wrapper +from gettext import gettext as _ +from types import ModuleType +from types import TracebackType + +from ._compat import _default_text_stderr +from ._compat import _default_text_stdout +from ._compat import _find_binary_writer +from ._compat import binary_streams +from ._compat import open_stream +from ._compat import should_strip_ansi +from ._compat import strip_ansi +from ._compat import text_streams +from ._compat import WIN +from .globals import resolve_color_default + +if t.TYPE_CHECKING: + import typing_extensions as te + + P = te.ParamSpec("P") + +R = t.TypeVar("R") + + +def _posixify(name: str) -> str: + return "-".join(name.split()).lower() + + +def _safecall(func: t.Callable[P, R]) -> t.Callable[P, R | None]: + """Wraps a function so that it swallows exceptions. + + :meta private: + """ + + def wrapper(*args: P.args, **kwargs: P.kwargs) -> R | None: + try: + return func(*args, **kwargs) + except Exception: + pass + return None + + return update_wrapper(wrapper, func) + + +def make_str(value: t.Any) -> str: + """Converts a value into a valid string.""" + if isinstance(value, bytes): + try: + return value.decode(sys.getfilesystemencoding()) + except UnicodeError: + return value.decode("utf-8", "replace") + return str(value) + + +def _make_default_short_help(help: str, max_length: int = 45) -> str: + """Returns a condensed version of help string. + + :meta private: + """ + # Consider only the first paragraph. + paragraph_end = help.find("\n\n") + + if paragraph_end != -1: + help = help[:paragraph_end] + + # Collapse newlines, tabs, and spaces. + words = help.split() + + if not words: + return "" + + # The first paragraph started with a "no rewrap" marker, ignore it. + if words[0] == "\b": + words = words[1:] + + total_length = 0 + last_index = len(words) - 1 + + for i, word in enumerate(words): + total_length += len(word) + (i > 0) + + if total_length > max_length: # too long, truncate + break + + if word[-1] == ".": # sentence end, truncate without "..." + return " ".join(words[: i + 1]) + + if total_length == max_length and i != last_index: + break # not at sentence end, truncate with "..." + else: + return " ".join(words) # no truncation needed + + # Account for the length of the suffix. + total_length += len("...") + + # remove words until the length is short enough + while i > 0: + total_length -= len(words[i]) + (i > 0) + + if total_length <= max_length: + break + + i -= 1 + + return " ".join(words[:i]) + "..." + + +class _LazyFile: + """A lazy file works like a regular file but it does not fully open + the file but it does perform some basic checks early to see if the + filename parameter does make sense. This is useful for safely opening + files for writing. + + :meta private: + """ + + name: str + mode: str + encoding: str | None + errors: str | None + atomic: bool + _f: t.IO[t.Any] | None + should_close: bool + + def __init__( + self, + filename: str | os.PathLike[str], + mode: str = "r", + encoding: str | None = None, + errors: str | None = "strict", + atomic: bool = False, + ) -> None: + self.name = os.fspath(filename) + self.mode = mode + self.encoding = encoding + self.errors = errors + self.atomic = atomic + + if self.name == "-": + self._f, self.should_close = open_stream(filename, mode, encoding, errors) + else: + if "r" in mode: + # Open and close the file in case we're opening it for + # reading so that we can catch at least some errors in + # some cases early. + open(filename, mode).close() + self._f = None + self.should_close = True + + def __getattr__(self, name: str) -> t.Any: + return getattr(self.open(), name) + + def __repr__(self) -> str: + if self._f is not None: + return repr(self._f) + return f"" + + def open(self) -> t.IO[t.Any]: + """Opens the file if it's not yet open. This call might fail with + a :exc:`FileError`. Not handling this error will produce an error + that Click shows. + """ + if self._f is not None: + return self._f + try: + rv, self.should_close = open_stream( + self.name, self.mode, self.encoding, self.errors, atomic=self.atomic + ) + except OSError as e: + from .exceptions import FileError + + raise FileError(self.name, hint=e.strerror) from e + self._f = rv + return rv + + def close(self) -> None: + """Closes the underlying file, no matter what.""" + if self._f is not None: + self._f.close() + + def close_intelligently(self) -> None: + """This function only closes the file if it was opened by the lazy + file wrapper. For instance this will never close stdin. + """ + if self.should_close: + self.close() + + def __enter__(self) -> _LazyFile: + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + tb: TracebackType | None, + ) -> None: + self.close_intelligently() + + def __iter__(self) -> cabc.Iterator[t.AnyStr]: + self.open() + return iter(self._f) # type: ignore + + +class _KeepOpenFile: + """Proxy a file object but keep it open across a ``with`` block. + + Wraps a borrowed file (such as ``sys.stdin`` or ``sys.stdout``) so that + leaving a ``with`` block does not close it, as used by :func:`open_file` + for the ``-`` filename. The caller stays responsible for the file: an + explicit :meth:`close` still passes through to the wrapped object. + + Dunder methods are proxied explicitly: implicit special-method lookups + bypass :meth:`__getattr__`, because Python resolves them on the type rather + than the instance. + + :meta private: + """ + + _file: t.IO[t.Any] + + def __init__(self, file: t.IO[t.Any]) -> None: + self._file = file + + def __getattr__(self, name: str) -> t.Any: + return getattr(self._file, name) + + def __enter__(self) -> _KeepOpenFile: + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + tb: TracebackType | None, + ) -> None: + pass + + def __repr__(self) -> str: + return repr(self._file) + + def __iter__(self) -> cabc.Iterator[t.AnyStr]: + return iter(self._file) + + +def echo( + message: object = None, + file: t.IO[t.Any] | None = None, + nl: bool = True, + err: bool = False, + color: bool | None = None, +) -> None: + """Print a message and newline to stdout or a file. This should be + used instead of :func:`print` because it provides better support + for different data, files, and environments. + + Compared to :func:`print`, this does the following: + + - Ensures that the output encoding is not misconfigured on Linux. + - Supports Unicode in the Windows console. + - Supports writing to binary outputs, and supports writing bytes + to text outputs. + - Removes ANSI color and style codes if the output does not look + like an interactive terminal. + - Always flushes the output. + + :param message: The string or bytes to output. Other objects are + converted to strings. + :param file: The file to write to. Defaults to ``stdout``. + :param err: Write to ``stderr`` instead of ``stdout``. + :param nl: Print a newline after the message. Enabled by default. + :param color: Force showing or hiding colors and other styles. By + default Click will remove color if the output does not look like + an interactive terminal. + + .. versionchanged:: 8.5.0 + Colorama is no longer used for color on Windows. + + .. versionchanged:: 6.0 + Support Unicode output on the Windows console. Click does not + modify ``sys.stdout``, so ``sys.stdout.write()`` and ``print()`` + will still not support Unicode. + + .. versionchanged:: 4.0 + Added the ``color`` parameter. + + .. versionadded:: 3.0 + Added the ``err`` parameter. + + .. versionchanged:: 2.0 + Support colors on Windows if colorama is installed. + """ + if file is None: + if err: + file = _default_text_stderr() + else: + file = _default_text_stdout() + + # There are no standard streams attached to write to. For example, + # pythonw on Windows. + if file is None: + return + + match message: + case str() | bytes() | bytearray(): + out = message + case None: + out = "" + case _: + out = str(message) + + if nl: + if isinstance(out, str): + out += "\n" + else: + out += b"\n" + + if not out: + file.flush() + return + + # If there is a message and the value looks like bytes, we manually + # need to find the binary stream and write the message in there. + # This is done separately so that most stream types will work as you + # would expect. Eg: you can write to StringIO for other cases. + if isinstance(out, (bytes, bytearray)): + binary_file = _find_binary_writer(file) + if binary_file is not None: + file.flush() + binary_file.write(out) + binary_file.flush() + return + + # ANSI style code support. For no message or bytes, nothing happens. + # When outputting to a file instead of a terminal, strip codes. + elif should_strip_ansi(file, resolve_color_default(color)): + out = strip_ansi(out) + + file.write(out) # type: ignore + file.flush() + + +def _get_binary_stream(name: t.Literal["stdin", "stdout", "stderr"]) -> t.BinaryIO: + """Returns a system stream for byte processing. + + .. deprecated:: 8.5.0 + Will be removed in Click 9.0. + + :param name: the name of the stream to open. Valid names are ``'stdin'``, + ``'stdout'`` and ``'stderr'`` + + :meta private: + """ + opener = binary_streams.get(name) + if opener is None: + raise TypeError(_("Unknown standard stream '{name}'").format(name=name)) + return opener() + + +def _get_text_stream( + name: t.Literal["stdin", "stdout", "stderr"], + encoding: str | None = None, + errors: str | None = "strict", +) -> t.TextIO: + """Returns a system stream for text processing. + + .. deprecated:: 8.5.0 + Will be removed in Click 9.0. + + This usually returns a wrapped stream around a binary stream returned from + :func:`get_binary_stream` but it also can take shortcuts for already + correctly configured streams. + + + :param name: the name of the stream to open. Valid names are ``'stdin'``, + ``'stdout'`` and ``'stderr'`` + :param encoding: overrides the detected default encoding. + :param errors: overrides the default error mode. + :meta private: + """ + opener = text_streams.get(name) + if opener is None: + raise TypeError(_("Unknown standard stream '{name}'").format(name=name)) + return opener(encoding, errors) + + +def open_file( + filename: str | os.PathLike[str], + mode: str = "r", + encoding: str | None = None, + errors: str | None = "strict", + lazy: bool = False, + atomic: bool = False, +) -> t.IO[t.Any]: + """Open a file, with extra behavior to handle ``'-'`` to indicate + a standard stream, lazy open on write, and atomic write. Similar to + the behavior of the :class:`~click.File` param type. + + If ``'-'`` is given to open ``stdout`` or ``stdin``, the stream is + wrapped so that using it in a context manager will not close it. + This makes it possible to use the function without accidentally + closing a standard stream: + + .. code-block:: python + + with open_file(filename) as f: + ... + + :param filename: The name or Path of the file to open, or ``'-'`` for + ``stdin``/``stdout``. + :param mode: The mode in which to open the file. + :param encoding: The encoding to decode or encode a file opened in + text mode. + :param errors: The error handling mode. + :param lazy: Wait to open the file until it is accessed. For read + mode, the file is temporarily opened to raise access errors + early, then closed until it is read again. + :param atomic: Write to a temporary file and replace the given file + on close. + + .. versionadded:: 3.0 + """ + if lazy: + return t.cast( + "t.IO[t.Any]", _LazyFile(filename, mode, encoding, errors, atomic=atomic) + ) + + f, should_close = open_stream(filename, mode, encoding, errors, atomic=atomic) + + if not should_close: + f = t.cast("t.IO[t.Any]", _KeepOpenFile(f)) + + return f + + +def format_filename( + filename: str | bytes | os.PathLike[str] | os.PathLike[bytes], + shorten: bool = False, +) -> str: + """Format a filename as a string for display. Ensures the filename can be + displayed by replacing any invalid bytes or surrogate escapes in the name + with the replacement character ``�``. + + Invalid bytes or surrogate escapes will raise an error when written to a + stream with ``errors="strict"``. This will typically happen with ``stdout`` + when the locale is something like ``en_GB.UTF-8``. + + Many scenarios *are* safe to write surrogates though, due to PEP 538 and + PEP 540, including: + + - Writing to ``stderr``, which uses ``errors="backslashreplace"``. + - The system has ``LANG=C.UTF-8``, ``C``, or ``POSIX``. Python opens + stdout and stderr with ``errors="surrogateescape"``. + - None of ``LANG/LC_*`` are set. Python assumes ``LANG=C.UTF-8``. + - Python is started in UTF-8 mode with ``PYTHONUTF8=1`` or ``-X utf8``. + Python opens stdout and stderr with ``errors="surrogateescape"``. + + :param filename: formats a filename for UI display. This will also convert + the filename into unicode without failing. + :param shorten: this optionally shortens the filename to strip of the + path that leads up to it. + """ + if shorten: + filename = os.path.basename(filename) + else: + filename = os.fspath(filename) + + if isinstance(filename, bytes): + filename = filename.decode(sys.getfilesystemencoding(), "replace") + else: + filename = filename.encode("utf-8", "surrogateescape").decode( + "utf-8", "replace" + ) + + return filename + + +def get_app_dir(app_name: str, roaming: bool = True, force_posix: bool = False) -> str: + r"""Returns the config folder for the application. The default behavior + is to return whatever is most appropriate for the operating system. + + To give you an idea, for an app called ``"Foo Bar"``, something like + the following folders could be returned: + + Mac OS X: + ``~/Library/Application Support/Foo Bar`` + Mac OS X (POSIX): + ``~/.foo-bar`` + Unix: + ``~/.config/foo-bar`` + Unix (POSIX): + ``~/.foo-bar`` + Windows (roaming): + ``C:\Users\\AppData\Roaming\Foo Bar`` + Windows (not roaming): + ``C:\Users\\AppData\Local\Foo Bar`` + + .. versionadded:: 2.0 + + :param app_name: the application name. This should be properly capitalized + and can contain whitespace. + :param roaming: controls if the folder should be roaming or not on Windows. + Has no effect otherwise. + :param force_posix: if this is set to `True` then on any POSIX system the + folder will be stored in the home folder with a leading + dot instead of the XDG config home or darwin's + application support folder. + """ + if WIN: + key = "APPDATA" if roaming else "LOCALAPPDATA" + folder = os.environ.get(key) + if folder is None: + folder = os.path.expanduser("~") + return os.path.join(folder, app_name) + if force_posix: + return os.path.join(os.path.expanduser(f"~/.{_posixify(app_name)}")) + if sys.platform == "darwin": + return os.path.join( + os.path.expanduser("~/Library/Application Support"), app_name + ) + return os.path.join( + os.environ.get("XDG_CONFIG_HOME", os.path.expanduser("~/.config")), + _posixify(app_name), + ) + + +class _PacifyFlushWrapper: + """This wrapper is used to catch and suppress BrokenPipeErrors resulting + from ``.flush()`` being called on broken pipe during the shutdown/final-GC + of the Python interpreter. Notably ``.flush()`` is always called on + ``sys.stdout`` and ``sys.stderr``. So as to have minimal impact on any + other cleanup code, and the case where the underlying file is not a broken + pipe, all calls and attributes are proxied. + + :meta private: + """ + + wrapped: t.IO[t.Any] + + def __init__(self, wrapped: t.IO[t.Any]) -> None: + self.wrapped = wrapped + + def flush(self) -> None: + try: + self.wrapped.flush() + except OSError as e: + import errno + + if e.errno != errno.EPIPE: + raise + + def __getattr__(self, attr: str) -> t.Any: + return getattr(self.wrapped, attr) + + +def _detect_program_name( + path: str | None = None, _main: ModuleType | None = None +) -> str: + """Determine the command used to run the program, for use in help + text. If a file or entry point was executed, the file name is + returned. If ``python -m`` was used to execute a module or package, + ``python -m name`` is returned. + + This doesn't try to be too precise, the goal is to give a concise + name for help text. Files are only shown as their name without the + path. ``python`` is only shown for modules, and the full path to + ``sys.executable`` is not shown. + + :param path: The Python file being executed. Python puts this in + ``sys.argv[0]``, which is used by default. + :param _main: The ``__main__`` module. This should only be passed + during internal testing. + + .. versionadded:: 8.0 + Based on command args detection in the Werkzeug reloader. + + :meta private: + """ + if _main is None: + _main = sys.modules["__main__"] + + if not path: + path = sys.argv[0] + + # The value of __package__ indicates how Python was called. It may + # not exist if a setuptools script is installed as an egg. It may be + # set incorrectly for entry points created with pip on Windows. + # It is set to "" inside a Shiv or PEX zipapp. + if getattr(_main, "__package__", None) in {None, ""} or ( + os.name == "nt" + and _main.__package__ == "" + and not os.path.exists(path) + and os.path.exists(f"{path}.exe") + ): + # Executed a file, like "python app.py". + return os.path.basename(path) + + # Executed a module, like "python -m example". + # Rewritten by Python from "-m script" to "/path/to/script.py". + # Need to look at main module to determine how it was executed. + py_module = t.cast(str, _main.__package__) + name = os.path.splitext(os.path.basename(path))[0] + + # A submodule like "example.cli". + if name != "__main__": + py_module = f"{py_module}.{name}" + + return f"python -m {py_module.lstrip('.')}" + + +def _expand_args( + args: cabc.Iterable[str], + *, + user: bool = True, + env: bool = True, + glob_recursive: bool = True, +) -> list[str]: + """Simulate Unix shell expansion with Python functions. + + See :func:`glob.glob`, :func:`os.path.expanduser`, and + :func:`os.path.expandvars`. + + This is intended for use on Windows, where the shell does not do any + expansion. It may not exactly match what a Unix shell would do. + + :param args: List of command line arguments to expand. + :param user: Expand user home directory. + :param env: Expand environment variables. + :param glob_recursive: ``**`` matches directories recursively. + + .. versionchanged:: 8.1 + Invalid glob patterns are treated as empty expansions rather + than raising an error. + + .. versionadded:: 8.0 + + :meta private: + """ + from glob import glob + + out = [] + + for arg in args: + if user: + arg = os.path.expanduser(arg) + + if env: + arg = os.path.expandvars(arg) + + try: + matches = glob(arg, recursive=glob_recursive) + except re.error: + matches = [] + + if not matches: + out.append(arg) + else: + out.extend(matches) + + return out + + +def __getattr__(name: str) -> object: + import warnings + + if name in { + "LazyFile", + "KeepOpenFile", + "make_default_short_help", + "PacifyFlushWrapper", + "safecall", + "get_text_stream", + "get_binary_stream", + }: + warnings.warn( + f"'click.utils.{name}' is deprecated and will be removed in Click 9.0.", + DeprecationWarning, + stacklevel=2, + ) + return globals()[f"_{name}"] + + raise AttributeError(name) diff --git a/.venv/lib/python3.14/site-packages/filelock-4.2.4.dist-info/INSTALLER b/.venv/lib/python3.14/site-packages/filelock-4.2.4.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/filelock-4.2.4.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/.venv/lib/python3.14/site-packages/filelock-4.2.4.dist-info/METADATA b/.venv/lib/python3.14/site-packages/filelock-4.2.4.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..826b5a93e0ebc4d46d1d2db52de9e2cce6cc2374 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/filelock-4.2.4.dist-info/METADATA @@ -0,0 +1,38 @@ +Metadata-Version: 2.5 +Name: filelock +Version: 4.2.4 +Summary: A platform independent file lock. +Project-URL: Documentation, https://py-filelock.readthedocs.io +Project-URL: Homepage, https://github.com/tox-dev/py-filelock +Project-URL: Source, https://github.com/tox-dev/py-filelock +Project-URL: Tracker, https://github.com/tox-dev/py-filelock/issues +Maintainer-email: Bernát Gábor +License-Expression: MIT +License-File: LICENSE +Keywords: application,cache,directory,log,user +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: MIT License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 3 :: Only +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 +Classifier: Programming Language :: Python :: 3.14 +Classifier: Programming Language :: Python :: 3.15 +Classifier: Topic :: Internet +Classifier: Topic :: Software Development :: Libraries +Classifier: Topic :: System +Requires-Python: >=3.11 +Description-Content-Type: text/markdown + +# filelock + +[![PyPI](https://img.shields.io/pypi/v/filelock)](https://pypi.org/project/filelock/) +[![Supported Python versions](https://img.shields.io/pypi/pyversions/filelock.svg)](https://pypi.org/project/filelock/) +[![Documentation status](https://readthedocs.org/projects/py-filelock/badge/?version=latest)](https://py-filelock.readthedocs.io/en/latest/?badge=latest) +[![Downloads](https://static.pepy.tech/badge/filelock/month)](https://pepy.tech/project/filelock) +[![check](https://github.com/tox-dev/py-filelock/actions/workflows/check.yaml/badge.svg)](https://github.com/tox-dev/py-filelock/actions/workflows/check.yaml) + +For more information checkout the [official documentation](https://py-filelock.readthedocs.io/en/latest/index.html). diff --git a/.venv/lib/python3.14/site-packages/filelock-4.2.4.dist-info/RECORD b/.venv/lib/python3.14/site-packages/filelock-4.2.4.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..9a16ae153221a0e7ed7f0487ba696a018c111dec --- /dev/null +++ b/.venv/lib/python3.14/site-packages/filelock-4.2.4.dist-info/RECORD @@ -0,0 +1,52 @@ +filelock-4.2.4.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +filelock-4.2.4.dist-info/METADATA,sha256=2ZwPcqerhMkktzQBiiilABUF_g-WKAxEYrhkMT7d21A,1976 +filelock-4.2.4.dist-info/RECORD,, +filelock-4.2.4.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87 +filelock-4.2.4.dist-info/licenses/LICENSE,sha256=YIyJ1QYK6ZIa3M8yNmlbxlSplG4SMj72wCHfoE4pTUg,1088 +filelock/__init__.py,sha256=LaXgCbdErp-ugbr3o5bzpYJMBwi1hg0_hO4_vrwP1lk,5693 +filelock/__pycache__/__init__.cpython-314.pyc,, +filelock/__pycache__/_api.cpython-314.pyc,, +filelock/__pycache__/_async.cpython-314.pyc,, +filelock/__pycache__/_async_read_write.cpython-314.pyc,, +filelock/__pycache__/_descriptor.cpython-314.pyc,, +filelock/__pycache__/_error.cpython-314.pyc,, +filelock/__pycache__/_identity.cpython-314.pyc,, +filelock/__pycache__/_lease.cpython-314.pyc,, +filelock/__pycache__/_marker.cpython-314.pyc,, +filelock/__pycache__/_read_write.cpython-314.pyc,, +filelock/__pycache__/_soft.cpython-314.pyc,, +filelock/__pycache__/_soft_protocol.cpython-314.pyc,, +filelock/__pycache__/_strict.cpython-314.pyc,, +filelock/__pycache__/_unix.cpython-314.pyc,, +filelock/__pycache__/_util.cpython-314.pyc,, +filelock/__pycache__/_windows.cpython-314.pyc,, +filelock/__pycache__/asyncio.cpython-314.pyc,, +filelock/__pycache__/version.cpython-314.pyc,, +filelock/_api.py,sha256=OUugdO27gcc_gdfVOJYo0C6Bv6lvXL8Fv84vT7le378,106170 +filelock/_async.py,sha256=jSqb-YUze8cjKaNrlJ2qlMdfN9s1DZbgRMnozp4kDZs,20487 +filelock/_async_read_write.py,sha256=Ts6N_p5-UQLDoYHJKE67NfFV6ZpIcpX0cuu3S55dXzQ,17947 +filelock/_descriptor.py,sha256=Z81x98R7RbN5aHgaRzu48i7c0Ktsheze6Yg5uvWrU0Y,3277 +filelock/_error.py,sha256=sZNfFV3wacnrGCrnOipucPbnHN78NPt1IDnlg1nON1k,3235 +filelock/_identity.py,sha256=s2Kk4SQI0C5en9lHsPkWOVvGzuyY_wgZl8JRSOn63_8,17183 +filelock/_lease.py,sha256=eK0xXpuccVNM8o1BXSQByBE4TaIVYQSbSvfbbS6dReY,19173 +filelock/_marker.py,sha256=dVjalDpcpwqIRF7VUvFK2tkVZ4OCthjmmIinKy9jhFE,7896 +filelock/_read_write.py,sha256=5e0gtCHfLArSNKYH0jGdBXuD60m7stQV8KhuYgUvqQU,68071 +filelock/_soft.py,sha256=fXLlCQKHzDHaXCuwU8bXZw6qgmF1LEKRWVifxnRhRyk,33064 +filelock/_soft_protocol.py,sha256=D64pnXMlM7ysKpdx0fNXhJJgiMcS9T2Qwsu2GpPs2i0,218 +filelock/_soft_rw/__init__.py,sha256=JAMF4NA010_Jq-nEgHHZgWHZ7Of9wI3PKHg24mDdX50,452 +filelock/_soft_rw/__pycache__/__init__.cpython-314.pyc,, +filelock/_soft_rw/__pycache__/_async.cpython-314.pyc,, +filelock/_soft_rw/__pycache__/_protocol.cpython-314.pyc,, +filelock/_soft_rw/__pycache__/_storage.cpython-314.pyc,, +filelock/_soft_rw/__pycache__/_sync.cpython-314.pyc,, +filelock/_soft_rw/_async.py,sha256=v3q2adfqb9vQRwqhlctYdKZYlltJ4XutZcJ7IXuDy4M,16939 +filelock/_soft_rw/_protocol.py,sha256=O-ENv_XstRpIIDh4GnFoPNfkIg-xsDc8tVpsfBhrklA,30996 +filelock/_soft_rw/_storage.py,sha256=SbhhE8HQp7RcPo7BPnUNSEi8di2b6M7JwXHBh4EkuSc,13691 +filelock/_soft_rw/_sync.py,sha256=qVgup1GUig8fDSWLkl16z8_9UXojAcPJ_nZWl5BzIaI,37638 +filelock/_strict.py,sha256=iaEnG4QmSESjubZIhz0aq4Fq5cWmBy3STeim-8F6AZs,58570 +filelock/_unix.py,sha256=BLotMhNLFKTrW6zQ_qoI9qZGo3YqYfhGO_qP5W_p6Lw,10985 +filelock/_util.py,sha256=XW3K2JNfJapekS7YgUYBGiOFYcSf8A1cBqnKfU6jxzY,19280 +filelock/_windows.py,sha256=cDFvJN8dJtdDYjHYPt_kplvJCklD_yI7_UiaJkV-V80,38880 +filelock/asyncio.py,sha256=x3-lsFBKzEnPQR_2bQDoSD9ErdmV_3lw5rPyRq7AZc4,42865 +filelock/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +filelock/version.py,sha256=v35Uwrw-IApU2AA-2B77yCvOvNQ1Vh7gXvxfaJ9Vd14,520 diff --git a/.venv/lib/python3.14/site-packages/filelock-4.2.4.dist-info/WHEEL b/.venv/lib/python3.14/site-packages/filelock-4.2.4.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..39dc805011177e5b2a77e7afb5ec7187aeb3e34d --- /dev/null +++ b/.venv/lib/python3.14/site-packages/filelock-4.2.4.dist-info/WHEEL @@ -0,0 +1,4 @@ +Wheel-Version: 1.0 +Generator: hatchling 1.32.4 +Root-Is-Purelib: true +Tag: py3-none-any diff --git a/.venv/lib/python3.14/site-packages/filelock-4.2.4.dist-info/licenses/LICENSE b/.venv/lib/python3.14/site-packages/filelock-4.2.4.dist-info/licenses/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..291919c0b6f41d014767f6c877af9f7595fcff99 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/filelock-4.2.4.dist-info/licenses/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 Bernát Gábor and contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/.venv/lib/python3.14/site-packages/filelock/__init__.py b/.venv/lib/python3.14/site-packages/filelock/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..15a6a55e2102657d1d2778790fbfd0d592cce3e3 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/filelock/__init__.py @@ -0,0 +1,174 @@ +""" +A platform independent file lock that supports the with-statement. + +.. autodata:: filelock.__version__ + :no-value: + +.. autodata:: filelock.OwnerMode + +.. autodata:: filelock.CompromiseReason + +""" + +from __future__ import annotations + +__lazy_modules__ = ["filelock._descriptor", "filelock._error", "importlib"] + +import sys +import warnings +from importlib import import_module +from typing import TYPE_CHECKING, Final + +from filelock._api import AcquireReturnProxy, BaseFileLock, CloseErrorPolicy, ContextErrorPolicy, LockOptions +from filelock._descriptor import lock_descriptor, unlock_descriptor +from filelock._error import LeaseSettingsMismatch, SoftFileLockLifetimeWarning, SoftFileLockProtocolError, Timeout +from filelock._unix import UnixFileLock +from filelock._unix import has_fcntl as _has_fcntl +from filelock._windows import WindowsFileLock + +# Keeps ``filelock.version`` the string 4.0.12 exposed rather than the submodule. +from filelock.version import version + +if TYPE_CHECKING: + from filelock import asyncio + from filelock._async_read_write import AsyncAcquireReadWriteReturnProxy, AsyncReadWriteLock + from filelock._lease import CompromiseReason, LeaseCompromise, SoftFileLease + from filelock._marker import MarkerSoftFileLock, OwnerMode, OwnerRecord + from filelock._read_write import ReadWriteLock + from filelock._soft import SoftFileLock + from filelock._soft_rw import AsyncAcquireSoftReadWriteReturnProxy, AsyncSoftReadWriteLock, SoftReadWriteLock + from filelock._strict import StrictSoftFileClaim, StrictSoftFileClaimState, StrictSoftFileLock + from filelock.asyncio import ( + AsyncAcquireReturnProxy, + AsyncSoftFileLease, + AsyncSoftFileLock, + AsyncStrictSoftFileLock, + AsyncUnixFileLock, + AsyncWindowsFileLock, + BaseAsyncFileLock, + ) + +#: version of the project as a string +__version__: Final[str] = version + +#: ``True`` when :func:`fcntl.flock` is available. Off Windows, :data:`FileLock` then resolves to +#: :class:`UnixFileLock`, otherwise to :class:`SoftFileLock`. Always ``False`` on Windows. +#: +#: :meta hide-value: +has_fcntl: Final[bool] = _has_fcntl + + +if sys.platform == "win32": # pragma: win32 cover + _FileLock: type[BaseFileLock] = WindowsFileLock +else: # pragma: win32 no cover # ruff:ignore[collapsible-else-if] # the else carries the win32 no-cover pragma + if has_fcntl: + _FileLock: type[BaseFileLock] = UnixFileLock + else: + from filelock._soft import SoftFileLock + + _FileLock = SoftFileLock + warnings.warn("only soft file lock is available", stacklevel=2) + +if TYPE_CHECKING: + FileLock = SoftFileLock + AsyncFileLock = AsyncSoftFileLock +else: + #: Alias for the lock, which should be used for the current platform. + FileLock = _FileLock + +# Importing asyncio and sqlite3 costs more than the rest of the package, so names that need them load on first access. +_LAZY_ATTRIBUTES: Final[dict[str, str]] = { + "AsyncAcquireReadWriteReturnProxy": "._async_read_write", + "AsyncAcquireReturnProxy": ".asyncio", + "AsyncAcquireSoftReadWriteReturnProxy": "._soft_rw", + "AsyncFileLock": ".asyncio", + "AsyncReadWriteLock": "._async_read_write", + "AsyncSoftFileLease": ".asyncio", + "AsyncSoftFileLock": ".asyncio", + "AsyncSoftReadWriteLock": "._soft_rw", + "AsyncStrictSoftFileLock": ".asyncio", + "AsyncUnixFileLock": ".asyncio", + "AsyncWindowsFileLock": ".asyncio", + "BaseAsyncFileLock": ".asyncio", + "CompromiseReason": "._lease", + "LeaseCompromise": "._lease", + "MarkerSoftFileLock": "._marker", + "OwnerMode": "._marker", + "OwnerRecord": "._marker", + "ReadWriteLock": "._read_write", + "SoftFileLease": "._lease", + "SoftFileLock": "._soft", + "SoftReadWriteLock": "._soft_rw", + "StrictSoftFileClaim": "._strict", + "StrictSoftFileClaimState": "._strict", + "StrictSoftFileLock": "._strict", + "asyncio": ".asyncio", +} + + +def __getattr__(name: str) -> object: + if (module_name := _LAZY_ATTRIBUTES.get(name)) is None: + msg = f"module {__name__!r} has no attribute {name!r}" + raise AttributeError(msg) + try: + module = import_module(module_name, __name__) + except ImportError: # pragma: lacks sqlite3 + if module_name not in {"._read_write", "._async_read_write"}: + raise + value = None + else: + if name == "asyncio": + value = module + else: + value = getattr(module, f"Async{_FileLock.__name__}" if name == "AsyncFileLock" else name) + globals()[name] = value + return value + + +def __dir__() -> list[str]: + return sorted({*globals(), *_LAZY_ATTRIBUTES}) + + +__all__ = [ + "AcquireReturnProxy", + "AsyncAcquireReadWriteReturnProxy", + "AsyncAcquireReturnProxy", + "AsyncAcquireSoftReadWriteReturnProxy", + "AsyncFileLock", + "AsyncReadWriteLock", + "AsyncSoftFileLease", + "AsyncSoftFileLock", + "AsyncSoftReadWriteLock", + "AsyncStrictSoftFileLock", + "AsyncUnixFileLock", + "AsyncWindowsFileLock", + "BaseAsyncFileLock", + "BaseFileLock", + "CloseErrorPolicy", + "CompromiseReason", + "ContextErrorPolicy", + "FileLock", + "LeaseCompromise", + "LeaseSettingsMismatch", + "LockOptions", + "MarkerSoftFileLock", + "OwnerMode", + "OwnerRecord", + "ReadWriteLock", + "SoftFileLease", + "SoftFileLock", + "SoftFileLockLifetimeWarning", + "SoftFileLockProtocolError", + "SoftReadWriteLock", + "StrictSoftFileClaim", + "StrictSoftFileClaimState", + "StrictSoftFileLock", + "Timeout", + "UnixFileLock", + "WindowsFileLock", + "__version__", + "asyncio", + "has_fcntl", + "lock_descriptor", + "unlock_descriptor", +] diff --git a/.venv/lib/python3.14/site-packages/filelock/__pycache__/__init__.cpython-314.pyc b/.venv/lib/python3.14/site-packages/filelock/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1b9735b2216eede81b0baa248ddd37ce79d45d6b Binary files /dev/null and b/.venv/lib/python3.14/site-packages/filelock/__pycache__/__init__.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/filelock/__pycache__/_api.cpython-314.pyc b/.venv/lib/python3.14/site-packages/filelock/__pycache__/_api.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c72646091c70e3d95eb06174991cf2d5fd2086f8 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/filelock/__pycache__/_api.cpython-314.pyc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77deb712aa05dd38c93603f3ee8c056bed8e15896c4e3791c3151699c8b3c31b +size 133516 diff --git a/.venv/lib/python3.14/site-packages/filelock/__pycache__/_async.cpython-314.pyc b/.venv/lib/python3.14/site-packages/filelock/__pycache__/_async.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..50e98e4c1496f935c8bb47b8a5912b4734676a10 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/filelock/__pycache__/_async.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/filelock/__pycache__/_async_read_write.cpython-314.pyc b/.venv/lib/python3.14/site-packages/filelock/__pycache__/_async_read_write.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ed7cfe55730f62b1560eba8c1c76458b0b3383af Binary files /dev/null and b/.venv/lib/python3.14/site-packages/filelock/__pycache__/_async_read_write.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/filelock/__pycache__/_descriptor.cpython-314.pyc b/.venv/lib/python3.14/site-packages/filelock/__pycache__/_descriptor.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b0cb3bd3925b3ed14e77b698fed153103db8670a Binary files /dev/null and b/.venv/lib/python3.14/site-packages/filelock/__pycache__/_descriptor.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/filelock/__pycache__/_error.cpython-314.pyc b/.venv/lib/python3.14/site-packages/filelock/__pycache__/_error.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6aa83bd92d41fa1e14f83ab44a94be031f2695d3 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/filelock/__pycache__/_error.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/filelock/__pycache__/_identity.cpython-314.pyc b/.venv/lib/python3.14/site-packages/filelock/__pycache__/_identity.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f8275627ce246ec36e1696a3aaf9cbde7f32e367 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/filelock/__pycache__/_identity.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/filelock/__pycache__/_lease.cpython-314.pyc b/.venv/lib/python3.14/site-packages/filelock/__pycache__/_lease.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0a38a45a31ef5ae5ba9ea11906b5f685754273ab Binary files /dev/null and b/.venv/lib/python3.14/site-packages/filelock/__pycache__/_lease.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/filelock/__pycache__/_marker.cpython-314.pyc b/.venv/lib/python3.14/site-packages/filelock/__pycache__/_marker.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ab307df637811adffdb3127750f67dc9d30663cb Binary files /dev/null and b/.venv/lib/python3.14/site-packages/filelock/__pycache__/_marker.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/filelock/__pycache__/_read_write.cpython-314.pyc b/.venv/lib/python3.14/site-packages/filelock/__pycache__/_read_write.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5fff576f83f0f248b364ee7279c59b4df6a3d892 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/filelock/__pycache__/_read_write.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/filelock/__pycache__/_soft.cpython-314.pyc b/.venv/lib/python3.14/site-packages/filelock/__pycache__/_soft.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e29f4edb10c662491575d862441974b5c1bb981b Binary files /dev/null and b/.venv/lib/python3.14/site-packages/filelock/__pycache__/_soft.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/filelock/__pycache__/_soft_protocol.cpython-314.pyc b/.venv/lib/python3.14/site-packages/filelock/__pycache__/_soft_protocol.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b040a7d13002278cd5efa4e3e7a46a8073d936b9 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/filelock/__pycache__/_soft_protocol.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/filelock/__pycache__/_strict.cpython-314.pyc b/.venv/lib/python3.14/site-packages/filelock/__pycache__/_strict.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..00412c5a3f71e0f7fceeefedd603fc47cd43eb40 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/filelock/__pycache__/_strict.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/filelock/__pycache__/_unix.cpython-314.pyc b/.venv/lib/python3.14/site-packages/filelock/__pycache__/_unix.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c2fc2318b4c0a35c550e00bb96dcb1a8881897e1 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/filelock/__pycache__/_unix.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/filelock/__pycache__/_util.cpython-314.pyc b/.venv/lib/python3.14/site-packages/filelock/__pycache__/_util.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..db73b32a26a0219312cc9943b32a71c80012d4f2 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/filelock/__pycache__/_util.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/filelock/__pycache__/_windows.cpython-314.pyc b/.venv/lib/python3.14/site-packages/filelock/__pycache__/_windows.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..18900da1c1b6e72081c4e9bc9f64c0c9759c6ce0 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/filelock/__pycache__/_windows.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/filelock/__pycache__/asyncio.cpython-314.pyc b/.venv/lib/python3.14/site-packages/filelock/__pycache__/asyncio.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d338c5838f00a68f681a2d93082cf86386517e14 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/filelock/__pycache__/asyncio.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/filelock/__pycache__/version.cpython-314.pyc b/.venv/lib/python3.14/site-packages/filelock/__pycache__/version.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a81585d9936e94c3afd3895baeaff2b25492c4ab Binary files /dev/null and b/.venv/lib/python3.14/site-packages/filelock/__pycache__/version.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/filelock/_api.py b/.venv/lib/python3.14/site-packages/filelock/_api.py new file mode 100644 index 0000000000000000000000000000000000000000..3dbf52ccbeb70abc99e4f43b1e0f1cf039560bfc --- /dev/null +++ b/.venv/lib/python3.14/site-packages/filelock/_api.py @@ -0,0 +1,2266 @@ +from __future__ import annotations + +__lazy_modules__ = [ + "atexit", + "decimal", + "filelock._error", + "filelock._util", + "inspect", + "math", + "numbers", + "secrets", + "types", + "warnings", +] + +import atexit +import contextlib +import inspect +import logging +import math +import os +import secrets +import stat +import sys +import time +import types +import warnings +from abc import ABCMeta, abstractmethod +from collections.abc import Callable, Hashable +from contextlib import contextmanager +from dataclasses import dataclass, fields +from decimal import Decimal +from functools import cache, partial +from itertools import count, starmap +from numbers import Real +from threading import TIMEOUT_MAX, Condition, RLock, Thread, current_thread, get_ident, local +from typing import TYPE_CHECKING, Final, Generic, Literal, NoReturn, TypedDict, TypeVar, cast +from weakref import WeakKeyDictionary, WeakValueDictionary, ref + +from filelock._error import SoftFileLockLifetimeWarning, Timeout +from filelock._util import ensure_directory_exists + +#: No explicit file permission mode was passed. Lock files then open with 0o666 so umask and default ACLs pick +#: the final permissions, and fchmod is skipped to preserve POSIX default ACL inheritance. +_UNSET_FILE_MODE: Final[int] = -1 + +_PERMISSION_BITS: Final[int] = stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO + +#: Ceiling on the retry counter used as a power of two, so a long contended wait cannot overflow the backoff multiply. +_MAX_BACKOFF_EXPONENT: Final[int] = 20 + +#: How a context manager reconciles a body failure with a release failure on exit (see the property of this name). +ContextErrorPolicy = Literal["chain", "group"] +_CONTEXT_ERROR_POLICIES: Final[frozenset[str]] = frozenset({"chain", "group"}) + +#: What a descriptor-owning backend does with an ``os.close`` failure after relinquishing ownership (see the property). +CloseErrorPolicy = Literal["default", "raise", "suppress"] +_CLOSE_ERROR_POLICIES: Final[frozenset[str]] = frozenset({"default", "raise", "suppress"}) + +if TYPE_CHECKING: + from collections.abc import Generator, Mapping + from threading import Lock + from types import FrameType, TracebackType + from typing import Protocol + + from _typeshed import Unused + + from filelock._lease import _LeaseClaim + from filelock._read_write import ReadWriteLock + from filelock._soft_rw import SoftReadWriteLock + + class _ForkResettable(Protocol): + def _reset_after_fork_in_child(self) -> None: ... + + class _ForkDescriptorOwner(Protocol): + def _descriptors_for_fork(self) -> tuple[tuple[int, tuple[int, int] | None], ...]: ... + + # Matched against the class object itself rather than `type[...]` of it. A metaclass supplies this method to the + # class while leaving instances without it, so a `type[_ForkResettableClass]` bound rejects `ReadWriteLock`. + class _ForkResettableClass(Protocol): + def _reset_class_after_fork(self) -> None: ... + + class _RegisterAtFork(Protocol): + def __call__( + self, + *, + before: Callable[[], None] | None = None, + after_in_parent: Callable[[], None] | None = None, + after_in_child: Callable[[], None] | None = None, + ) -> None: ... + + from typing import Self + +_LOGGER: Final[logging.Logger] = logging.getLogger("filelock") +_REGISTER_AT_FORK: Final[_RegisterAtFork | None] = cast("_RegisterAtFork | None", getattr(os, "register_at_fork", None)) +_HAS_REGISTER_AT_FORK: Final[bool] = _REGISTER_AT_FORK is not None + +_ExtraValue = TypeVar("_ExtraValue") +_MarkerValue = TypeVar("_MarkerValue") +_SubclassValue = TypeVar("_SubclassValue") +_LockInitValue = float | int | bool | str | None | Callable[[int], None] + + +class LockOptions(TypedDict, total=False): + """Every option the metaclass forwards, so a subclass adding its own can still type what it passes through.""" + + timeout: float + mode: int + thread_local: bool + blocking: bool + is_singleton: bool + poll_interval: float + lifetime: float | None + context_error_policy: ContextErrorPolicy + close_error_policy: CloseErrorPolicy + fallback_to_soft: bool + preserve_lock_file: bool + on_acquired: Callable[[int], None] | None + + +def _is_exception_group(error: BaseException) -> bool: + return isinstance(error, BaseExceptionGroup) + + +def _raise_grouped_errors( + message: str, + first_error: BaseException, + second_error: BaseException, + *additional_errors: BaseException, + marker: tuple[str, _MarkerValue] | None = None, +) -> NoReturn: + errors = (first_error, second_error, *additional_errors) + _detach_grouped_contexts(errors) + group: Final = BaseExceptionGroup(message, errors) + if marker is not None: + setattr(group, marker[0], marker[1]) + raise group from None + + +def _detach_grouped_contexts(errors: tuple[BaseException, ...]) -> None: + seen: set[int] = set() + pending = list(errors) + while pending: + error = pending.pop() + if id(error) in seen: + continue + seen.add(id(error)) + if (context := error.__context__) is not None and ( + context is error + or _same_exception_tree(error, context) + or any(context is root or _contains_exception(root, context) for root in errors) + ): + error.__context__ = None + elif context is not None: + pending.append(context) + if error.__cause__ is not None: + pending.append(error.__cause__) + if _is_exception_group(error): + pending.extend(cast("_ExceptionGroupProtocol", error).exceptions) + + +def _same_exception_tree(first: BaseException, second: BaseException) -> bool: + pending = [(first, second)] + seen: set[tuple[int, int]] = set() + while pending: + first_error, second_error = pending.pop() + if first_error is second_error: + continue + if (pair := (id(first_error), id(second_error))) in seen: + continue + seen.add(pair) + if ( + type(first_error) is not type(second_error) + or not _is_exception_group(first_error) + or not _is_exception_group(second_error) + ): + return False + first_group = cast("_ExceptionGroupProtocol", first_error) + second_group = cast("_ExceptionGroupProtocol", second_error) + if first_group.message != second_group.message or len(first_group.exceptions) != len(second_group.exceptions): + return False + pending.extend(zip(first_group.exceptions, second_group.exceptions, strict=True)) + return True + + +def _contains_exception(error: BaseException, target: BaseException | None) -> bool: + if target is None or not _is_exception_group(error): + return False + pending = list(cast("_ExceptionGroupProtocol", error).exceptions) + seen: set[int] = set() + while pending: + child = pending.pop() + if child is target: + return True + if id(child) in seen: + continue + seen.add(id(child)) + if _is_exception_group(child): + pending.extend(cast("_ExceptionGroupProtocol", child).exceptions) + return False + + +def _append_exception_context(error: BaseException, context: BaseException) -> None: + if _exception_graph_contains(error, context) or _exception_graph_contains(context, error): + return + if error.__context__ is None: + error.__context__ = context + return + tail = error + seen: set[int] = set() + while id(tail) not in seen: + seen.add(id(tail)) + if (next_error := tail.__cause__ if tail.__cause__ is not None else tail.__context__) is None: + tail.__context__ = context + return + tail = next_error + + +def _exception_graph_contains(error: BaseException, target: BaseException) -> bool: + pending = [error] + seen: set[int] = set() + while pending: + current = pending.pop() + if current is target: + return True + if id(current) in seen: # pragma: no cover - arbitrary caller exceptions can contain cycles + continue + seen.add(id(current)) + if current.__cause__ is not None: + pending.append(current.__cause__) + if current.__context__ is not None: + pending.append(current.__context__) + if _is_exception_group(current): + pending.extend(cast("_ExceptionGroupProtocol", current).exceptions) + return False + + +def _grouped_errors( + error: BaseException, message: str, marker: tuple[str, _MarkerValue] +) -> tuple[BaseException, ...] | None: + if not _is_exception_group(error): + return None + group = cast("_ExceptionGroupProtocol", error) + return group.exceptions if group.message == message and getattr(group, marker[0], None) is marker[1] else None + + +if TYPE_CHECKING: + + class _ExceptionGroupProtocol(Protocol): + @property + def message(self) -> str: ... + + @property + def exceptions(self) -> tuple[BaseException, ...]: ... + + +def _raise_chained_errors(first_error: BaseException, second_error: BaseException | None = None) -> NoReturn: + if second_error is None: + first_context = first_error.__context__ + try: + raise first_error # ruff:ignore[raise-within-try] # the handler restores caller-supplied context before propagation + except BaseException: + first_error.__context__ = first_context + raise + if (second_context := second_error.__context__) is not None and second_context is not first_error: + _detach_exception_context(second_context, first_error) + _append_exception_context(first_error, second_context) + first_context = first_error.__context__ + try: + raise first_error # ruff:ignore[raise-within-try] # the second raise needs this error as implicit context + except BaseException: # ruff:ignore[blind-except] # first_error may be a control-flow exception + first_error.__context__ = first_context + try: + raise second_error # ruff:ignore[raise-within-try] # the handler makes the chain interpreter-independent + except BaseException: + second_error.__context__ = first_error + first_error.__context__ = first_context + raise + + +def _detach_exception_context(error: BaseException, target: BaseException) -> None: + pending = [error] + seen: set[int] = set() + while pending: + current = pending.pop() + if id(current) in seen: + continue + seen.add(id(current)) + if current.__context__ is target: + current.__context__ = None + elif current.__context__ is not None: + pending.append(current.__context__) + if current.__cause__ is not None: + pending.append(current.__cause__) + if _is_exception_group(current): + pending.extend(cast("_ExceptionGroupProtocol", current).exceptions) + + +def _raise_body_and_release(body_error: BaseException, release_error: BaseException) -> NoReturn: + # Group mode: surface the body failure and the release failure as sibling leaves instead of letting one hide in the + # other's __context__. BaseExceptionGroup returns a plain ExceptionGroup when both leaves subclass Exception, so + # ``except*`` and ``except Exception`` still catch them; a BaseException leaf (KeyboardInterrupt, CancelledError) + # keeps the group outside ordinary handlers. ``from None`` stops the group itself gaining a redundant __context__. + _raise_grouped_errors("lock body and release both failed", body_error, release_error) + + +def _raise_cleanup_errors( + message: str, + primary_error: BaseException, + *cleanup_errors: BaseException | None, +) -> NoReturn: + _raise_grouped_errors( + message, + primary_error, + *(error for error in cleanup_errors if error is not None), + ) + + +# On Windows os.path.realpath calls CreateFileW with share_mode=0, which blocks concurrent DeleteFileW and causes +# livelocks under threaded contention with SoftFileLock. os.path.abspath is purely string-based and avoids this. +_resolve_dir: Final[Callable[[str], str]] = os.path.abspath if sys.platform == "win32" else os.path.realpath + + +def _canonical(path: str | os.PathLike[str]) -> str: + """ + Return one stable key for *path*, collapsing equivalent spellings without following a final symlink. + + Relative, absolute, and ``./`` spellings of one lock file must map to a single singleton instance, deadlock-registry + entry, and removal key. Resolving the whole path with ``realpath`` would follow a final symlink and alias a lock + target the backend deliberately rejects, so the registry identity would differ from the backend's. Resolving only + the parent directory and re-appending the literal final component collapses the equivalent spellings while keeping a + final symlink a distinct key. On Windows the parent is resolved with ``abspath`` so junctions and reparse points are + not followed either. + """ + parent, name = os.path.split(os.fspath(path)) + return os.path.join(_resolve_dir(parent or os.curdir), name) # ruff:ignore[os-path-join] # string join matches abspath/realpath + + +class _ThreadLocalRegistry(local): + def __init__(self) -> None: + super().__init__() + self.held: dict[Hashable, int] = {} + + +_registry: Final[_ThreadLocalRegistry] = _ThreadLocalRegistry() + + +_T = TypeVar("_T", bound="BaseFileLock") +_LockT = TypeVar("_LockT", bound="BaseFileLock | ReadWriteLock | SoftReadWriteLock") + + +class FileLockMeta(ABCMeta): + _instances: WeakValueDictionary[str, BaseFileLock] + _instances_lock: RLock + _instances_under_construction: set[str] + + def __call__( # ruff:ignore[too-many-arguments] # forwards the public constructor's documented parameters + cls: type[_T], + lock_file: str | os.PathLike[str], + timeout: float = -1, + mode: int = _UNSET_FILE_MODE, + thread_local: bool = True, # ruff:ignore[boolean-type-hint-positional-argument, boolean-default-value-positional-argument] # public API: positional bool kept for backwards compatibility + *, + blocking: bool = True, + is_singleton: bool = False, + poll_interval: float = 0.05, + lifetime: float | None = None, + context_error_policy: ContextErrorPolicy = "chain", + close_error_policy: CloseErrorPolicy = "default", + fallback_to_soft: bool = True, + preserve_lock_file: bool = False, + on_acquired: Callable[[int], None] | None = None, + **kwargs: _ExtraValue, + ) -> _T: + _ensure_current_process() + _resolve_lazy_imports_before_shutdown() + lifetime = _resolve_lifetime(lifetime, cls, stacklevel=cls._constructor_lifetime_warning_stacklevel) + poll_interval = _resolve_poll_interval(poll_interval) + # Validate before building the instance: a raise inside __init__ would leave a half-constructed object whose + # __del__ then trips over the missing context. + _check_blocking(blocking) + # nan is left to acquire, which knows whether the call blocks; a non-blocking call ignores the timeout. + timeout = _seconds("timeout", timeout) + _check_mode(mode, cls.__name__) + context_error_policy = _resolve_context_error_policy(context_error_policy) + close_error_policy = _resolve_close_error_policy(close_error_policy) + preserve_lock_file = _resolve_preserve_lock_file( + preserve=preserve_lock_file, supported=cls._preserve_lock_file_supported, cls_name=cls.__name__ + ) + on_acquired = _resolve_on_acquired(on_acquired, supported=cls._on_acquired_supported, cls_name=cls.__name__) + # Checked before the singleton lookup so a cache hit rejects a misspelled option just like a fresh build does. + params: Final = _supported_init_params( + cls, + { + "timeout": timeout, + "mode": mode, + "thread_local": thread_local, + "blocking": blocking, + "is_singleton": is_singleton, + "poll_interval": poll_interval, + "lifetime": lifetime, + "context_error_policy": context_error_policy, + "close_error_policy": close_error_policy, + "fallback_to_soft": fallback_to_soft, + "preserve_lock_file": preserve_lock_file, + "on_acquired": on_acquired, + **kwargs, + }, + ) + if not is_singleton: + return super().__call__(lock_file, **params) + + # Look up, build and store under one lock. Without it two threads racing the first construction for a + # path both miss the cache and each build their own instance, so callers relying on is_singleton for + # reentrant locking across instances end up with two "singletons" and acquire()'s deadlock check then + # rejects a legitimate reentrant acquire; the unguarded writes to the WeakValueDictionary are a data + # race besides. ReadWriteLock and SoftReadWriteLock already guard their singleton caches this way. + # Key the cache on the canonical form so equivalent spellings of one path share a singleton, and it matches the + # deadlock-registry key acquire() uses. + singleton_key = _canonical(lock_file) + with cls._instances_lock: + # A cached lock resolves its relative path at each acquisition and pins it for the hold. After a chdir, or + # once a parent symlink points elsewhere, an unheld lock may name another file than this key; a held one + # names the file it pinned. + if (instance := cls._instances.get(singleton_key)) is None or _canonical( + instance._lock_path # ruff:ignore[private-member-access] # the metaclass owns the instance it returns + ) != singleton_key: + if singleton_key in cls._instances_under_construction: # pragma: needs fork + msg = f"Singleton lock construction is already active for {lock_file!s}" + raise RuntimeError(msg) + construction_registry = cls._instances_under_construction + construction_pid = os.getpid() + construction_registry.add(singleton_key) + try: + instance = super().__call__(lock_file, **params) + finally: + construction_registry.discard(singleton_key) + if os.getpid() != construction_pid: # pragma: needs fork + msg = "Lock construction cannot continue after fork; construct a new lock in the child" + raise RuntimeError(msg) + cls._instances[singleton_key] = instance + instance._singleton_key = singleton_key # ruff:ignore[private-member-access] # the cache entry the finalizer looks up + return instance + + non_matching_params = { + name: (str(passed_param), str(set_param)) + for name, (passed_param, set_param) in { + "thread_local": (thread_local, instance.is_thread_local()), + # Compare the wait, not the value: nan never equals itself, and -1, -5 and inf all mean no limit. + "timeout": ( + _timeout_behavior(timeout, any_negative=True), + _timeout_behavior(instance.timeout, any_negative=True), + ), + "mode": (mode, instance._context.mode), # ruff:ignore[private-member-access] # compares against the managed instance's own context + "blocking": (blocking, instance.blocking), + "poll_interval": (poll_interval, instance.poll_interval), + "lifetime": (lifetime, instance.lifetime), + "context_error_policy": (context_error_policy, instance.context_error_policy), + "close_error_policy": (close_error_policy, instance.close_error_policy), + "fallback_to_soft": (fallback_to_soft, instance.fallback_to_soft), + "preserve_lock_file": (preserve_lock_file, instance.preserve_lock_file), + }.items() + if passed_param != set_param + } + # The metaclass owns the instance it returns, so reading the subclass hook stays inside this module's contract. + non_matching_params.update(instance._singleton_extra_mismatches(kwargs)) # ruff:ignore[private-member-access] + # Callables compare by identity, not equality: two equal callables can close over different state, so a + # singleton must reject a different hook object even if it compares equal. Keep it out of the scalar dict above. + hook_mismatch = on_acquired is not instance.on_acquired + if not non_matching_params and not hook_mismatch: + return instance # ty: ignore[invalid-return-type] # https://github.com/astral-sh/ty/issues/3231 + + msg = "Singleton lock instances cannot be initialized with differing arguments" + msg += "\nNon-matching arguments: " + for param_name, (passed_param, set_param) in non_matching_params.items(): + msg += f"\n\t{param_name} (existing lock has {set_param} but {passed_param} was passed)" + if hook_mismatch: + msg += f"\n\ton_acquired (existing lock has {instance.on_acquired} but {on_acquired} was passed)" + raise ValueError(msg) + + +def _resolve_lifetime(lifetime: float | None, cls: type[BaseFileLock], *, stacklevel: int) -> float | None: + """ + Validate ``lifetime`` and drop a value the backend cannot honor. + + ``lifetime`` is a deliberate age-based lease: a lock file older than ``lifetime`` is broken even while its holder is + still alive. Existence locks (:class:`SoftFileLock`) implement that behavior by unlinking a reclaimable pathname, + which can overlap a live holder. A native OS lock lives on the inode, so unlinking the pathname by age cannot revoke + the kernel lock; a contender would lock a fresh inode and overlap the live holder (#590). Ignore the request with a + warning rather than accept a setting that breaks mutual exclusion. + """ + if lifetime is None: + return None + # An int stays exact: no float holds every int 4.0.12 accepted here. + if isinstance(lifetime, bool) or not isinstance(lifetime, int): + lifetime = _seconds("lifetime", lifetime) + if lifetime < 0 or (isinstance(lifetime, float) and not math.isfinite(lifetime)): + msg = f"lifetime must be finite and non-negative, not {lifetime!r}" + raise ValueError(msg) + if not cls._lifetime_supported: + warnings.warn( + f"lifetime is ignored for {cls.__name__}: {cls._lifetime_unsupported_reason}; " + f"only SoftFileLock supports lifetime-based expiry", + stacklevel=stacklevel, + ) + return None + if cls._lifetime_replacements is not None: + strict_lock, lease = cls._lifetime_replacements + warnings.warn( + f"{cls.__name__}(lifetime=...) uses age-based expiry and can overlap a live holder; " + f"use {lease} for expiry or {strict_lock} for fail-closed locking", + SoftFileLockLifetimeWarning, + stacklevel=stacklevel, + ) + return lifetime + + +def _resolve_poll_interval(poll_interval: float) -> float: + """Validate ``poll_interval`` used for ``time.sleep`` / ``asyncio.sleep`` retries.""" + if (poll_interval := _seconds("poll_interval", poll_interval)) < 0 or not math.isfinite(poll_interval): + msg = f"poll_interval must be finite and non-negative, not {poll_interval!r}" + raise ValueError(msg) + _check_timeout_max("poll_interval", poll_interval) + return poll_interval + + +def _check_blocking(blocking: bool) -> None: # ruff:ignore[boolean-type-hint-positional-argument] # validates the option's own value + if not isinstance(blocking, bool): + msg = f"blocking must be a bool, not {type(blocking).__name__}" + raise TypeError(msg) + + +def _seconds(name: str, value: float) -> float: + # 4.0.12 accepted Decimal and Fraction; a bool is a typo, and a string would only fail later. + if isinstance(value, bool) or not isinstance(value, (Real, Decimal)): + msg = f"{name} must be a number of seconds, not {type(value).__name__}" + raise TypeError(msg) + # float() refuses a signaling NaN, which means no number of seconds just as a quiet one does. + if isinstance(value, Decimal) and value.is_snan(): + return math.nan + try: + seconds: Final = float(value) + except OverflowError: + msg = f"{name} is too large to be a number of seconds: {value!r}" + raise ValueError(msg) from None + # float() rounds a tiny negative to -0.0, which compares as zero, yet timeout and lifetime checks read the sign. + return -math.ulp(0.0) if seconds == 0 and value < 0 else seconds + + +def _check_mode(mode: int, cls_name: str) -> None: + if mode == _UNSET_FILE_MODE: + return + if isinstance(mode, bool) or not isinstance(mode, int): + msg = f"mode must be an int, not {type(mode).__name__}" + raise TypeError(msg) + # A lock file is never executed, so setuid, setgid and sticky have no meaning on it and only widen its permissions. + if not 0 <= mode <= _PERMISSION_BITS: + msg = f"mode={mode:#o} must be within 0o000-0o777 for {cls_name}" + raise ValueError(msg) + # A lock reopens, reads, or deletes the files it creates, so a mode that denies the owner read or write fails later + # and for good. + if ~mode & (stat.S_IRUSR | stat.S_IWUSR): + msg = f"mode={mode:#o} must grant the owner read and write for {cls_name}" + raise ValueError(msg) + + +def _resolve_context_error_policy(policy: str) -> ContextErrorPolicy: + if policy not in _CONTEXT_ERROR_POLICIES: + msg = f"context_error_policy must be 'chain' or 'group', got {policy!r}" + raise ValueError(msg) + return cast("ContextErrorPolicy", policy) + + +def _resolve_close_error_policy(policy: str) -> CloseErrorPolicy: + if policy not in _CLOSE_ERROR_POLICIES: + msg = f"close_error_policy must be 'default', 'raise', or 'suppress', got {policy!r}" + raise ValueError(msg) + return cast("CloseErrorPolicy", policy) + + +def _resolve_preserve_lock_file(*, preserve: bool, supported: bool, cls_name: str) -> bool: + # An existence lock unlinks its marker to release, so preserving the pathname would defeat unlocking. Reject the + # request rather than silently ignore it, since a caller asking for a stable identity must know it cannot be kept. + if preserve and not supported: + msg = f"preserve_lock_file=True is not supported by {cls_name}: unlinking its marker is how it releases" + raise ValueError(msg) + return preserve + + +def _resolve_on_acquired( + on_acquired: Callable[[int], None] | None, *, supported: bool, cls_name: str +) -> Callable[[int], None] | None: + if on_acquired is None: + return None + # An existence lock stores protocol state in its marker, so a caller writing through the descriptor would corrupt + # stale detection and ownership metadata; only native locks lend out the descriptor. + if not supported: + msg = f"on_acquired is not supported by {cls_name}: only native locks expose the lock descriptor" + raise ValueError(msg) + return on_acquired + + +def _supported_init_params( + cls: type[BaseFileLock], params: dict[str, _LockInitValue | _ExtraValue] +) -> dict[str, _LockInitValue | _ExtraValue]: + if (model := _init_parameter_model(cls)).accepts_kwargs: + return params + if unsupported := sorted( + name + for name, value in params.items() + if name not in model.accepted_params + and ((parameter := model.default_params.get(name)) is None or value != parameter.default) + ): + msg = f"{cls.__name__} does not support non-default lock options: {', '.join(unsupported)}" + raise TypeError(msg) + # virtualenv narrows a BaseFileLock descendant's signature; omit base defaults it does not accept (#340). + return {name: value for name, value in params.items() if name in model.accepted_params} + + +def _timeout_behavior(timeout: float, *, any_negative: bool) -> float | str: + # inf, and for a BaseFileLock any negative value, wait without a limit like -1. + if math.isnan(timeout): + return "nan" + return -1.0 if timeout == math.inf or (any_negative and timeout < 0) else timeout + + +def _check_timeout_max(name: str, value: float) -> None: + # Lock.acquire, Event.wait and time.sleep raise OverflowError past this bound, which is only ~49.7 days on Windows. + if value > TIMEOUT_MAX: + msg = f"{name} must not exceed threading.TIMEOUT_MAX ({TIMEOUT_MAX}), got {value!r}" + raise ValueError(msg) + + +_INIT_PARAMETER_MODELS: Final[WeakKeyDictionary[type[BaseFileLock], _InitParameterModel]] = WeakKeyDictionary() + + +def _init_parameter_model(cls: type[BaseFileLock]) -> _InitParameterModel: + # A strong cache would keep dynamically created subclasses alive for the process lifetime. + with _fork_transition(), _FORK_STATE.parameter_models_lock: + if (model := _INIT_PARAMETER_MODELS.get(cls)) is None: + parameters = inspect.signature(cls.__init__).parameters.values() + model = _InitParameterModel( + accepted_params=frozenset( + parameter.name + for parameter in parameters + if parameter.kind in {inspect.Parameter.POSITIONAL_OR_KEYWORD, inspect.Parameter.KEYWORD_ONLY} + ), + accepts_kwargs=any(parameter.kind is inspect.Parameter.VAR_KEYWORD for parameter in parameters), + default_params={ + name: parameter + for name, parameter in inspect.signature(type(cls).__call__).parameters.items() + if parameter.default is not inspect.Parameter.empty + }, + ) + _INIT_PARAMETER_MODELS[cls] = model + return model + + +@dataclass(frozen=True) +class _InitParameterModel: + accepted_params: frozenset[str] + accepts_kwargs: bool + default_params: dict[str, inspect.Parameter] + + +class BaseFileLock(contextlib.ContextDecorator, metaclass=FileLockMeta): # ruff:ignore[too-many-public-methods] # public config properties + """ + Abstract base class for a file lock object. + + Provides the common reentrant API and state management. Subclasses implement the locking mechanism + (:class:`UnixFileLock `, :class:`WindowsFileLock `, + :class:`SoftFileLock `). + + """ + + _instances: WeakValueDictionary[str, BaseFileLock] + _instances_lock: RLock + _instances_under_construction: set[str] + + #: How the cross-instance deadlock message names the conflicting holder; the async subclass says "task". + _deadlock_holder_desc: str = "FileLock instance in this thread" + + #: Whether an age-based :attr:`lifetime` lease may break this lock. Only existence locks set it (they reclaim by + #: unlinking a pathname); native OS locks leave it ``False`` since a kernel lock cannot be revoked by file age. + _lifetime_supported: bool = False + + #: Strict-lock and lease replacements for a backend with legacy age-based expiry. + _lifetime_replacements: tuple[str, str] | None = None + + #: Why a backend that refuses ``lifetime`` cannot honor it, named in the warning that drops the value. + _lifetime_unsupported_reason: str = "a native OS lock cannot be broken safely by file age" + + #: Async construction adds one metaclass frame before lifetime validation. + _constructor_lifetime_warning_stacklevel: int = 3 + + #: Whether :attr:`preserve_lock_file` may be ``True``. Native locks keep the pathname on release, so they support + #: it; existence locks unlink their marker to release and reject it. + _preserve_lock_file_supported: bool = True + + #: Whether an :attr:`on_acquired` hook may be set. Native locks lend the descriptor out; existence locks keep + #: protocol state in the marker and reject it. + _on_acquired_supported: bool = True + + #: Ceiling in seconds on the jittered backoff between contended acquisition retries. ``0`` keeps the fixed + #: poll cadence; a multi-file backend sets it so contending processes desynchronize instead of livelocking. + _poll_backoff_cap: float = 0.0 + + #: The singleton cache key of this instance, or ``None`` outside the cache. + _singleton_key: str | None = None + + def __init_subclass__(cls, **kwargs: _SubclassValue) -> None: + """Give each lock subclass its own singleton registry and lock.""" + super().__init_subclass__(**kwargs) + cls._instances = WeakValueDictionary() + cls._instances_lock = RLock() + cls._instances_under_construction = set() + _register_fork_class(cls) + + @classmethod + def _reset_class_after_fork(cls) -> None: # pragma: forked child + cls._instances = WeakValueDictionary() + cls._instances_lock = RLock() + cls._instances_under_construction = set() + + def __init__( # ruff:ignore[too-many-arguments] # public constructor: one parameter per documented lock option + self, + lock_file: str | os.PathLike[str], + timeout: float = -1, + mode: int = _UNSET_FILE_MODE, + thread_local: bool = True, # ruff:ignore[boolean-type-hint-positional-argument, boolean-default-value-positional-argument] # public API: positional bool kept for backwards compatibility + *, + blocking: bool = True, + is_singleton: bool = False, + poll_interval: float = 0.05, + lifetime: float | None = None, + context_error_policy: ContextErrorPolicy = "chain", + close_error_policy: CloseErrorPolicy = "default", + fallback_to_soft: bool = True, + preserve_lock_file: bool = False, + on_acquired: Callable[[int], None] | None = None, + ) -> None: + """ + Create a new lock object. + + :param lock_file: path to the file + :param timeout: default timeout when acquiring the lock, in seconds. It will be used as fallback value in the + acquire method, if no timeout value (``None``) is given. If you want to disable the timeout, set it to a + negative value or ``inf``. A timeout of 0 means that there is exactly one attempt to acquire the file lock. + :param mode: file permissions for the lockfile. When not specified, the OS controls permissions via umask and + default ACLs, preserving POSIX default ACL inheritance in shared directories. Construction raises + :class:`ValueError` unless it grants the owner read and write, which the lock needs for its own files. On + Windows a native lock creates its file with an owner-only ACL when the mode has no group and other bits; any + other mode, and a soft lock, leave the file the ACL its directory hands down. A Unix lock file that already + exists keeps its permissions. + :param thread_local: Whether this object's internal context should be thread local or not. If this is set to + ``False`` then the lock will be reentrant across threads. When ``True`` (the default), **all fields of the + lock's internal context are per-thread**, including the configuration values ``poll_interval``, ``timeout``, + ``blocking``, ``mode``, and ``lifetime``. Setting one of these properties from one thread does not change + the value seen by another thread; threads that did not perform the write continue to see the value supplied + at construction time. ``mode`` has no setter, so construction is the only place it is ever set. If you need + configuration values to be visible across threads, construct the lock with ``thread_local=False``. + :param blocking: whether the lock should be blocking or not + :param is_singleton: If this is set to ``True`` then only one instance of this class will be created per lock + file. This is useful if you want to use the lock object for reentrant locking without needing to pass the + same object around. + :param poll_interval: default interval for polling the lock file, in seconds. It will be used as fallback value + in the acquire method, if no poll_interval value (``None``) is given. + :param lifetime: for :class:`SoftFileLock`, the age in seconds after which a waiting process may delete the + marker, even while its holder remains alive. This legacy expiry mode does not provide strict mutual + exclusion. ``None`` (the default) disables age-based expiry. Native OS locks (:class:`FileLock`) cannot be + revoked by file age and ignore a non-``None`` ``lifetime`` with a warning. + :param context_error_policy: how a context manager reconciles a failure in its body with a failure while + releasing on exit. ``"chain"`` (the default) keeps Python's behavior: the release error propagates with the + body error in its ``__context__``. ``"group"`` raises a :class:`BaseExceptionGroup` holding the body error + first and the release error second, so neither hides the other. + :param close_error_policy: what to do with an ``os.close`` failure after relinquishing descriptor ownership. + ``"default"`` keeps each backend's historical behavior (Unix native locks drop a FUSE/Docker ``EIO``; + Windows native locks and :class:`SoftFileLock` propagate); ``"raise"`` always propagates the ``OSError``; + ``"suppress"`` always ignores it. Held state is released either way. It does not affect unlock failures or + lock-file deletion. + :param fallback_to_soft: for :class:`UnixFileLock`, whether to switch to :class:`SoftFileLock` when the + filesystem's ``flock`` returns ``ENOSYS``. ``True`` (the default) keeps the historical fallback; + ``False`` fails closed, letting the ``ENOSYS`` propagate so a caller that needs kernel-enforced + locking is never silently downgraded. It has no effect on Windows or :class:`SoftFileLock`. + :param preserve_lock_file: for native locks (:class:`FileLock`), whether filelock promises not to unlink the + lock pathname on release. ``False`` (the default) keeps each backend's cleanup: Windows removes the lock + file, Unix already leaves it. ``True`` keeps a stable file identity for ACLs, auditing, and holder metadata: + Windows skips its post-release unlink and Unix refuses to enter the ``ENOSYS`` soft fallback (which releases + by unlinking). :class:`SoftFileLock` rejects ``True``. The promise covers filelock's own release path only; + it cannot stop another process or the filesystem from removing the pathname. + :param on_acquired: for native locks (:class:`FileLock`), a callable invoked with the borrowed lock descriptor + once per physical acquisition, after filelock holds the native lock and finished backend initialization but + before :meth:`~BaseFileLock.acquire` returns. Recursive acquisitions do not call it again. The callback may + read, write, seek, truncate, or set metadata through ``os`` on the descriptor, but must not close, unlock, + or take ownership of it, and filelock does not fsync its writes. If it raises, filelock releases the lock + and re-raises. :class:`SoftFileLock` rejects the hook. + + :raises TypeError: if *timeout* or *poll_interval* is a :class:`bool` or not a real number, *blocking* is not a + :class:`bool`, or *mode* is not an :class:`int` + :raises ValueError: if *poll_interval* is negative, not finite, or above :data:`threading.TIMEOUT_MAX`, if + *timeout* is too large to convert to :class:`float`, or if *mode* falls outside ``0o000``-``0o777`` or + denies the owner read and write + + """ + self._creator_pid = os.getpid() + #: Holder id in the deadlock registry; a stand-in that finishes the finalizer has another id(self). + self._holder_id = id(self) + self._transition_lock = RLock() + self._is_thread_local = thread_local + self._is_singleton = is_singleton + self._context_error_policy = context_error_policy # already validated by the metaclass + self._close_error_policy = close_error_policy # already validated by the metaclass + self._fallback_to_soft = fallback_to_soft + self._preserve_lock_file = preserve_lock_file # already validated by the metaclass + self._on_acquired = on_acquired # already validated by the metaclass + + self._context: FileLockContext = (ThreadLocalFileContext if thread_local else FileLockContext)( + lock_file=os.fspath(lock_file), + timeout=timeout, + mode=mode, + blocking=blocking, + poll_interval=poll_interval, + lifetime=lifetime, + ) + #: Each thread's committed hold on a thread-local lock, so another thread can release it. + self._thread_holds: dict[int, tuple[FileLockContext, Thread]] = {} + #: Each thread's uncommitted acquisition, so a backend can finish it for an exited thread. + self._thread_pending: dict[int, tuple[FileLockContext, Thread]] = {} + _register_fork_object(self) + + def _singleton_extra_mismatches( # ruff:ignore[no-self-use] # the base class adds no options of its own + self, _kwargs: Mapping[str, _ExtraValue], / + ) -> dict[str, tuple[str, str]]: + # A subclass's own options reach the metaclass only as kwargs, and only the subclass knows their defaults and + # how __init__ resolves them, so each override compares its options and merges what super() reports. + return {} + + def is_thread_local(self) -> bool: + """:returns: a flag indicating if this lock is thread local or not""" + return self._is_thread_local + + @property + def is_singleton(self) -> bool: + """ + A flag indicating if this lock is singleton or not. + + .. versionadded:: 3.13.0 + + """ + return self._is_singleton + + @property + def context_error_policy(self) -> ContextErrorPolicy: + """ + How a context manager reconciles a body failure with a release failure on exit. + + .. versionadded:: 3.30.0 + + """ + return self._context_error_policy + + @property + def close_error_policy(self) -> CloseErrorPolicy: + """ + What a lock does with an ``os.close`` failure after relinquishing descriptor ownership. + + .. versionadded:: 3.30.0 + + """ + return self._close_error_policy + + def _close_released_fd(self, fd: int, *, default_suppresses: bool) -> None: + # CPython never retries close() after EINTR because the descriptor number may already be reused, so neither does + # this. close_error_policy decides the error's fate after the backend relinquishes descriptor ownership. + try: + os.close(fd) + except OSError: + if not self._suppresses_close_error(default_suppresses=default_suppresses): + raise + + def _suppresses_close_error(self, *, default_suppresses: bool) -> bool: + return self._close_error_policy == "suppress" or (self._close_error_policy == "default" and default_suppresses) + + @property + def fallback_to_soft(self) -> bool: + """ + Whether a :class:`FileLock` falls back to :class:`SoftFileLock` when the filesystem lacks ``flock``. + + Only :class:`UnixFileLock` acts on it: when ``False`` an ``ENOSYS`` from ``flock`` propagates instead of + switching to existence-lock semantics. + + .. versionadded:: 3.30.0 + + """ + return self._fallback_to_soft + + @property + def preserve_lock_file(self) -> bool: + """ + Whether filelock promises not to unlink the lock pathname on release. + + When ``True``, Windows skips its post-release unlink and Unix refuses the ``ENOSYS`` soft fallback. + :class:`SoftFileLock` rejects ``True`` because unlinking its marker is how it releases. + + .. versionadded:: 3.30.0 + + """ + return self._preserve_lock_file + + @property + def on_acquired(self) -> Callable[[int], None] | None: + """ + The callback run with the borrowed lock descriptor once per physical acquisition, or ``None``. + + Native locks only. It runs after the native lock is held and backend initialization finished, before + :meth:`~BaseFileLock.acquire` returns; a raise rolls back the acquisition. :class:`SoftFileLock` rejects it. + + .. versionadded:: 3.30.0 + + """ + return self._on_acquired + + @property + def lock_file(self) -> str: + """Path to the lock file.""" + return self._context.lock_file + + @property + def _lock_path(self) -> str: + # The OS resolves a relative lock_file against the working directory at each call. A chdir during an + # acquisition or a hold would otherwise make the backend wait on, lock or remove a different file. + return self._context.claim_root or self._context.lock_file + + @property + def timeout(self) -> float: + """ + The default timeout value, in seconds. + + .. versionadded:: 2.0.0 + + """ + return self._context.timeout + + @timeout.setter + def timeout(self, value: float | str | bytes | bytearray) -> None: + """ + Change the default timeout value. + + :param value: the new value, in seconds; a negative value or ``inf`` waits without a limit + + :raises TypeError: if *value* is a :class:`bool` or not a real number + :raises ValueError: if *value* is too large to convert to :class:`float` + + .. deprecated:: 4.0.13 + + Passing a numeric string; pass the number instead. + + """ + if isinstance(value, (str, bytes, bytearray)): + warnings.warn( + "setting timeout from a string is deprecated and will be removed; pass a number", + DeprecationWarning, + stacklevel=2, + ) + value = float(value) + self._context.timeout = _seconds("timeout", value) + + @property + def blocking(self) -> bool: + """ + Whether the locking is blocking or not. + + .. versionadded:: 3.14.0 + + """ + return self._context.blocking + + @blocking.setter + def blocking(self, value: bool) -> None: + """ + Change the default blocking value. + + :param value: the new value as bool + + :raises TypeError: if *value* is not a bool + + """ + _check_blocking(value) + self._context.blocking = value + + @property + def poll_interval(self) -> float: + """ + The default polling interval, in seconds. + + .. versionadded:: 3.24.0 + + """ + return self._context.poll_interval + + @poll_interval.setter + def poll_interval(self, value: float) -> None: + """ + Change the default polling interval. + + :param value: the new value, in seconds + + :raises ValueError: if *value* is negative, not finite, or above :data:`threading.TIMEOUT_MAX` + :raises TypeError: if *value* is a :class:`bool` or not a real number + + """ + self._context.poll_interval = _resolve_poll_interval(value) + + @property + def lifetime(self) -> float | None: + """ + The soft marker age in seconds that permits expiry, or ``None`` to disable age-based expiry. + + A non-``None`` value permits a waiter to enter while the previous holder remains active, so it does not provide + strict mutual exclusion. Native locks ignore the value with a warning. + + .. versionadded:: 3.24.0 + + """ + return self._context.lifetime + + @lifetime.setter + def lifetime(self, value: float | None) -> None: + """ + Change the legacy age-based expiry threshold. + + :param value: the new value in seconds, or ``None`` to disable expiration + + :raises ValueError: if *value* is negative or not finite + :raises TypeError: if *value* is not ``None`` and not a real number + + """ + self._context.lifetime = _resolve_lifetime(value, type(self), stacklevel=3) + + @property + def mode(self) -> int: + """ + The file permissions requested for the lock file. + + Without an explicit ``mode`` this reports ``0o644`` for backwards compatibility, but the file is created with + ``0o666`` filtered through the umask and default ACLs, so with a ``002`` umask it ends up ``0o664``. Check + :attr:`has_explicit_mode` before relying on the value, or ``os.stat`` the file for its actual permissions. + + """ + return 0o644 if self._context.mode == _UNSET_FILE_MODE else self._context.mode + + @property + def has_explicit_mode(self) -> bool: + """Whether the file permissions were explicitly set.""" + return self._context.mode != _UNSET_FILE_MODE + + def _open_mode(self) -> int: + """Mode for ``os.open``: 0o666 when unset so umask and ACLs decide, otherwise the explicit mode.""" + return 0o666 if self._context.mode == _UNSET_FILE_MODE else self._context.mode + + @property + def is_locked(self) -> bool: + """ + A boolean indicating if the lock file is holding the lock currently. + + .. versionchanged:: 2.0.0 + + This was previously a method and is now a property. + + """ + _ensure_current_process() + return self._context.lock_file_fd is not None + + @property + def lock_counter(self) -> int: + """The number of times this lock has been acquired (but not yet released).""" + _ensure_current_process() + return self._context.lock_counter + + def __enter__(self) -> Self: + """ + Acquire the lock. + + :returns: the lock object + + """ + self.acquire() + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: + """Release the lock, reconciling a release failure with any body failure per :attr:`context_error_policy`.""" + self._release_in_context(exc_value) + + def _release_in_context(self, body_error: BaseException | None) -> None: + # Release from a context-manager exit. "chain" lets a release failure propagate with the body error already in + # its __context__ (Python's default); "group" raises both as sibling leaves so neither one hides the other. + try: + self.release() + except BaseException as release_error: + if body_error is None or self._context_error_policy == "chain": + raise + _raise_body_and_release(body_error, release_error) + + def __del__(self) -> None: + """Force-release so a dropped reference never leaks a held lock, including every thread's hold.""" + if vars(self).get("_creator_pid") != os.getpid(): + return # pragma: forked child + # CPython runs __del__ before it clears weak references. Until then the singleton cache still hands this lock + # out, and a thread that takes a hold through it would lose that hold to a release here. The cycle collector + # clears the references first, and at exit no other thread runs. + if (key := self._singleton_key) is not None and not sys.is_finalizing() and self._instances.get(key) is self: + self._release_once_unreachable() + else: + self._release_abandoned_holds() + + def _release_abandoned_holds(self) -> None: + # A finalizer must not raise. A release error during garbage collection would otherwise surface as an + # unraisable-exception warning, attributed to whichever code triggered collection. The dropped lock still gets + # best-effort cleanup; an explicit release() reports the same error to a caller who can act on it. + with contextlib.suppress(Exception): + self.release(force=True) + # Nothing references the lock any more, so no thread can still be inside it through this instance. + with contextlib.suppress(Exception): + self._release_thread_holds(exited_only=False) + + def _release_once_unreachable(self) -> None: + # The callback runs once the lock is gone, so it reads the state through a holder. The state alone keeps the + # holder alive: a reference from the callback would keep the state, and through it the lock, alive. + self._finalizer_state = _FinalizerState(vars(self)) + _UNREACHABLE_LOCKS.add(ref(self, partial(_finalize_unreachable, ref(self._finalizer_state), type(self)))) + + def acquire( + self, + timeout: float | None = None, + poll_interval: float | None = None, + *, + poll_intervall: float | None = None, + blocking: bool | None = None, + cancel_check: Callable[[], bool] | None = None, + ) -> AcquireReturnProxy[Self]: + """ + Try to acquire the file lock. + + :param timeout: maximum wait time for acquiring the lock, ``None`` means use the default :attr:`~timeout` is and + if ``timeout < 0``, there is no timeout and this method will block until the lock could be acquired + :param poll_interval: interval of trying to acquire the lock file, ``None`` means use the default + :attr:`~poll_interval` + :param poll_intervall: deprecated, kept for backwards compatibility, use ``poll_interval`` instead + :param blocking: whether to retry until this call holds the lock or the timeout expires, ``None`` means use + the default :attr:`~blocking`. ``False`` makes one attempt and raises :class:`~Timeout` on failure. + :param cancel_check: a callable returning ``True`` when the acquisition should be canceled. Checked on each poll + iteration. When triggered, raises :class:`~Timeout` just like an expired timeout. + + :returns: a context object that will unlock the file when the context is exited + + :raises Timeout: if fails to acquire lock within the timeout period + :raises ValueError: if a blocking call gets a ``nan`` timeout, or *poll_interval* exceeds + :data:`threading.TIMEOUT_MAX` + + .. code-block:: python + + # You can use this method in the context manager (recommended) + with lock.acquire(): + pass + + # Or use an equivalent try-finally construct: + lock.acquire() + try: + pass + finally: + lock.release() + + .. versionchanged:: 2.0.0 + + This method returns now a *proxy* object instead of *self*, so that it can be used in a with statement + without side effects. + + """ + self._raise_if_inherited() + if timeout is None: + timeout = self._context.timeout + + if blocking is None: + blocking = self._context.blocking + + if poll_intervall is not None: + msg = "use poll_interval instead of poll_intervall" + warnings.warn(msg, DeprecationWarning, stacklevel=2) + poll_interval = poll_intervall + + poll_interval = poll_interval if poll_interval is not None else self._context.poll_interval + poll_interval = _resolve_poll_interval(poll_interval) + timeout = _resolve_timeout(timeout, blocking=blocking) + + start_time = time.perf_counter() + # Wait for admission before touching any state: a caller refused entry must leave the counter, the registry and + # the descriptor exactly as it found them. + with self._transition_admission( + blocking=blocking, + cancel_check=cancel_check, + timeout=timeout, + poll_interval=poll_interval, + start_time=start_time, + ): + # The interrupted acquire cannot run until the handler returns, so a second one here would wait on it + # forever or take the lock out from under it. Once the lock is held, as in an on_acquired hook, it nests. + if not self.is_locked and _on_this_stack(self._context.acquiring): + msg = f"Deadlock: lock '{self.lock_file}' is acquired again while this thread's acquire of it waits" + raise RuntimeError(msg) + # An interrupted release commits only the hold it started with and ignores a release above it on the + # stack. A hold taken here would nest into one the OS dropped, or stay held for good. + if _on_this_stack(self._context.releasing): + msg = f"lock '{self.lock_file}' is acquired again while this thread's release of it runs" + raise RuntimeError(msg) + canonical = _canonical(self.lock_file) + self._context.lock_counter += 1 + + self._raise_if_would_deadlock(canonical, timeout=timeout, blocking=blocking) + if not self.is_locked: + self._pin_lock_path(canonical) + + try: + self._poll_until_acquired( + blocking=blocking, + cancel_check=cancel_check, + timeout=timeout, + poll_interval=poll_interval, + start_time=start_time, + ) + except BaseException: + self._reconcile_failed_acquire(canonical) + raise + self._commit_acquire(canonical) + return AcquireReturnProxy(lock=self) + + @contextlib.contextmanager + def _transition_admission( + self, + *, + blocking: bool, + cancel_check: Callable[[], bool] | None, + timeout: float, + poll_interval: float, + start_time: float, + ) -> Generator[None]: + # One thread at a time drives the physical transition of a shared instance: its counter, descriptor and claim + # state are plain attributes every thread reads and writes, so an unserialized loser clears the winner's + # descriptor or double-closes it. Waiting in slices keeps cancel_check and the timeout live. A thread-local + # context is never shared, so it skips the gate. + if self._is_thread_local: + yield + return + deadline: Final = None if timeout < 0 else start_time + timeout + wait: Final = poll_interval if blocking else 0.0 + # wait leads the min() so a non-blocking call, whose nan timeout goes unchecked, waits 0 rather than nan. + while not self._transition_lock.acquire( + timeout=wait if deadline is None else min(wait, max(deadline - time.perf_counter(), 0.0)) + ): + if not blocking or (cancel_check is not None and cancel_check()): + raise Timeout(self.lock_file) + if deadline is not None and time.perf_counter() >= deadline: + raise Timeout(self.lock_file) + try: + yield + finally: + self._transition_lock.release() + + def release(self, force: bool = False) -> None: # ruff:ignore[boolean-type-hint-positional-argument, boolean-default-value-positional-argument] # public API: positional bool kept for backwards compatibility + """ + Release the file lock. The lock is only completely released when the lock counter reaches 0. The lock file + itself may be deleted automatically, the behavior is platform-specific. + + :param force: If true, the lock counter is ignored and the lock is released in every case. On a thread-local + lock it also releases the holds of threads that exited without releasing; a hold of a thread still running + stays with that thread. Dropping the last reference to the lock releases every thread's hold. + + """ + counter_before: int | None = None + try: + counter_before = self._context.lock_counter + self._release_current(force=force) + except BaseException as release_error: + # A handler that raised after release() began and before the backend changed anything leaves the counter as + # it was; only then does a second attempt release the hold the caller gave up. A failed system call keeps it + # for an explicit retry, and a changed counter means a nested level or the backend's own retry already ran. + if ( + counter_before is not None + and counter_before == self._context.lock_counter + and self._context.lock_file_fd is not None + and not _is_system_call_failure(release_error) + ): + with contextlib.suppress(Exception): + self._release_current(force=force) + raise + if force: + self._release_thread_holds(exited_only=True) + + def _release_current(self, *, force: bool) -> None: + # A shared instance releases under the same gate its acquisition ran through, so a thread entering the lock + # never observes a partially torn-down owner. Only an acquirer that found the lock free holds the gate for + # long, so an unheld lock returns without waiting behind it. + if self._context.lock_file_fd is None: + # Only a committed hold registers, so an entry without a descriptor is a release a signal handler stopped + # after the unlock; left alone it refuses every other instance on this path in this thread. + if self._context.lock_file_registry is not None: + self._finish_interrupted_release() + return + with contextlib.nullcontext() if self._is_thread_local else self._transition_lock: + state: Final = self._hold_state() + # A signal handler can land between the backend's unlock and its clearing of the descriptor, where a second + # release would close the number twice, maybe hitting a file that reused it. The flag holds the releasing + # frame, so only a release still on this stack stops it, and a flag a cut-short frame left set does not. + if self._creator_pid != os.getpid() or state.lock_file_fd is None or _on_this_stack(state.releasing): + return + if not force and state.lock_counter > 1: + state.lock_counter -= 1 + return + state.releasing = inspect.currentframe() + try: + _LOGGER.debug("Attempting to release lock %s on %s", id(self), self.lock_file) + self._release_with_fork_tracking() + finally: + # A failure after the OS unlock (during close or unlink) still released the lock: the backend cleared + # its descriptor, so commit the counter and registry to released even as the cleanup error propagates. + # A failure that left the lock held keeps the counter so a later release can retry the OS unlock. + if state.lock_file_fd is None: # pragma: no branch # a release that returns cleared the descriptor + self._commit_release() + state.releasing = None + _LOGGER.debug("Lock %s released on %s", id(self), self.lock_file) + + def _finish_interrupted_release(self) -> None: + # An unheld lock never waits behind an acquirer, whose own commit takes the entry over anyway. + if not self._transition_lock.acquire(blocking=False): + return # pragma: no cover # only a concurrent acquire of a shared instance holds the gate + try: + if self._context.lock_file_fd is None: # pragma: no branch # only a racing acquire fills it + self._commit_release() + finally: + self._transition_lock.release() + + def _hold_state(self) -> FileLockContext: + return self._context.current if isinstance(self._context, ThreadLocalFileContext) else self._context + + def _release_thread_holds(self, *, exited_only: bool) -> None: + if not isinstance(context := self._context, ThreadLocalFileContext): + return + # Serialized, or two threads would both close an exited thread's descriptor; copy() is atomic free-threaded. + with self._transition_lock: + own: Final = context.current + for held, thread in self._thread_holds.copy().values(): + if exited_only and thread.is_alive(): + continue + context.current = held + try: + self._release_current(force=True) + finally: + context.current = own + + def _raise_if_inherited(self) -> None: + if self._creator_pid != os.getpid(): # pragma: forked child + msg = f"{type(self).__name__} on {self.lock_file} was inherited across fork; construct a new instance" + raise RuntimeError(msg) + + def _mark_descriptor_owned(self, fd: int, identity: tuple[int, int] | None = None) -> None: + state: Final = self._hold_state() + # Held first, pending cleared second: an interrupt between the two then leaves the descriptor recorded. + state.lock_file_fd = fd + state.lock_file_fd_identity = identity + state.pending_lock_file_fd = None + state.pending_lock_file_fd_identity = None + if isinstance(self._context, ThreadLocalFileContext): + self._thread_pending.pop(id(state), None) + self._thread_holds[id(state)] = state, current_thread() + + def _mark_descriptor_pending(self, fd: int, identity: tuple[int, int] | None = None) -> None: + state: Final = self._hold_state() + state.pending_lock_file_fd = fd + state.pending_lock_file_fd_identity = identity + if isinstance(self._context, ThreadLocalFileContext): + self._thread_pending[id(state)] = state, current_thread() + + def _mark_descriptor_released(self) -> None: + state: Final = self._hold_state() + state.pending_lock_file_fd = None + state.pending_lock_file_fd_identity = None + state.lock_file_fd = None + state.lock_file_fd_identity = None + if isinstance(self._context, ThreadLocalFileContext): + self._thread_pending.pop(id(state), None) + self._thread_holds.pop(id(state), None) + + def _reset_after_fork_in_child(self) -> None: # pragma: forked child + # fork copies the lock in whatever state the parent's threads left it, so give the child an unheld one. + self._transition_lock = RLock() + if isinstance(self._context, ThreadLocalFileContext): + self._thread_holds.clear() + self._thread_pending.clear() + self._context.owner_claim_paths = () + self._context.claim_root = None + self._context.lock_file_fd = None + self._context.lock_file_fd_token = None + self._context.lock_file_fd_identity = None + self._context.pending_lock_file_fd = None + self._context.pending_lock_file_fd_identity = None + self._context.lock_counter = 0 + self._context.releasing = None + self._context.acquiring = None + self._context.lock_file_key = None + self._context.lock_file_registry = None + + def _descriptors_for_fork(self) -> tuple[tuple[int, tuple[int, int] | None], ...]: # pragma: needs fork + descriptors: list[tuple[int, tuple[int, int] | None]] = [] + if self._context.lock_file_fd is not None and self._context.lock_file_fd_token is None: + descriptors.append((self._context.lock_file_fd, self._context.lock_file_fd_identity)) + if self._context.pending_lock_file_fd is not None: + descriptors.append((self._context.pending_lock_file_fd, self._context.pending_lock_file_fd_identity)) + return tuple(descriptors) + + def _raise_if_would_deadlock(self, canonical: str, *, timeout: float, blocking: bool) -> None: + """ + Fail fast when a *different* live instance already holds this path in the current deadlock scope. + + Only the first, indefinitely-blocking acquire can self-deadlock this way: waiting in the OS primitive would + block on a lock this flow already owns. A finite timeout or ``blocking=False`` keeps the normal Timeout path. + """ + would_block = self._context.lock_counter == 1 and not self.is_locked and timeout < 0 and blocking + if would_block and _registry.held.get(self._registry_key(canonical)) not in {None, self._holder_id}: + self._context.lock_counter -= 1 + msg = ( + f"Deadlock: lock '{self.lock_file}' is already held by a different {self._deadlock_holder_desc}. " + f"Use is_singleton=True to enable reentrant locking across instances." + ) + raise RuntimeError(msg) + + def _registry_key(self, canonical: str) -> Hashable: + return canonical if (scope := self._deadlock_scope()) is None else (scope, canonical) + + @staticmethod + def _deadlock_scope() -> Hashable | None: + """ + Execution unit whose own hold would deadlock a new acquire. + + ``None`` scopes holders to the thread, which the thread-local registry already separates. Async locks + override it with the running task, since one event loop thread runs many tasks and only a reacquire from + the *same* task can self-deadlock. + """ + return None + + def _poll_until_acquired( + self, + *, + blocking: bool, + cancel_check: Callable[[], bool] | None, + timeout: float, + poll_interval: float, + start_time: float, + ) -> None: + lock_id = id(self) + lock_filename = self.lock_file + attempt = 0 + self._context.acquiring = inspect.currentframe() + try: + # Created once, so a directory that disappears while the acquisition waits fails it, not builds a second + # lock file beside the holder's. + ensure_directory_exists(self._lock_path) + while True: + self._raise_if_inherited() + if not self.is_locked: + self._try_break_expired_lock() + _LOGGER.debug("Attempting to acquire lock %s on %s", lock_id, lock_filename) + self._acquire_with_fork_tracking() + self._raise_if_inherited() + if self.is_locked: + _LOGGER.debug("Lock %s acquired on %s", lock_id, lock_filename) + return + if self._check_give_up( + blocking=blocking, + cancel_check=cancel_check, + timeout=timeout, + start_time=start_time, + ): + raise Timeout(lock_filename, self._unprobeable_holder()) + attempt += 1 + delay = self._poll_delay(poll_interval, attempt) + if timeout >= 0: + delay = min(delay, max(start_time + timeout - time.perf_counter(), 0.0)) + msg = "Lock %s not acquired on %s, waiting %s seconds ..." + _LOGGER.debug(msg, lock_id, lock_filename, delay) + time.sleep(delay) + finally: + self._context.acquiring = None + + def _poll_delay(self, poll_interval: float, attempt: int) -> float: + # A single-file lock retries on a fixed cadence. A backend that publishes several files per acquisition sets a + # cap, and then contending processes back off across a jittered, exponentially widening window instead of + # colliding on every poll; poll_interval stays the floor so a lone waiter is still responsive. + if not self._poll_backoff_cap: + return poll_interval + # Cap the exponent before doubling: under heavy contention attempt reaches the thousands, and 2**attempt would + # overflow the float multiply long before the window itself stops growing past the cap. + window = min( + self._poll_backoff_cap, poll_interval * 2 ** min(attempt, _MAX_BACKOFF_EXPONENT) + ) # pragma: needs hard-link + return max(poll_interval, secrets.randbelow(int(window * 1_000_000) + 1) / 1_000_000) # pragma: needs hard-link + + def _reconcile_failed_acquire(self, canonical: str) -> None: + # An acquire that raised while still holding the native lock (a hook that failed and whose rollback could not + # release) must keep the registry entry so a later release can retry the OS unlock; otherwise roll the counter + # back. is_locked was already reconciled by whichever release ran. + if self.is_locked: + self._commit_acquire(canonical) + else: + self._undo_acquire() + + def _invoke_on_acquired(self) -> None: + # The wrapper runs in the backend executor for async locks, preserving the callback's documented thread. + if self._on_acquired is None or self._context.lock_counter != 1: + return + try: + self._on_acquired(cast("int", self._context.lock_file_fd)) + except BaseException as callback_error: # arbitrary caller code; roll back on any failure + callback_context = callback_error.__context__ + try: + self._release_with_fork_tracking() + except BaseException as release_error: # ruff:ignore[blind-except] # both errors surface via the group below + _raise_body_and_release(callback_error, release_error) + callback_error.__context__ = callback_context + raise + + def _acquire_with_fork_tracking(self) -> None: + with _fork_transition(self): + try: + self._acquire() + except BaseException as acquisition_error: + self._rollback_failed_acquire(acquisition_error) + raise + try: + self._register_context_descriptor() + except BaseException as registration_error: # pragma: needs fork + self._rollback_failed_registration(registration_error) + raise + if self.is_locked: + self._invoke_on_acquired() + + def _rollback_failed_acquire(self, acquisition_error: BaseException) -> None: + if not self.is_locked: + return + registration_error: BaseException | None = None + tracking_error: BaseException | None = None + try: + self._register_context_descriptor() + except BaseException as error: # ruff:ignore[blind-except] # preserve registration and acquisition failures + registration_error = error + try: + # Rollback may fail too; retain the fd so a child can close it without another identity probe. + self._register_unverified_context_descriptor() + except BaseException as error: # ruff:ignore[blind-except] # pragma: no cover - allocation/control-flow during fallback + tracking_error = error + try: + self._release_with_fork_tracking() + except BaseException as rollback_error: # ruff:ignore[blind-except] # preserve rollback and acquisition failures + _raise_cleanup_errors( + "lock acquisition cleanup failed", + acquisition_error, + registration_error, + tracking_error, + rollback_error, + ) + if registration_error is not None: # pragma: needs fork + _raise_cleanup_errors( + "lock acquisition cleanup failed", acquisition_error, registration_error, tracking_error + ) + + def _rollback_failed_registration(self, registration_error: BaseException) -> None: # pragma: needs fork + tracking_error: BaseException | None = None + try: + # Rollback may fail too; retain the fd so a child can close it without another identity probe. + self._register_unverified_context_descriptor() + except BaseException as error: # ruff:ignore[blind-except] # pragma: no cover - allocation/control-flow during fallback + tracking_error = error + try: + self._release_with_fork_tracking() + except BaseException as rollback_error: # ruff:ignore[blind-except] # preserve rollback and registration failures + _raise_cleanup_errors( + "descriptor registration cleanup failed", registration_error, tracking_error, rollback_error + ) + if tracking_error is not None: # pragma: no cover - requires failed in-memory fallback + _raise_cleanup_errors("descriptor registration cleanup failed", registration_error, tracking_error) + + def _release_with_fork_tracking(self) -> None: + with _fork_transition(self): + try: + self._release() + except BaseException as release_error: + # A backend unlocks first and clears the descriptor second. A signal handler that raises in between + # leaves a hold the OS dropped. A failed system call alone shows the lock is still held, so it keeps it. + if self._context.lock_file_fd is not None and not _is_system_call_failure(release_error): + with contextlib.suppress(Exception): + self._release() + raise + finally: + self._unregister_released_descriptor() + + def _register_context_descriptor(self) -> None: + if self._context.lock_file_fd is not None and self._context.lock_file_fd_token is None: + self._context.lock_file_fd_token = _register_owned_descriptor( + self._context.lock_file_fd, + self._context.lock_file_fd_identity, + ) + + def _register_unverified_context_descriptor(self) -> None: + # The rollback only reaches here still holding a descriptor it never managed to register. + if self._context.lock_file_fd is not None and self._context.lock_file_fd_token is None: # pragma: no branch + self._context.lock_file_fd_token = _register_unverified_owned_descriptor(self._context.lock_file_fd) + + def _unregister_released_descriptor(self) -> None: + if self._context.lock_file_fd is None: + if (token := self._context.lock_file_fd_token) is not None: # pragma: needs fork + _unregister_owned_descriptor(token) + self._context.lock_file_fd_token = None + self._context.lock_file_fd_identity = None + + def _undo_acquire(self) -> None: + """Roll back the counter after a failed acquire, dropping the registry entry once nothing holds the path.""" + self._context.lock_counter = max(0, self._context.lock_counter - 1) + if self._context.lock_counter == 0: + self._drop_registry_entry() + self._context.claim_root = None + + def _pin_lock_path(self, canonical: str) -> None: + # A path without a final component names no file. Its canonical form is a directory, and pinning that would + # change the error the backend raises. + self._context.claim_root = None if canonical.endswith(os.sep) else canonical + + def _commit_acquire(self, canonical: str) -> None: + """Record this instance as the holder once the first acquire succeeds, so peers can detect the deadlock.""" + # Keyed on the registry rather than the counter: an on_acquired hook that acquires again bumps the counter + # past 1 before this first commit runs, and must not leave the hold unregistered. + if self._context.lock_file_registry is None: + key = self._registry_key(canonical) + # The holder scope is resolved once at commit so a later release from another flow drops the right entry. + self._context.lock_file_key = key + # A shared instance may be released from another thread, which must clear the entry in the acquirer's + # registry: the releaser's own thread-local registry never held it. + self._context.lock_file_registry = _registry.held + _registry.held[key] = self._holder_id + + def _drop_registry_entry(self) -> None: + """Forget the key owned by this hold without resolving a mutable path again.""" + # Ordered so a run an interrupt cuts short repeats safely: the entry goes only while it is still ours, and the + # key outlives the registry it indexes, so a later commit sees no registry and records the hold afresh. + key, registry = self._context.lock_file_key, self._context.lock_file_registry + if registry is not None and registry.get(key) == self._holder_id: + del registry[key] + self._context.lock_file_registry = None + self._context.lock_file_key = None + + def _commit_release(self) -> None: + """Record the lock as fully released: reset the recursion counter and drop the deadlock-registry entry.""" + self._context.lock_counter = 0 + self._drop_registry_entry() + self._context.claim_root = None + + def _try_break_expired_lock(self) -> None: + """Remove the lock file once it is older than :attr:`lifetime`; only an existence lock accepts one.""" + + def _check_give_up( + self, + *, + blocking: bool, + cancel_check: Callable[[], bool] | None, + timeout: float, + start_time: float, + ) -> bool: + lock_id, lock_filename = id(self), self.lock_file + if blocking is False: + _LOGGER.debug("Failed to immediately acquire lock %s on %s", lock_id, lock_filename) + return True + if cancel_check is not None and cancel_check(): + _LOGGER.debug("Cancellation requested for lock %s on %s", lock_id, lock_filename) + return True + if 0 <= timeout < time.perf_counter() - start_time: + _LOGGER.debug("Timeout on acquiring lock %s on %s", lock_id, lock_filename) + return True + return False + + def _unprobeable_holder(self) -> str | None: # ruff:ignore[no-self-use] # existence locks override it + return None + + @abstractmethod + def _acquire(self) -> None: + """If the file lock could be acquired, self._context.lock_file_fd holds the file descriptor of the lock file.""" + raise NotImplementedError + + @abstractmethod + def _release(self) -> None: + """Releases the lock and sets self._context.lock_file_fd to None.""" + raise NotImplementedError + + +def _resolve_read_write_timeout(timeout: float, *, blocking: bool) -> float: + """Validate a read-write lock's acquire ``timeout``: ``-1`` or a non-negative number, ``inf`` meaning ``-1``.""" + if (timeout := _resolve_timeout(timeout, blocking=blocking)) < 0 and timeout != -1 and blocking: + msg = f"timeout must be a non-negative number or -1, not {timeout!r}" + raise ValueError(msg) + return timeout + + +def _acquire_thread_lock( + lock: Lock, *, blocking: bool, timeout: float, poll_interval: float, cancel_check: Callable[[], bool] | None +) -> bool: + """Take a read-write lock's in-process lock; with *cancel_check*, ask it every *poll_interval* while waiting.""" + if not blocking: + return lock.acquire(blocking=False) + if cancel_check is None: + # Lock.acquire raises OverflowError past TIMEOUT_MAX, which is 49.7 days on Windows. + return lock.acquire() if timeout == -1 else lock.acquire(timeout=min(timeout, TIMEOUT_MAX)) + deadline = None if timeout == -1 else time.perf_counter() + timeout + while not lock.acquire( + timeout=poll_interval if deadline is None else min(poll_interval, max(deadline - time.perf_counter(), 0)) + ): + if cancel_check() or (deadline is not None and time.perf_counter() >= deadline): + return False + return True + + +def _resolve_timeout(timeout: float, *, blocking: bool) -> float: + """Validate an acquire ``timeout``; every negative value waits without a limit.""" + if math.isnan(timeout := _seconds("timeout", timeout)) and blocking: + msg = "timeout must be a number of seconds, not nan" + raise ValueError(msg) + # The fail-fast deadlock check only sees a negative timeout as unlimited, so spell inf that way. + return -1 if timeout == math.inf else timeout + + +@dataclass(slots=True, weakref_slot=True) +class _FinalizerState: + """A finalized lock's state, held for the release that runs once the lock is unreachable.""" + + state: dict[str, object] | None + + +#: Weak references whose callback finishes a singleton's finalizer once the cache no longer finds it. +_UNREACHABLE_LOCKS: Final[set[ref[BaseFileLock]]] = set() + + +def _finalize_unreachable(holder: ref[_FinalizerState], cls: type[BaseFileLock], reference: ref[BaseFileLock]) -> None: + _UNREACHABLE_LOCKS.discard(reference) + if (finalizer_state := holder()) is None or (state := finalizer_state.state) is None: + return # pragma: no cover # only a collector freeing a resurrected lock as part of a cycle drops the holder + # The lock's dictionary points back at the holder; that cycle ends here. + finalizer_state.state = None + # The release methods only read and write this state, so a fresh instance takes it over. Dropping that instance + # runs __del__ again, and the cache no longer finds the lock. + stand_in = object.__new__(cls) + stand_in.__dict__ = state + del stand_in + + +def _is_system_call_failure(error: BaseException) -> bool: + # A failed system call sets errno. A bare TimeoutError raised by a signal handler is an OSError without one. + if _is_exception_group(error): + return all(map(_is_system_call_failure, cast("_ExceptionGroupProtocol", error).exceptions)) + return isinstance(error, OSError) and error.errno is not None + + +def _on_this_stack(frame: FrameType | None) -> bool: + if frame is None: + return False + caller = inspect.currentframe() + while caller is not None and caller is not frame: + caller = caller.f_back + return caller is not None + + +# acquire() returns this wrapper instead of self so entering the with-statement does not call __enter__ a second +# time; returning self would re-acquire the lock in BaseFileLock.__enter__ without a matching release (issue #37). +class AcquireReturnProxy(Generic[_LockT]): + """A context-aware object that will release the lock file when exiting.""" + + def __init__(self, lock: _LockT) -> None: + self.lock: _LockT = lock + + def __enter__(self) -> _LockT: + return self.lock + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: + # Widen to the bound: ty narrows the else branch to the type variable minus BaseFileLock, not to the union rest. + lock: Final = cast("BaseFileLock | ReadWriteLock | SoftReadWriteLock", self.lock) + if isinstance(lock, BaseFileLock): + lock._release_in_context(exc_value) # ruff:ignore[private-member-access] # forwards __exit__ to the owned lock's context release + else: # a reader/writer lock does not carry a context_error_policy + lock.release() + + +@dataclass +class FileLockContext: + """Holds the context for a ``BaseFileLock`` object.""" + + # A separate class so ThreadLocalFileContext can make the whole context thread-local. + + lock_file: str + timeout: float + mode: int + blocking: bool + poll_interval: float + + #: The lock lifetime in seconds; ``None`` means the lock never expires. + lifetime: float | None = None + + #: File descriptor from os.open for the lock file; not None while the lock is held. + lock_file_fd: int | None = None + + #: Registry token for the descriptor owned by this thread's context. + lock_file_fd_token: int | None = None + + #: Identity captured by a backend that already inspected the descriptor. + lock_file_fd_identity: tuple[int, int] | None = None + + #: Descriptor opened by a backend but not yet committed as the held lock. + pending_lock_file_fd: int | None = None + + #: Identity captured for a descriptor whose acquisition has not committed. + pending_lock_file_fd_identity: tuple[int, int] | None = None + + #: Depth of nested acquisitions; the lock is released only when it returns to 0. + lock_counter: int = 0 + + #: Canonical registry key captured when the first physical acquisition commits. + lock_file_key: Hashable | None = None + + #: The acquiring thread's deadlock registry holding :attr:`lock_file_key`. + lock_file_registry: dict[Hashable, int] | None = None + + #: Claim pathnames this owner published, removed by name on release so no holder ever unlinks a peer's claim. + owner_claim_paths: tuple[str, ...] = () + + #: The frame of the :meth:`BaseFileLock.release` tearing this hold down, so a re-entrant call from a signal handler + #: on the same thread returns instead of closing the descriptor a second time. + releasing: FrameType | None = None + + #: The frame of an acquire still polling for this hold, so one a signal handler starts on this thread fails fast. + acquiring: FrameType | None = None + + #: Canonical lock path, resolved when an acquisition starts and kept until its hold ends. A waiter on a relative + #: path stays in the directory it started in, and the release finds the file there, after a chdir in between. + claim_root: str | None = None + + #: The lease claim of this hold, kept with it so a cross-thread release stops the right heartbeat. + lease_claim: _LeaseClaim | None = None + + +class ThreadLocalFileContext(FileLockContext, local): + """ + A thread local version of the ``FileLockContext`` class. + + Each thread's state lives in an ordinary :class:`FileLockContext` at :attr:`current`, which the lock also keeps, so + ``release(force=True)`` and the finalizer can unlock a hold whose thread exited. + """ + + current: FileLockContext + + # threading.local runs __init__ again with these arguments the first time each thread touches the context. + def __init__( # ruff:ignore[too-many-arguments] # mirrors the FileLockContext fields set at construction + self, + *, + lock_file: str, + timeout: float, + mode: int, + blocking: bool, + poll_interval: float, + lifetime: float | None = None, + ) -> None: + self.current = FileLockContext( + lock_file=lock_file, + timeout=timeout, + mode=mode, + blocking=blocking, + poll_interval=poll_interval, + lifetime=lifetime, + ) + + +def _delegate_to_current(name: str) -> property: + return property( + lambda context: getattr(context.current, name), + lambda context, value: setattr(context.current, name, value), + ) + + +for _field in fields(FileLockContext): + setattr(ThreadLocalFileContext, _field.name, _delegate_to_current(_field.name)) + + +@dataclass(frozen=True) +class _OwnedDescriptor: + fd: int + creator_pid: int + device: int | None + inode: int | None + + +class _ForkState: + def __init__(self) -> None: + self.gate = Condition(RLock()) + self.registry_lock = RLock() + self.parameter_models_lock = RLock() + self.transitions: dict[int, dict[int, _ForkDescriptorOwner | None]] = {} + self.active_transitions = 0 + self.admission_closed = False + self.fork_owner_depths: dict[int, int] = {} + self.pinned_objects: dict[int, list[tuple[_ForkResettable, ...]]] = {} + self.pinned_classes: dict[int, list[tuple[_ForkResettableClass, ...]]] = {} + self.provisional_descriptor_tokens: dict[int, list[tuple[int, ...]]] = {} + self.pid = os.getpid() + + def reset_synchronization(self) -> None: # pragma: forked child + self.gate = Condition(RLock()) + self.registry_lock = RLock() + self.parameter_models_lock = RLock() + self.transitions = {} + self.active_transitions = 0 + self.admission_closed = False + self.fork_owner_depths = {} + self.pinned_objects = {} + self.pinned_classes = {} + self.provisional_descriptor_tokens = {} + + +_FORK_OBJECTS: Final[WeakValueDictionary[int, _ForkResettable]] = WeakValueDictionary() +_FORK_CLASSES: Final[WeakValueDictionary[int, _ForkResettableClass]] = WeakValueDictionary() +_OWNED_DESCRIPTORS: Final[dict[int, _OwnedDescriptor]] = {} +_DESCRIPTOR_TOKENS: Final[count[int]] = count() +_TRANSITION_TOKENS: Final[count[int]] = count() +_FORK_STATE: Final = _ForkState() +_FORK_AUDIT_EVENTS: Final[frozenset[str]] = frozenset({"os.fork", "os.forkpty"}) +# CPython before 3.12 guards sys.settrace with one process-wide flag that stays set while audit hooks run, so a thread +# switch inside a Python hook makes a concurrent settrace raise. Coverage and debuggers call it on every thread start. +_SETTRACE_SURVIVES_AUDIT_HOOKS: Final[bool] = sys.implementation.name != "cpython" or sys.version_info >= (3, 12) + + +@cache +def _resolve_lazy_imports_before_shutdown() -> None: + if hasattr(types, "LazyImportType"): # pragma: >=3.15 cover + atexit.register(_resolve_lazy_imports) + + +def _resolve_lazy_imports() -> None: # pragma: >=3.15 cover + # Interpreter shutdown clears sys.modules and sys.meta_path before it finalizes a lock still alive at exit, so a + # lazy import (PEP 810) that the release path in __del__ reaches first can no longer load. Load them while it can. + for name, module in list(sys.modules.items()): + if name == "filelock" or name.startswith("filelock."): + for attribute, value in list(vars(module).items()): + if type(value) is types.LazyImportType: + getattr(module, attribute) + + +def _register_fork_hooks() -> None: + if _REGISTER_AT_FORK is None: + return # pragma: lacks fork + if _SETTRACE_SURVIVES_AUDIT_HOOKS: # pragma: needs fork # pragma: needs settrace-safe-audit-hooks + sys.addaudithook(_audit_fork_safety) + _REGISTER_AT_FORK( # pragma: needs fork + before=_pin_fork_objects, + after_in_parent=_resume_parent_after_fork, + after_in_child=_reset_child_after_fork, + ) + + +@contextmanager +def _fork_transition(descriptor_owner: _ForkDescriptorOwner | None = None) -> Generator[None]: + # No fork follows exit, and a daemon thread stopped inside this bookkeeping holds its lock forever. + if not _HAS_REGISTER_AT_FORK or sys.is_finalizing(): + yield # pragma: lacks fork + return # pragma: lacks fork + _ensure_current_process() # pragma: needs fork + creator_pid = os.getpid() # pragma: needs fork + # A generator holding this transition can be finalized on another thread, which must close the entering thread's. + thread_id: Final = get_ident() # pragma: needs fork + token: Final = _enter_fork_transition(descriptor_owner, thread_id) # pragma: needs fork + try: # pragma: needs fork + yield + finally: + if os.getpid() == creator_pid: # pragma: needs fork + _leave_fork_transition(token, thread_id) + + +def _register_fork_object(instance: _ForkResettable) -> None: + if not _HAS_REGISTER_AT_FORK: + return # pragma: lacks fork + with _fork_transition(), _FORK_STATE.registry_lock: # pragma: needs fork + _FORK_OBJECTS[id(instance)] = instance + _refresh_owner_pins() + + +def _register_fork_class(cls: _ForkResettableClass) -> None: + if not _HAS_REGISTER_AT_FORK: + return # pragma: lacks fork + with _fork_transition(), _FORK_STATE.registry_lock: # pragma: needs fork + _FORK_CLASSES[id(cls)] = cls + _refresh_owner_pins() + + +def _register_owned_descriptor(fd: int, identity: tuple[int, int] | None = None) -> int | None: + if not _HAS_REGISTER_AT_FORK: + return None # pragma: lacks fork + with _fork_transition(): # pragma: needs fork + if identity is None: + stat_result = os.fstat(fd) + identity = stat_result.st_dev, stat_result.st_ino + return _record_owned_descriptor(fd, identity) + + +def _register_unverified_owned_descriptor(fd: int) -> int | None: + if not _HAS_REGISTER_AT_FORK: + return None # pragma: lacks fork + with _fork_transition(): # pragma: needs fork + return _record_owned_descriptor(fd, None) + + +def _record_owned_descriptor(fd: int, identity: tuple[int, int] | None) -> int: # pragma: needs fork + with _FORK_STATE.registry_lock: + token = next(_DESCRIPTOR_TOKENS) + _OWNED_DESCRIPTORS[token] = _OwnedDescriptor( + fd=fd, + creator_pid=os.getpid(), + device=None if identity is None else identity[0], + inode=None if identity is None else identity[1], + ) + return token + + +def _unregister_owned_descriptor(token: int) -> None: # pragma: needs fork + # No fork follows exit, and a daemon thread stopped inside this bookkeeping holds its lock forever. + if sys.is_finalizing(): # pragma: no cover # only exit reaches it, after coverage stops measuring + return + with _fork_transition(), _FORK_STATE.registry_lock: + _OWNED_DESCRIPTORS.pop(token, None) + + +def _pin_fork_objects() -> None: # pragma: needs fork + _ensure_current_process() + thread_id = get_ident() + provisional_descriptor_tokens: list[int] = [] + # A signal handler that raises during the wait does not stop the fork. The after hooks pop one snapshot per + # admission closed here, so the finally records the snapshot on failure too. + try: + with _FORK_STATE.gate: + _FORK_STATE.fork_owner_depths[thread_id] = _FORK_STATE.fork_owner_depths.get(thread_id, 0) + 1 + _FORK_STATE.admission_closed = True + while _FORK_STATE.active_transitions > len(_FORK_STATE.transitions.get(thread_id, ())): + _FORK_STATE.gate.wait() + transition_owners = tuple(_FORK_STATE.transitions.get(thread_id, {}).values()) + _verify_unverified_descriptors() + owners = {id(owner): owner for owner in transition_owners if owner is not None} + provisional_descriptor_tokens.extend( + starmap( + _snapshot_descriptor_for_fork, + (descriptor for owner in owners.values() for descriptor in owner._descriptors_for_fork()), # ruff:ignore[private-member-access] # snapshots each owner's own fork descriptors + ) + ) + finally: + with _FORK_STATE.registry_lock: + _FORK_STATE.provisional_descriptor_tokens.setdefault(thread_id, []).append( + tuple(provisional_descriptor_tokens) + ) + _FORK_STATE.pinned_objects.setdefault(thread_id, []).append(tuple(_FORK_OBJECTS.values())) + _FORK_STATE.pinned_classes.setdefault(thread_id, []).append(tuple(_FORK_CLASSES.values())) + + +def _resume_parent_after_fork() -> None: # pragma: needs fork + thread_id = get_ident() + try: + with _FORK_STATE.registry_lock: + for token in _FORK_STATE.provisional_descriptor_tokens[thread_id].pop(): + _OWNED_DESCRIPTORS.pop(token, None) + _FORK_STATE.pinned_objects[thread_id].pop() + _FORK_STATE.pinned_classes[thread_id].pop() + if not _FORK_STATE.provisional_descriptor_tokens[thread_id]: # pragma: needs fork + del _FORK_STATE.provisional_descriptor_tokens[thread_id] + del _FORK_STATE.pinned_objects[thread_id] + del _FORK_STATE.pinned_classes[thread_id] + finally: + # No other code reopens admission. A failure above would leave the other threads waiting for good. + with _FORK_STATE.gate: + # A handler that raised before the depth was stored left no entry to decrement. + if (depth := _FORK_STATE.fork_owner_depths.get(thread_id, 1)) <= 1: + _FORK_STATE.fork_owner_depths.pop(thread_id, None) + else: # pragma: no cover - earlier at-fork callbacks may deadlock first + _FORK_STATE.fork_owner_depths[thread_id] = depth - 1 + _FORK_STATE.admission_closed = bool(_FORK_STATE.fork_owner_depths) + _FORK_STATE.gate.notify_all() + + +def _ensure_current_process() -> None: + _reset_child_after_fork() + + +def _reset_child_after_fork() -> None: # pragma: forked child + if (pid := os.getpid()) == _FORK_STATE.pid: + return + thread_id = get_ident() + pinned_objects = _FORK_STATE.pinned_objects.get(thread_id, [()])[-1] + pinned_classes = _FORK_STATE.pinned_classes.get(thread_id, [()])[-1] + descriptors: list[_OwnedDescriptor] = [] + for token, descriptor in tuple(_OWNED_DESCRIPTORS.items()): + if descriptor.creator_pid != pid: + descriptors.append(_OWNED_DESCRIPTORS.pop(token)) + _FORK_STATE.reset_synchronization() + _FORK_STATE.pid = pid + _INIT_PARAMETER_MODELS.clear() + _detach_child_state(descriptors, pinned_objects, pinned_classes) + + +def _enter_fork_transition(descriptor_owner: _ForkDescriptorOwner | None, thread_id: int) -> int: # pragma: needs fork + with _FORK_STATE.gate: + while ( + _FORK_STATE.admission_closed + and thread_id not in _FORK_STATE.fork_owner_depths + and thread_id not in _FORK_STATE.transitions + ): + _FORK_STATE.gate.wait() # pragma: no cover - exercised in isolated interpreter + token = next(_TRANSITION_TOKENS) + _FORK_STATE.transitions.setdefault(thread_id, {})[token] = descriptor_owner + _FORK_STATE.active_transitions += 1 + return token + + +def _leave_fork_transition(token: int, thread_id: int) -> None: # pragma: needs fork + with _FORK_STATE.gate: + owned: Final = _FORK_STATE.transitions[thread_id] + del owned[token] + if not owned: + del _FORK_STATE.transitions[thread_id] + _FORK_STATE.active_transitions -= 1 + _FORK_STATE.gate.notify_all() + + +def _verify_unverified_descriptors() -> None: # pragma: needs fork + with _FORK_STATE.registry_lock: + for token, descriptor in tuple(_OWNED_DESCRIPTORS.items()): + if descriptor.creator_pid != os.getpid() or descriptor.device is not None: + continue + try: + stat_result = os.fstat(descriptor.fd) + except OSError: + continue + _OWNED_DESCRIPTORS[token] = _OwnedDescriptor( + fd=descriptor.fd, + creator_pid=descriptor.creator_pid, + device=stat_result.st_dev, + inode=stat_result.st_ino, + ) + + +def _snapshot_descriptor_for_fork(fd: int, identity: tuple[int, int] | None) -> int: # pragma: needs fork + if identity is None: # pragma: needs fork + try: + stat_result = os.fstat(fd) + except OSError: + pass + else: + identity = stat_result.st_dev, stat_result.st_ino + return _record_owned_descriptor(fd, identity) + + +def _detach_child_state( # pragma: forked child + descriptors: list[_OwnedDescriptor], + pinned_objects: tuple[_ForkResettable, ...], + pinned_classes: tuple[_ForkResettableClass, ...], +) -> None: + for descriptor in descriptors: + if descriptor.device is None: + continue + try: + stat_result = os.fstat(descriptor.fd) + except OSError: + continue + if (stat_result.st_dev, stat_result.st_ino) != (descriptor.device, descriptor.inode): + continue + with contextlib.suppress(OSError): + os.close(descriptor.fd) + for instance in pinned_objects: + instance._reset_after_fork_in_child() # ruff:ignore[private-member-access] # resets each pinned instance in the fork child + for cls in pinned_classes: + cls._reset_class_after_fork() + _registry.held.clear() + + +def _refresh_owner_pins() -> None: # pragma: needs fork + with _FORK_STATE.gate: + fork_owner_thread_ids = tuple(_FORK_STATE.fork_owner_depths) + if get_ident() not in fork_owner_thread_ids: # pragma: no cover - at-fork callbacks disable tracing + return + objects = tuple(_FORK_OBJECTS.values()) + classes = tuple(_FORK_CLASSES.values()) + for thread_id in fork_owner_thread_ids: + if object_snapshots := _FORK_STATE.pinned_objects.get(thread_id): # pragma: needs fork + object_snapshots[:] = [objects] * len(object_snapshots) + _FORK_STATE.pinned_classes[thread_id][:] = [classes] * len(object_snapshots) + + +# The defaults capture the module globals: the hook outlives them at interpreter shutdown, where CPython wipes the +# module dict to None before the final audit events fire. +def _audit_fork_safety( # pragma: no cover - CPython disables tracing while Python audit hooks run + event: str, + _args: Unused, + *, + _fork_events: frozenset[str] = _FORK_AUDIT_EVENTS, + _state: _ForkState = _FORK_STATE, + _get_ident: Callable[[], int] = get_ident, +) -> None: + if event in _fork_events: + if _get_ident() in _state.transitions or _state.fork_owner_depths: + msg = f"{event} is unsafe while filelock is changing descriptor ownership" + raise RuntimeError(msg) + elif event == "_posixsubprocess.fork_exec" and _get_ident() in _state.transitions: + msg = "fork_exec is unsafe while filelock is changing descriptor ownership" + raise RuntimeError(msg) + + +_register_fork_hooks() + + +__all__ = [ + "_UNSET_FILE_MODE", + "AcquireReturnProxy", + "BaseFileLock", + "CloseErrorPolicy", + "ContextErrorPolicy", + "FileLockContext", + "FileLockMeta", + "LockOptions", + "ThreadLocalFileContext", + "_acquire_thread_lock", + "_append_exception_context", + "_canonical", + "_check_timeout_max", + "_delegate_to_current", + "_ensure_current_process", + "_fork_transition", + "_grouped_errors", + "_raise_body_and_release", + "_raise_chained_errors", + "_raise_cleanup_errors", + "_raise_grouped_errors", + "_register_fork_class", + "_register_fork_object", + "_register_owned_descriptor", + "_resolve_poll_interval", + "_resolve_read_write_timeout", + "_resolve_timeout", + "_seconds", + "_timeout_behavior", + "_unregister_owned_descriptor", +] diff --git a/.venv/lib/python3.14/site-packages/filelock/_async.py b/.venv/lib/python3.14/site-packages/filelock/_async.py new file mode 100644 index 0000000000000000000000000000000000000000..a7a8fc2aacd8989132e919189d1362617de63308 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/filelock/_async.py @@ -0,0 +1,507 @@ +"""Separate caller cancellation from backend task and executor-future results.""" + +from __future__ import annotations + +__lazy_modules__ = ["asyncio", "collections", "concurrent.futures", "filelock._error"] + +import asyncio +import contextlib +import time +from collections import deque +from concurrent.futures import Future as ConcurrentFuture +from dataclasses import dataclass +from threading import Lock +from typing import TYPE_CHECKING, Final, Generic, Literal, NoReturn, TypeVar, cast +from weakref import WeakKeyDictionary + +from filelock._api import ( + _append_exception_context, + _fork_transition, + _raise_chained_errors, + _resolve_read_write_timeout, +) +from filelock._error import Timeout + +if TYPE_CHECKING: + from collections.abc import AsyncGenerator, Awaitable, Callable + from concurrent.futures import Executor + + from filelock._read_write import ReadWriteLock + from filelock._soft_rw import SoftReadWriteLock + +_T = TypeVar("_T") + +#: The holder an async wrapper records its backend level under. No thread has this identifier, so sync code on the +#: executor thread that took the level cannot re-enter or release it. +_TASKS_THREAD_ID: Final[int] = 0 + + +class _AsyncTransitionUnavailableError(Exception): + pass + + +@dataclass(frozen=True) +class _BackendOutcome(Generic[_T]): + value: _T | None = None + error: BaseException | None = None + + +class _AsyncTransitionGate: + """ + First-in, first-out admission for the calls that transition one lock context. + + Each call queues a ticket, and the head of the queue holds the gate. A call that gives up removes its own ticket, + so abandoned calls do not pile up behind a long holder. + """ + + def __init__(self) -> None: + self._queue_lock: Final[Lock] = Lock() + self._queue: Final[deque[ConcurrentFuture[None]]] = deque() + + @contextlib.asynccontextmanager + async def hold(self) -> AsyncGenerator[None]: + ticket: ConcurrentFuture[None] = ConcurrentFuture() + try: + self._join(ticket) + if not ticket.done(): + await self._wait_for_turn(ticket, blocking=True, cancel_check=None, deadline=None, poll_interval=0.05) + yield + finally: + self._leave(ticket) + + @contextlib.asynccontextmanager + async def hold_for_acquire( + self, + *, + blocking: bool, + cancel_check: Callable[[], bool] | None, + deadline: float | None, + poll_interval: float, + ) -> AsyncGenerator[None]: + ticket: ConcurrentFuture[None] = ConcurrentFuture() + try: + self._join(ticket) + if not ticket.done(): + await self._wait_for_turn( + ticket, + blocking=blocking, + cancel_check=cancel_check, + deadline=deadline, + poll_interval=poll_interval, + ) + yield + finally: + self._leave(ticket) + + async def _wait_for_turn( + self, + ticket: ConcurrentFuture[None], + *, + blocking: bool, + cancel_check: Callable[[], bool] | None, + deadline: float | None, + poll_interval: float, + ) -> None: + if not blocking: + raise _AsyncTransitionUnavailableError + waiter = asyncio.wrap_future(ticket) + while not ticket.done(): + if cancel_check is not None and cancel_check(): + raise _AsyncTransitionUnavailableError + wait_interval = poll_interval + if deadline is not None: + if (remaining := deadline - time.perf_counter()) <= 0: + raise _AsyncTransitionUnavailableError + wait_interval = min(wait_interval, remaining) + await asyncio.wait((waiter,), timeout=wait_interval) + # A hand-off that a signal handler's exception cut short leaves the head undone, so the head settles itself. + self._settle_head() + + def _join(self, ticket: ConcurrentFuture[None]) -> None: + with self._queue_lock: + self._queue.append(ticket) + if len(self._queue) == 1: + ticket.set_result(None) + + def _leave(self, ticket: ConcurrentFuture[None]) -> None: + with self._queue_lock: + try: + self._queue.remove(ticket) + except ValueError: + # The join never completed, or this ticket already left. + return + self._settle_head_locked() + + def _settle_head(self) -> None: + with self._queue_lock: + self._settle_head_locked() + + def _settle_head_locked(self) -> None: + if self._queue and not (head := self._queue[0]).done(): + head.set_result(None) + + +@dataclass +class _TaskHold: + """ + One task's hold, as the nesting count per wrapper it entered through. + + A task's hold ends when its last wrapper level is released or closed. Closing one wrapper ends only the levels + taken through it. + """ + + depths: dict[object, int] + + +class _TaskOwners: + """ + Per-task ownership in front of a sync reader/writer lock that tracks threads. + + An async wrapper runs each call on an executor thread, and that thread does not identify the calling task. Waiting + writers block new readers so a stream of readers cannot starve them. A thread lock guards the table because tasks on + different event loops share it. The table stands for one backend level, taken once for all tasks. Sync code can + end that level directly, with ``close()`` or a forced release on a shared singleton. The next admission then + forgets the holds instead of letting more tasks join them. + """ + + def __init__(self) -> None: + self._lock: Final[Lock] = Lock() + self._mode: Literal["read", "write"] | None = None + self._holds: dict[asyncio.Task[object], _TaskHold] = {} + self._transitioning = False + self._waiting_writers = 0 + #: One ticket per waiting call, settled on the next state change. A call that stops waiting takes its own out, + #: so one that times out or is canceled leaves nothing behind. + self._waiters: dict[ConcurrentFuture[None], None] = {} + + async def acquire( # ruff:ignore[too-many-arguments] # one keyword per piece of wrapper state the table consults + self, + mode: Literal["read", "write"], + *, + timeout: float, + blocking: bool, + lock_file: str, + via: object, + enter: Callable[[float], Awaitable[None]], + joinable: Callable[[], bool], + ) -> None: + task = _current_task() + timeout = _resolve_read_write_timeout(timeout, blocking=blocking) + deadline = None if not blocking or timeout < 0 else time.perf_counter() + timeout + waiting_writer = False + try: + while isinstance( + admission := self._admit( + task, mode, lock_file, via=via, joinable=joinable, waiting_writer=waiting_writer + ), + ConcurrentFuture, + ): + try: + if not blocking or (deadline is not None and time.perf_counter() >= deadline): + raise Timeout(lock_file) + if mode == "write" and not waiting_writer: + waiting_writer = True + with self._lock: + self._waiting_writers += 1 + wait = None if deadline is None else deadline - time.perf_counter() + await asyncio.wait((asyncio.wrap_future(admission),), timeout=wait) + finally: + self._forget(admission) + finally: + if waiting_writer: + with self._lock: + self._waiting_writers -= 1 + self._notify() + if admission == "held": + return + try: + await enter(-1 if deadline is None else max(0.0, deadline - time.perf_counter())) + except BaseException: + self._finish_transition(mode=None) + raise + self._finish_transition(mode=mode, holds={task: _TaskHold(depths={via: 1})}) + + def _admit( # ruff:ignore[too-many-arguments] # the acquire's state, passed through under the table lock + self, + task: asyncio.Task[object], + mode: Literal["read", "write"], + lock_file: str, + *, + via: object, + joinable: Callable[[], bool], + waiting_writer: bool, + ) -> Literal["held", "enter"] | ConcurrentFuture[None]: + with self._lock: + # A "held" answer trusts the backend level. Once the backend lost it, the holds are gone and the next task + # has to take the lock again. + if self._mode is not None and not self._transitioning and not joinable(): + self._holds.clear() + self._mode = None + if (hold := self._holds.get(task)) is not None: + if self._mode != mode: + msg = ( + f"Cannot acquire {mode} lock on {lock_file}: already holding a " + f"{'write' if mode == 'read' else 'read'} lock " + f"({'downgrade' if mode == 'read' else 'upgrade'} not allowed)" + ) + raise RuntimeError(msg) + hold.depths[via] = hold.depths.get(via, 0) + 1 + return "held" + writers_ahead = self._waiting_writers - waiting_writer + if not self._transitioning: + if mode == "read" and self._mode == "read" and not writers_ahead: + self._holds[task] = _TaskHold(depths={via: 1}) + return "held" + if not self._holds and (mode == "write" or not writers_ahead): + self._transitioning, self._mode = True, mode + return "enter" + return self._ticket() + + async def release( + self, + *, + force: bool, + lock_file: str, + via: object, + leave: Callable[[], asyncio.Future[_BackendOutcome[None]]], + still_held: Callable[[], bool], + ) -> None: + task = _current_task() + with self._lock: + if (hold := self._holds.get(task)) is None or via not in hold.depths: + if force: + return + msg = f"Cannot release a lock on {lock_file} that is not held by this task through this wrapper" + raise RuntimeError(msg) + if not force and hold.depths[via] > 1: + hold.depths[via] -= 1 + return + removed = hold.depths.pop(via) + if hold.depths: + return + del self._holds[task] + if self._holds: + return + self._transitioning = True + mode = self._mode + if not still_held(): + self._finish_transition(mode=None) + msg = f"Cannot release a lock on {lock_file} that is not held by this task" + raise RuntimeError(msg) + try: + cancellation = await _drain_cancellation(leave()) + except BaseException: + hold.depths[via] = removed + # Keep the hold for a retried release only while the sync lock still has the level. Restored after the sync + # lock let go, it would stand for a lock nobody holds. + self._finish_transition(mode=mode if still_held() else None, holds={task: hold}) + raise + self._finish_transition(mode=None) + if cancellation is not None: + # The release succeeded, so a restored hold would be a phantom no backend call can clear. + raise cancellation + + async def retire( + self, + via: object | None, + *, + leave: Callable[[], asyncio.Future[_BackendOutcome[None]]], + still_held: Callable[[], bool], + ) -> None: + """ + End the holds taken through *via*, or every hold when it is ``None``, as one transition. + + Leaves the backend through *leave* when that ends the last hold, and always when *via* is ``None``. Arrivals + wait while it runs, so none joins a hold the backend is letting go of. A transition already under way is waited + out first: a task entering through *via* must not get a hold after the wrapper is gone. A cancellation is raised + once the holds are settled. + """ + cancellation: asyncio.CancelledError | None = None + while isinstance(settled := self._drop_holds_via(via), ConcurrentFuture): + try: + await _wait_until_done(asyncio.wrap_future(settled)) + except asyncio.CancelledError as error: + cancellation = error + finally: + self._forget(settled) + mode, dropped = settled + if via is None or (mode is not None and still_held()): + try: + cancellation = await _drain_cancellation(leave()) or cancellation + except BaseException: + # A close that failed with the backend still holding stays retryable, as a failed release does. + self._finish_transition(mode=mode if still_held() else None, holds=dropped) + raise + self._finish_transition(mode=None) + elif mode is not None: + # Sync code already ended the level taken for tasks, so there is nothing to give back. + self._finish_transition(mode=None) + if cancellation is not None: + raise cancellation + + def _drop_holds_via( + self, via: object | None + ) -> tuple[Literal["read", "write"] | None, dict[asyncio.Task[object], _TaskHold]] | ConcurrentFuture[None]: + # Returns the future a running transition settles, or else the mode to leave (None when other holds keep the + # backend level) and the dropped holds. The table stays transitioning while that mode is left. + with self._lock: + if self._transitioning: + return self._ticket() + if via is None: + dropped = dict(self._holds) + self._holds.clear() + else: + dropped = {} + for task, hold in list(self._holds.items()): + if (count := hold.depths.pop(via, None)) is not None: + dropped[task] = _TaskHold(depths={via: count}) + if not hold.depths: + del self._holds[task] + if self._holds and via is not None: + return None, dropped + mode = self._mode + self._transitioning = mode is not None or via is None + return mode, dropped + + def _finish_transition( + self, *, mode: Literal["read", "write"] | None, holds: dict[asyncio.Task[object], _TaskHold] | None = None + ) -> None: + with self._lock: + self._transitioning, self._mode = False, mode + if mode is not None and holds: + for task, hold in holds.items(): + self._holds.setdefault(task, _TaskHold(depths={})).depths.update(hold.depths) + self._notify() + + def _ticket(self) -> ConcurrentFuture[None]: + # Called under the table lock, in the section that decided to wait, so no state change slips in between. + ticket: ConcurrentFuture[None] = ConcurrentFuture() + self._waiters[ticket] = None + return ticket + + def _forget(self, ticket: ConcurrentFuture[None]) -> None: + with self._lock: + self._waiters.pop(ticket, None) + + def _notify(self) -> None: + waiters, self._waiters = self._waiters, {} + for ticket in waiters: + ticket.set_result(None) + + +_TASK_OWNERS: Final[WeakKeyDictionary[ReadWriteLock | SoftReadWriteLock, _TaskOwners]] = WeakKeyDictionary() +_TASK_OWNERS_LOCK: Final[Lock] = Lock() + + +def _task_owners_for(sync_lock: ReadWriteLock | SoftReadWriteLock) -> _TaskOwners: + """Singleton wrappers share one sync lock, so they must share its task holds too.""" + with _fork_transition(), _TASK_OWNERS_LOCK: + if (owners := _TASK_OWNERS.get(sync_lock)) is None: + owners = _TASK_OWNERS[sync_lock] = _TaskOwners() + return owners + + +def _current_task() -> asyncio.Task[object]: + if (task := asyncio.current_task()) is None: + msg = "an async reader/writer lock must be used from inside an asyncio task" + raise RuntimeError(msg) + return task + + +async def _drain_cancellation(future: asyncio.Future[_BackendOutcome[None]]) -> asyncio.CancelledError | None: + # Returned, not raised, so the caller can tell a call that completed under cancellation from one that failed. + try: + await _wait_until_done(future) + except asyncio.CancelledError as cancellation: + try: + await _drain_future(future) + except BaseException as error: # ruff:ignore[blind-except] # reported with the cancellation below + _raise_cancelled_error(cancellation, error) + return cancellation + _future_result(future) + return None + + +async def _drain_future(future: asyncio.Future[_BackendOutcome[_T]]) -> _T: + while not future.done(): + with contextlib.suppress(asyncio.CancelledError): + await _wait_until_done(future) + return _future_result(future) + + +async def _wait_until_done(future: asyncio.Future[_T]) -> None: + if not future.done(): + await asyncio.wait((future,)) + + +def _future_result(future: asyncio.Future[_BackendOutcome[_T]]) -> _T: + outcome = future.result() + if (error := outcome.error) is None: + return cast("_T", outcome.value) + context = error.__context__ + try: + raise error # ruff:ignore[raise-within-try] # the handler restores context changed across the async boundary + except BaseException: + error.__context__ = context + raise + + +def _run_in_executor( + loop: asyncio.AbstractEventLoop, executor: Executor | None, func: Callable[[], _T] +) -> asyncio.Future[_BackendOutcome[_T]]: + outcome: asyncio.Future[_BackendOutcome[_T]] = loop.create_future() + + def settle(call: asyncio.Future[_BackendOutcome[_T]]) -> None: + try: + result = call.result() + except asyncio.CancelledError: + # shutdown(cancel_futures=True) dropped the call unrun: a CancelledError would cancel a task nobody canceled + # and hide that the lock state never changed. + msg = "the executor shut down before running the lock backend call" + result = _BackendOutcome(error=RuntimeError(msg)) + outcome.set_result(result) + + loop.run_in_executor(executor, _capture_call, func).add_done_callback(settle) + return outcome + + +def _capture_call(func: Callable[[], _T]) -> _BackendOutcome[_T]: + try: + return _BackendOutcome(value=func()) + except BaseException as error: # ruff:ignore[blind-except] # backend control-flow exceptions are operation results + return _BackendOutcome(error=error) + + +def _raise_cancelled_error(cancellation: asyncio.CancelledError, error: BaseException) -> NoReturn: + # A reconciliation step failed while unwinding a cancellation, so keep both exception chains. Splice the error's + # existing context onto the cancellation, then make the cancellation the error's context, so both the failure and + # the cancellation that triggered it survive. Shared by the async wrappers so cancellations report the same way. + if (context := error.__context__) is not None and context is not cancellation: + if (cancellation_context := cancellation.__context__) is not None: + _append_exception_context(context, cancellation_context) + cancellation.__context__ = context + error.__context__ = cancellation + _raise_chained_errors(error) + + +async def _capture_awaitable(awaitable: Awaitable[_T]) -> _BackendOutcome[_T]: + try: + return _BackendOutcome(value=await awaitable) + except BaseException as error: # ruff:ignore[blind-except] # backend cancellation must remain distinct from caller cancellation + return _BackendOutcome(error=error) + + +__all__ = [ + "_TASKS_THREAD_ID", + "_AsyncTransitionGate", + "_AsyncTransitionUnavailableError", + "_BackendOutcome", + "_capture_awaitable", + "_capture_call", + "_drain_future", + "_future_result", + "_raise_cancelled_error", + "_run_in_executor", + "_task_owners_for", + "_wait_until_done", +] diff --git a/.venv/lib/python3.14/site-packages/filelock/_async_read_write.py b/.venv/lib/python3.14/site-packages/filelock/_async_read_write.py new file mode 100644 index 0000000000000000000000000000000000000000..5920457397ac02dabb9a5a184c55cbd3a1f31488 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/filelock/_async_read_write.py @@ -0,0 +1,413 @@ +"""Async wrapper around :class:`ReadWriteLock` for use with ``asyncio``.""" + +from __future__ import annotations + +__lazy_modules__ = ["asyncio", "concurrent.futures", "filelock._async", "filelock._error", "functools", "threading"] + +import asyncio +import functools +import os +import sqlite3 +import threading +from concurrent.futures import ThreadPoolExecutor +from concurrent.futures import thread as executor_threads +from contextlib import asynccontextmanager +from typing import TYPE_CHECKING, Final, Literal, ParamSpec, TypeVar, cast + +from filelock._api import ( + _append_exception_context, + _ensure_current_process, + _fork_transition, + _register_fork_object, +) +from filelock._async import ( + _TASKS_THREAD_ID, + _BackendOutcome, + _capture_call, + _drain_future, + _future_result, + _raise_cancelled_error, + _task_owners_for, + _wait_until_done, +) +from filelock._error import Timeout +from filelock._read_write import ReadWriteLock + +if TYPE_CHECKING: + from collections.abc import AsyncGenerator, Callable, MutableMapping + from concurrent import futures + from types import TracebackType + + from filelock._api import AcquireReturnProxy + + +_P = ParamSpec("_P") +_R = TypeVar("_R") + + +class AsyncReadWriteLock: + """ + Async wrapper around :class:`ReadWriteLock` for use in ``asyncio`` applications. + + This wrapper dispatches every blocking SQLite operation to a thread pool via ``loop.run_in_executor()`` because + Python's :mod:`sqlite3` module has no async API. Each ``asyncio`` task owns its own hold. Tasks share a read lock, + and a task asking for the write lock waits until the other holders release, with waiting writers ahead of new + readers. As with :class:`ReadWriteLock`, upgrading or downgrading a held lock raises :class:`RuntimeError`. + Singleton wrappers over one path share their task holds, and closing one ends only the holds taken through it. + + :param lock_file: path to the SQLite database file used as the lock + :param timeout: maximum wait time in seconds; ``-1`` waits up to SQLite's busy-timeout cap of about 23 days + :param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately when the lock is unavailable + :param is_singleton: if ``True``, reuse existing :class:`ReadWriteLock` instances for the same resolved path + :param loop: event loop for ``run_in_executor``; ``None`` uses the running loop + :param executor: executor for ``run_in_executor``. When ``None`` this lock creates and owns a dedicated + single-thread executor so every operation runs on the same thread (SQLite affinity requires this) and shuts it + down in :meth:`close`. This lock uses a caller-supplied executor as-is and never shuts it down, so after passing + no executor call :meth:`close` to release the owned one. + + .. versionadded:: 3.21.0 + + """ + + def __init__( # ruff:ignore[too-many-arguments] # public constructor: one parameter per documented lock option + self, + lock_file: str | os.PathLike[str], + timeout: float = -1, + *, + blocking: bool = True, + is_singleton: bool = True, + loop: asyncio.AbstractEventLoop | None = None, + executor: futures.Executor | None = None, + ) -> None: + creator_pid = os.getpid() + self._creator_pid = creator_pid + self._fork_invalidated = False + self._closed = False + self._owns_backend = not is_singleton + #: Marks the task holds taken through this wrapper, which its close() ends. + self._via: Final = object() + _register_fork_object(self) + with _fork_transition(): + self._lock = ReadWriteLock(lock_file, timeout, blocking=blocking, is_singleton=is_singleton) + self._owners = _task_owners_for(self._lock) + self._loop = loop + self._owns_executor = executor is None + self._executor = executor or ThreadPoolExecutor(max_workers=1) + if os.getpid() != creator_pid: # pragma: forked child + msg = "AsyncReadWriteLock construction cannot continue after fork" + raise RuntimeError(msg) + + @property + def lock_file(self) -> str: + """The path to the lock file.""" + return self._lock.lock_file + + @property + def timeout(self) -> float: + """The default timeout.""" + return self._lock.timeout + + @property + def blocking(self) -> bool: + """Whether blocking is enabled by default.""" + return self._lock.blocking + + @property + def loop(self) -> asyncio.AbstractEventLoop | None: + """The event loop (or ``None`` for the running loop).""" + return self._loop + + @property + def executor(self) -> futures.Executor: + """The executor used for ``run_in_executor`` (a dedicated single-thread one if none was supplied).""" + return self._executor + + @asynccontextmanager + async def read_lock(self, timeout: float | None = None, *, blocking: bool | None = None) -> AsyncGenerator[None]: + """ + Async context manager that acquires and releases a shared read lock. + + Falls back to instance defaults for *timeout* and *blocking* when ``None``. + + :param timeout: maximum wait time in seconds, or ``None`` to use the instance default + :param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default + + """ + await self.acquire_read(timeout, blocking=blocking) + body_error: BaseException | None = None + try: + yield + except BaseException as error: + body_error = error + raise + finally: + await self._release_in_context(body_error) + + @asynccontextmanager + async def write_lock(self, timeout: float | None = None, *, blocking: bool | None = None) -> AsyncGenerator[None]: + """ + Async context manager that acquires and releases an exclusive write lock. + + Falls back to instance defaults for *timeout* and *blocking* when ``None``. + + :param timeout: maximum wait time in seconds, or ``None`` to use the instance default + :param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default + + """ + await self.acquire_write(timeout, blocking=blocking) + body_error: BaseException | None = None + try: + yield + except BaseException as error: + body_error = error + raise + finally: + await self._release_in_context(body_error) + + async def _release_in_context(self, body_error: BaseException | None) -> None: + try: + await self.release() + except BaseException as release_error: + if body_error is not None: + _append_exception_context(release_error, body_error) + raise + + async def acquire_read( + self, timeout: float | None = None, *, blocking: bool | None = None + ) -> AsyncAcquireReadWriteReturnProxy: + """ + Acquire a shared read lock. + + See :meth:`ReadWriteLock.acquire_read` for full semantics. + + :param timeout: seconds to wait; ``None`` uses the instance setting; ``-1`` waits up to SQLite's + busy-timeout cap of about 23 days + :param blocking: if ``False``, raise :class:`~filelock.Timeout` on contention; + ``None`` uses the instance setting + + :returns: a proxy that can be used as an async context manager to release the lock + + :raises RuntimeError: if the calling task already holds the write lock + :raises Timeout: if the lock cannot be acquired within *timeout* seconds + :raises TypeError: if *timeout* is a :class:`bool` or not a real number + :raises ValueError: if a blocking call gets a ``nan`` *timeout* or a negative one other than ``-1`` + + """ + self._raise_if_unusable() + await self._acquire("read", timeout, blocking=blocking) + return AsyncAcquireReadWriteReturnProxy(lock=self) + + async def acquire_write( + self, timeout: float | None = None, *, blocking: bool | None = None + ) -> AsyncAcquireReadWriteReturnProxy: + """ + Acquire an exclusive write lock. + + See :meth:`ReadWriteLock.acquire_write` for full semantics. + + :param timeout: seconds to wait; ``None`` uses the instance setting; ``-1`` waits up to SQLite's + busy-timeout cap of about 23 days + :param blocking: if ``False``, raise :class:`~filelock.Timeout` on contention; + ``None`` uses the instance setting + + :returns: a proxy that can be used as an async context manager to release the lock + + :raises RuntimeError: if the calling task already holds the read lock + :raises Timeout: if the lock cannot be acquired within *timeout* seconds + :raises TypeError: if *timeout* is a :class:`bool` or not a real number + :raises ValueError: if a blocking call gets a ``nan`` *timeout* or a negative one other than ``-1`` + + """ + self._raise_if_unusable() + await self._acquire("write", timeout, blocking=blocking) + return AsyncAcquireReadWriteReturnProxy(lock=self) + + async def _acquire(self, mode: Literal["read", "write"], timeout: float | None, *, blocking: bool | None) -> None: + timeout = self.timeout if timeout is None else timeout + blocking = self.blocking if blocking is None else blocking + + async def enter(remaining: float) -> None: + # A close() may have landed while this task waited for the table. + self._raise_if_unusable() + await self._run_acquire( + functools.partial( + self._lock._acquire_level_of, # ruff:ignore[private-member-access] # the wrapper owns this backend hold + _TASKS_THREAD_ID, + mode, + remaining, + blocking=blocking, + ) + ) + + def joinable() -> bool: + self._raise_if_unusable() + return self._backend_holds() + + await self._owners.acquire( + mode, + timeout=timeout, + blocking=blocking, + lock_file=self.lock_file, + via=self._via, + enter=enter, + joinable=joinable, + ) + + async def release(self, *, force: bool = False) -> None: + """ + Release one level of the calling task's hold. + + The wrapper releases the database lock once no task holds it. + + :param force: if ``True``, end every level the calling task took through this wrapper. Levels taken through + other wrappers on the same path stay. A wrapper that holds no level for the task releases nothing. + + :raises RuntimeError: if the calling task holds no level through this wrapper and *force* is ``False`` + + """ + _ensure_current_process() + if self._inherited: # pragma: needs fork + return + await self._owners.release( + force=force, + lock_file=self.lock_file, + via=self._via, + leave=functools.partial(self._submit, self._leave_backend), + still_held=self._backend_holds, + ) + + async def close(self) -> None: + """ + End the holds of the tasks that acquired through this instance and release the lock once no task holds it. + + After calling this method, the lock instance is no longer usable. With ``is_singleton=False`` it also closes the + underlying SQLite connection. A singleton wrapper leaves the shared :class:`ReadWriteLock` open, so tasks that + acquired through another wrapper and synchronous users of the same path keep their holds. The next acquire + notices a synchronous ``close()`` or forced release of that shared lock. One that lands in the few bytecodes + between that check and the admission still admits the task into the ended hold. + + """ + _ensure_current_process() + if self._inherited: # pragma: needs fork + return + if self._closed: + return + # Refuse acquires from here on, so none is admitted into a hold the backend is letting go of. + self._closed = True + try: + await self._owners.retire( + None if self._owns_backend else self._via, + leave=functools.partial(self._submit, self._lock.close if self._owns_backend else self._leave_backend), + still_held=self._backend_holds, + ) + except BaseException as error: + # Only a failed backend call can leave the level held, and then a retried close() has to find it. + if not isinstance(error, asyncio.CancelledError) and self._backend_holds(): + self._closed = False + else: + self._shutdown_owned_executor() + raise + # Wait for the worker to exit rather than letting it drain in the background: a caller that forks right + # after closing deserves a single-threaded process, and os.fork warns about any surviving thread. + if self._owns_executor: + await asyncio.to_thread(functools.partial(self._executor.shutdown, wait=True)) + + async def _run_acquire(self, acquire: Callable[..., AcquireReturnProxy]) -> None: + abandoned = threading.Event() + acquire_future = self._submit(acquire, cancel_check=abandoned.is_set) + try: + await _wait_until_done(acquire_future) + except asyncio.CancelledError as cancellation: + # Stops a backend wait on another process's hold, which would otherwise outlast the caller by as long. + abandoned.set() + try: + await _drain_future(acquire_future) + except Timeout: + # Nothing was taken, so the caller's cancellation is the whole outcome. + raise cancellation from None + except BaseException as error: # ruff:ignore[blind-except] # reported with the cancellation below + _raise_cancelled_error(cancellation, error) + try: + await _drain_future(self._submit(self._leave_backend)) + except BaseException as error: # ruff:ignore[blind-except] # reported with the cancellation below + _raise_cancelled_error(cancellation, error) + raise + _future_result(acquire_future) + + def _backend_holds(self) -> bool: + # A sync close() or forced release on a shared singleton ends the tasks' level without the table taking part. + return self._lock._holds_level_of(_TASKS_THREAD_ID) # ruff:ignore[private-member-access] # the wrapper owns this backend hold + + def _leave_backend(self) -> None: + # Only the tasks' level: a sync user of the same singleton keeps its own. + self._lock._release_level_of(_TASKS_THREAD_ID) # ruff:ignore[private-member-access] # the wrapper owns this backend hold + + def _submit( + self, func: Callable[_P, _R], *args: _P.args, **kwargs: _P.kwargs + ) -> asyncio.Future[_BackendOutcome[_R]]: + return (self._loop or asyncio.get_running_loop()).run_in_executor( + self._executor, + _capture_call, + functools.partial(func, *args, **kwargs), + ) + + def _shutdown_owned_executor(self) -> None: + if self._owns_executor: + self._executor.shutdown(wait=False) + + @property + def _inherited(self) -> bool: + return self._fork_invalidated or os.getpid() != self._creator_pid + + def _raise_if_unusable(self) -> None: + _ensure_current_process() + if self._inherited: # pragma: needs fork + msg = f"AsyncReadWriteLock on {self.lock_file} was invalidated by fork(); construct a new instance" + raise RuntimeError(msg) + if self._closed: + msg = "Cannot operate on a closed database." + raise sqlite3.ProgrammingError(msg) + + def _reset_after_fork_in_child(self) -> None: # pragma: forked child + self._fork_invalidated = True + if self._owns_executor: + # The worker did not survive the fork, yet before 3.12 the child's exit still wakes it through its queue. On + # PyPy that queue is pure Python, and its lock may have been held by the worker at fork time, so exit hangs. + # CPython 3.12+ forgets every inherited worker in the child; do the same for the one this lock owns. + # typeshed declares the registry read-only; it is a WeakKeyDictionary the exit hook iterates. + registry = cast( + "MutableMapping[threading.Thread, object]", + executor_threads._threads_queues, # ruff:ignore[private-member-access] # the exit hook's registry + ) + for worker in cast("ThreadPoolExecutor", self._executor)._threads: # ruff:ignore[private-member-access] # only the executor lists its workers + registry.pop(worker, None) + + def __del__(self) -> None: + # Safety net when close() was never called: shut down the executor we own so its worker thread does not + # outlive the lock. shutdown(wait=False) never blocks. + if os.getpid() == getattr(self, "_creator_pid", None) and getattr(self, "_owns_executor", False): + self._executor.shutdown(wait=False) + + +class AsyncAcquireReadWriteReturnProxy: + """Context-aware object that releases the async read/write lock on exit.""" + + def __init__(self, lock: AsyncReadWriteLock) -> None: + self.lock = lock + + async def __aenter__(self) -> AsyncReadWriteLock: + return self.lock + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: + await self.lock.release() + + +__all__ = [ + "AsyncAcquireReadWriteReturnProxy", + "AsyncReadWriteLock", +] diff --git a/.venv/lib/python3.14/site-packages/filelock/_descriptor.py b/.venv/lib/python3.14/site-packages/filelock/_descriptor.py new file mode 100644 index 0000000000000000000000000000000000000000..61450ff9fae19481ddd7213ce8e5bf5f2d3c32f2 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/filelock/_descriptor.py @@ -0,0 +1,76 @@ +"""A minimal native lock over a caller-owned file descriptor, contending with :class:`FileLock` on the same file.""" + +from __future__ import annotations + +__lazy_modules__ = ["filelock._unix", "threading", "typing"] + +import sys +import time +from threading import TIMEOUT_MAX +from typing import Final + +from filelock._api import _seconds + +if sys.platform == "win32": # pragma: win32 cover + from filelock._windows import _lock_fd_nonblocking, _unlock_fd +else: # pragma: win32 no cover + from filelock._unix import _lock_fd_nonblocking, _unlock_fd + + +def lock_descriptor(fd: int, *, blocking: bool = True, poll_interval: float = 0.05) -> bool: + """ + Take the native OS lock on *fd*, a file descriptor the caller opened and owns. + + This is the same one-byte exclusive lock :class:`FileLock` uses, so a descriptor lock and a path lock on the same + file contend with each other. Unlike :class:`FileLock` it adds no path handling: it never opens, truncates, closes, + unlinks, chmods, canonicalizes, or falls back. The caller owns *fd* before, during, and after the call, and must + close it. On Windows *fd* must be a synchronous descriptor (its handle not opened with ``FILE_FLAG_OVERLAPPED``). + + For timeout, reentrancy, singleton, lifetime, or stale-break behavior, use :class:`FileLock`. There is no async + wrapper. Run this in an executor, or drive ``blocking=False`` from your own polling loop. + + :param fd: an open file descriptor the caller owns. + :param blocking: when ``True`` (default), retry the nonblocking attempt every *poll_interval* seconds until it + succeeds; when ``False``, make one attempt. + :param poll_interval: positive seconds between attempts while blocking, at most :data:`threading.TIMEOUT_MAX`; + ignored when *blocking* is ``False``. + + :returns: ``True`` once the lock is held, or ``False`` on contention when ``blocking`` is ``False``. + + :raises OSError: for a permanent native failure, such as an invalid descriptor, or with ``errno.ENOSYS`` when the + Python build lacks the native locking primitive. The descriptor is left open. + :raises ValueError: if a blocking call receives a *poll_interval* outside that range, including ``nan``. + + .. versionadded:: 3.30.0 + + """ + if not blocking: + return _lock_fd_nonblocking(fd) + # time.sleep raises OverflowError past TIMEOUT_MAX; nan fails both comparisons. + if not 0 < (poll_interval := _seconds("poll_interval", poll_interval)) <= TIMEOUT_MAX: + msg: Final[str] = f"poll_interval must be greater than 0 and at most {TIMEOUT_MAX}, got {poll_interval}" + raise ValueError(msg) + while not _lock_fd_nonblocking(fd): + time.sleep(poll_interval) + return True + + +def unlock_descriptor(fd: int) -> None: + """ + Release the native OS lock on *fd* without touching the descriptor. + + :param fd: the descriptor a prior :func:`lock_descriptor` locked; the caller still owns and must close it. + + :raises OSError: if the native unlock fails, including ``errno.ENOSYS`` when the Python build lacks the native + locking primitive; the caller may retry on the same descriptor. + + .. versionadded:: 3.30.0 + + """ + _unlock_fd(fd) + + +__all__ = [ + "lock_descriptor", + "unlock_descriptor", +] diff --git a/.venv/lib/python3.14/site-packages/filelock/_error.py b/.venv/lib/python3.14/site-packages/filelock/_error.py new file mode 100644 index 0000000000000000000000000000000000000000..fb5831306a90d2e1369c6242c7a16f52fa26fddc --- /dev/null +++ b/.venv/lib/python3.14/site-packages/filelock/_error.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +__lazy_modules__ = ["typing"] + +from typing import Final + + +class Timeout(TimeoutError): # ruff:ignore[error-suffix-on-exception-name] # public exception name; renaming breaks the API + """Raised when the lock could not be acquired in *timeout* seconds.""" + + def __init__(self, lock_file: str, holder: str | None = None) -> None: + super().__init__() + self._lock_file = lock_file + self._holder = holder + + def __reduce__(self) -> tuple[type[Timeout], tuple[str], dict[str, object]]: + # holder rides in the state so a subclass taking only lock_file, and a 4.0.12 reader, can still unpickle it. + return self.__class__, (self._lock_file,), self.__dict__ + + def __str__(self) -> str: # pragma: needs hard-link + message: Final = f"The file lock '{self._lock_file}' could not be acquired." + if self._holder is None: + return message + return f"{message} It is held by {self._holder}, which this host cannot probe." + + def __repr__(self) -> str: + return f"{self.__class__.__name__}({self.lock_file!r})" + + @property + def lock_file(self) -> str: + """The path of the file lock.""" + return self._lock_file + + +class SoftFileLockLifetimeWarning(FutureWarning): + """ + The configured soft-lock lifetime permits overlapping live holders after expiry. + + A :class:`FutureWarning` shows by default wherever the lock is configured; a :class:`DeprecationWarning` shows only + in ``__main__``. + + """ + + +class LeaseSettingsMismatch(ValueError): # ruff:ignore[error-suffix-on-exception-name] # public exception name; renaming breaks the API + """A lease contender disagrees with the published claim about how long the lease lasts.""" + + +class SoftFileLockProtocolError(OSError): + """Raised when strict soft-lock state cannot be interpreted without risking overlap.""" + + def __init__(self, lock_file: str, claim_name: str | None, reason: str) -> None: + self._lock_file = lock_file + self._claim_name = claim_name + self._reason = reason + super().__init__(self.__str__()) + + def __reduce__( + self, + ) -> tuple[ + type[SoftFileLockProtocolError], tuple[str, str | None, str], dict[str, object] + ]: # pragma: needs hard-link + return self.__class__, (self._lock_file, self._claim_name, self._reason), self.__dict__ + + def __str__(self) -> str: + location = self._lock_file if self._claim_name is None else f"{self._lock_file}: claim {self._claim_name!r}" + return f"Invalid strict soft-lock state at {location}: {self._reason}" + + @property + def lock_file(self) -> str: # pragma: needs hard-link + """The requested lock path.""" + return self._lock_file + + @property + def claim_name(self) -> str | None: # pragma: needs hard-link + """The claim that caused the error, if scanning identified one.""" + return self._claim_name + + @property + def reason(self) -> str: # pragma: needs hard-link + """The protocol validation failure.""" + return self._reason + + +__all__ = [ + "LeaseSettingsMismatch", + "SoftFileLockLifetimeWarning", + "SoftFileLockProtocolError", + "Timeout", +] diff --git a/.venv/lib/python3.14/site-packages/filelock/_identity.py b/.venv/lib/python3.14/site-packages/filelock/_identity.py new file mode 100644 index 0000000000000000000000000000000000000000..4abc60d0406be2be484b5f2a918952c6d3cd14e7 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/filelock/_identity.py @@ -0,0 +1,328 @@ +from __future__ import annotations + +__lazy_modules__ = ["hashlib", "pathlib", "socket", "typing"] + +import hashlib +import os +import socket +import sys +from errno import EPERM, ESRCH +from pathlib import Path +from typing import Final + +#: Every marker format in this package caps the hostname at the RFC 1123 limit and reads a longer one as malformed. +_HOST_NAME_LIMIT: Final[int] = 253 +#: The bytes those formats carry verbatim: printable non-space ASCII, less the ``?`` reserved for the escape. +_HOST_NAME_VERBATIM: Final[frozenset[int]] = frozenset(range(0x21, 0x7F)) - {ord("?")} +#: The characters at the end of an over-long name that carry a digest of the part cut off. +_HOST_NAME_DIGEST_LEN: Final[int] = 8 +#: The kernel ABI fixes the initial PID namespace's inode (``PID_NS_INIT_INO`` in ``linux/nsfs.h``); every namespace +#: created later draws one from ``0xF0000000`` up, so this value names the host's own namespace. +_INITIAL_PID_NAMESPACE: Final[int] = 0xEFFFFFFC +_NAMESPACE_MARK: Final[str] = "?pidns-" + + +def host_name() -> str: + """ + The hostname recorded alongside an owner, so a marker written on another machine is never probed here. + + Marker formats here hold the field to printable non-space ASCII, while ``socket.gethostname()`` reports whatever + the kernel stores: a space, a newline that forges an extra marker line, or a byte no codec encodes, which Python + hands back as a surrogate. A holder that publishes one raw writes a marker it cannot read back, and loses the lock + to the first peer that ages it out as malformed. ``?`` is illegal in a hostname, so escaping it along with every + out-of-grammar byte as ``?`` leaves a real name untouched and still keeps two hosts apart, which + :func:`owner_is_stale` relies on to refuse to probe a foreign PID. + + Linux appends the PID namespace, as Mercurial's lock prefix does. Two containers in one pod share the hostname and + the boot id but not the PID namespace, so a PID recorded in one names another process, or none, in the other. The + suffix starts with a ``?`` no escape can produce, so ``a`` in a namespace never reads as a host literally named + ``a?pidns-...``. The initial namespace adds nothing, which keeps a host's name what older releases recorded. + + An over-long name ends in a digest of the part cut to fit the limit. A bare cut gives every name that shares the + kept prefix one identity, and :func:`owner_is_stale` would then probe a local PID for another host's holder. + """ + name = ( + "".join( + chr(byte) if byte in _HOST_NAME_VERBATIM else f"?{byte:02x}" + for byte in socket.gethostname().encode("utf-8", "surrogateescape") + ) + + _pid_namespace_suffix() + ) + if len(name) > _HOST_NAME_LIMIT: + keep = _HOST_NAME_LIMIT - _HOST_NAME_DIGEST_LEN - 1 + name = f"{name[:keep]}-{hashlib.sha256(name[keep:].encode()).hexdigest()[:_HOST_NAME_DIGEST_LEN]}" + # An escape is three characters and a kept byte is never '?', so a bare '?' can only mean an empty hostname. + return name or "?" + + +def _pid_namespace_suffix() -> str: + try: + namespace: Final = Path("/proc/self/ns/pid").stat().st_ino + except OSError: # only Linux 3.8+ has the file, and a sandbox may hide it; nothing else tells namespaces apart + return "" + return "" if namespace == _INITIAL_PID_NAMESPACE else f"{_NAMESPACE_MARK}{namespace:x}" + + +def owner_is_stale(pid: int, hostname: str, start_token: int | None) -> bool: + """ + Whether the recorded owner is provably gone, so reclaiming its marker cannot detach a live holder. + + Fail closed: return ``True`` only when this process can prove the exact recorded owner is dead. A marker from + another host cannot be probed; a live PID whose start token still matches, or whose token cannot be read, is the + holder or is indistinguishable from it; a live PID whose start token differs is a recycled PID, so the process that + wrote the marker is gone. PostgreSQL, Qt ``QLockFile`` and Mercurial all break a stale lock only on proof of death + and treat an unreadable or foreign owner as still holding. + """ + if hostname != host_name(): + return False + if process_alive(pid) and ( + start_token is None or (current := process_start_token(pid)) is None or _same_start(current, start_token) + ): + return False + # Only a bare hostname can be an older release's, written from a container that shares this host's name. + return _NAMESPACE_MARK in hostname or not _owner_runs_elsewhere(pid, start_token) + + +def owner_is_current_process(pid: int, hostname: str, start_token: int | None) -> bool: + """ + Whether the recorded owner is this process, so a marker a recycled PID left behind never reads as ours. + + Fail closed: a recorded start token must match this process's. One that differs was written by an earlier process + that reused this PID, and when this process cannot read its own token nothing proves the marker is ours. A marker + without a token falls back to the PID and hostname. + """ + if pid != os.getpid() or hostname != host_name(): + return False + return start_token is None or start_token == process_start_token(pid) + + +def _same_start(current: int, recorded: int) -> bool: + # A Linux token carries the boot id in its high 64 bits, or 0 there when the process could not read the boot id. + # systemd's ProcSubset=pid hides /proc/sys from one unit, so one process may lack the boot id while its peer has + # it. A missing boot id then proves nothing, and only starttime is compared. Other platforms' tokens fit in the + # low bits. + return current == recorded or (0 in {current >> 64, recorded >> 64} and current % 2**64 == recorded % 2**64) + + +if sys.platform == "win32": # pragma: win32 cover + import ctypes + from ctypes import wintypes + + _KERNEL32: Final[ctypes.WinDLL] = ctypes.WinDLL("kernel32", use_last_error=True) + _KERNEL32.CloseHandle.argtypes = [wintypes.HANDLE] + _KERNEL32.CloseHandle.restype = wintypes.BOOL + _KERNEL32.OpenProcess.argtypes = [wintypes.DWORD, wintypes.BOOL, wintypes.DWORD] + _KERNEL32.OpenProcess.restype = wintypes.HANDLE + _KERNEL32.GetProcessTimes.argtypes = [ + wintypes.HANDLE, + ctypes.POINTER(wintypes.FILETIME), + ctypes.POINTER(wintypes.FILETIME), + ctypes.POINTER(wintypes.FILETIME), + ctypes.POINTER(wintypes.FILETIME), + ] + _KERNEL32.GetProcessTimes.restype = wintypes.BOOL + + _WIN_SYNCHRONIZE: Final[int] = 0x100000 + _WIN_PROCESS_QUERY_LIMITED_INFORMATION: Final[int] = 0x1000 + _WIN_ERROR_INVALID_PARAMETER: Final[int] = 87 + _WIN_INHERIT_HANDLE: Final[bool] = False + + def process_alive(pid: int) -> bool: + """Whether a process with this PID exists, treating an access denial as proof it does.""" + handle = _KERNEL32.OpenProcess(_WIN_SYNCHRONIZE, _WIN_INHERIT_HANDLE, pid) + if handle: + _KERNEL32.CloseHandle(handle) + return True + return ctypes.get_last_error() != _WIN_ERROR_INVALID_PARAMETER + + def process_start_token(pid: int) -> int | None: + """The process creation FILETIME as a 100ns tick count, or ``None`` when it cannot be read.""" + handle = _KERNEL32.OpenProcess(_WIN_PROCESS_QUERY_LIMITED_INFORMATION, _WIN_INHERIT_HANDLE, pid) + if not handle: + return None + creation, exit_time, kernel_time, user_time = (wintypes.FILETIME() for _ in range(4)) + try: + if not _KERNEL32.GetProcessTimes( + handle, + ctypes.byref(creation), + ctypes.byref(exit_time), + ctypes.byref(kernel_time), + ctypes.byref(user_time), + ): + return None # pragma: no cover # win32 GetProcessTimes failure path; not reproducible on a live handle + finally: + _KERNEL32.CloseHandle(handle) + return (creation.dwHighDateTime << 32) | creation.dwLowDateTime + +else: # pragma: win32 no cover + + def process_alive(pid: int) -> bool: + """Whether a process with this PID exists, treating an access denial (``EPERM``) as proof it does.""" + try: + os.kill(pid, 0) + except OSError as error: + if error.errno == ESRCH: + return False + if error.errno == EPERM: + return True + raise + return True + + if sys.platform in {"linux", "android"}: # pragma: linux cover + # Termux/Android reports sys.platform == "android" but runs the Linux kernel, so /proc//stat and the boot + # id are the same reliable start-time source; treat it exactly like Linux rather than the tokenless fallback. + # comm (field 2) is wrapped in parentheses and may itself contain spaces or a ')', so the fixed fields start + # after the final ')'. starttime is field 22 overall, the twentieth of those trailing fields (index 19). + _STARTTIME_INDEX: Final[int] = 19 + + def _read_boot_id() -> int: + # starttime is measured in clock ticks since boot, so on its own it repeats across a reboot. Folding the + # boot id into the high bits makes the Linux token reboot-safe like the absolute clocks macOS and Windows + # expose, while staying a single integer so a 3.29 reader still parses the third marker line. 0 when this + # process cannot read the boot id leaves the bare starttime, which _same_start compares alone. + try: + boot_id = Path("/proc/sys/kernel/random/boot_id").read_text(encoding="ascii") + return int(boot_id.strip().replace("-", ""), 16) + except (OSError, ValueError): + return 0 + + _BOOT_ID: Final[int] = _read_boot_id() + #: ``TIME_NS_INIT_INO`` in ``linux/proc_ns.h``, fixed by the kernel ABI like the initial PID namespace's inode. + _INITIAL_TIME_NAMESPACE: Final[int] = 0xEFFFFFFA + + def process_start_token(pid: int) -> int | None: + """ + The ``/proc//stat`` ``starttime`` folded with the boot id. + + ``None`` when the process is gone, or when this process runs in a time namespace with a boottime offset. + Since Linux 5.6 the kernel shifts ``starttime`` by the reader's offset, so a value read there differs + from what a peer outside reads. + """ + return _read_start_token(pid) if _starttime_unshifted() else None + + def _starttime_unshifted() -> bool: + # setns can move this process into another time namespace, so ask on every read. + try: + namespace: Final = Path("/proc/self/ns/time").stat().st_ino + except FileNotFoundError: # no entry means no time namespaces on this kernel + return True + except OSError: # any other failure leaves the namespace unknown, so starttime is not comparable + return False + if namespace == _INITIAL_TIME_NAMESPACE: + return True + # A container runtime may give each container a time namespace without offsets. timens_offsets describes + # the namespace this process's children get, its own unless it unshared one since. + try: + if Path("/proc/self/ns/time_for_children").stat().st_ino != namespace: + return False + offsets: Final = Path("/proc/self/timens_offsets").read_text(encoding="ascii") + except OSError: + return False + # The clock goes by name since Linux 5.8 and by clock id before. + return any(line.split() in (["boottime", "0", "0"], ["7", "0", "0"]) for line in offsets.splitlines()) + + def _read_start_token(pid: int) -> int | None: + try: + data = Path(f"/proc/{pid}/stat").read_bytes() + except OSError: + return None + # psutil identifies a process by the same (pid, starttime) pair; the boot id extends that across reboots. + fields = data[data.rfind(b")") + 1 :].split() + if len(fields) <= _STARTTIME_INDEX: # pragma: no cover # a truncated /proc read, never seen in practice + return None + try: + starttime = int(fields[_STARTTIME_INDEX]) + except ValueError: # pragma: no cover # /proc always renders starttime as an integer + return None + return (_BOOT_ID << 64) | starttime + + def _owner_runs_elsewhere(pid: int, start_token: int | None) -> bool: + """ + Whether the owner runs in another PID namespace as *pid*. + + An older release in a container sharing this hostname recorded its in-container PID, which means nothing + here. Readers without a boottime offset read the same starttime and boot id. A task with the recorded + start token whose innermost PID is *pid* is therefore the owner; one that started in the same tick is not. + A reader with an offset cannot compare start tokens, so the innermost PID alone keeps the marker. + """ + if start_token is None or (_BOOT_ID and start_token >> 64 not in {0, _BOOT_ID}): + return False + unshifted: Final = _starttime_unshifted() + try: + with os.scandir("/proc") as entries: + return any( + entry.name.isdigit() + and (task := int(entry.name)) != pid + and ( + not unshifted + or ((token := _read_start_token(task)) is not None and _same_start(token, start_token)) + ) + and _innermost_pid(task) == pid + for entry in entries + ) + except OSError: + # A /proc this process cannot list proves nothing either way, which leaves 4.0.12's verdict. + return False + + def _innermost_pid(task: int) -> int | None: + try: + status: Final = Path(f"/proc/{task}/status").read_bytes() + except OSError: # pragma: no cover # the task exited between the stat read and this one + return None + # The kernel ends a line only with "\n". A task names itself, and its Name line keeps "\r", "\v" and the + # other separators that a text-mode read and str.splitlines() split at. A name could then forge a line. + return next( + (int(line.split()[-1]) for line in status.split(b"\n") if line.startswith(b"NSpid:")), + None, # kernels before 4.1 record no namespace PIDs + ) + + elif sys.platform == "darwin": # pragma: darwin cover + import ctypes + import struct + + _LIBC: Final[ctypes.CDLL] = ctypes.CDLL(None, use_errno=True) + _CTL_KERN: Final[int] = 1 + _KERN_PROC: Final[int] = 14 + _KERN_PROC_PID: Final[int] = 1 + # kinfo_proc opens with kp_proc.p_starttime (a struct timeval) at offset 0: int64 seconds, int32 microseconds. + # The offset is fixed by the struct chain kinfo_proc -> extern_proc -> p_un, so a read at 0 is not a guess. + _TIMEVAL_AT_ZERO: Final[str] = " int | None: + """The process start time in microseconds from ``sysctl(KERN_PROC_PID)``, or ``None`` when it is gone.""" + mib = (ctypes.c_int * 4)(_CTL_KERN, _KERN_PROC, _KERN_PROC_PID, pid) + length = ctypes.c_size_t(0) + # The size probe reports the kinfo_proc size for any PID, so the read below, not the probe, tells a live + # process from a gone one: a gone PID leaves the fetch a zero-length success, so re-check the length after. + if _LIBC.sysctl(mib, 4, None, ctypes.byref(length), None, 0) != 0: + return None + buffer = (ctypes.c_char * length.value)() + if _LIBC.sysctl(mib, 4, buffer, ctypes.byref(length), None, 0) != 0 or length.value < _TIMEVAL_SIZE: + return None + seconds, microseconds = struct.unpack_from(_TIMEVAL_AT_ZERO, buffer.raw, 0) + return seconds * 1_000_000 + microseconds + + else: # pragma: no cover # a POSIX platform without a proven start-time source falls back to fail-closed liveness + + def process_start_token(pid: int) -> int | None: + """No proven start-time source, so the owner carries no token and liveness rests on the PID alone.""" + del pid + return None + + +if sys.platform not in {"linux", "android"}: # pragma: linux no cover # pragma: android no cover + + def _owner_runs_elsewhere(pid: int, start_token: int | None) -> bool: + """Without Linux PID namespaces, a PID that is gone or recycled leaves its owner nowhere else to run.""" + del pid, start_token + return False + + +__all__ = [ + "host_name", + "owner_is_current_process", + "owner_is_stale", + "process_alive", + "process_start_token", +] diff --git a/.venv/lib/python3.14/site-packages/filelock/_lease.py b/.venv/lib/python3.14/site-packages/filelock/_lease.py new file mode 100644 index 0000000000000000000000000000000000000000..d328eaf4be6f35b93d81a64cc95a640ba23a63c3 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/filelock/_lease.py @@ -0,0 +1,402 @@ +from __future__ import annotations + +__lazy_modules__ = [ + "contextlib", + "filelock._error", + "filelock._identity", + "filelock._soft", + "filelock._util", + "math", + "secrets", + "threading", + "weakref", +] + +import os +import secrets +import sys +import time +from contextlib import suppress +from dataclasses import dataclass +from math import isfinite +from threading import Event, Thread, current_thread +from typing import TYPE_CHECKING, Final, Literal, cast +from weakref import WeakMethod + +from filelock._api import _check_timeout_max, _seconds +from filelock._error import LeaseSettingsMismatch +from filelock._identity import owner_is_stale +from filelock._marker import MarkerSoftFileLock, OwnerMode, OwnerRecord, parse_marker +from filelock._soft import _read_lock_file +from filelock._util import pin_inode, touch + +if TYPE_CHECKING: + from collections.abc import Callable, Mapping + from typing import Unpack + + from filelock._api import LockOptions, _ExtraValue + +#: ``evicted`` is a :class:`~filelock.SoftReadWriteLock` loss: a peer waited out the stale threshold and committed a +#: generation without this holder. +CompromiseReason = Literal["marker-missing", "owner-changed", "refresh-failed", "evicted"] + +_RefreshOutcome = Literal["ok", "transient", "marker-missing", "owner-changed"] +_DEFAULT_LEASE_DURATION: Final[float] = 30.0 + + +@dataclass(frozen=True) +class LeaseCompromise: + """Why a held lease stopped being this process's to hold, and the error that ended its refresh, if any.""" + + lock_file: str + token: str + reason: CompromiseReason + error: Exception | None = None + + +@dataclass(frozen=True) +class _Heartbeat: + """A running heartbeat and the event that stops it, which only ever exist together.""" + + thread: Thread + stop: Event + + +@dataclass +class _LeaseClaim: + """The state of one claim: its token, its heartbeat, and how that claim was lost.""" + + token: str | None = None + compromise: LeaseCompromise | None = None + heartbeat: _Heartbeat | None = None + + +class SoftFileLease(MarkerSoftFileLock): + """ + Existence lock whose claim expires, so a peer may take it while the previous holder still runs. + + A lease trades mutual exclusion for progress. The holder publishes a claim and refreshes it every + ``heartbeat_interval`` seconds; a contender takes the marker once it has seen no refresh for ``lease_duration`` + seconds of its own monotonic clock, so clock skew between hosts cannot expire a live claim. Nothing stops the + expired holder: it keeps running, and it keeps using whatever the lock protects. Treat the lease as a hint about + who *should* be working, not as a guarantee that only one worker is. + + To make a protected resource reject a superseded holder, that resource must be linearizable and must fence on a + monotonic generation it controls. :attr:`token` names a claim; it does not fence one. Where overlap is unacceptable, + use :class:`StrictSoftFileLock ` instead. + + Every contender for a path must agree on ``lease_duration``. A contender that finds a claim published under a + different duration raises :class:`LeaseSettingsMismatch ` rather than apply its own + expiry to a peer that never agreed to it. + + Expiry reclaims less on Windows, which refuses to rename or delete a file another process holds open. A peer there + takes an expired claim only once the previous holder's process exits and its handle closes; a holder that lives on + but stops refreshing keeps the marker. Unix reclaims the marker either way. + + ``on_compromise`` fires from the heartbeat thread when a refresh fails, or when the marker vanishes or names another + owner. The holder should stop touching the protected resource when it runs. Because it runs on that thread, a + ``release()`` inside it only takes effect when the lease was built with ``thread_local=False``; the default + thread-local context hides the claim from every thread but the one that acquired it, so the release does nothing. + Signal the acquiring thread instead when the context stays thread-local. + + .. versionadded:: 3.30.0 + + """ + + _owner_mode: OwnerMode = "lease" + + #: lease_duration replaces the legacy age-based lifetime, so accepting both would give one lock two expiry clocks. + _lifetime_supported: bool = False + _lifetime_unsupported_reason: str = "lease_duration sets when a lease expires" + + def __init__( + self, + lock_file: str | os.PathLike[str], + *, + lease_duration: float = _DEFAULT_LEASE_DURATION, + heartbeat_interval: float | None = None, + on_compromise: Callable[[LeaseCompromise], None] | None = None, + **kwargs: Unpack[LockOptions], + ) -> None: + """ + Create a lease. + + :param lease_duration: seconds a contender must see the marker go unrefreshed before it may take the claim. + Every contender for the path must pass the same value. + :param heartbeat_interval: seconds between refreshes. Defaults to a third of ``lease_duration``, leaving room + for two missed refreshes before a peer may take the claim. Must be shorter than ``lease_duration`` and at + most :data:`threading.TIMEOUT_MAX`. + :param on_compromise: called from the heartbeat thread with a :class:`LeaseCompromise` when the claim is lost. + :param kwargs: every other :class:`BaseFileLock ` option, ``timeout`` and ``mode`` among + them. The metaclass passes them all by keyword, and taking them here lets + :class:`AsyncSoftFileLease ` add the async plumbing a fixed signature would + hide. + + """ + # A plain float: the marker records its repr, which NumPy's float64 spells in a form no parser reads. + if not isfinite(lease_duration := _seconds("lease_duration", lease_duration)) or lease_duration <= 0: + msg = f"lease_duration must be positive and finite, got {lease_duration!r}" + raise ValueError(msg) + heartbeat_interval = ( + lease_duration / 3 if heartbeat_interval is None else _seconds("heartbeat_interval", heartbeat_interval) + ) + if not 0 < heartbeat_interval < lease_duration: + msg = f"heartbeat_interval must be positive and below lease_duration, got {heartbeat_interval!r}" + raise ValueError(msg) + _check_timeout_max("heartbeat_interval", heartbeat_interval) + super().__init__(lock_file, **kwargs) + self._lease_duration = lease_duration + self._heartbeat_interval = heartbeat_interval + self._on_compromise = on_compromise + # Two threads lazily creating a shared claim could each get one, and release() would miss the heartbeat's. + if not self.is_thread_local(): + self._context.lease_claim = _LeaseClaim() + + def _singleton_extra_mismatches(self, kwargs: Mapping[str, _ExtraValue], /) -> dict[str, tuple[str, str]]: + mismatches = super()._singleton_extra_mismatches(kwargs) + lease_duration: Final = _seconds( + "lease_duration", cast("float", kwargs.get("lease_duration", _DEFAULT_LEASE_DURATION)) + ) + if lease_duration != self._lease_duration: + mismatches["lease_duration"] = (str(lease_duration), str(self._lease_duration)) + # An omitted heartbeat_interval resolves as in __init__; a differing duration is already reported above. + heartbeat_interval: Final = ( + self._lease_duration / 3 + if (passed := kwargs.get("heartbeat_interval")) is None + else _seconds("heartbeat_interval", cast("float", passed)) + ) + if heartbeat_interval != self._heartbeat_interval: + mismatches["heartbeat_interval"] = (str(passed), str(self._heartbeat_interval)) + # A callback compares by identity, as on_acquired does: two equal callables can close over different state. + if (on_compromise := kwargs.get("on_compromise")) is not self._on_compromise: + mismatches["on_compromise"] = (str(on_compromise), str(self._on_compromise)) + return mismatches + + @property + def _claim(self) -> _LeaseClaim: + # Per hold, so another thread's failed acquire cannot stop the holder's heartbeat. + state: Final = self._hold_state() + if (claim := state.lease_claim) is None: + claim = state.lease_claim = _LeaseClaim() + return claim + + @property + def lease_duration(self) -> float: + """The staleness in seconds after which a contender may take this claim.""" + return self._lease_duration + + @property + def token(self) -> str | None: + """ + The token naming the claim this process published. + + :returns: the token while the lease is held, ``None`` otherwise. It identifies a claim; it does not fence one. + + """ + return self._claim.token + + @property + def compromise(self) -> LeaseCompromise | None: + """ + The loss of claim the heartbeat observed. + + The value outlives ``release()``, so a holder can check after leaving the ``with`` block whether its work may + have overlapped a successor; the next acquisition resets it, as :attr:`SoftReadWriteLock.compromise + ` does. + + :returns: the :class:`LeaseCompromise` of the latest claim, or ``None`` while that claim holds or held to its + release + + """ + return self._claim.compromise + + def _acquire(self) -> None: + claim = self._claim + self._stop_heartbeat() # no earlier claim's heartbeat outlives the acquisition of the next one + # The published record reads the token, so it exists before the marker is written and goes back on failure. + claim.token = token = secrets.token_hex(16) + claim.compromise = None + try: + super()._acquire() + except BaseException: + claim.token = None + raise + # The context is thread-local by default, so the heartbeat thread cannot read the descriptor this one just + # published, nor the claim this one owns. Hand it the fd, the inode it verified and the claim instead. + if (fd := self._context.lock_file_fd) is not None and ( + identity := self._context.lock_file_fd_identity + ) is not None: + self._start_heartbeat(claim, fd, identity, token) + else: + claim.token = None + + def _release(self) -> None: + self._stop_heartbeat() + self._claim.token = None + super()._release() + + def _published_record(self) -> OwnerRecord: + return super()._published_record()._replace(token=self._claim.token, lease_duration=self._lease_duration) + + def _try_break_stale_lock(self) -> None: + # A break can fail for reasons a contender must ride out rather than raise on: a peer broke the marker first, + # or Windows refuses to rename a file whose holder still has it open. Poll again instead. + with suppress(OSError), pin_inode(self._lock_path) as pinned: + self._try_break_pinned_lock(pinned) + + def _try_break_pinned_lock(self, pinned: int | None) -> None: + if (peer := self._read_peer()) is None: + # Not a readable protocol 2 lease record: a partial write, a foreign or legacy protocol 1 marker, or the + # strict sentinel. The base self-heal evicts a genuinely malformed marker once it ages past the grace + # window and leaves a legitimate legacy or strict holder in place, so a corrupt marker no longer wedges + # every lease contender until its own timeout. + super()._try_break_stale_lock() + return + owner, marker = peer + if pinned not in {None, marker.st_ino}: # pragma: win32 no cover # Windows pins nothing + return # replaced before the read, so nothing keeps the number read from reuse + # Only a peer that published a lease agreed to be superseded by age; an exclusive owner is reclaimed only once + # it is provably dead, and a contract this version does not know never is. Raise the mismatch outside the read + # so the suppression cannot swallow it. + if owner.mode == "lease" and owner.lease_duration != self._lease_duration: + msg = ( + f"{self.lock_file} holds a lease of {owner.lease_duration!r}s but this contender configured " + f"{self._lease_duration!r}s; every contender for a path must agree on lease_duration" + ) + raise LeaseSettingsMismatch(msg) + # A dead or recycled owner is reclaimed at once, whether it held a lease or an exclusive marker; a live lease + # owner whose marker stops being refreshed is superseded on the schedule every contender agreed to. + if owner.mode != "unknown" and owner_is_stale(owner.pid, owner.hostname, owner.start): + self._take_over(marker, owner.pid) + return + if owner.mode == "lease" and self._marker_unchanged_for(marker) >= self._lease_duration: + self._take_over(marker, None) + + def _read_peer(self) -> tuple[OwnerRecord, os.stat_result] | None: + with suppress(OSError, ValueError): + content, marker = _read_lock_file(self._lock_path) + if (owner := parse_marker(content)) is not None: + return owner, marker + return None + + def _start_heartbeat(self, claim: _LeaseClaim, fd: int, identity: tuple[int, int], token: str) -> None: + # The thread watches the event it was handed rather than whatever the claim names later: a heartbeat that + # outlives its join timeout would otherwise adopt the next acquisition's event and never stop. + stop: Final = Event() + loop: Final = _RefreshLoop( + # A strong reference from a running thread would keep a dropped lease alive, so __del__ would never release + # it and its marker would outlive the process. The callback stops a heartbeat that __del__ could not reach. + report=WeakMethod(self._report_compromise, lambda _: stop.set()), + lock_file=self._lock_path, + claim=claim, + fd=fd, + identity=identity, + token=token, + stop=stop, + interval=self._heartbeat_interval, + duration=self._lease_duration, + ) + thread: Final = Thread(target=loop.run, name=f"filelock-lease-{os.getpid()}", daemon=True) + # Record the heartbeat before starting the thread so a release racing this acquire on a shared, + # non-thread-local claim always sees it and sets the stop event; the thread then exits at its first wait + # instead of outliving the release. A start that raises leaves the unstarted thread for _stop_heartbeat. + claim.heartbeat = _Heartbeat(thread, stop) + thread.start() + + def _stop_heartbeat(self) -> None: + claim = self._claim + if (heartbeat := claim.heartbeat) is None: + return + heartbeat.stop.set() + claim.heartbeat = None + # An unstarted thread has nothing to join, on_compromise may release from the heartbeat thread itself, and + # during finalization a daemon thread is frozen, so joining it raises PythonFinalizationError (3.14+). + if heartbeat.thread.ident is not None and heartbeat.thread is not current_thread() and not sys.is_finalizing(): + heartbeat.thread.join(timeout=self._heartbeat_interval) + + def _report_compromise( + self, + claim: _LeaseClaim, + reason: CompromiseReason, + error: OSError | None, + token: str, + ) -> None: + # Record it on the claim this heartbeat serves, not on self._claim: a thread-local claim read from the + # heartbeat thread is a different, empty one, so the holder would never see the loss it is being told about. + # The token is the one this thread published, not claim.token, which a release may already have cleared. + claim.compromise = LeaseCompromise(lock_file=self.lock_file, token=token, reason=reason, error=error) + if self._on_compromise is not None: + self._on_compromise(claim.compromise) + + +@dataclass(frozen=True) +class _RefreshLoop: + """What a heartbeat thread needs to refresh one claim, holding the lease itself only weakly.""" + + report: WeakMethod[Callable[[_LeaseClaim, CompromiseReason, OSError | None, str], None]] + lock_file: str + claim: _LeaseClaim + fd: int + identity: tuple[int, int] + token: str + stop: Event + interval: float + duration: float + + def run(self) -> None: + # A transient error (ESTALE / EIO on the NFS-style filesystems a lease targets) is not a loss. Like restic, call + # the claim unrefreshable once failures run long enough that a contender could take it before the next success. + last_success, missing = time.monotonic(), False + while not self.stop.wait(self.interval): + outcome, error = self._refresh() + if outcome == "ok": + last_success, missing = time.monotonic(), False + continue + # A peer's stale break exchanges a live marker aside for the length of its check, so only a second miss or + # change in a row is a loss. + if outcome in {"marker-missing", "owner-changed"} and not missing: + missing = True + continue + if outcome == "transient" and time.monotonic() - last_success < self.duration - self.interval: + continue + if (report := self.report()) is not None: + report(self.claim, "refresh-failed" if outcome == "transient" else outcome, error, self.token) + return + + def _refresh(self) -> tuple[_RefreshOutcome, OSError | None]: + try: + st = os.lstat(self.lock_file) + except FileNotFoundError as error: + return self._refresh_unnamed(error) + except OSError as error: + return "transient", error + # A peer that took the expired claim replaced the marker, so the pathname now names its inode, not ours. + if (st.st_dev, st.st_ino) != self.identity: + return "owner-changed", None + return self._touch() + + def _refresh_unnamed(self, name_error: FileNotFoundError) -> tuple[_RefreshOutcome, OSError | None]: + # A renamed directory takes the marker with it; the name is gone but the inode behind the descriptor keeps its + # link. + try: + linked = os.fstat(self.fd).st_nlink > 0 + except OSError as error: + return "transient", error + if not linked: + return "marker-missing", name_error + return self._touch() + + def _touch(self) -> tuple[_RefreshOutcome, OSError | None]: + try: + touch(self.lock_file, fd=self.fd) + except OSError as error: + return "transient", error + return "ok", None + + +__all__ = [ + "CompromiseReason", + "LeaseCompromise", + "SoftFileLease", + "_LeaseClaim", +] diff --git a/.venv/lib/python3.14/site-packages/filelock/_marker.py b/.venv/lib/python3.14/site-packages/filelock/_marker.py new file mode 100644 index 0000000000000000000000000000000000000000..2a7c694c5382de3b2235c62feb24bac25fd9c497 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/filelock/_marker.py @@ -0,0 +1,189 @@ +from __future__ import annotations + +__lazy_modules__ = ["contextlib", "filelock._identity", "filelock._util", "math"] + +import math +import os +import re +from contextlib import suppress +from typing import Final, Literal, NamedTuple + +from filelock._identity import host_name, owner_is_current_process, owner_is_stale, process_start_token +from filelock._soft import SoftFileLock, _read_lock_file, parse_decimal +from filelock._util import pin_inode, write_all + +#: Protocol 1 is the legacy ``\n\n[\n]`` marker that :class:`SoftFileLock` still writes. +#: Protocol 2 carries the owner mode and the lease claim. A protocol 1 reader treats a protocol 2 marker as malformed +#: and evicts it after its grace period, so the two never guarantee mutual exclusion against each other. +_PROTOCOL: Final[str] = "filelock/2" + +_MAX_PID: Final[int] = 2**31 - 1 + +#: The forms ``repr`` gives a non-negative finite float or int; ``float()`` would also accept whitespace, underscores +#: and non-ASCII digits. +_DURATION: Final[re.Pattern[str]] = re.compile(r"[0-9]+(?:\.[0-9]+)?(?:e[+-][0-9]+)?", re.ASCII) + +#: Preserve unknown contracts so contenders cannot mistake them for malformed, reclaimable markers. +OwnerMode = Literal["lease", "exclusive", "unknown"] + + +class MarkerSoftFileLock(SoftFileLock): + """An existence lock whose marker carries a protocol 2 owner record.""" + + #: Age-based expiry requires a lease contract; an exclusive holder grants none. + _owner_mode: OwnerMode = "exclusive" + + @property + def owner(self) -> OwnerRecord | None: + """ + The owner named by the marker on disk. + + :returns: the published record, or ``None`` when no marker exists or its record is malformed or protocol 1 + + """ + return self._read_owner() + + @property + def pid(self) -> int | None: + """ + The PID of the process holding this lock, read from the marker. + + :returns: the PID, or ``None`` when no marker exists or its record is unreadable + + """ + return None if (owner := self._read_owner()) is None else owner.pid + + @property + def is_lock_held_by_us(self) -> bool: + """ + Whether the marker on disk names this process. + + :returns: ``True`` when the marker names this process's PID, hostname and, when recorded, start token + + """ + owner = self._read_owner() + return owner is not None and owner_is_current_process(owner.pid, owner.hostname, owner.start) + + def force_break(self) -> None: + """ + Remove the marker whoever holds it, so a later contender can acquire. + + Forced breaking voids mutual exclusion: the previous holder keeps running and keeps using whatever the lock + protects. Reserve it for an operator clearing a marker whose holder is known to be gone. + """ + self.break_lock() + + def _try_break_stale_lock(self) -> None: + with suppress(OSError, ValueError), pin_inode(self._lock_path) as pinned: + content, marker = _read_lock_file(self._lock_path) + owner: Final[OwnerRecord | None] + if pinned not in {None, marker.st_ino}: # pragma: win32 no cover # Windows pins nothing + return # replaced before the read, so nothing keeps the number read from reuse + if (owner := parse_marker(content)) is None: + super()._try_break_stale_lock() + elif owner.mode != "unknown" and owner_is_stale(owner.pid, owner.hostname, owner.start): + self._take_over(marker, owner.pid) + + @classmethod + def _recorded_holder(cls, content: str | None) -> tuple[int, str] | None: + if (owner := parse_marker(content)) is not None: + return owner.pid, owner.hostname + return super()._recorded_holder(content) + + @classmethod + def _marker_is_live(cls, content: str | None, breaker_pid: int) -> bool: # pragma: win32 no cover + if (owner := parse_marker(content)) is None: + return super()._marker_is_live(content, breaker_pid) + return owner.mode == "unknown" or ( + (owner.pid != breaker_pid or owner.hostname != host_name()) + and not owner_is_stale(owner.pid, owner.hostname, owner.start) + ) + + def _read_owner(self) -> OwnerRecord | None: + with suppress(OSError, ValueError): + return parse_marker(_read_lock_file(self._lock_path)[0]) + return None + + def _write_lock_info(self, fd: int) -> None: + write_all(fd, encode_marker(self._published_record())) + + def _published_record(self) -> OwnerRecord: + return OwnerRecord( + pid=os.getpid(), + hostname=host_name(), + mode=self._owner_mode, + start=process_start_token(os.getpid()), + ) + + +class OwnerRecord(NamedTuple): + """Keep optional metadata when reading unknown lock modes.""" + + pid: int + hostname: str + mode: OwnerMode + token: str | None = None + lease_duration: float | None = None + start: int | None = None + + +def encode_marker(record: OwnerRecord) -> bytes: + """Render an owner record as the bytes a protocol 2 marker holds.""" + lines = [_PROTOCOL, f"pid={record.pid}", f"host={record.hostname}", f"mode={record.mode}"] + if record.token is not None: + lines.append(f"token={record.token}") + if record.lease_duration is not None: + lines.append(f"duration={record.lease_duration!r}") + if record.start is not None: + lines.append(f"start={record.start}") + return "".join(f"{line}\n" for line in lines).encode() + + +def parse_marker(content: str | None) -> OwnerRecord | None: + """Return the owner a protocol 2 marker names, or ``None`` when the record is malformed or protocol 1.""" + # A torn write may end in a prefix like "mode=exclu", an unknown contract nobody reclaims; malformed, it ages out. + if not content or content[-1] != "\n" or not (lines := content.strip().splitlines()) or lines[0] != _PROTOCOL: + return None + fields: dict[str, str] = {} + for line in lines[1:]: + key, separator, value = line.partition("=") + if not separator: + return None + fields[key] = value + return _build_record(fields) + + +def _build_record(fields: dict[str, str]) -> OwnerRecord | None: + # An unknown key is a field a newer filelock published, so ignore it rather than read the record as malformed. An + # unrecognized mode is the same story one level up: a contract this version does not implement. Reading it as + # malformed would age the marker out of a live owner's hands, so keep it and let the caller refuse to reclaim it. + # A record naming no mode at all states no contract and stays malformed. + if (published := fields.get("mode")) is None: + return None + mode: Final[OwnerMode] = published if published in {"lease", "exclusive"} else "unknown" + hostname = fields.get("host") + if not hostname or "pid" not in fields or ("duration" in fields and not _DURATION.fullmatch(fields["duration"])): + return None + try: + pid = parse_decimal(fields["pid"]) + duration = float(fields["duration"]) if "duration" in fields else None + start = parse_decimal(fields["start"]) if "start" in fields else None + except ValueError: + return None + if not 1 <= pid <= _MAX_PID: + return None + token = fields.get("token") + # "1e400" passes the grammar but overflows to inf, which mismatches every configured duration and so wedges reclaim, + # where a malformed marker ages out. + if mode == "lease" and (not token or duration is None or not (math.isfinite(duration) and duration > 0)): + return None + return OwnerRecord(pid=pid, hostname=hostname, mode=mode, token=token, lease_duration=duration, start=start) + + +__all__ = [ + "MarkerSoftFileLock", + "OwnerMode", + "OwnerRecord", + "encode_marker", + "parse_marker", +] diff --git a/.venv/lib/python3.14/site-packages/filelock/_read_write.py b/.venv/lib/python3.14/site-packages/filelock/_read_write.py new file mode 100644 index 0000000000000000000000000000000000000000..bc377d558d59fdd9115aff6ef1cee8ee239b997f --- /dev/null +++ b/.venv/lib/python3.14/site-packages/filelock/_read_write.py @@ -0,0 +1,1504 @@ +from __future__ import annotations + +__lazy_modules__ = [ + "collections", + "filelock._error", + "filelock._identity", + "hashlib", + "itertools", + "pathlib", + "secrets", + "tempfile", +] + +import hashlib +import logging +import os +import pathlib +import secrets +import sqlite3 +import stat +import sys +import tempfile +import threading +import time +from collections import deque +from contextlib import contextmanager, suppress +from dataclasses import dataclass, field +from itertools import count +from typing import TYPE_CHECKING, ClassVar, Final, Literal, NamedTuple, TypeAlias, cast +from weakref import WeakValueDictionary + +from filelock._api import ( + AcquireReturnProxy, + _acquire_thread_lock, + _canonical, + _ensure_current_process, + _fork_transition, + _raise_chained_errors, + _register_fork_class, + _register_fork_object, + _register_owned_descriptor, + _resolve_lazy_imports_before_shutdown, + _resolve_read_write_timeout, + _seconds, + _timeout_behavior, + _unregister_owned_descriptor, +) +from filelock._error import Timeout +from filelock._identity import host_name, owner_is_stale, process_start_token + +if sys.platform == "win32": # pragma: win32 cover + from filelock._windows import ( + _directory_is_private, + _identity_at, + _identity_of_fd, + _make_private_directory, + _open_non_reparse_fd, + _pinned, + ) + +if TYPE_CHECKING: + from collections.abc import Callable, Generator + from typing import Self + + from _typeshed import Unused + +_LOGGER: Final[logging.Logger] = logging.getLogger("filelock") +_GETPID: Final[Callable[[], int]] = os.getpid +_IS_PYPY: Final[bool] = sys.implementation.name == "pypy" +_NEEDS_CONNECTION_ESCROW: Final[bool] = ( + hasattr(os, "register_at_fork") and sys.implementation.name == "cpython" and sys.version_info < (3, 12) +) +_ConnectionParameter: TypeAlias = ( + str | bytes | os.PathLike[str] | os.PathLike[bytes] | float | int | type[sqlite3.Connection] | None +) +_DatabaseIdentity: TypeAlias = tuple[int, int] + +# sqlite3_busy_timeout() accepts a C int, max 2_147_483_647 on 32-bit. Use a lower value to be safe (~23 days). +_MAX_SQLITE_TIMEOUT_MS: Final[int] = 2_000_000_000 - 1 +# How long a wait that its caller may abandon runs before it checks whether it should stop. +_CANCEL_POLL_INTERVAL: Final[float] = 0.1 +_UNSAFE_FORK_EXIT_STATUS: Final[int] = 70 +#: The longest a contended acquisition waits for the database inside one fork transition, in seconds. +_MAX_TRANSITION_WAIT: Final[float] = 0.05 +# O_NONBLOCK keeps an open from blocking on a FIFO planted at the path; the regular-file check then rejects it. +_DB_OPEN_FLAGS: Final[int] = os.O_RDWR | os.O_CREAT | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0) +_DIRECTORY_FLAGS: Final[int] = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0) +# Header bytes 18 and 19 hold the file format versions, 2 in WAL mode: https://www.sqlite.org/fileformat.html +_SQLITE_MAGIC: Final[bytes] = b"SQLite format 3\x00" +_FORMAT_VERSIONS: Final[slice] = slice(18, 20) +_HEADER_SIZE: Final[int] = 20 +_WAL_FORMAT_VERSION: Final[int] = 2 + + +class _SQLiteTransitionContext(threading.local): + depth: int = 0 + + +_SQLITE_TRANSITION_CONTEXT: Final = _SQLiteTransitionContext() + + +class _ConnectionEscrow: + def __init__(self) -> None: + self._lock = threading.RLock() + self._functions: tuple[Callable[[sqlite3.Connection], None], Callable[[sqlite3.Connection], None]] | None = None + + def functions( + self, + ) -> tuple[Callable[[sqlite3.Connection], None], Callable[[sqlite3.Connection], None]] | None: + if not _NEEDS_CONNECTION_ESCROW: + return None # pragma: >=3.12 cover + with self._lock: # pragma: <3.12 cover # pragma: needs fork + if self._functions is None: + import ctypes # ruff:ignore[import-outside-top-level] # keep optional ctypes and its audited dlsym out of ordinary imports + + function_type = ctypes.PYFUNCTYPE(None, ctypes.py_object) + increment_address = ctypes.cast(ctypes.pythonapi.Py_IncRef, ctypes.c_void_p).value + decrement_address = ctypes.cast(ctypes.pythonapi.Py_DecRef, ctypes.c_void_p).value + if increment_address is None or decrement_address is None: # pragma: no cover - resolved CPython API + msg = "CPython reference functions have no address" + raise RuntimeError(msg) + self._functions = ( + cast("Callable[[sqlite3.Connection], None]", function_type(increment_address)), + cast("Callable[[sqlite3.Connection], None]", function_type(decrement_address)), + ) + return self._functions + + def _reset_after_fork_in_child(self) -> None: # pragma: forked child + self._lock = threading.RLock() + + +_CONNECTION_ESCROW: Final = _ConnectionEscrow() + + +class _ForkedDatabaseRegistry: + def __init__(self) -> None: + self._lock = threading.RLock() + self._paths: set[pathlib.Path] = set() + self._identities: set[_DatabaseIdentity] = set() + self._sqlite_used = False + self._all_paths_poisoned = False + + def raise_if_poisoned(self, path: pathlib.Path) -> None: + identity = self.identity(path) + with self._lock: + all_paths_poisoned = self._all_paths_poisoned + poisoned = ( + all_paths_poisoned or path in self._paths or (identity is not None and identity in self._identities) + ) + if poisoned: # pragma: needs fork + msg = ( + "ReadWriteLock is unavailable in a PyPy fork child; exec or exit before using it" + if all_paths_poisoned + else f"SQLite database {path!s} was active across fork(); exec or exit before using it in the child" + ) + raise RuntimeError(msg) + + def poison_after_fork(self, path: pathlib.Path, identity: _DatabaseIdentity | None) -> None: + self._paths.add(path) + if identity is not None: + self._identities.add(identity) + + def note_sqlite_use(self) -> None: + if _IS_PYPY: + with self._lock: + self._sqlite_used = True + + def _reset_after_fork_in_child(self) -> None: # pragma: forked child + self._lock = threading.RLock() + self._all_paths_poisoned = self._all_paths_poisoned or (_IS_PYPY and self._sqlite_used) + self._sqlite_used = False + + @staticmethod + def identity(path: pathlib.Path) -> _DatabaseIdentity | None: + try: + stat_result = path.stat() + except OSError: + return None + return stat_result.st_dev, stat_result.st_ino + + +_FORKED_DATABASES: Final = _ForkedDatabaseRegistry() + +_LINK_PREFIX: Final[str] = ".filelock-" +_LINK_NAME: Final[str] = "lock.db" +_SWEPT_DATABASES: Final[set[_DatabaseIdentity]] = set() + + +# POSIX locks belong to the process and the inode, so closing any descriptor on the inode drops the locks every other +# connection of this process holds there. SQLite defers its own closes for this reason (section 2.2 of +# https://www.sqlite.org/howtocorrupt.html); the probes stay open until the last connection on their inode closes. +class _ProbeDescriptors: + def __init__(self) -> None: + self._lock = threading.Lock() + self._inodes: dict[_DatabaseIdentity, _InodeProbes] = {} + self._releases: deque[_Probe] = deque() + self._leases = count() + + def adopt(self, database: str, claimed: list[_Probe]) -> _Probe | None: # pragma: win32 no cover + # Reuse a probe no live connection holds: each connect would otherwise keep another descriptor open until the + # inode's last close. + try: + stat_result: Final = os.lstat(database) + except OSError: + return None + identity: Final = stat_result.st_dev, stat_result.st_ino + with self._lock: + if not stat.S_ISREG(stat_result.st_mode) or not (probes := self._inodes.get(identity)) or not probes.idle: + return None + claimed.append(adopted := _Probe(identity, probes.idle.pop(), next(self._leases))) + probes.leases.add(adopted.lease) + return adopted + + def retain(self, fd: int, claimed: list[_Probe]) -> _Probe: # pragma: win32 no cover + stat_result: Final = os.fstat(fd) + identity: Final = stat_result.st_dev, stat_result.st_ino + token: Final = _register_owned_descriptor(fd, identity) + with self._lock: + (probes := self._inodes.setdefault(identity, _InodeProbes())).descriptors.append((fd, token)) + claimed.append(retained := _Probe(identity, fd, next(self._leases))) + probes.leases.add(retained.lease) + return retained + + def release(self, probe: _Probe) -> None: # pragma: win32 no cover + # A finalizer may run this on a thread already inside the lock, so it never waits; the lock holder drains. + self._releases.append(probe) + self.drain() + + def drain(self) -> None: # pragma: win32 no cover + while self._releases and self._lock.acquire(blocking=False): + try: + # Dequeued only once applied, so an exception leaves the rest of the release for the next drain. + while self._releases: + self._release_locked(self._releases[0]) + self._releases.popleft() + finally: + self._lock.release() + + def _release_locked(self, probe: _Probe) -> None: # pragma: win32 no cover + if (probes := self._inodes.get(probe.identity)) is None: + return + if probe.lease in probes.leases: + probes.leases.remove(probe.lease) + if probes.leases: + probes.idle.append(probe.fd) + if probes.leases: + return + # No connect may adopt a probe about to close, even one an interrupt leaves listed. + probes.idle.clear() + # Closing under the lock keeps a concurrent connect from taking a lock on this inode before the close. + while probes.descriptors: + fd, token = probes.descriptors[-1] + if token is not None: # pragma: needs fork + _unregister_owned_descriptor(token) + # Forgotten before the close: after an interrupt, a leaked descriptor beats closing a reused number. + probes.descriptors.pop() + # The descriptor is gone even when close reports an error, and no lock of this process depends on it. + with suppress(OSError): + os.close(fd) + del self._inodes[probe.identity] + + def _reset_after_fork_in_child(self) -> None: # pragma: forked child + # The child holds no POSIX locks, and the owned-descriptor registry closes the inherited probes. + self._lock = threading.Lock() + self._inodes = {} + self._releases = deque() + + +class _Operation(NamedTuple): + pid: int + start_time: float + #: Asked between bounded waits, so an abandoned acquire stops waiting; ``None`` waits in one call. + cancel_check: Callable[[], bool] | None + + +class _Probe(NamedTuple): + identity: _DatabaseIdentity + fd: int + # One per connection, so a release applied again after an interrupt kept it queued counts once. + lease: int + + +@dataclass(slots=True) +class _InodeProbes: + leases: set[int] = field(default_factory=set) + descriptors: list[tuple[int, int | None]] = field(default_factory=list) + idle: list[int] = field(default_factory=list) + + +_PROBE_DESCRIPTORS: Final = _ProbeDescriptors() + + +class _ForkSafeConnection(sqlite3.Connection): + _creator_pid: int + _decrement_escrow: Callable[[sqlite3.Connection], None] | None + _database_directory: _LinkDirectory | None + database_identity: _DatabaseIdentity | None = None + _probe: _Probe | None = None + + def __new__( + cls, + *_args: _ConnectionParameter, + **_kwargs: _ConnectionParameter, + ) -> Self: + connection = super().__new__(cls) + connection._creator_pid = _GETPID() + connection._decrement_escrow = None + connection._database_directory = None + return connection + + def close(self) -> None: + with _sqlite_transition(): + if _GETPID() != self._creator_pid: # pragma: needs fork + return + with _fork_transition(): + sqlite3.Connection.close(self) + try: + if (directory := self._database_directory) is not None: # pragma: needs posix-hard-link + self._database_directory = None + _remove_link_directory(directory) + finally: + if (probe := self._probe) is not None: # pragma: win32 no cover + self._probe = self.database_identity = None + _PROBE_DESCRIPTORS.release(probe) + if (decrement := self._decrement_escrow) is not None: # pragma: <3.12 cover # pragma: needs fork + self._decrement_escrow = None + decrement(self) + + def retain_link(self, directory: _LinkDirectory | None) -> None: # pragma: win32 cover + """Keep the private link directory, if there is one, until SQLite no longer uses it.""" + self._database_directory = directory + + def retain_until_close(self, directory: _LinkDirectory | None, probe: _Probe) -> None: # pragma: win32 no cover + """Keep the private link and this connection's probe descriptor until SQLite no longer uses them.""" + self._database_directory = directory + self._probe = probe + self.database_identity = probe.identity + + def acquire_escrow( # pragma: <3.12 cover # pragma: needs fork + self, + functions: tuple[Callable[[sqlite3.Connection], None], Callable[[sqlite3.Connection], None]] | None, + ) -> None: + # The caller only reaches here holding the escrow functions; it skips the call entirely without them. + if functions is not None: # pragma: no branch + increment, decrement = functions + increment(self) + self._decrement_escrow = decrement + + def __del__(self) -> None: + with suppress(sqlite3.Error, RuntimeError, OSError): + self.close() + + +@dataclass(slots=True) +class _OwnedConnection: + """ + The connection of one lock, closed when the lock is collected. + + CPython runs ``__del__`` before it clears weak references. A finalizer on the lock itself would close the + connection while the singleton cache still hands the lock out, and a thread given the lock would nest into a hold + with no database lock behind it. The lock alone refers to this holder, so the holder is collected once the cache no + longer finds the lock. + """ + + connection: _ForkSafeConnection | None = None + + def __del__(self) -> None: + if (connection := self.connection) is not None: + with suppress(sqlite3.Error, RuntimeError): + connection.close() + + +class _ReadWriteLockMeta(type): + """ + Resolve singleton instances for ``is_singleton=True`` construction. + + This logic lives here rather than in ReadWriteLock.get_lock so ``ReadWriteLock(path)`` returns cached instances + without a 2-arg ``super()`` call that type checkers cannot verify. + + """ + + _instances: WeakValueDictionary[pathlib.Path, ReadWriteLock] + _instances_lock: threading.RLock + _instances_pid: int + _instances_under_construction: set[pathlib.Path] + + def __call__( + cls, + lock_file: str | os.PathLike[str], + timeout: float = -1, + *, + blocking: bool = True, + is_singleton: bool = True, + ) -> ReadWriteLock: + _ensure_current_process() + _resolve_lazy_imports_before_shutdown() + timeout = _seconds("timeout", timeout) + if cls._instances_pid != _GETPID(): + cls._reset_class_after_fork() + construction_pid = _GETPID() + if not is_singleton: + instance = super().__call__(lock_file, timeout, blocking=blocking, is_singleton=is_singleton) + if _GETPID() != construction_pid: # pragma: forked child + msg = "ReadWriteLock construction cannot continue after fork" + raise RuntimeError(msg) + return instance + + # Resolve only the parent, so a symlink at the lock path stays its own key instead of aliasing its target. + normalized = pathlib.Path(_canonical(lock_file)) + with cls._instances_lock: + # One lookup, since another thread can drop the last reference between a membership check and a read. + if (instance := cls._instances.get(normalized)) is None: + if normalized in cls._instances_under_construction: # pragma: no cover - exercised in an audit callback + msg = f"Singleton lock construction is already active for {lock_file!s}" + raise RuntimeError(msg) + construction_registry = cls._instances_under_construction + construction_registry.add(normalized) + try: + instance = super().__call__(lock_file, timeout, blocking=blocking, is_singleton=is_singleton) + finally: + if _GETPID() == construction_pid: + construction_registry.discard(normalized) + if _GETPID() != construction_pid: + msg = "ReadWriteLock construction cannot continue after fork" + raise RuntimeError(msg) + cls._instances[normalized] = instance + + if ( + _timeout_behavior(instance.timeout, any_negative=False) + != _timeout_behavior(timeout, any_negative=False) + or instance.blocking != blocking + ): + msg = ( + f"Singleton lock created with timeout={instance.timeout}, blocking={instance.blocking}," + f" cannot be changed to timeout={timeout}, blocking={blocking}" + ) + raise ValueError(msg) + return instance + + def _reset_class_after_fork(cls) -> None: # pragma: forked child + cls._instances = WeakValueDictionary() + cls._instances_lock = threading.RLock() + cls._instances_pid = _GETPID() + cls._instances_under_construction = set() + + +class ReadWriteLock(metaclass=_ReadWriteLockMeta): + """ + Cross-process read-write lock backed by SQLite. + + Allows concurrent shared readers or a single exclusive writer. The lock is reentrant within the same mode (multiple + ``acquire_read`` calls nest, as do multiple ``acquire_write`` calls from the same thread), but upgrading from read + to write or downgrading from write to read raises :class:`RuntimeError`. Write locks are pinned to the thread that + acquired them, and each thread sharing a read lock can release only the levels it acquired. + + An instance coordinates processes, not threads: a thread asking for the opposite mode of another thread's hold + raises :class:`RuntimeError` instead of waiting. Threads that must wait for each other each need their own + instance (``is_singleton=False``). + + By default, ``is_singleton=True``: calling ``ReadWriteLock(path)`` with the same resolved path returns the same + instance. A ``.db`` extension is a convention rather than a requirement; the filesystem must be one the active + SQLite VFS supports. Give the lock a file of its own: a WAL-mode database, or one with a schema larger than a lock + needs, raises :class:`ValueError`. A database directory that other accounts can write must also be writable by + this one and support hard links; the protection against other accounts ends once one of them can replace the + database directory or a directory above it. + + :param lock_file: path to the SQLite database file used as the lock + :param timeout: maximum wait time in seconds; ``-1`` waits up to SQLite's busy-timeout cap of about 23 days + :param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately when the lock is unavailable + :param is_singleton: if ``True``, reuse existing instances for the same resolved path + + .. versionadded:: 3.21.0 + + """ + + _instances: WeakValueDictionary[pathlib.Path, ReadWriteLock] = WeakValueDictionary() + _instances_lock = threading.RLock() + _instances_pid = _GETPID() + _instances_under_construction: ClassVar[set[pathlib.Path]] = set() + + def __init_subclass__(cls) -> None: + super().__init_subclass__() + cls._instances = WeakValueDictionary() + cls._instances_lock = threading.RLock() + cls._instances_pid = _GETPID() + cls._instances_under_construction = set() + _register_fork_class(cls) + + @classmethod + def get_lock( + cls, lock_file: str | os.PathLike[str], timeout: float = -1, *, blocking: bool = True + ) -> ReadWriteLock: + """ + Return the singleton :class:`ReadWriteLock` for *lock_file*. + + :param lock_file: path to the SQLite database file used as the lock + :param timeout: maximum wait time in seconds; ``-1`` waits up to SQLite's busy-timeout cap of about 23 days + :param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately when the lock is unavailable + + :returns: the singleton lock instance + + :raises ValueError: if an instance already exists for this path with different *timeout* or *blocking* values + + """ + return cls(lock_file, timeout, blocking=blocking) + + def __init__( + self, + lock_file: str | os.PathLike[str], + timeout: float = -1, + *, + blocking: bool = True, + is_singleton: bool = True, # ruff:ignore[unused-method-argument] # consumed by _ReadWriteLockMeta.__call__ + ) -> None: + self.lock_file = os.fspath(lock_file) + self._canonical_path = pathlib.Path(_canonical(lock_file)) + _FORKED_DATABASES.raise_if_poisoned(self._canonical_path) + self.timeout = timeout + self.blocking = blocking + self._transaction_lock = threading.Lock() # serializes the (possibly blocking) SQLite transaction work + self._internal_lock = threading.Lock() # protects _lock_level / _current_mode updates and rollback + self._lock_level = 0 + self._current_mode: Literal["read", "write"] | None = None + self._write_thread_id: int | None = None + self._read_depths: dict[int, int] = {} + self._acquisition_thread_ids: set[int] = set() + self._owned_connection = _OwnedConnection() + self._connection_transaction_released = True + self._connection_identity: _DatabaseIdentity | None = None + self._closed = False + self._creator_pid = _GETPID() + self._fork_invalidated = False + _register_fork_object(self) + with _fork_transition(), _sqlite_transition(): + validation_connection = self._open_connection(sqlite_timeout=5.0) + validation_connection.close() + + def acquire_read(self, timeout: float | None = None, *, blocking: bool | None = None) -> AcquireReturnProxy: + """ + Acquire a shared read lock. + + If this instance already holds a read lock, the lock level is incremented (reentrant). Attempting to acquire a + read lock while holding a write lock raises :class:`RuntimeError` (downgrade not allowed). + + :param timeout: seconds to wait; ``None`` uses the instance setting; ``-1`` waits up to SQLite's + busy-timeout cap of about 23 days + :param blocking: if ``False``, raise :class:`~filelock.Timeout` on contention; + ``None`` uses the instance setting + + :returns: a proxy that can be used as a context manager to release the lock + + :raises RuntimeError: if a write lock is already held on this instance + :raises Timeout: if the lock cannot be acquired within *timeout* seconds + :raises TypeError: if *timeout* is a :class:`bool` or not a real number + :raises ValueError: if a blocking call gets a ``nan`` *timeout* or a negative one other than ``-1`` + + """ + return self._acquire( + "read", + self.timeout if timeout is None else timeout, + blocking=self.blocking if blocking is None else blocking, + ) + + def acquire_write(self, timeout: float | None = None, *, blocking: bool | None = None) -> AcquireReturnProxy: + """ + Acquire an exclusive write lock. + + If this instance already holds a write lock from the same thread, the lock level is incremented (reentrant). + Attempting to acquire a write lock while holding a read lock raises :class:`RuntimeError` (upgrade not allowed). + Write locks are pinned to the acquiring thread: a different thread trying to re-enter also raises + :class:`RuntimeError`. + + :param timeout: seconds to wait; ``None`` uses the instance setting; ``-1`` waits up to SQLite's + busy-timeout cap of about 23 days + :param blocking: if ``False``, raise :class:`~filelock.Timeout` on contention; + ``None`` uses the instance setting + + :returns: a proxy that can be used as a context manager to release the lock + + :raises RuntimeError: if a read lock is already held, or a write lock is held by a different thread + :raises Timeout: if the lock cannot be acquired within *timeout* seconds + :raises TypeError: if *timeout* is a :class:`bool` or not a real number + :raises ValueError: if a blocking call gets a ``nan`` *timeout* or a negative one other than ``-1`` + + """ + return self._acquire( + "write", + self.timeout if timeout is None else timeout, + blocking=self.blocking if blocking is None else blocking, + ) + + @contextmanager + def read_lock(self, timeout: float | None = None, *, blocking: bool | None = None) -> Generator[None]: + """ + Context manager that acquires and releases a shared read lock. + + Falls back to instance defaults for *timeout* and *blocking* when ``None``. + + :param timeout: maximum wait time in seconds, or ``None`` to use the instance default + :param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default + + """ + self.acquire_read(timeout, blocking=blocking) + try: + yield + finally: + self.release() + + @contextmanager + def write_lock(self, timeout: float | None = None, *, blocking: bool | None = None) -> Generator[None]: + """ + Context manager that acquires and releases an exclusive write lock. + + Falls back to instance defaults for *timeout* and *blocking* when ``None``. + + :param timeout: maximum wait time in seconds, or ``None`` to use the instance default + :param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default + + """ + self.acquire_write(timeout, blocking=blocking) + try: + yield + finally: + self.release() + + def release(self, *, force: bool = False) -> None: + """ + Release one level of the current lock. + + When the lock level reaches zero the underlying SQLite transaction is rolled back, releasing the database lock. + + :param force: if ``True``, release the lock completely regardless of the current lock level + + :raises RuntimeError: if no lock is currently held and *force* is ``False``, or if the calling thread holds + no level of the current lock and *force* is ``False`` + + """ + with _fork_transition(): + _ensure_current_process() + if self._inherited: # pragma: needs fork + return + self._raise_if_acquiring("release") + self._release(force=force, close=False) + + def close(self) -> None: + """ + Release the lock (if held) and close the underlying SQLite connection. + + After calling this method, the lock instance is no longer usable. Unlike :meth:`release`, it ends every + thread's hold on this instance, whichever thread calls it. + + """ + with _fork_transition(): + _ensure_current_process() + if self._inherited: # pragma: needs fork + return + self._raise_if_acquiring("close") + self._release(force=True, close=True) + + def _acquire_level_of( + self, + holder: int, + mode: Literal["read", "write"], + timeout: float, + *, + blocking: bool, + cancel_check: Callable[[], bool], + ) -> AcquireReturnProxy: + # Records the level under *holder* instead of the calling thread: an async wrapper's executor thread does not + # identify the tasks it holds for. Gives up with Timeout once *cancel_check* is true. + return self._acquire(mode, timeout, blocking=blocking, holder=holder, cancel_check=cancel_check) + + def _release_level_of(self, thread_id: int) -> None: + with _fork_transition(): + _ensure_current_process() + self._raise_if_acquiring("release") + self._release(force=False, close=False, thread_id=thread_id) + + def _holds_level_of(self, thread_id: int) -> bool: + # Read without the internal lock. The async wrapper asks from its event loop, which must not wait out a release + # on another thread. A stale answer is what a caller asking a moment earlier would have had. A level whose + # transaction ended is one an interrupted release left behind; it counts as gone, as _holds_level() treats it. + if thread_id != self._write_thread_id and thread_id not in self._read_depths: + return False + try: + return not self._transaction_ended() + except sqlite3.ProgrammingError: + # The connection closed between the read of _con and the in_transaction call, so the transaction is over. + return False + + def _release(self, *, force: bool, close: bool, thread_id: int | None = None) -> None: + thread_id = threading.get_ident() if thread_id is None else thread_id + with self._transaction_lock, self._internal_lock: + if not force: + self._raise_if_not_holder(thread_id) + if self._lock_level == 0: + if force and self._con is None: + if close: + self._closed = True + return + if not force: + msg = f"Cannot release a lock on {self.lock_file} (lock id: {id(self)}) that is not held" + raise RuntimeError(msg) + if not force and self._lock_level > 1: + self._leave_one_level(thread_id) + return + try: + self._finish_connection() + except BaseException: + # Once the rollback ran, the level is gone whatever stopped the cleanup, a signal handler's exception + # included. Kept, it would let the next acquire nest into a transaction that no longer exists. + if self._transaction_ended(): + self._clear_lock_state() + raise + self._clear_lock_state() + if close: + self._closed = True + + def _transaction_ended(self) -> bool: + if self._connection_transaction_released or (connection := self._con) is None: + return True + # An interrupt can land between the rollback and the flag that records it. + return not connection.in_transaction + + def _raise_if_not_holder(self, thread_id: int) -> None: + # The connection runs with check_same_thread=False, so a release from another thread would end the owner's + # transaction under it and let a second writer in. + if self._current_mode == "write" and thread_id != self._write_thread_id: + msg = ( + f"Cannot release write lock on {self.lock_file} (lock id: {id(self)}) " + f"from thread {thread_id} while it is held by thread {self._write_thread_id}" + ) + raise RuntimeError(msg) + # Readers share that one transaction too, so a thread that holds no read level would end another's hold. + if self._current_mode == "read" and thread_id not in self._read_depths: + msg = ( + f"Cannot release read lock on {self.lock_file} (lock id: {id(self)}) " + f"from thread {thread_id}, which does not hold it" + ) + raise RuntimeError(msg) + + def _leave_one_level(self, thread_id: int) -> None: + self._lock_level -= 1 + if self._current_mode == "read" and (depth := self._read_depths.pop(thread_id) - 1): + self._read_depths[thread_id] = depth + + def _clear_lock_state(self) -> None: + self._lock_level = 0 + self._current_mode = None + self._write_thread_id = None + self._read_depths = {} + + @property + def _con(self) -> _ForkSafeConnection | None: + return self._owned_connection.connection + + @_con.setter + def _con(self, connection: _ForkSafeConnection | None) -> None: + self._owned_connection.connection = connection + + def _reset_after_fork_in_child(self) -> None: # pragma: forked child + if self._con is not None: + _FORKED_DATABASES.poison_after_fork(self._canonical_path, self._connection_identity) + self._con = None + self._connection_transaction_released = True + self._connection_identity = None + self._transaction_lock = threading.Lock() + self._internal_lock = threading.Lock() + self._clear_lock_state() + self._acquisition_thread_ids = set() + self._fork_invalidated = True + + @property + def _inherited(self) -> bool: + return self._fork_invalidated or _GETPID() != self._creator_pid + + def _raise_if_unusable(self) -> None: + _ensure_current_process() + if self._inherited: # pragma: needs fork + msg = f"ReadWriteLock on {self.lock_file} was invalidated by fork(); construct a new instance" + raise RuntimeError(msg) + if self._closed: + msg = "Cannot operate on a closed database." + raise sqlite3.ProgrammingError(msg) + + def _acquire( + self, + mode: Literal["read", "write"], + timeout: float, + *, + blocking: bool, + holder: int | None = None, + cancel_check: Callable[[], bool] | None = None, + ) -> AcquireReturnProxy: + timeout = _resolve_read_write_timeout(timeout, blocking=blocking) + start_time: Final = time.monotonic() + remaining = timeout # -1 waits up to SQLite's busy-timeout cap + while True: + # A fork waits for the open transitions and admits no new one meanwhile. A wait for the database inside one + # stalls the forking thread and, behind it, each lock call in the process. With no other thread to fork, + # the wait runs in one piece, which keeps SQLite's writer preference. threading does not count a + # thread started through _thread, so a fork from one of those still waits. + final = ( + not blocking + or 0 <= remaining <= _MAX_TRANSITION_WAIT + or not hasattr(os, "register_at_fork") + or threading.active_count() == 1 + ) + try: + return self._attempt_acquire( + mode, + remaining if final else _MAX_TRANSITION_WAIT, + blocking=blocking, + holder=holder, + cancel_check=cancel_check, + ) + except Timeout: + if final or (cancel_check is not None and cancel_check()): + raise + # Only a runtime with fork handlers waits in more than one attempt. + if timeout != -1: # pragma: needs register-at-fork + remaining = max(timeout - (time.monotonic() - start_time), 0) + + def _attempt_acquire( + self, + mode: Literal["read", "write"], + timeout: float, + *, + blocking: bool, + holder: int | None = None, + cancel_check: Callable[[], bool] | None = None, + ) -> AcquireReturnProxy: + with _fork_transition(): + self._raise_if_unusable() + operation_pid = _GETPID() + thread_id = threading.get_ident() + holder = thread_id if holder is None else holder + with self._internal_lock: + if self._holds_level(): + return self._validate_reentrant(mode, holder) + if thread_id in self._acquisition_thread_ids: # pragma: no cover - exercised in an audit callback + msg = f"Cannot acquire ReadWriteLock on {self.lock_file} while acquisition is active in this thread" + raise RuntimeError(msg) + self._acquisition_thread_ids.add(thread_id) + try: + start_time = time.perf_counter() + self._acquire_transaction_lock(blocking=blocking, timeout=timeout, cancel_check=cancel_check) + try: + self._raise_if_unusable() + return self._do_acquire_inner( + mode, + timeout, + holder, + blocking=blocking, + operation=_Operation(operation_pid, start_time, cancel_check), + ) + finally: + self._transaction_lock.release() + finally: + with self._internal_lock: + self._acquisition_thread_ids.discard(thread_id) + + def _do_acquire_inner( + self, + mode: Literal["read", "write"], + timeout: float, + holder: int, + *, + blocking: bool, + operation: _Operation, + ) -> AcquireReturnProxy: + operation_pid, start_time, _ = operation + # Double-check: another thread may have acquired the lock while we waited on _transaction_lock. + with self._internal_lock: + if self._holds_level(): + return self._validate_reentrant(mode, holder) + if self._con is not None: + self._finish_connection() + try: + self._open_for_acquisition( + timeout, + blocking=blocking, + operation_pid=operation_pid, + start_time=start_time, + ) + self._configure_and_begin(mode, timeout, blocking=blocking, operation=operation) + self._raise_if_process_changed(operation_pid) + except BaseException as error: + acquisition_error: BaseException + if isinstance(error, sqlite3.OperationalError) and "database is locked" in str(error): + acquisition_error = Timeout(self.lock_file) + else: + acquisition_error = error + try: + self._finish_connection() + except sqlite3.Error as cleanup_error: + _raise_chained_errors(acquisition_error, cleanup_error) + if acquisition_error is not error: + raise acquisition_error from None + raise + with self._internal_lock: + self._raise_if_process_changed(operation_pid) + self._current_mode = mode + self._lock_level = 1 + if mode == "write": + self._write_thread_id = holder + else: + self._read_depths = {holder: 1} + return AcquireReturnProxy(lock=self) + + def _open_for_acquisition(self, timeout: float, *, blocking: bool, operation_pid: int, start_time: float) -> None: + with _sqlite_transition(): + sqlite_timeout = ( + timeout_for_sqlite( + timeout, + blocking=blocking, + already_waited=time.perf_counter() - start_time, + ) + / 1000 + ) + connection = self._open_connection(sqlite_timeout=sqlite_timeout) + self._con, self._connection_transaction_released, self._connection_identity = ( + connection, + False, + _FORKED_DATABASES.identity(self._canonical_path), + ) + self._raise_if_process_changed(operation_pid) + + def _configure_and_begin( + self, + mode: Literal["read", "write"], + timeout: float, + *, + blocking: bool, + operation: _Operation, + ) -> None: + with _sqlite_transition(): + connection = cast("_ForkSafeConnection", self._con) + # SQLite opened the file the name held at connect time, and its first statement would replay a journal + # beside that file, so the name is checked before any statement runs. + self._raise_if_database_replaced(connection) + with _bounded_schema(connection, self.lock_file): + self._raise_if_process_changed(operation.pid) + # Use legacy journal mode (not WAL) because WAL does not block readers while a concurrent EXCLUSIVE + # write transaction is active, which makes read-write locking impossible without modifying table data. + # MEMORY is safe here since no writes happen, so a crash cannot corrupt the DB. + # See https://sqlite.org/lang_transaction.html#deferred_immediate_and_exclusive_transactions + self._execute_waiting( + connection, "PRAGMA journal_mode=MEMORY;", timeout, blocking=blocking, operation=operation + ) + self._raise_if_process_changed(operation.pid) + statements = "BEGIN EXCLUSIVE TRANSACTION;" if mode == "write" else "BEGIN TRANSACTION;" + if mode == "read": + # SQLite takes the SHARED lock only when a statement reads; BEGIN alone stays deferred. + # https://www.sqlite.org/lockingv3.html#transaction_control + # sqlite_schema is an alias added in SQLite 3.33.0; sqlite_master works on every version. + statements += " SELECT name FROM sqlite_master LIMIT 1;" + self._execute_waiting(connection, statements, timeout, blocking=blocking, operation=operation) + self._raise_if_database_replaced(connection) + + @staticmethod + def _execute_waiting( + connection: _ForkSafeConnection, statements: str, timeout: float, *, blocking: bool, operation: _Operation + ) -> None: + # SQLite's busy wait cannot be interrupted. An acquire its caller may abandon waits in bounded slices and asks + # between them whether to stop. + slice_ms = int(_CANCEL_POLL_INTERVAL * 1000) + while True: + busy_ms = timeout_for_sqlite( + timeout, blocking=blocking, already_waited=time.perf_counter() - operation.start_time + ) + cancel_check = operation.cancel_check if busy_ms > slice_ms else None + try: + connection.executescript( + f"PRAGMA busy_timeout={busy_ms if cancel_check is None else slice_ms}; {statements}" + ).close() + except sqlite3.OperationalError as error: + if cancel_check is None or "database is locked" not in str(error) or cancel_check(): + raise + # A read's BEGIN succeeds before its SELECT waits, so the retry has to start a fresh transaction. + if connection.in_transaction: + connection.rollback() + else: + return + + def _raise_if_database_replaced(self, connection: _ForkSafeConnection) -> None: + # Windows pins the name: SQLite opens the database without delete sharing, so it cannot be unlinked or renamed. + if (identity := connection.database_identity) is None: # pragma: win32 cover + return + try: # pragma: win32 no cover + current = self._canonical_path.stat(follow_symlinks=False) + except FileNotFoundError: # pragma: win32 no cover + current = None + # The lock lives on the inode, so one taken after the path moved on excludes no later opener. + if current is None or (current.st_dev, current.st_ino) != identity: # pragma: win32 no cover + msg = f"lock database {self.lock_file!r} was unlinked or replaced while acquiring; leave it in place" + raise OSError(msg) + + def _open_connection(self, *, sqlite_timeout: float) -> _ForkSafeConnection: + with _sqlite_transition(): + creator_pid = _GETPID() + functions = _CONNECTION_ESCROW.functions() + if _GETPID() != creator_pid: # pragma: forked child + msg = "SQLite connection construction cannot continue after fork" + raise RuntimeError(msg) + try: + connection = _connect( + os.fspath(self._canonical_path), + factory=_ForkSafeConnection, + timeout=sqlite_timeout, + ) + except BlockingIOError: + raise Timeout(self.lock_file) from None + if functions is not None: # pragma: <3.12 cover # pragma: needs fork + connection.acquire_escrow(functions) + if _GETPID() != creator_pid: # pragma: forked child + _FORKED_DATABASES.poison_after_fork( + self._canonical_path, + _FORKED_DATABASES.identity(self._canonical_path), + ) + msg = "SQLite connection construction cannot continue after fork" + raise RuntimeError(msg) + return connection + + def _finish_connection(self) -> None: + with _sqlite_transition(): + if (connection := self._con) is None: + return + rollback_error: sqlite3.Error | None = None + if not self._connection_transaction_released: + if connection.in_transaction: + try: + connection.rollback() + except sqlite3.Error as error: + if connection.in_transaction: + raise + self._connection_transaction_released = True + rollback_error = error + else: + self._connection_transaction_released = True + else: + self._connection_transaction_released = True + try: + connection.close() + except sqlite3.Error as close_error: + if rollback_error is not None: + _raise_chained_errors(rollback_error, close_error) + raise + self._con = None + self._connection_transaction_released = True + self._connection_identity = None + if rollback_error is not None: + raise rollback_error + + def _holds_level(self) -> bool: + # A release interrupted after its rollback, but before it recorded that, leaves a level with no transaction. + if self._lock_level > 0 and self._transaction_ended(): + self._clear_lock_state() + return self._lock_level > 0 + + def _validate_reentrant(self, mode: Literal["read", "write"], cur: int) -> AcquireReturnProxy: + if self._current_mode != mode: + opposite = "write" if mode == "read" else "read" + if cur == self._write_thread_id or cur in self._read_depths: + direction = "downgrade" if mode == "read" else "upgrade" + reason = f"already holding a {opposite} lock ({direction} not allowed)" + else: + reason = ( + f"another thread holds a {opposite} lock on this instance, which coordinates processes rather " + "than threads; give each thread its own instance with is_singleton=False" + ) + msg = f"Cannot acquire {mode} lock on {self.lock_file} (lock id: {id(self)}): {reason}" + raise RuntimeError(msg) + if mode == "write" and cur != self._write_thread_id: + msg = ( + f"Cannot acquire write lock on {self.lock_file} (lock id: {id(self)}) " + f"from thread {cur} while it is held by thread {self._write_thread_id}" + ) + raise RuntimeError(msg) + if mode == "read": + self._read_depths[cur] = self._read_depths.get(cur, 0) + 1 + self._lock_level += 1 + return AcquireReturnProxy(lock=self) + + def _acquire_transaction_lock( + self, *, blocking: bool, timeout: float, cancel_check: Callable[[], bool] | None + ) -> None: + if not _acquire_thread_lock( + self._transaction_lock, + blocking=blocking, + timeout=timeout, + poll_interval=_CANCEL_POLL_INTERVAL, + cancel_check=cancel_check, + ): + raise Timeout(self.lock_file) from None + + def _raise_if_acquiring(self, operation: Literal["acquire", "close", "release"]) -> None: + with self._internal_lock: + active_in_current_thread = threading.get_ident() in self._acquisition_thread_ids + if active_in_current_thread: # pragma: no cover - exercised in an audit callback + msg = f"Cannot {operation} ReadWriteLock on {self.lock_file} while acquisition is active in this thread" + raise RuntimeError(msg) + + def _raise_if_process_changed(self, operation_pid: int) -> None: + if _GETPID() != operation_pid or self._inherited: # pragma: forked child + msg = f"ReadWriteLock on {self.lock_file} was invalidated by fork(); construct a new instance" + raise RuntimeError(msg) + + +def _connect(database: str, *, factory: type[_ForkSafeConnection], timeout: float) -> _ForkSafeConnection: + _FORKED_DATABASES.note_sqlite_use() + # A symlink at the path would make SQLite open, lock, or create its target, so connect through a descriptor that + # refuses one. + if sys.platform == "win32": # pragma: win32 cover + fd: Final = _open_lock_database(database, oplock_wait=timeout) + link: _LinkDirectory | None = None + # A Windows lock belongs to the handle that took it, so closing this one leaves SQLite's locks in place. + try: + _raise_if_wal(fd, database) + target = database + if not _directory_is_private(os.fspath(pathlib.Path(database).parent)): + link = _link_beside_on_windows(database, _identity_of_fd(fd)) + target = os.fspath(link.database) + try: + connection = sqlite3.connect( + target, check_same_thread=False, factory=factory, cached_statements=0, timeout=timeout + ) + except BaseException: + if link is not None: + _remove_link_directory(link) + raise + connection.retain_link(link) + return connection + finally: + os.close(fd) + return _connect_through_descriptor( # pragma: win32 no cover + database, factory=factory, timeout=timeout, owner=os.geteuid() + ) + + +def _connect_through_descriptor( # pragma: win32 no cover + database: str, *, factory: type[_ForkSafeConnection], timeout: float, owner: int +) -> _ForkSafeConnection: + # Listed before its lease is taken and cleared once a connection owns it, so any exception between, an interrupt + # included, releases the lease instead of pinning the inode's probes. + claimed: Final[list[_Probe]] = [] + link: _LinkDirectory | None = None + try: + if (probe := _PROBE_DESCRIPTORS.adopt(database, claimed)) is None: + probe = _PROBE_DESCRIPTORS.retain(_open_lock_database(database), claimed) + identity, fd, _ = probe + # Releases a finalizer queued while the table was locked. + _PROBE_DESCRIPTORS.drain() + _raise_if_wal(fd, database) + target = database + # SQLite opens the name it is given, following symlinks, and uses the -journal, -wal and -shm names beside it, + # so the database name is safe only where no process of another account can create names. A descriptor path + # such as /proc/self/fd/ is no better: SQLite reads it back into the name the kernel reports. + if not _names_are_private(pathlib.Path(database).parent, owner): # pragma: needs posix-hard-link + link = _link_beside(database, identity, owner) + target = os.fspath(link.database) + connection = sqlite3.connect( + target, check_same_thread=False, factory=factory, cached_statements=0, timeout=timeout + ) + connection.retain_until_close(link, probe) + claimed.clear() + return connection + finally: + for unowned in claimed: + _PROBE_DESCRIPTORS.release(unowned) + if claimed and link is not None: # pragma: needs posix-hard-link + # A connection that already took the link removes it too when it closes. + _remove_link_directory(link) + + +def _names_are_private(directory: pathlib.Path, owner: int) -> bool: # pragma: win32 no cover + status: Final = os.lstat(directory) + # The sticky bit still lets others create names, so a directory anyone else may write to never qualifies. + return status.st_uid in {owner, 0} and not status.st_mode & (stat.S_IWGRP | stat.S_IWOTH) + + +class _LinkDirectory(NamedTuple): # pragma: needs posix-hard-link + path: pathlib.Path + identity: _DatabaseIdentity + # Made through a descriptor of path, which nobody else can list, so a directory another process swaps in under + # path cannot hold this name and SQLite fails to open instead of following the swap. + name: str + + @property + def database(self) -> pathlib.Path: + return self.path / self.name / _LINK_NAME + + +def _link_beside( # pragma: needs posix-hard-link + database: str, identity: _DatabaseIdentity, owner: int +) -> _LinkDirectory: + # Beside the database because a link cannot cross filesystems, and TMPDIR is often on another one. + parent: Final = pathlib.Path(database).parent + _sweep_links_of_the_dead(parent, identity, owner) + path: Final = pathlib.Path(tempfile.mkdtemp(prefix=_link_prefix(), dir=parent)) + fd: Final = os.open(path, _DIRECTORY_FLAGS) + try: + made: Final = os.fstat(fd) + # Whoever can rename in the parent can put another directory under the new name before it opens. + if made.st_uid != owner or made.st_mode & 0o077: + msg = f"link directory {os.fspath(path)!r} was replaced while opening" + raise OSError(msg) + link: Final = _LinkDirectory(path, (made.st_dev, made.st_ino), secrets.token_hex(8)) + try: + os.mkdir(link.name, 0o700, dir_fd=fd) + os.link(database, link.database, follow_symlinks=False) + linked: Final = link.database.stat(follow_symlinks=False) + except BaseException: + _remove_link_directory(link) + raise + finally: + os.close(fd) + if (linked.st_dev, linked.st_ino) != identity: + _remove_link_directory(link) + msg = f"lock database changed while opening: {database!r}" + raise OSError(msg) + return link + + +def _sweep_links_of_the_dead( + parent: pathlib.Path, identity: _DatabaseIdentity, owner: int +) -> None: # pragma: needs posix-hard-link + # A process killed while it held a link leaves the directory, and an extra link to the database, behind for good. + # Two threads sweeping at once only both try the same removals, which a lock would cost a fork-safety reset for. + if identity in _SWEPT_DATABASES: + return + _SWEPT_DATABASES.add(identity) + try: + with os.scandir(parent) as entries: + names: Final = [ + entry.name + for entry in entries + if entry.name.startswith(_LINK_PREFIX) and entry.is_dir(follow_symlinks=False) + ] + except OSError: + return + for name in names: + # A peer sweeping alongside, or a directory this user cannot remove, must not fail the connect. + with suppress(OSError): + _remove_dead_links(parent / name, identity, owner) + + +def _remove_dead_links( + path: pathlib.Path, identity: _DatabaseIdentity, owner: int +) -> None: # pragma: needs posix-hard-link + if sys.platform == "win32": # pragma: win32 cover + _remove_dead_links_on_windows(path, identity) + return + # Any directory can carry a dead holder's name after a rename in the parent, so only hard links to this database + # go, and the directory only once that leaves it empty. + fd: Final = os.open(path, _DIRECTORY_FLAGS) + try: + # Only a directory this account made carries a holder of its own; a name another account chose is never + # parsed, so no PID is probed on its behalf. + if os.fstat(fd).st_uid != owner or not _owner_is_gone(path.name): + return + with os.scandir(fd) as entries: + names: Final = [entry.name for entry in entries] + for entry in names: + # Releases up to 4.2.0 linked the database right inside the directory. + if entry == _LINK_NAME: + _unlink_link_to(identity, fd) + continue + try: + nested = os.open(entry, _DIRECTORY_FLAGS, dir_fd=fd) + except OSError: + continue + try: + _unlink_link_to(identity, nested) + finally: + os.close(nested) + with suppress(OSError): + os.rmdir(entry, dir_fd=fd) + finally: + os.close(fd) + with suppress(OSError): + path.rmdir() + + +def _unlink_link_to(identity: _DatabaseIdentity, dir_fd: int) -> None: # pragma: needs posix-hard-link + try: + linked: Final = os.stat(_LINK_NAME, dir_fd=dir_fd, follow_symlinks=False) + except OSError: + return + if (linked.st_dev, linked.st_ino) == identity: + os.unlink(_LINK_NAME, dir_fd=dir_fd) + + +def _owner_is_gone(name: str) -> bool: # pragma: needs posix-hard-link + match name.removeprefix(_LINK_PREFIX).split("-"): + case [pid, start, digest, _] if ( + pid.isascii() and pid.isdigit() and (not start or (start.isascii() and start.isdigit())) + ): + if not 1 <= (owner_pid := int(pid)) <= 2**31 - 1: + return False + host: Final = host_name() + return digest == _host_digest(host) and owner_is_stale(owner_pid, host, int(start) if start else None) + case _: + return False + + +def _link_prefix() -> str: # pragma: needs posix-hard-link + # The owner rides in the name, which mkdtemp creates in one step, so no directory ever exists without it. + start: Final = process_start_token(pid := os.getpid()) + return f"{_LINK_PREFIX}{pid}-{'' if start is None else start}-{_host_digest(host_name())}-" + + +def _host_digest(host: str) -> str: # pragma: needs posix-hard-link + # SHA-256 as host_name does: some runtimes, GraalPy's among them, have no BLAKE2 with a custom digest size. + return hashlib.sha256(host.encode()).hexdigest()[:16] + + +def _remove_link_directory(link: _LinkDirectory) -> None: # pragma: needs posix-hard-link + if sys.platform == "win32": # pragma: win32 cover + _remove_link_directory_on_windows(link) + return + # A rename in the parent may have put any directory under this name, so entries go through a descriptor checked + # to be the directory made, and the name goes only through rmdir, which refuses a non-empty directory. + try: + fd: Final = os.open(link.path, _DIRECTORY_FLAGS) + except OSError: + return + try: + opened: Final = os.fstat(fd) + if (opened.st_dev, opened.st_ino) != link.identity: + return + for suffix in ("", "-journal", "-wal", "-shm"): + with suppress(FileNotFoundError): + os.unlink(f"{link.name}/{_LINK_NAME}{suffix}", dir_fd=fd) + with suppress(FileNotFoundError): + os.rmdir(link.name, dir_fd=fd) + finally: + os.close(fd) + # The link is gone, so a failure here leaves one stray empty directory and is not worth failing a release for. + with suppress(OSError): + link.path.rmdir() + + +if sys.platform == "win32": # pragma: win32 cover + + def _link_beside_on_windows(database: str, identity: _DatabaseIdentity) -> _LinkDirectory: + parent: Final = pathlib.Path(database).parent + # No uid exists here: a directory is a dead holder's only if its ACL admits nobody but the trusted accounts. + _sweep_links_of_the_dead(parent, identity, 0) + path: Final = parent / f"{_link_prefix()}{secrets.token_hex(8)}" + _make_private_directory(os.fspath(path)) + link: _LinkDirectory | None = None + try: + # The handle keeps a rename in the parent from swapping the directory while the link is made inside it. + with _pinned(os.fspath(path)) as directory_identity: + link = _LinkDirectory(path, directory_identity, secrets.token_hex(8)) + _fill_link_directory(link, database, identity) + except BaseException: + if link is not None: # pragma: no branch # a pinned directory always gets its link + _remove_link_directory(link) + raise + return link + + def _fill_link_directory(link: _LinkDirectory, database: str, identity: _DatabaseIdentity) -> None: + if not _directory_is_private(os.fspath(link.path)): + msg = f"link directory {os.fspath(link.path)!r} was replaced while opening" + raise OSError(msg) + _make_private_directory(os.fspath(link.path / link.name)) + os.link(database, link.database) + # The open database handle pins its path against a rename, so only a file index that is not unique can differ. + if _identity_at(os.fspath(link.database)) != identity: # pragma: no cover + msg = f"lock database changed while opening: {database!r}" + raise OSError(msg) + + def _remove_dead_links_on_windows(path: pathlib.Path, identity: _DatabaseIdentity) -> None: + with _pinned(os.fspath(path)): + if not _directory_is_private(os.fspath(path)) or not _owner_is_gone(path.name): + return + with os.scandir(path) as entries: + nested: Final = [pathlib.Path(entry.path) for entry in entries] + for directory in nested: + _unlink_link_to_on_windows(identity, directory / _LINK_NAME) + with suppress(OSError): + directory.rmdir() + with suppress(OSError): + path.rmdir() + + def _unlink_link_to_on_windows(identity: _DatabaseIdentity, path: pathlib.Path) -> None: + with suppress(OSError): + if _identity_at(os.fspath(path)) == identity: + path.unlink() + + def _remove_link_directory_on_windows(link: _LinkDirectory) -> None: + # A removal never fails a release: SQLite or a scanner may still hold a name, and the next sweep retries. + with suppress(OSError): + with _pinned(os.fspath(link.path)) as directory_identity: + if directory_identity != link.identity: + return + for suffix in ("", "-journal", "-wal", "-shm"): + with suppress(FileNotFoundError): + pathlib.Path(f"{link.database}{suffix}").unlink() + with suppress(FileNotFoundError): + pathlib.Path(link.database.parent).rmdir() + link.path.rmdir() + + +def _open_lock_database(database: str, *, oplock_wait: float = 0) -> int: + if sys.platform == "win32": # pragma: win32 cover + # The handle refuses a reparse point and omits delete sharing, so the name cannot be swapped while it is open. + if (fd := _open_non_reparse_fd(database, 0o600, oplock_wait=oplock_wait)) is None: + msg = f"lock database {database!r} is held without sharing or pending deletion" + raise PermissionError(msg) + return fd + fd = os.open(database, _DB_OPEN_FLAGS, 0o600) # pragma: win32 no cover + if not stat.S_ISREG(os.fstat(fd).st_mode): # pragma: win32 no cover + os.close(fd) + msg = f"refusing a non-regular lock database: {database!r}" + raise OSError(msg) + return fd # pragma: win32 no cover + + +def _raise_if_wal(fd: int, database: str) -> None: + # SQLite derives the -wal name from the path it connects through, a descriptor name on POSIX, and WAL never blocks + # readers behind a writer, so the lock cannot work on such a database. + # pread: an adopted probe is shared, so its file offset is not this connect's to move. + header: Final = os.pread(fd, _HEADER_SIZE, 0) if hasattr(os, "pread") else os.read(fd, _HEADER_SIZE) + if header.startswith(_SQLITE_MAGIC) and _WAL_FORMAT_VERSION in header[_FORMAT_VERSIONS]: + msg = f"refusing a WAL-mode lock database: {database!r}; give ReadWriteLock a file of its own" + raise ValueError(msg) + + +@contextmanager +def _bounded_schema(connection: sqlite3.Connection, database: str) -> Generator[None]: + # The first statement loads the file's complete schema, which no timeout bounds, and a lock database has none. + # Waiting for a lock runs no virtual machine steps, so a step budget bounds only that load. + calls: Final = count() + max_calls: Final = 10 + connection.setlimit(sqlite3.SQLITE_LIMIT_LENGTH, 4096) + connection.set_progress_handler(lambda: next(calls) >= max_calls, 1000) + try: + yield + except sqlite3.DatabaseError as error: + # An interrupted schema load reports a malformed schema rather than the interrupt. + if isinstance(error, sqlite3.DataError) or next(calls) > max_calls: + msg = ( + f"refusing a lock database whose schema is too large to load: {database!r}; " + "give ReadWriteLock a file of its own" + ) + raise ValueError(msg) from error + raise + finally: + connection.set_progress_handler(None, 0) + + +@contextmanager +def _sqlite_transition() -> Generator[None]: + _SQLITE_TRANSITION_CONTEXT.depth += 1 + try: + yield + finally: + _SQLITE_TRANSITION_CONTEXT.depth -= 1 + + +def _abort_forked_sqlite_transition() -> None: # pragma: forked child + if _SQLITE_TRANSITION_CONTEXT.depth: + os._exit(_UNSAFE_FORK_EXIT_STATUS) # inherited SQLite handles cannot be used or closed safely + + +def _track_sqlite_use(event: str, _args: Unused) -> None: + if event == "sqlite3.connect": + _FORKED_DATABASES.note_sqlite_use() + + +def timeout_for_sqlite(timeout: float, *, blocking: bool, already_waited: float) -> int: + if blocking is False: + return 0 + + if timeout == -1: + return _MAX_SQLITE_TIMEOUT_MS + + if timeout < 0: + msg = "timeout must be a non-negative number or -1" + raise ValueError(msg) + + timeout_ms = int((max(timeout - already_waited, 0) if timeout > 0 else timeout) * 1000) + if timeout_ms > _MAX_SQLITE_TIMEOUT_MS or timeout_ms < 0: + _LOGGER.warning("timeout %s is too large for SQLite, using %s ms instead", timeout, _MAX_SQLITE_TIMEOUT_MS) + return _MAX_SQLITE_TIMEOUT_MS + return timeout_ms + + +_register_fork_object(_CONNECTION_ESCROW) +_register_fork_object(_FORKED_DATABASES) +_register_fork_object(_PROBE_DESCRIPTORS) +_register_fork_class(ReadWriteLock) +if _IS_PYPY: + sys.addaudithook(_track_sqlite_use) # pragma: pypy cover +if hasattr(os, "register_at_fork"): # pragma: needs fork + os.register_at_fork(after_in_child=_abort_forked_sqlite_transition) + +__all__ = [ + "ReadWriteLock", + "timeout_for_sqlite", +] diff --git a/.venv/lib/python3.14/site-packages/filelock/_soft.py b/.venv/lib/python3.14/site-packages/filelock/_soft.py new file mode 100644 index 0000000000000000000000000000000000000000..d54fffa70d700a341ed84d6c551c4f59c585da6e --- /dev/null +++ b/.venv/lib/python3.14/site-packages/filelock/_soft.py @@ -0,0 +1,643 @@ +from __future__ import annotations + +__lazy_modules__ = [ + "contextlib", + "filelock._identity", + "filelock._soft_protocol", + "filelock._util", + "itertools", + "pathlib", + "re", + "secrets", + "typing", +] + +import os +import re +import secrets +import stat +import sys +import threading +import time +from contextlib import suppress +from errno import EACCES, EEXIST, EPERM +from itertools import islice +from pathlib import Path +from typing import Final + +from filelock._api import BaseFileLock, _raise_grouped_errors +from filelock._identity import host_name, owner_is_current_process, owner_is_stale, process_start_token +from filelock._soft_protocol import STRICT_SOFT_SENTINEL_RECORD +from filelock._util import ( + break_lock_file, + exchange_names, + exchange_supported, + pin_inode, + raise_on_not_writable_file, + remove_own_file, + write_all, +) + +if sys.platform == "win32": # pragma: win32 cover + from filelock._windows import open_read_without_oplock_wait + from filelock._windows import unlink_without_oplock_wait as unlink_path +else: # pragma: win32 no cover + from filelock._util import unlink_path + +MALFORMED_LOCK_AGE_THRESHOLD: Final[float] = 2.0 +_MAX_LOCK_FILE_SIZE: Final[int] = 1024 +_UNLINK_MAX_RETRIES: Final[int] = 10 +_MARKER_WITH_START_TOKEN_LINE_COUNT: Final[int] = 3 +#: Contenders act on the record in the marker, so no other account may write it, whatever the mode or umask. +_OTHERS_WRITE: Final[int] = stat.S_IWGRP | stat.S_IWOTH +#: All a SoftFileLock before 3.22 writes, and what a current one shows between creating its marker and writing it. +_RECORDLESS_MARKER: Final[str] = "" +_CLAIMS_IN_PROGRESS: Final[set[str]] = set() +_CLAIMS_LOCK: Final[threading.Lock] = threading.Lock() + + +class SoftFileLock(BaseFileLock): + """ + Cooperative file lock based on a shared existence marker. + + Unlike :class:`UnixFileLock ` and :class:`WindowsFileLock `, this + lock does not use OS-level locking primitives. Instead, it creates the lock file with ``O_CREAT | O_EXCL`` and + treats its existence as the lock indicator. The filesystem must provide coherent exclusive creation and directory + updates to each participating process. A crash can leave the marker behind. + + The marker contains the holder's PID and hostname. A contender may remove it when it can no longer find a same-host + process with that PID. A configured :attr:`~filelock.BaseFileLock.lifetime` also permits removal based on marker + age, including while the holder remains alive. Age-based expiry can overlap protected operations and does not + provide strict mutual exclusion. + + """ + + #: Existence locks reclaim by unlinking a pathname, so an age-based lease may break one; a native inode lock cannot. + _lifetime_supported: bool = True + + #: Age-based expiry preserves historical behavior but does not provide strict mutual exclusion. + _lifetime_replacements: tuple[str, str] | None = ("StrictSoftFileLock", "SoftFileLease") + + #: An existence lock unlinks its marker to release, so it cannot promise to keep the pathname. + _preserve_lock_file_supported: bool = False + + #: An existence lock keeps protocol state in its marker, so it cannot lend the descriptor to an on_acquired hook. + _on_acquired_supported: bool = False + + #: The marker's (mtime, inode, ctime) when this instance last saw it change, and when, on its monotonic clock. + _marker_seen: tuple[tuple[float, int, int], float] | None = None + + #: The identity of a marker a Windows release relinquished but has not unlinked yet. + _marker_cleanup: tuple[int, int] | None = None + + #: The descriptor of that marker while its close has not run yet, as a list so one C call can take it. + _marker_fds: list[int] + + def _acquire(self) -> None: + self._finish_marker_cleanup() + raise_on_not_writable_file(self._lock_path) + # O_CREAT | O_EXCL makes the create fail with EEXIST when the file already exists, so a successful open + # means this process now holds the lock. + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_TRUNC + if (o_nofollow := getattr(os, "O_NOFOLLOW", None)) is not None: # pragma: needs o-nofollow + flags |= o_nofollow + try: + fd = os.open(self._lock_path, flags, self._open_mode() & ~_OTHERS_WRITE) + except OSError as exception: + if not ( + exception.errno == EEXIST or (exception.errno == EACCES and sys.platform == "win32") + ): # pragma: win32 no cover + raise + self._try_break_stale_lock() + return + # Publish held state only once the record is on disk. On failure, cancellation included, close the descriptor + # and unlink the path only while it still names the file we opened: after an expired lease a successor's + # marker may sit at the same path. The guard starts at the open, so a signal handler that raises before + # _mark_descriptor_owned leaves no marker behind. + identity: tuple[int, int] | None = None + try: + identity = _file_identity(os.fstat(fd)) + self._mark_descriptor_pending(fd) + self._apply_explicit_mode(fd) + self._write_lock_info(fd) + self._mark_descriptor_owned(fd, identity) + except BaseException: + self._mark_descriptor_released() + try: + if sys.platform != "win32": # pragma: win32 no cover + # Before the close: our open descriptor keeps the inode allocated, so a successor's marker cannot + # carry its number. + with suppress(OSError): + remove_own_file(self._lock_path, fd) + finally: + try: + os.close(fd) + finally: + if sys.platform == "win32": # pragma: win32 cover + self._unlink_held_marker(identity) + raise + + def _apply_explicit_mode(self, fd: int) -> None: + # os.open filters the mode through the umask, so an explicit mode lands as given only through fchmod, the way + # UnixFileLock applies it. Windows maps the mode to a read-only bit that os.open already set. + if self.has_explicit_mode and sys.platform != "win32": # pragma: win32 no cover + with suppress(PermissionError): + os.fchmod(fd, self._context.mode & ~_OTHERS_WRITE) + + def _try_break_stale_lock(self) -> None: + with suppress(OSError, ValueError), pin_inode(self._lock_path) as pinned: + content, marker = _read_lock_file(self._lock_path) + if pinned not in {None, marker.st_ino}: # pragma: win32 no cover # Windows pins nothing + return # another node replaced the pinned one before the read, and nothing keeps its number from reuse + if content == STRICT_SOFT_SENTINEL_RECORD: # pragma: needs hard-link + return + # Such a marker is held for as long as its owner runs, so being unchanged proves nobody gone. + if content == _RECORDLESS_MARKER: + return + holder = _parse_lock_holder(content) + + if holder is None: + # Unparsable: wrong line count, a non-integer PID or start token, empty, oversized or not UTF-8. + # Self-heal only once the file is clearly not a half-written fresh lock (a peer between O_EXCL and + # _write_lock_info), so the brief create-then-write window is never mistaken for a stale lock. + if self._marker_unchanged_for(marker) >= MALFORMED_LOCK_AGE_THRESHOLD: + self._take_over(marker, _placeholder_breaker(content)) + return + + if owner_is_stale(*holder): + self._take_over(marker, holder[0]) + + def _try_break_expired_lock(self) -> None: + if (lifetime := self._context.lifetime) is None: + return + with suppress(OSError), pin_inode(self._lock_path) as pinned: + # _read_lock_file takes the lstat, not stat: a symlink swapped in at the path must report its own mtime. + content, marker = _read_lock_file(self._lock_path) + if pinned not in {None, marker.st_ino} or time.time() - marker.st_mtime < lifetime: + return + holder = self._recorded_holder(content) + self._take_over(marker, None if holder is None else holder[0]) + + def _take_over(self, marker: os.stat_result, breaker_pid: int | None) -> None: + # Swap the judged marker for a complete marker of ours in one exchange, so the lock path never stands empty. + # Once the file swapped out proves to be the one judged, this lock holds the path; the caller's pin keeps that + # file's inode number from naming anything else. Only a peer's exchange can move our marker after that, and + # the peer puts it back because what it took is not the file it judged. Every failure before the handoff takes + # our marker off the path and puts back what the exchange took. Without an exchange, the portable break decides. + if exchange_supported(): # pragma: win32 no cover + self._take_over_by_exchange(marker, breaker_pid) + else: + break_lock_file(self._lock_path, marker.st_mtime, marker.st_ino) + + def _take_over_by_exchange(self, marker: os.stat_result, breaker_pid: int | None) -> None: # pragma: win32 no cover + if (current := os.lstat(self._lock_path)).st_mtime != marker.st_mtime or current.st_ino != marker.st_ino: + return + if breaker_pid is not None and self._restore_stranded_marker(breaker_pid, marker): + return + aside: Final = f"{self._lock_path}.break.{os.getpid()}.{secrets.token_hex(16)}" + flags: Final = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0) + fd: Final = os.open(aside, flags, self._open_mode() & ~_OTHERS_WRITE) + self._mark_descriptor_pending(fd) + exchanged = held = False + try: + identity: Final = _file_identity(os.fstat(fd)) + self._apply_explicit_mode(fd) + self._write_lock_info(fd) + if not exchange_names(aside, self._lock_path): + return # NFS, SMB and the like refuse it; renaming the marker aside would empty the path + exchanged = True + if (taken := os.lstat(aside)).st_mtime != marker.st_mtime or taken.st_ino != marker.st_ino: + return # a peer recreated the marker, which the cleanup below swaps back + Path(aside).unlink() + exchanged = False + self._mark_descriptor_owned(fd, identity) + held = True + finally: + if not held: + self._abandon_take_over(fd, aside, put_back=exchanged) + + def _restore_stranded_marker(self, breaker_pid: int, marker: os.stat_result) -> bool: # pragma: win32 no cover + # A breaker killed between its exchange and the put-back leaves its own marker at the path and the live + # holder's marker under its break name. A recoverer first renames that record to a name of its own. Only one + # contender wins the rename, so two of them never exchange the same record and reverse each other. A record + # a dead recoverer holds is claimable again. A claim of this process that no thread is working on was + # abandoned by a failed put-back, and the next poll takes it up again. Where the marker cannot be put back + # the lock counts as held, as it does wherever no exchange is possible. + if (candidates := self._stranded_candidates(breaker_pid)) is None: + return True + return any(self._recover_claimed(path, breaker_pid, marker, adopt=abandoned) for path, abandoned in candidates) + + def _stranded_candidates(self, breaker_pid: int) -> list[tuple[str, bool]] | None: # pragma: win32 no cover + directory, name = os.path.split(self._lock_path) + stranded: Final = re.compile(rf"{re.escape(name)}\.break\.{breaker_pid}\.[0-9a-f]{{32}}") + claimed: Final = re.compile(rf"{re.escape(name)}\.restore\.(\d+)\.[0-9a-f]{{32}}") + candidates: list[tuple[str, bool]] = [] + with os.scandir(directory or os.curdir) as entries: + for entry in entries: + if not entry.is_file(follow_symlinks=False): + continue + path = str(Path(directory) / entry.name) + if stranded.fullmatch(entry.name): + candidates.append((path, False)) + elif found := claimed.fullmatch(entry.name): + ours = int(found[1]) == os.getpid() + if not (ours or owner_is_stale(int(found[1]), host_name(), None)): + return None + candidates.append((path, ours)) + return sorted(candidates) + + def _recover_claimed( + self, path: str, breaker_pid: int, marker: os.stat_result, *, adopt: bool + ) -> bool: # pragma: win32 no cover + # One try/finally owns the registration from the first step to the last, so no interruption can leave it set. + # Registered before the rename publishes the name, so a thread that scans meanwhile sees a claim in progress. + own: Final = path if adopt else f"{self._lock_path}.restore.{os.getpid()}.{secrets.token_hex(16)}" + if not _begin_claim(own): + return True + try: + if not adopt: + try: + Path(path).rename(own) + except OSError: # a peer that won the rename now holds the record, so the lock counts as held + return True + return self._restore_claimed(own, breaker_pid, marker) + finally: + _end_claim(own) + + def _restore_claimed(self, own: str, breaker_pid: int, marker: os.stat_result) -> bool: # pragma: win32 no cover + try: + content, _ = _read_lock_file(own) + except OSError: + self._return_claim(own, breaker_pid) + return True + if not self._marker_is_live(content, breaker_pid): + with suppress(OSError): + Path(own).unlink() + return False + # The inode of the marker judged dead must still be at the path, or a peer has acted since. + with suppress(OSError): + current = os.lstat(self._lock_path) + if (current.st_ino, current.st_mtime) == (marker.st_ino, marker.st_mtime) and exchange_names( + own, self._lock_path + ): + if os.lstat(own).st_ino == marker.st_ino: + Path(own).unlink() + return True + # A peer replaced the path after the check, so the exchange took its marker: swap it straight back. + exchange_names(own, self._lock_path) + self._return_claim(own, breaker_pid) + return True + + def _return_claim(self, own: str, breaker_pid: int) -> None: # pragma: win32 no cover + # A refused rename leaves the claim under this process's PID, and the next poll retries it. + with suppress(OSError): + Path(own).rename(f"{self._lock_path}.break.{breaker_pid}.{secrets.token_hex(16)}") + + @classmethod + def _marker_is_live(cls, content: str | None, breaker_pid: int) -> bool: # pragma: win32 no cover + if content == STRICT_SOFT_SENTINEL_RECORD: # pragma: needs hard-link + return True + if content == _RECORDLESS_MARKER: + # An empty marker is a pre-3.22 holder for as long as it exists, as the stale check treats it. + return True + # The breaker's own marker names its PID on this host; the same number on another host is a holder. + return ( + (holder := _parse_lock_holder(content)) is not None + and (holder[0] != breaker_pid or holder[1] != host_name()) + and not owner_is_stale(*holder) + ) + + def _abandon_take_over(self, fd: int, aside: str, *, put_back: bool) -> None: # pragma: win32 no cover + self._mark_descriptor_released() + try: + if put_back: + exchange_names(aside, self._lock_path) + finally: + try: + # Whichever name still holds our marker loses it: the descriptor keeps its inode number ours. + own: Final = os.fstat(fd) + for name in (self._lock_path, aside): + with suppress(FileNotFoundError): + if os.path.samestat(os.lstat(name), own): + Path(name).unlink() + finally: + os.close(fd) + + def _marker_unchanged_for(self, marker: os.stat_result) -> float: + # The mtime is the writer's or file server's clock, so against time.time() a contender whose clock runs ahead + # would expire a fresh marker. Only watch it for change, timed on this process's monotonic clock. The owner of a + # file sets its mtime, and ext4 reuses a freed inode number, so a new marker can repeat both. The change time + # cannot be set and moves with every create, which keeps a new marker from inheriting an old one's age. + now: Final = time.monotonic() + key: Final = marker.st_mtime, marker.st_ino, marker.st_ctime_ns + if (seen := self._marker_seen) is None or seen[0] != key: + self._marker_seen = key, now + return 0.0 + return now - seen[1] + + def _unprobeable_holder(self) -> str | None: + # This host cannot prove such a holder dead, so whoever decides whether it is gone needs its name. + with suppress(OSError, ValueError): + if (content := _read_lock_file(self._lock_path)[0]) == _RECORDLESS_MARKER: + return "an owner that recorded no PID (a filelock release before 3.22, or one still writing its record)" + if (holder := self._recorded_holder(content)) and holder[1] != host_name(): + return f"pid {holder[0]} on {holder[1]}" + return None + + @classmethod + def _recorded_holder(cls, content: str | None) -> tuple[int, str] | None: + return None if (holder := _parse_lock_holder(content)) is None else holder[:2] + + @staticmethod + def _write_lock_info(fd: int) -> None: + # No suppression: a write failure must reach the acquisition rollback so it never publishes a half-written + # marker as held state. The optional third line is this process's start token, absent when the platform + # exposes no proven start time, in which case a reader falls back to PID-only liveness. + info = f"{os.getpid()}\n{host_name()}\n" + if (token := process_start_token(os.getpid())) is not None: + info += f"{token}\n" + write_all(fd, info.encode()) + + @property + def pid(self) -> int | None: + """ + The PID of the process holding this lock, read from the lock file. + + :returns: the PID as an integer, or ``None`` if the lock file does not exist or cannot be parsed + + """ + with suppress(OSError, ValueError): + holder = _parse_lock_holder(_read_lock_file(self._lock_path)[0]) + if holder is not None: + return holder[0] + return None + + @property + def is_lock_held_by_us(self) -> bool: + """ + Whether this lock is held by the current process. + + :returns: ``True`` if the lock file exists and names the current process's PID, hostname and, when recorded, + start token + + """ + with suppress(OSError, ValueError): + if (holder := _parse_lock_holder(_read_lock_file(self._lock_path)[0])) is not None: + return owner_is_current_process(*holder) + return False + + def break_lock(self) -> None: + """Forcibly break the lock by removing the lock file, regardless of who holds it.""" + with suppress(OSError): + Path(self._lock_path).unlink() + + def _release_current(self, *, force: bool) -> None: + try: + super()._release_current(force=force) + finally: + # A release a handler cut short after the descriptor closed still owes the unlink on Windows. + self._finish_marker_cleanup() + + def _release(self) -> None: + fd = self._context.lock_file_fd + assert fd is not None # ruff:ignore[assert] # _release runs only while held, so the descriptor is set + # Capture the held file's identity before closing so cleanup can refuse to unlink a successor's marker. A + # supported lifetime lease lets a peer break our expired marker and create its own at this path before we + # release; unlinking by path alone would then delete the successor's lock. + identity: tuple[int, int] | None = None + with suppress(OSError): + identity = _file_identity(os.fstat(fd)) + if sys.platform == "win32": # pragma: win32 cover + self._release_then_unlink(fd, identity) + else: # pragma: win32 no cover + # Our open descriptor keeps the inode allocated, so its number cannot name a successor's marker yet, and an + # interrupt before the unlink lands leaves the lock held for the retried release to finish. + unlinked: Final = self._unlink_open_marker(fd, identity) + # A failed close may already have released and recycled the descriptor number. Relinquish it before the + # one close attempt so no later release can close an unrelated descriptor that reused the same integer. + self._mark_descriptor_released() + try: + self._close_released_fd(fd, default_suppresses=False) + finally: + # A share that refuses to delete an open file gets the unlink after the close; the marker still exists + # until then, so its inode cannot be reused either. + if not unlinked: + self._unlink_held_marker(identity) + + def _release_then_unlink(self, fd: int, identity: tuple[int, int] | None) -> None: # pragma: win32 cover + # Windows cannot unlink an open file, so the unlink outlives the hold until it lands, here or on a later acquire + # or release. Only the gate's holder touches the pending record, so two threads never overwrite each other's. + with self._transition_lock: + self._marker_fds = [fd] + self._marker_cleanup = identity + self._mark_descriptor_released() + try: + self._close_marker_fd() + # Marker cleanup must also run for control-flow exceptions, and both failures must remain observable. + except BaseException as close_error: + try: + self._finish_marker_cleanup() + except BaseException as cleanup_error: # ruff:ignore[blind-except] # keep control-flow failures + _raise_grouped_errors( + "lock descriptor close and marker cleanup both failed", + close_error, + cleanup_error, + ) + raise + self._finish_marker_cleanup() + + def _marker_names(self) -> tuple[str, ...]: + # After a rename of the lock's directory only the caller's spelling, resolved through the working directory, + # still reaches the marker this hold opened. + return tuple(dict.fromkeys((self._lock_path, self.lock_file))) + + def _unlink_open_marker(self, fd: int, identity: tuple[int, int] | None) -> bool: # pragma: win32 no cover + if identity is None: + return True + replaced = False + for name in self._marker_names(): + try: + if _file_identity(os.lstat(name)) == identity: + Path(name).unlink() + return True + replaced = True + except FileNotFoundError: + continue + except OSError: + return False + if replaced: + try: + _leave_released_record(fd) + except OSError: + return False + return True + + def _finish_marker_cleanup(self) -> None: + # Never waits: whoever holds the gate is acquiring or releasing this lock and finishes the cleanup itself. + if self._marker_cleanup is not None and self._transition_lock.acquire(blocking=False): # pragma: win32 cover + try: + # A close error here belongs to the release an interrupt cut short, not to this caller. + with suppress(OSError): + self._close_marker_fd() + if (identity := self._marker_cleanup) is not None: # pragma: no branch # the gate's holder clears it + self._windows_unlink_if_ours(identity) + finally: + self._transition_lock.release() + + def _close_marker_fd(self) -> None: # pragma: win32 cover + # Popping and closing is one chain of C calls that no signal handler can split, so a retried release closes the + # descriptor at most once; a second close could hit an unrelated file that reused the number. + try: + list(map(os.close, islice(iter(vars(self).setdefault("_marker_fds", []).pop, None), 1))) + except IndexError: + return + except OSError: + if not self._suppresses_close_error(default_suppresses=False): + raise + + def _unlink_held_marker(self, identity: tuple[int, int] | None) -> None: + if sys.platform == "win32": # pragma: win32 cover + if identity is not None: + self._windows_unlink_if_ours(identity) + else: # pragma: win32 no cover + for name in self._marker_names(): + with suppress(OSError): + if _file_identity(os.lstat(name)) == identity: + Path(name).unlink() + return + + def _windows_unlink_if_ours(self, identity: tuple[int, int]) -> None: # pragma: win32 cover + retry_delay = 0.001 + for attempt in range(_UNLINK_MAX_RETRIES): + # Windows doesn't immediately release file handles after close, causing EACCES/EPERM on unlink. Recheck + # identity each attempt: a failed unlink leaves a window for a successor to replace the marker at this path. + try: + for name in self._marker_names(): + with suppress(FileNotFoundError): + if _file_identity(os.lstat(name)) == identity: + # Forgotten before the unlink: Windows does not promise unique file IDs, so no retry may + # reach a successor that reused ours. + self._marker_cleanup = None + unlink_path(name) + break + except OSError as exc: + if exc.errno in {EACCES, EPERM}: + # The file is still ours, so the next acquire or release tries again if these attempts run out. + self._marker_cleanup = identity + if attempt < _UNLINK_MAX_RETRIES - 1: + time.sleep(retry_delay) + retry_delay *= 2 + continue + self._marker_cleanup = None + return + + +def _file_identity(st: os.stat_result) -> tuple[int, int]: + # (st_dev, st_ino) names the concrete inode behind a path, so a marker recreated at the same pathname after an + # expired lease reads as a different file. CPython populates both on Windows from the volume serial and file index. + return st.st_dev, st.st_ino + + +def _begin_claim(name: str | Path) -> bool: # pragma: win32 no cover + # Keyed by the file name alone: its random token identifies the claim, and the spelling of the directory varies. + with _CLAIMS_LOCK: + if (key := Path(name).name) in _CLAIMS_IN_PROGRESS: + return False + _CLAIMS_IN_PROGRESS.add(key) + return True + + +def _end_claim(name: str | Path) -> None: # pragma: win32 no cover + with _CLAIMS_LOCK: + _CLAIMS_IN_PROGRESS.discard(Path(name).name) + + +def _placeholder_breaker(content: str | None) -> int | None: # pragma: win32 no cover + # The placeholder a break leaves at the path while it works names the breaker that may have died there. + if content is None or not content.startswith(placeholder := "filelock/break\n") or not content.endswith("\n"): + return None + with suppress(ValueError): + return parse_decimal(content.removeprefix(placeholder)[:-1]) + return None + + +def _leave_released_record(fd: int) -> None: + # A peer's exchange holds our marker under its break name and will put it back. Overwriting the record lets + # contenders age it out instead of waiting on this live PID. The write comes before the truncate so the file is + # never empty. + if sys.platform != "win32": # pragma: win32 no cover + os.pwrite(fd, released := b"filelock/released\n", 0) + os.ftruncate(fd, len(released)) + + +def _read_lock_file(path: str) -> tuple[str | None, os.stat_result]: + # A legitimate lock file is always a regular file. Classify the path with lstat first, so any other node (symlink, + # FIFO, socket, device) is reported as a malformed lock the caller can evict, without an os.open that would follow + # a symlink, stall on a FIFO, or fail on a socket and leave acquisition wedged. The stat result still flows back + # for the identity-checked stale break. lstat, not stat, so a hostile symlink is never followed onto its target. + st = os.lstat(path) + if not stat.S_ISREG(st.st_mode): # pragma: needs fifo + return None, st + # Re-check on the opened handle: O_NOFOLLOW refuses a symlink swapped in after the lstat, O_NONBLOCK stops a FIFO + # swapped in from stalling the open, and the fstat catches any other non-regular replacement race before we read. + # The capped read stops a huge regular file (e.g. one filled from /dev/zero) from exhausting memory. + if sys.platform == "win32": # pragma: win32 cover + # os.open would wait for an oplock holder to answer its break, past the caller's deadline. + fd = open_read_without_oplock_wait(path) + else: # pragma: win32 no cover + fd = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0)) + try: + st = os.fstat(fd) + if not stat.S_ISREG(st.st_mode): # pragma: no cover # only a non-regular node swapped in after the lstat + return None, st + data = os.read(fd, _MAX_LOCK_FILE_SIZE + 1) + finally: + os.close(fd) + if len(data) <= _MAX_LOCK_FILE_SIZE: + with suppress(UnicodeDecodeError): + return data.decode("utf-8"), st + return None, st + + +def _parse_lock_holder(content: str | None) -> tuple[int, str, int | None] | None: + # A well-formed lock file is "\n\n" with an optional "\n" third line naming the + # holder's process start instant (a filelock 3.29 marker wrote this only on Windows; every platform writes it now). + # Anything else (wrong line count, a non-integer PID or start token, empty or unreadable content) is unparsable; + # returning None lets the caller treat it as a malformed lock to self-heal rather than a holder. + # Every writer since 3.22 ends each line with "\n", so a record without one was cut by a crash or read mid-write. As + # a holder it would name a foreign host no one reclaims, or a start token that lets a contender break a live lock. + if not content or not content.endswith("\n") or len(lines := content.strip().splitlines()) not in {2, 3}: + return None + try: + pid = parse_decimal(lines[0]) + start_token = parse_decimal(lines[2]) if len(lines) == _MARKER_WITH_START_TOKEN_LINE_COUNT else None + except ValueError: + return None + # A pid outside the valid range is a malformed lock, not a holder. Without this, a non-positive pid + # reaches os.kill() where 0 / -1 mean "the caller's own process group / every process" so a dead + # holder reads as alive and the lock is never reclaimed, while an oversized pid raises OverflowError + # (not OSError/ValueError) out of the self-heal path. _parse_marker_bytes already enforces this range. + if not 1 <= pid <= 2**31 - 1: + return None + return pid, lines[1], start_token + + +def parse_decimal(text: str) -> int: + """ + Read a PID or start token as a writer formats it: ASCII digits only. + + :raises ValueError: for anything else ``int()`` would accept, such as ``1_000``, surrounding whitespace, a sign or + non-ASCII digits; no filelock writes those + + """ + if not (text.isascii() and text.isdigit()): + msg = f"not a decimal integer: {text!r}" + raise ValueError(msg) + return int(text) + + +__all__ = [ + "MALFORMED_LOCK_AGE_THRESHOLD", + "SoftFileLock", + "parse_decimal", +] diff --git a/.venv/lib/python3.14/site-packages/filelock/_soft_protocol.py b/.venv/lib/python3.14/site-packages/filelock/_soft_protocol.py new file mode 100644 index 0000000000000000000000000000000000000000..43f048a6d49b4a35449f36aa10094a309c3f43bf --- /dev/null +++ b/.venv/lib/python3.14/site-packages/filelock/_soft_protocol.py @@ -0,0 +1,11 @@ +from __future__ import annotations + +__lazy_modules__ = ["typing"] + +from typing import Final + +STRICT_SOFT_SENTINEL_RECORD: Final[str] = "1\nfilelock-strict-v1\x00\n0\n" + +__all__ = [ + "STRICT_SOFT_SENTINEL_RECORD", +] diff --git a/.venv/lib/python3.14/site-packages/filelock/_soft_rw/__init__.py b/.venv/lib/python3.14/site-packages/filelock/_soft_rw/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9401ecc9ba9b637ab8ea8131273fd22fee474dbd --- /dev/null +++ b/.venv/lib/python3.14/site-packages/filelock/_soft_rw/__init__.py @@ -0,0 +1,14 @@ +"""Cross-process and cross-host reader/writer lock over a generation log of immutable snapshots.""" + +from __future__ import annotations + +__lazy_modules__ = ["filelock._soft_rw._async"] + +from filelock._soft_rw._async import AsyncAcquireSoftReadWriteReturnProxy, AsyncSoftReadWriteLock +from filelock._soft_rw._sync import SoftReadWriteLock + +__all__ = [ + "AsyncAcquireSoftReadWriteReturnProxy", + "AsyncSoftReadWriteLock", + "SoftReadWriteLock", +] diff --git a/.venv/lib/python3.14/site-packages/filelock/_soft_rw/__pycache__/__init__.cpython-314.pyc b/.venv/lib/python3.14/site-packages/filelock/_soft_rw/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..07d613629c0a531bea7349e3f5b0c43ee64a4535 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/filelock/_soft_rw/__pycache__/__init__.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/filelock/_soft_rw/__pycache__/_async.cpython-314.pyc b/.venv/lib/python3.14/site-packages/filelock/_soft_rw/__pycache__/_async.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..aef01550a5e3a23f6562995804526ec389c321f9 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/filelock/_soft_rw/__pycache__/_async.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/filelock/_soft_rw/__pycache__/_protocol.cpython-314.pyc b/.venv/lib/python3.14/site-packages/filelock/_soft_rw/__pycache__/_protocol.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..65d505d5331a8e195d8204e3cf3d746823f4e1e4 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/filelock/_soft_rw/__pycache__/_protocol.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/filelock/_soft_rw/__pycache__/_storage.cpython-314.pyc b/.venv/lib/python3.14/site-packages/filelock/_soft_rw/__pycache__/_storage.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..387f4fc080c29ad1f4d6df886e07d90861379329 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/filelock/_soft_rw/__pycache__/_storage.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/filelock/_soft_rw/__pycache__/_sync.cpython-314.pyc b/.venv/lib/python3.14/site-packages/filelock/_soft_rw/__pycache__/_sync.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..610ecae74819caf98c012d78047486bda34ba562 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/filelock/_soft_rw/__pycache__/_sync.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/filelock/_soft_rw/_async.py b/.venv/lib/python3.14/site-packages/filelock/_soft_rw/_async.py new file mode 100644 index 0000000000000000000000000000000000000000..f751d6b95b73f910dd6e3ad12b9729bca820c063 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/filelock/_soft_rw/_async.py @@ -0,0 +1,373 @@ +"""Async wrapper around :class:`SoftReadWriteLock` for use with ``asyncio``.""" + +from __future__ import annotations + +__lazy_modules__ = ["asyncio", "filelock._async", "filelock._error", "functools", "threading"] + +import asyncio +import functools +import os +import threading +from contextlib import asynccontextmanager +from typing import TYPE_CHECKING, Final, Literal, ParamSpec, TypeVar + +from filelock._async import ( + _TASKS_THREAD_ID, + _BackendOutcome, + _capture_call, + _drain_future, + _future_result, + _raise_cancelled_error, + _task_owners_for, + _wait_until_done, +) +from filelock._error import Timeout +from filelock._soft_rw._sync import SoftReadWriteLock, _lost_hold_error + +if TYPE_CHECKING: + from collections.abc import AsyncGenerator, Callable + from concurrent import futures + from types import TracebackType + + from filelock._api import AcquireReturnProxy + from filelock._lease import LeaseCompromise + +_P = ParamSpec("_P") +_R = TypeVar("_R") + + +class AsyncSoftReadWriteLock: + """ + Async wrapper around :class:`SoftReadWriteLock` for ``asyncio`` applications. + + The sync class's blocking filesystem operations run on a thread pool via ``loop.run_in_executor()``. Each + ``asyncio`` task owns its own hold. Tasks share a read lock, and a task asking for the write lock waits until the + other holders release, with waiting writers ahead of new readers. Upgrading or downgrading a held lock raises + :class:`RuntimeError`. The underlying :class:`SoftReadWriteLock` handles forks, the heartbeat, and stale eviction. + Singleton wrappers share one sync lock and its task holds, and closing one ends only the holds taken through it. + + :param lock_file: path to the lock file; the protocol directory lives next to it as ``.rw`` + :param timeout: maximum wait time in seconds; ``-1`` means block indefinitely + :param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately on contention + :param is_singleton: if ``True``, reuse existing :class:`SoftReadWriteLock` instances per resolved path + :param heartbeat_interval: seconds between heartbeat refreshes; default 30 s + :param stale_threshold: seconds a holder record may stay unchanged before a contender evicts it; defaults to + ``3 * heartbeat_interval`` + :param poll_interval: seconds between acquire retries under contention; default 0.25 s + :param on_compromise: called from the heartbeat thread with a :class:`~filelock.LeaseCompromise` when the hold is + lost + :param loop: event loop for ``run_in_executor``; ``None`` uses the running loop + :param executor: executor for ``run_in_executor``; ``None`` uses the default executor + + .. versionadded:: 3.27.0 + + """ + + def __init__( # ruff:ignore[too-many-arguments] # public constructor: one parameter per documented lock option + self, + lock_file: str | os.PathLike[str], + timeout: float = -1, + *, + blocking: bool = True, + is_singleton: bool = True, + heartbeat_interval: float = 30.0, + stale_threshold: float | None = None, + poll_interval: float = 0.25, + on_compromise: Callable[[LeaseCompromise], None] | None = None, + loop: asyncio.AbstractEventLoop | None = None, + executor: futures.Executor | None = None, + ) -> None: + self._creator_pid = os.getpid() + self._closed = False + self._owns_backend = not is_singleton + #: Marks the task holds taken through this wrapper, which its close() ends. + self._via: Final = object() + self._lock = SoftReadWriteLock( + lock_file, + timeout, + blocking=blocking, + is_singleton=is_singleton, + heartbeat_interval=heartbeat_interval, + stale_threshold=stale_threshold, + poll_interval=poll_interval, + on_compromise=on_compromise, + ) + self._owners = _task_owners_for(self._lock) + self._loop = loop + self._executor = executor + + @property + def lock_file(self) -> str: + """The path to the lock file passed to the constructor.""" + return self._lock.lock_file + + @property + def timeout(self) -> float: + """The default timeout applied when ``acquire_read`` / ``acquire_write`` is called without one.""" + return self._lock.timeout + + @property + def blocking(self) -> bool: + """Whether ``acquire_*`` defaults to blocking; ``False`` makes contention raise immediately.""" + return self._lock.blocking + + @property + def generation(self) -> int | None: + """The generation the current hold was granted at, a fencing token; ``None`` when no lock is held.""" + return self._lock.generation + + @property + def compromise(self) -> LeaseCompromise | None: + """How the current hold was lost, or ``None`` while it stands; acquiring raises until the hold is released.""" + return self._lock.compromise + + @property + def loop(self) -> asyncio.AbstractEventLoop | None: + """The event loop used for ``run_in_executor``, or ``None`` for the running loop.""" + return self._loop + + @property + def executor(self) -> futures.Executor | None: + """The executor used for ``run_in_executor``, or ``None`` for the default executor.""" + return self._executor + + @asynccontextmanager + async def read_lock(self, timeout: float | None = None, *, blocking: bool | None = None) -> AsyncGenerator[None]: + """ + Async context manager that acquires and releases a shared read lock. + + :param timeout: maximum wait time in seconds, or ``None`` to use the instance default + :param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default + + :raises RuntimeError: if the calling task already holds the write lock + :raises Timeout: if the lock cannot be acquired within *timeout* seconds + :raises TypeError: if *timeout* is a :class:`bool` or not a real number + :raises ValueError: if a blocking call gets a ``nan`` *timeout* or a negative one other than ``-1`` + + """ + await self.acquire_read(timeout, blocking=blocking) + try: + yield + finally: + await self.release() + + @asynccontextmanager + async def write_lock(self, timeout: float | None = None, *, blocking: bool | None = None) -> AsyncGenerator[None]: + """ + Async context manager that acquires and releases an exclusive write lock. + + :param timeout: maximum wait time in seconds, or ``None`` to use the instance default + :param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default + + :raises RuntimeError: if the calling task already holds the read lock + :raises Timeout: if the lock cannot be acquired within *timeout* seconds + :raises TypeError: if *timeout* is a :class:`bool` or not a real number + :raises ValueError: if a blocking call gets a ``nan`` *timeout* or a negative one other than ``-1`` + + """ + await self.acquire_write(timeout, blocking=blocking) + try: + yield + finally: + await self.release() + + async def acquire_read( + self, timeout: float | None = None, *, blocking: bool | None = None + ) -> AsyncAcquireSoftReadWriteReturnProxy: + """ + Acquire a shared read lock. + + See :meth:`SoftReadWriteLock.acquire_read` for reentrancy / upgrade / fork semantics. The blocking work runs + inside ``run_in_executor`` so other coroutines on the same loop keep progressing while this call waits. + + :param timeout: maximum wait time in seconds, or ``None`` to use the instance default + :param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default + + :returns: a proxy usable as an async context manager to release the lock + + :raises RuntimeError: if the calling task already holds the write lock, if this instance was invalidated by + :func:`os.fork`, or if :meth:`close` was called + :raises Timeout: if the lock cannot be acquired within *timeout* seconds + :raises TypeError: if *timeout* is a :class:`bool` or not a real number + :raises ValueError: if a blocking call gets a ``nan`` *timeout* or a negative one other than ``-1`` + + """ + self._raise_if_unusable() + await self._acquire("read", timeout, blocking=blocking) + return AsyncAcquireSoftReadWriteReturnProxy(lock=self) + + async def acquire_write( + self, timeout: float | None = None, *, blocking: bool | None = None + ) -> AsyncAcquireSoftReadWriteReturnProxy: + """ + Acquire an exclusive write lock. + + See :meth:`SoftReadWriteLock.acquire_write` for the writer-preferring semantics. The blocking work runs inside + ``run_in_executor``. + + :param timeout: maximum wait time in seconds, or ``None`` to use the instance default + :param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default + + :returns: a proxy usable as an async context manager to release the lock + + :raises RuntimeError: if the calling task already holds the read lock, if this instance was invalidated by + :func:`os.fork`, or if :meth:`close` was called + :raises Timeout: if the lock cannot be acquired within *timeout* seconds + :raises TypeError: if *timeout* is a :class:`bool` or not a real number + :raises ValueError: if a blocking call gets a ``nan`` *timeout* or a negative one other than ``-1`` + + """ + self._raise_if_unusable() + await self._acquire("write", timeout, blocking=blocking) + return AsyncAcquireSoftReadWriteReturnProxy(lock=self) + + async def _acquire(self, mode: Literal["read", "write"], timeout: float | None, *, blocking: bool | None) -> None: + blocking = self._lock.blocking if blocking is None else blocking + + async def enter(remaining: float) -> None: + # A close() may have landed while this task waited for the table. + self._raise_if_unusable() + await self._run_acquire( + functools.partial( + self._lock._acquire_level_of, # ruff:ignore[private-member-access] # the wrapper owns this backend hold + _TASKS_THREAD_ID, + mode, + remaining, + blocking=blocking, + ) + ) + + def joinable() -> bool: + self._raise_if_unusable() + if (compromise := self._lock.compromise) is not None: + raise _lost_hold_error(self.lock_file, mode, compromise) + return self._backend_holds() + + await self._owners.acquire( + mode, + timeout=self._lock.timeout if timeout is None else timeout, + blocking=blocking, + lock_file=self.lock_file, + via=self._via, + enter=enter, + joinable=joinable, + ) + + async def release(self, *, force: bool = False) -> None: + """ + Release one level of the calling task's hold. + + The wrapper releases the lock once no task holds it. + + :param force: if ``True``, end every level the calling task took through this wrapper. Levels taken through + other wrappers on the same path stay. A wrapper that holds no level for the task releases nothing. + + :raises RuntimeError: if the calling task holds no level through this wrapper and *force* is ``False`` + + """ + if self._creator_pid == os.getpid(): + await self._owners.release( + force=force, + lock_file=self.lock_file, + via=self._via, + leave=functools.partial(self._submit, self._leave_backend), + still_held=self._backend_holds, + ) + + async def close(self) -> None: + """ + End the holds of the tasks that acquired through this instance and release the lock once no task holds it. + + Idempotent. After calling this method the instance can no longer acquire locks. With ``is_singleton=False`` it + also closes the underlying :class:`SoftReadWriteLock`. A singleton wrapper leaves the shared one open, so tasks + that acquired through another wrapper and synchronous users of the same path keep their holds. The next acquire + notices a synchronous ``close()`` or forced release of that shared lock. One that lands in the few bytecodes + between that check and the admission still admits the task into the ended hold. + + """ + if self._creator_pid != os.getpid() or self._closed: + return + # Refuse acquires from here on, so none is admitted into a hold the backend is letting go of. + self._closed = True + try: + await self._owners.retire( + None if self._owns_backend else self._via, + leave=functools.partial(self._submit, self._lock.close if self._owns_backend else self._leave_backend), + still_held=self._backend_holds, + ) + except BaseException as error: + # Only a failed backend call can leave the level held, and then a retried close() has to find it. + if not isinstance(error, asyncio.CancelledError) and self._backend_holds(): + self._closed = False + raise + + def _raise_if_unusable(self) -> None: + if self._creator_pid != os.getpid(): # pragma: forked child + msg = f"AsyncSoftReadWriteLock on {self.lock_file} was inherited across fork; construct a new instance" + raise RuntimeError(msg) + if self._closed: + msg = f"AsyncSoftReadWriteLock on {self.lock_file} has been closed" + raise RuntimeError(msg) + + def _backend_holds(self) -> bool: + # A sync close() or forced release on a shared singleton ends the tasks' level without the table taking part. + return self._lock._holds_level_of(_TASKS_THREAD_ID) # ruff:ignore[private-member-access] # the wrapper owns this backend hold + + def _leave_backend(self) -> None: + # Only the tasks' level: a sync user of the same singleton keeps its own. + self._lock._release_level_of(_TASKS_THREAD_ID) # ruff:ignore[private-member-access] # the wrapper owns this backend hold + + async def _run_acquire(self, acquire: Callable[..., AcquireReturnProxy]) -> None: + # run_in_executor cannot recall work the pool already started. A sync acquire that won the lock before it saw + # the cancellation has published its claim and started the heartbeat, which keeps the record fresh so no peer + # evicts it. Wait the call out and hand the claim back, as AsyncReadWriteLock does. + abandoned = threading.Event() + acquire_future = self._submit(acquire, cancel_check=abandoned.is_set) + try: + await _wait_until_done(acquire_future) + except asyncio.CancelledError as cancellation: + # Stops a backend wait on another process's hold, which would otherwise outlast the caller by as long. + abandoned.set() + try: + await _drain_future(acquire_future) + except Timeout: + # Nothing was taken, so the caller's cancellation is the whole outcome. + raise cancellation from None + except BaseException as error: # ruff:ignore[blind-except] # reported with the cancellation below + _raise_cancelled_error(cancellation, error) + try: + await _drain_future(self._submit(self._leave_backend)) + except BaseException as error: # ruff:ignore[blind-except] # reported with the cancellation below + _raise_cancelled_error(cancellation, error) + raise + _future_result(acquire_future) + + def _submit( + self, func: Callable[_P, _R], *args: _P.args, **kwargs: _P.kwargs + ) -> asyncio.Future[_BackendOutcome[_R]]: + loop = self._loop or asyncio.get_running_loop() + return loop.run_in_executor(self._executor, _capture_call, functools.partial(func, *args, **kwargs)) + + +class AsyncAcquireSoftReadWriteReturnProxy: + """Async context-aware object that releases an :class:`AsyncSoftReadWriteLock` on exit.""" + + def __init__(self, lock: AsyncSoftReadWriteLock) -> None: + self.lock = lock + + async def __aenter__(self) -> AsyncSoftReadWriteLock: + return self.lock + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: + await self.lock.release() + + +__all__ = [ + "AsyncAcquireSoftReadWriteReturnProxy", + "AsyncSoftReadWriteLock", +] diff --git a/.venv/lib/python3.14/site-packages/filelock/_soft_rw/_protocol.py b/.venv/lib/python3.14/site-packages/filelock/_soft_rw/_protocol.py new file mode 100644 index 0000000000000000000000000000000000000000..565f9bcf7994c0808aa2d6d02a239cd96055b50d --- /dev/null +++ b/.venv/lib/python3.14/site-packages/filelock/_soft_rw/_protocol.py @@ -0,0 +1,638 @@ +""" +The generation-log protocol behind :class:`~filelock.SoftReadWriteLock`. + +The lock's whole state is a sequence of immutable snapshots, ``gen/``, each naming the writer and the readers that +hold the lock at generation ``N``. Every transition is one atomic publication of the next snapshot: a participant reads +the latest generation, derives the successor it wants, and links a fully written temporary file to ``gen/``. The +link fails if a peer got there first, so the participant re-reads and tries again. No step is ever half done: a crash +leaves at most an unlinked temporary file, never a state that another participant must repair before it can proceed. + +Liveness rests on a nonce, not a clock. Each participant keeps ``holders/`` and rewrites its nonce every +heartbeat. A contender records the bytes it read there and the moment it read them on its own monotonic clock; a +member whose bytes have not changed for ``stale_threshold`` is evicted by committing a snapshot without it. Two hosts +never compare clocks, so a skewed file server or a suspended virtual machine cannot make a live holder look dead. + +The generation number is monotonic and unique to one transition within one incarnation of the log, so the generation at +which a hold was granted fences it: a resource that rejects anything carrying a lower generation than the highest it has +accepted refuses a holder that paused past its expiry and resumed. A random epoch file names the incarnation, so a +participant that outlives a removal of the log reads the new one instead of continuing the old numbering into it. +""" + +from __future__ import annotations + +__lazy_modules__ = ["contextlib", "filelock._error", "filelock._identity", "math", "pathlib", "secrets"] + +import os +import secrets +from contextlib import suppress +from dataclasses import dataclass +from math import isfinite +from pathlib import Path +from typing import TYPE_CHECKING, Final, Literal, Protocol + +from filelock._error import SoftFileLockProtocolError +from filelock._identity import host_name + +if TYPE_CHECKING: + from collections.abc import Callable + +Mode = Literal["read", "write"] +HeartbeatOutcome = Literal["ok", "lost", "transient"] + +_PROTOCOL: Final[str] = "filelock-rw/1" +_GENERATION_DIGITS: Final[int] = 20 +_TOKEN_HEX_LENGTH: Final[int] = 32 +#: A snapshot line is ``reader=`` plus a token, so this bounds a hostile record without capping real reader counts. +_MAX_RECORD_SIZE: Final[int] = 1 << 20 +#: Snapshots older than this many generations behind the latest are removed by whoever commits. The window is also how +#: far ``latest`` probes past a gap before it trusts that nothing newer exists; ``gen/HEAD``, not the window, is what +#: keeps a listing served further behind from passing an old generation off as the head. +_RETAINED_GENERATIONS: Final[int] = 64 +_GENERATIONS_DIRECTORY: Final[str] = "gen" +#: A random token created once per log: removing ``gen/`` removes it, so a participant can tell a log that was wiped and +#: restarted from the one it remembers. +_EPOCH_NAME: Final[str] = "epoch" +#: The epoch plus a copy of the newest snapshot, renamed into place after each commit. NFS revalidates a file opened by +#: name but may serve a listing from cache, so this is the head a stale client cannot miss; it may lag, never lead. +_HEAD_NAME: Final[str] = "HEAD" +_HOLDERS_DIRECTORY: Final[str] = "holders" +_TEMPORARY_PREFIX: Final[str] = ".commit-" + + +class TreeReplacedError(RuntimeError): + """ + Raised by a ``Files`` implementation when a protocol directory is not a real directory (a symlink, a junction, a + file) or belongs to another account. + """ + + +class Files(Protocol): + """The filesystem operations the protocol runs on; :mod:`._storage` implements them over :mod:`os`.""" + + def read(self, path: str) -> bytes | None: + """The file's bytes, ``None`` when it does not exist, or ``b""`` when it is not a regular file.""" + + def create(self, path: str, data: bytes, *, durable: bool = False) -> tuple[int, int]: + """ + Create the file exclusively with the whole record written; raise ``FileExistsError`` when it exists. + + The returned device and inode (the kernel's identity for a file, independent of its name) let ``overwrite`` + check that the name still leads to this file. + + *durable* flushes it to stable storage first, for a record that outlives the host that wrote it: a generation a + crash left empty would wedge every other participant. A holder record dies with its writer, so it skips that. + """ + + def link(self, source: str, target: str) -> bool: + """Hard-link *source* to *target* without replacing; ``True`` when *target* names *source*'s file afterwards.""" + + def replace(self, source: str, target: str) -> bool: + """Rename *source* over *target* atomically; ``False`` when the platform refused because *target* is in use.""" + + def overwrite(self, path: str, data: bytes, identity: tuple[int, int]) -> bool: + """ + Rewrite from offset zero the file ``create`` returned *identity* for. + + ``False`` when *path* no longer leads to that file. + """ + + def unlink(self, path: str) -> None: + """Remove the file, tolerating one that is already gone.""" + + def listdir(self, path: str) -> list[str]: + """The names in the directory, or an empty list when it does not exist.""" + + def prepare(self, root: str, *, force: bool = False) -> None: + """ + Create the protocol directories under *root*, refusing anything at those paths that is not a directory. + + Done once per lock; *force* repeats it after a write found a directory missing. + """ + + +@dataclass(frozen=True) +class Snapshot: + """The lock's state at one generation: which token writes, which tokens read.""" + + generation: int + writer: str | None + readers: frozenset[str] + + @property + def members(self) -> frozenset[str]: + """Every token this generation names.""" + return self.readers if self.writer is None else self.readers | {self.writer} + + def encode(self) -> bytes: + lines = [_PROTOCOL, f"generation={self.generation}"] + if self.writer is not None: + lines.append(f"writer={self.writer}") + lines.extend(f"reader={token}" for token in sorted(self.readers)) + return "".join(f"{line}\n" for line in lines).encode("ascii") + + +def parse_snapshot(data: bytes) -> Snapshot | None: + """The snapshot a ``gen/`` record encodes, or ``None`` when the record is malformed.""" + if not data or len(data) > _MAX_RECORD_SIZE: + return None + try: + lines = data.decode("ascii").split("\n") + except UnicodeDecodeError: + return None + if lines[0] != _PROTOCOL or lines[-1]: + return None + generation: int | None = None + writer: str | None = None + readers: set[str] = set() + for line in lines[1:-1]: + key, _, value = line.partition("=") + if ( + key == "generation" + and generation is None + and len(value) <= _GENERATION_DIGITS + and value.isdigit() + and str(int(value)) == value + ): + generation = int(value) + elif key == "writer" and writer is None and is_token(value): + writer = value + elif key == "reader" and is_token(value) and value not in readers: + readers.add(value) + else: + return None + return None if generation is None else Snapshot(generation=generation, writer=writer, readers=frozenset(readers)) + + +def is_token(value: str) -> bool: + """Whether *value* is a claim token: exactly 32 lowercase hex digits.""" + return len(value) == _TOKEN_HEX_LENGTH and all(character in "0123456789abcdef" for character in value) + + +def new_token() -> str: + """A fresh claim token.""" + return secrets.token_hex(_TOKEN_HEX_LENGTH // 2) + + +def encode_holder(token: str, nonce: str, lease: float) -> bytes: + """ + The ``holders/`` record; only the nonce changes between heartbeats, so the length stays constant. + + *lease* is the writer's own ``stale_threshold``: a peer configured tighter must not read this holder's longer + heartbeat interval as death, so evictors wait the longer of their threshold and this one. + """ + return ( + f"{_PROTOCOL}\ntoken={token}\npid={os.getpid()}\nhost={host_name()}\nlease={float(lease)!r}\nnonce={nonce}\n" + ).encode("ascii") + + +def holder_lease(record: bytes | None) -> float | None: + """The ``lease`` a holder record carries, or ``None`` when it has none (an older release) or it is malformed.""" + try: + text = next( + line for line in (record or b"").decode("ascii").split("\n") if line.startswith("lease=") + ).removeprefix("lease=") + lease = float(text) + except (StopIteration, ValueError): # UnicodeDecodeError is a ValueError + return None + # Only the exact spelling repr() writes: float() alone also takes "1_000", " 1", and non-ASCII digits. + return lease if isfinite(lease) and lease > 0 and repr(lease) == text else None + + +class Ledger: + """ + How long each observed value has stayed the same, measured on this observer's monotonic clock. + + A value that has not changed for ``stale_threshold`` belongs to a participant that stopped refreshing it. The clock + is the observer's own and only ever measures the gap between two of its own observations, which is what lets two + hosts agree on liveness without agreeing on the time. + """ + + def __init__(self, clock: Callable[[], float]) -> None: + self._clock = clock + self._seen: dict[str, tuple[bytes | None, float]] = {} + + def observe(self, key: str, value: bytes | None, *, asked: float | None = None) -> float: + """ + Record *value* for *key* and return for how long it has stayed the same. + + *asked* is the clock reading taken before the caller fetched *value*. An unchanged value counts as unchanged up + to *asked*, so a pause between the fetch and this call adds no time. + """ + now = self._clock() + if (seen := self._seen.get(key)) is None or seen[0] != value: + self._seen[key] = (value, now) + return 0.0 + return (now if asked is None else asked) - seen[1] + + def forget(self, key: str) -> None: + self._seen.pop(key, None) + + def retain(self, keys: set[str]) -> None: + """Drop every key not in *keys*: peers that left would otherwise stay in the ledger for the lock's lifetime.""" + self._seen = {key: seen for key, seen in self._seen.items() if key in keys} + + +class GenerationLog: + """The ordered snapshots under ``gen/``, found by listing and probing, advanced by linking the next one.""" + + def __init__(self, files: Files, lock_file: str, root: str) -> None: + self._files = files + self._lock_file = lock_file + self._directory = Path(root, _GENERATIONS_DIRECTORY) + self._epoch_path = str(self._directory / _EPOCH_NAME) + self._head_path = str(self._directory / _HEAD_NAME) + self._epoch: bytes | None = None + self._latest: Snapshot | None = None + + def latest(self) -> Snapshot: + """ + The newest snapshot. + + Starts from the newest readable generation among those ``gen/HEAD``, the listing, or memory name, then probes + forward by name across the retained window. ``HEAD`` only names a generation, and that generation must still + read back: a committer stalled before its rename can leave ``HEAD`` on a generation compacted since. A + generation compacted between the listing and the read, and a hole left by a committer that died before it + compacted, are covered by the probe. Memory and ``HEAD`` are bound to the log's epoch, so a log wiped and + restarted is never continued from a generation it no longer holds. Only a log that names generations, now or as + remembered, of which none can be read is refused, because that is a client so far behind the log that nothing + it reads can be trusted. + """ + # When everything the first pass named is gone, peers compacted past it while this client looked, or the log was + # replaced; one more pass settles whether this client cannot see the head at all. + if (latest := self._find()) is None and (latest := self._find()) is None: + raise SoftFileLockProtocolError(self._lock_file, None, "generation log names no readable snapshot") + self._latest = latest + return latest + + def _find(self) -> Snapshot | None: + if (epoch := self._incarnation()) != self._epoch: + self._epoch, self._latest = epoch, None + named = set(self._list()) + # Generation 0 is the empty log, which has no record to read back. + if self._latest is not None and self._latest.generation: + named.add(self._latest.generation) + if head := self._head_generation(): + named.add(head) + for generation in sorted(named, reverse=True): + if (start := self._read(generation)) is not None: + # Compaction keeps the surviving generations contiguous, so the first missing name past a readable HEAD + # generation is the end; probing the whole window on every poll cost 64 failed opens. + return self._probe(start, patience=1 if generation == head else _RETAINED_GENERATIONS) + newest = max(named, default=0) + found = self._probe(Snapshot(generation=newest, writer=None, readers=frozenset())) + return found if found.generation > newest or not named else None + + def _head_generation(self) -> int | None: + # HEAD is never flushed, so a crash can leave it unreadable; readers verify what it names anyway, so a torn one + # only sends them to the window probe. + if (record := self._head_record()) is None or (snapshot := parse_snapshot(record)) is None: + return None + return snapshot.generation + + def _head_record(self) -> bytes | None: + if (data := self._files.read(self._head_path)) is None: + return None + # A HEAD from another epoch landed after its log was removed; a log whose epoch was never written runs without. + epoch, newline, record = data.partition(b"\n") + return record if self._valid_epoch() == epoch + newline else None + + def _valid_epoch(self) -> bytes | None: + epoch = self._epoch + return epoch if epoch is not None and is_token(epoch.decode("ascii", "replace").removesuffix("\n")) else None + + def _incarnation(self) -> bytes | None: + if (epoch := self._files.read(self._epoch_path)) is None: + # Exclusive creation, not a link: reading the log must keep working where os.link does not exist. A peer + # that reads the file before its token is written takes the empty value for the epoch, and its next commit + # sees the change and re-reads, so a partial read costs one retry. + try: + self._files.create(self._epoch_path, f"{new_token()}\n".encode("ascii"), durable=True) + except FileExistsError: + pass + except FileNotFoundError: + # No gen/ yet: there is no log to tell apart, and the first participant's prepare() creates it. + return None + epoch = self._files.read(self._epoch_path) + return epoch + + def _list(self) -> list[int]: + listed = self._listed() + for generation in listed[:-_RETAINED_GENERATIONS]: + self._files.unlink(self._path(generation)) + return listed[-_RETAINED_GENERATIONS:] + + def _listed(self) -> list[int]: + # isdigit() alone admits Unicode digits: superscripts int() rejects, fullwidth ones it parses to another number. + return sorted( + int(name) + for name in self._files.listdir(str(self._directory)) + if len(name) == _GENERATION_DIGITS and name.isascii() and name.isdigit() + ) + + def _probe(self, start: Snapshot, patience: int = _RETAINED_GENERATIONS) -> Snapshot: + latest = start + generation = start.generation + gap = 0 + while gap < patience: + generation += 1 + if (snapshot := self._read(generation)) is None: + gap += 1 + continue + latest = snapshot + gap = 0 + return latest + + def _read(self, generation: int) -> Snapshot | None: + if (data := self._files.read(self._path(generation))) is None: + return None + if (snapshot := parse_snapshot(data)) is None or snapshot.generation != generation: + name = self._name(generation) + raise SoftFileLockProtocolError(self._lock_file, name, "malformed generation record") + return snapshot + + def commit(self, successor: Snapshot) -> bool: + """ + Publish *successor* as the next generation; ``False`` when a peer published that generation first. + + The record is complete before it gets its public name, and a no-replace hard link is the compare-and-swap: only + one link to ``gen/`` can ever succeed. The link's return code is not trusted, because an NFS retry can + report a link that did land as failed; the file's identity after the call is what decides. A log whose epoch + changed since the snapshot was read is a different log, so the commit is refused as lost and the caller + re-reads; that narrows a wipe racing this commit to the moment between the epoch check and the link. A slot + compaction freed is refused the same way, before the link and again after it, since a committer can stall in + between. After the link a surviving predecessor proves little: it may be a record another stalled committer + linked into its own freed slot and has yet to remove. A slot comes free once the log is a whole window past it, + and the head stays, so a record the listing or ``HEAD`` leads by a window is refused as well. A record refused + after it landed is removed when nothing was built on it. One that peers lead by less than a window stays, or its + slot would come free early. + """ + if self._files.read(self._epoch_path) != self._epoch or not self._extends_the_log(successor.generation): + return False + if not self._publish( + self._path(successor.generation), data := successor.encode(), self._files.link, durable=True + ): + return False + lead = max([*self._listed(), self._head_generation() or 0]) - successor.generation + if lead >= _RETAINED_GENERATIONS or not self._extends_the_log(successor.generation): + if not 0 < lead < _RETAINED_GENERATIONS: + self._files.unlink(self._path(successor.generation)) + return False + self._latest = successor + # Readers verify the generation HEAD names, so a failed rename costs a probe, not the commit. A late one would + # move HEAD back past later commits, so HEAD only ever moves forward. + with suppress(OSError): + if ( + (epoch := self._valid_epoch()) is not None + and self._files.read(self._epoch_path) == epoch + and (self._head_generation() or 0) < successor.generation + ): + self._publish(self._head_path, epoch + data, self._files.replace, durable=False) + if (expired := successor.generation - _RETAINED_GENERATIONS - 1) > 0: + self._compact_through(expired) + return True + + def _extends_the_log(self, generation: int) -> bool: + # Compaction removes oldest first, so a missing predecessor means compaction freed this slot. The first + # generation has none; a HEAD of this epoch means the log already moved past it. + if generation > 1: + return self._files.read(self._path(generation - 1)) is not None + return self._head_record() is None + + def _publish(self, target: str, data: bytes, place: Callable[[str, str], bool], *, durable: bool) -> bool: + temporary = str(self._directory / f"{_TEMPORARY_PREFIX}{new_token()}") + try: + self._files.create(temporary, data, durable=durable) + except FileNotFoundError: + # No gen/: the log was removed, or nobody has prepared it yet. Either way there is nothing to publish into + # until a participant's prepare() creates it, and the caller re-reads whatever log exists by then. + return False + try: + return place(temporary, target) + finally: + self._files.unlink(temporary) + + def _compact_through(self, expired: int) -> None: + # Oldest first, so survivors stay contiguous: a stalled committer's leftover would otherwise sit below a hole. + oldest = expired + while oldest > 1 and self._files.read(self._path(oldest - 1)) is not None: + oldest -= 1 + for generation in range(oldest, expired + 1): + self._files.unlink(self._path(generation)) + + def _path(self, generation: int) -> str: + return str(self._directory / self._name(generation)) + + @staticmethod + def _name(generation: int) -> str: + return f"{generation:0{_GENERATION_DIGITS}d}" + + +class Participant: + """ + One acquisition, from publishing a claim through holding the lock to leaving. + + ``advance`` is polled until it reports the lock granted, ``heartbeat`` runs on the holder's schedule, and ``leave`` + commits the participant out. None of them sleep: the caller owns the deadline and the poll interval. The ledger and + the log outlive one acquisition: how long a peer's record has stayed unchanged is knowledge every attempt against + the same lock shares, or a caller whose attempts are each shorter than the stale threshold could never evict. + """ + + def __init__( # ruff:ignore[too-many-arguments] # one value per protocol input; the lock builds this once per acquisition + self, + files: Files, + root: str, + mode: Mode, + *, + stale_threshold: float, + clock: Callable[[], float], + ledger: Ledger, + log: GenerationLog, + ) -> None: + self.token: Final[str] = new_token() + self.mode: Final[Mode] = mode + self.generation: int | None = None + self._files = files + self._root = root + self._stale_threshold = stale_threshold + self._clock = clock + self._ledger = ledger + self._log = log + self._holder = self._holder_path(self.token) + self._holder_identity: tuple[int, int] | None = None + self._entered = False + self._next_sweep = clock() + + def publish(self) -> None: + """Create the holder record peers watch for liveness; it exists before the token appears in any snapshot.""" + self._files.prepare(self._root) + self._create_holder() + + def _create_holder(self) -> None: + try: + self._holder_identity = self._files.create(self._holder, self._record()) + except FileNotFoundError: + # Removing the whole log takes the directories with it. + self._files.prepare(self._root, force=True) + self._holder_identity = self._files.create(self._holder, self._record()) + + def _record(self) -> bytes: + return encode_holder(self.token, new_token(), self._stale_threshold) + + def advance(self) -> bool: + """ + Take the next step toward holding the lock; ``True`` once the lock is granted. + + A reader enters as soon as no live writer is named. A writer enters as soon as no live writer is named, which + blocks every new reader, then waits for the named readers to leave. Stale members are evicted in the same commit + that admits this participant, and a writer's final admission is itself a commit, so no admission can rest on a + snapshot a peer is about to supersede. + """ + # A lost race is not a reason to wait: re-read and try again before handing the poll interval back to the + # caller. + for _ in range(4): + latest = self._log.latest() + self._keep_claim_fresh(latest) + self._sweep(latest) + stale = frozenset(token for token in latest.members - {self.token} if self._is_stale(token)) + writer = None if latest.writer in stale else latest.writer + readers = latest.readers - stale + if not self._entered: + if writer is not None: + # A live writer blocks entry in both modes; only an eviction is worth committing. + if stale: + self._commit(latest, writer, readers, stale) + return False + writer, readers = (self.token, readers) if self.mode == "write" else (None, readers | {self.token}) + granted = self.mode == "read" or not readers + if self._entered and not stale and not granted: + return False + if (generation := self._commit(latest, writer, readers, stale)) is None: + continue + self._entered = True + if granted: + self.generation = generation + return granted + return False + + def _keep_claim_fresh(self, latest: Snapshot) -> None: + # The snapshot is the truth about membership: a commit the filesystem reported lost may have landed, and a peer + # may have evicted this participant while it waited to drain, so both directions are taken from it. A wait can + # outlast the stale threshold, so the nonce changes on every poll; a peer then never reads a patient contender + # as a corpse. A record that has gone missing was taken by an evictor or a sweeper that did, or by a removal of + # the whole log. + self._entered = self.token in latest.members + if not self._refresh_nonce(): + with suppress(FileExistsError): + self._create_holder() + + def _sweep(self, latest: Snapshot) -> None: + # A participant that crashed before its first commit leaves a record no snapshot names, and a commit that + # crashed between creating and unlinking its temporary file leaves that too. Neither blocks anyone, so they are + # collected on the same unchanged-for-a-threshold rule as a member, and only every half threshold at most. + if (now := self._clock()) < self._next_sweep: + return + self._next_sweep = now + self._stale_threshold / 2 + watched = set(latest.members) + holders = Path(self._root, _HOLDERS_DIRECTORY) + for name in self._files.listdir(str(holders)): + if name != self.token and name not in latest.members and is_token(name): + watched.add(key := f"holder:{name}") + self._collect(key, str(holders / name)) + generations = Path(self._root, _GENERATIONS_DIRECTORY) + for name in self._files.listdir(str(generations)): + if name.startswith(_TEMPORARY_PREFIX): + watched.add(key := f"temporary:{name}") + self._collect(key, str(generations / name)) + self._ledger.retain(watched) + + def _collect(self, key: str, path: str) -> None: + if self._unchanged_for_its_lease(key, path): + self._files.unlink(path) + self._ledger.forget(key) + + def _commit( + self, latest: Snapshot, writer: str | None, readers: frozenset[str], stale: frozenset[str] + ) -> int | None: + successor = Snapshot(generation=latest.generation + 1, writer=writer, readers=readers) + if not self._log.commit(successor): + return None + for token in stale: + self._files.unlink(self._holder_path(token)) + self._ledger.forget(token) + return successor.generation + + def _is_stale(self, token: str) -> bool: + return self._unchanged_for_its_lease(token, self._holder_path(token)) + + def _unchanged_for_its_lease(self, key: str, path: str) -> bool: + asked = self._clock() + record = self._files.read(path) + return self._ledger.observe(key, record, asked=asked) >= max(self._stale_threshold, holder_lease(record) or 0.0) + + def _holder_path(self, token: str) -> str: + return str(Path(self._root, _HOLDERS_DIRECTORY, token)) + + def _refresh_nonce(self) -> bool: + if (identity := self._holder_identity) is None: + return False + return self._files.overwrite(self._holder, self._record(), identity) + + def heartbeat(self) -> tuple[HeartbeatOutcome, OSError | None]: + """ + Refresh the nonce and confirm the latest snapshot still names this participant. + + ``lost`` means a peer evicted this participant or removed its holder record: the lock is no longer held, and the + holder must stop using what it protects. ``transient`` is a filesystem error refreshing the nonce, worth + retrying on the next tick. A failure to read the log or to sweep decides nothing about this holder's liveness, + so it is not counted against it. + """ + try: + if not self._refresh_nonce(): + return "lost", None + except TreeReplacedError: + return "lost", None + except OSError as error: + return "transient", error + try: + latest = self._log.latest() + self._sweep(latest) + except (OSError, TreeReplacedError): + return "ok", None + return ("ok" if self.token in latest.members else "lost"), None + + def leave(self) -> None: + """Commit this participant out of the latest snapshot, then remove its holder record.""" + # The record goes even when the log cannot be read: a contender that never entered then leaves nothing behind, + # and a member still named ages out the same way with or without one. + # A tree that was swapped out leaves no record this participant can reach, so there is nothing to commit out of. + with suppress(TreeReplacedError): + try: + while self.token in (latest := self._log.latest()).members: + successor = Snapshot( + generation=latest.generation + 1, + writer=None if latest.writer == self.token else latest.writer, + readers=latest.readers - {self.token}, + ) + if self._log.commit(successor): + break + finally: + self._entered = False + self.generation = None + self._files.unlink(self._holder) + + +__all__ = [ + "Files", + "GenerationLog", + "HeartbeatOutcome", + "Ledger", + "Mode", + "Participant", + "Snapshot", + "TreeReplacedError", + "encode_holder", + "holder_lease", + "is_token", + "new_token", + "parse_snapshot", +] diff --git a/.venv/lib/python3.14/site-packages/filelock/_soft_rw/_storage.py b/.venv/lib/python3.14/site-packages/filelock/_soft_rw/_storage.py new file mode 100644 index 0000000000000000000000000000000000000000..a6ff6c64efab66b9879c35b714ce22a97e1c4cb5 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/filelock/_soft_rw/_storage.py @@ -0,0 +1,278 @@ +"""The :class:`~._protocol.Files` implementation over :mod:`os`.""" + +from __future__ import annotations + +__lazy_modules__ = ["filelock._soft_rw._protocol", "filelock._strict", "filelock._util", "pathlib"] + +import errno +import os +import stat +import sys +import time +from contextlib import contextmanager, suppress +from errno import ELOOP, ENOENT, ENOTDIR, ENXIO, ESTALE +from pathlib import Path +from typing import TYPE_CHECKING, Final + +from filelock._soft_rw._protocol import ( + _GENERATIONS_DIRECTORY, + _HOLDERS_DIRECTORY, + _MAX_RECORD_SIZE, + TreeReplacedError, +) +from filelock._strict import _link_no_follow, _pin_directories, _raise_if_hard_links_unsupported +from filelock._util import ensure_directory_exists, foreign_directory_owner, write_all + +if TYPE_CHECKING: + from collections.abc import Generator + +_O_NOFOLLOW: Final[int] = getattr(os, "O_NOFOLLOW", 0) +#: Probed once at import, as :mod:`filelock._strict` does, because a per-call check flips when a test mocks the +#: function. The set lists os.rename, whose implementation os.replace shares. +_SUPPORTS_DIR_FD: Final[bool] = ( + hasattr(os, "link") + and {os.link, os.open, os.stat, os.mkdir, os.rename, os.unlink} <= os.supports_dir_fd + and os.scandir in os.supports_fd +) +#: Windows opens descriptors in text mode by default, which rewrites newlines; every record here is exact bytes. +_O_BINARY: Final[int] = getattr(os, "O_BINARY", 0) +#: ESTALE is what an NFS client reports for a file a peer unlinked out from under its cached handle: gone, not broken. +_MISSING_ERRNOS: Final[frozenset[int]] = frozenset({ENOENT, ESTALE}) +#: Windows refuses an open with EACCES while another process is deleting the file or holds it open without sharing; +#: both clear within moments, so an open is retried this long before the refusal is taken as real. +_WINDOWS_OPEN_GRACE: Final[float] = 0.5 + + +class OsFiles: + """Record files under a lock's protocol root, opened without following symlinks and owned by this UID alone.""" + + def __init__(self, lock_file: str) -> None: + self._lock_file = lock_file + self._prepared = False + self._identities: dict[str, tuple[int, int]] = {} + + def read(self, path: str) -> bytes | None: + # O_NONBLOCK keeps an open of a FIFO planted at the path from blocking; the regular-file check below rejects it. + if (fd := self._open(path, os.O_RDONLY | _O_NOFOLLOW | getattr(os, "O_NONBLOCK", 0) | _O_BINARY)) is None: + return None + try: + if not stat.S_ISREG(os.fstat(fd).st_mode): + return b"" + chunks: list[bytes] = [] + remaining = _MAX_RECORD_SIZE + 1 + while remaining > 0 and (chunk := os.read(fd, remaining)): + chunks.append(chunk) + remaining -= len(chunk) + return b"".join(chunks) + finally: + os.close(fd) + + def create(self, path: str, data: bytes, *, durable: bool = False) -> tuple[int, int]: + with self._relative(path) as (name, dir_fd): + fd = os.open(name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | _O_NOFOLLOW | _O_BINARY, 0o600, dir_fd=dir_fd) + try: + write_all(fd, data) + if durable: + os.fsync(fd) + created = os.fstat(fd) + except BaseException: + os.close(fd) + # The name is unique to this call, so removing it cannot take a record from anyone else. + self.unlink(path) + raise + os.close(fd) + return created.st_dev, created.st_ino + + def link(self, source: str, target: str) -> bool: + failure: NotImplementedError | OSError | None = None + with self._relative(source) as (source_name, source_fd), self._relative(target) as (target_name, target_fd): + try: + _link_no_follow(source_name, target_name, src_dir_fd=source_fd, dst_dir_fd=target_fd) + except FileExistsError: + pass + except (NotImplementedError, OSError) as error: + _raise_if_hard_links_unsupported(self._lock_file, error) + failure = error + # The file's identity decides, not the call's result: an NFS retransmit can report a link that landed as + # failed, and an EEXIST can be this very link answered twice. Any other error on a link that did not land is + # a fault, not a peer winning the race, and a retry would spin. + if (identity := _identity(source_name, source_fd)) is not None and identity == _identity( + target_name, target_fd + ): + return True + if failure is not None: + raise failure + return False + + def replace(self, source: str, target: str) -> bool: + try: + with self._relative(source) as (source_name, source_fd), self._relative(target) as (target_name, target_fd): + os.replace(source_name, target_name, src_dir_fd=source_fd, dst_dir_fd=target_fd) + except PermissionError: + # Windows refuses to rename over a file a reader holds open; callers tolerate a target one rename behind. + return False + return True + + def overwrite(self, path: str, data: bytes, identity: tuple[int, int]) -> bool: + # O_NONBLOCK keeps an open of a FIFO (a named pipe) at the path from blocking, as in read. With no reader the + # open fails with ENXIO. + try: + fd = self._open(path, os.O_WRONLY | _O_NOFOLLOW | getattr(os, "O_NONBLOCK", 0) | _O_BINARY) + except OSError as error: + if error.errno != ENXIO: + raise + return False # pragma: needs fifo + if fd is None: + return False + try: + # O_NOFOLLOW refuses a symlink but not a hard link (a second name for the same file) or another file renamed + # over the name, and a write through either lands in a file that is not the record. The protocol does not + # link holder records, so a second name marks a file that inherited a removed record's inode number. + opened = os.fstat(fd) + if not stat.S_ISREG(opened.st_mode) or (opened.st_dev, opened.st_ino) != identity or opened.st_nlink != 1: + return False + write_all(fd, data) + finally: + os.close(fd) + return True + + def unlink(self, path: str) -> None: + try: + with self._relative(path) as (name, dir_fd): + os.unlink(name, dir_fd=dir_fd) + except OSError as error: + # Windows refuses to remove a file another process holds open; the record then outlives its owner until + # that process exits, which only delays the cleanup a later participant repeats. + if error.errno not in _MISSING_ERRNOS and not isinstance(error, PermissionError): + raise + + def listdir(self, path: str) -> list[str]: + try: + with self._directory(path) as dir_fd, os.scandir(path if dir_fd is None else dir_fd) as entries: + return [entry.name for entry in entries] + except FileNotFoundError: + return [] + + def prepare(self, root: str, *, force: bool = False) -> None: + if self._prepared and not force: + return + if force: + self._identities.clear() + ensure_directory_exists(root) + for directory, depth in ( + (Path(root), 0), + (Path(root, _GENERATIONS_DIRECTORY), 1), + (Path(root, _HOLDERS_DIRECTORY), 1), + ): + with self._relative(str(directory), depth) as (name, dir_fd): + with suppress(FileExistsError): + os.mkdir(name, 0o700, dir_fd=dir_fd) + # mkdir has no O_NOFOLLOW, so check what the name resolves to before the lock creates anything in it. + found = os.stat(name, dir_fd=dir_fd, follow_symlinks=False) + parent = directory.parent.stat() if dir_fd is None else os.fstat(dir_fd) + if ( + not stat.S_ISDIR(found.st_mode) + or getattr(found, "st_file_attributes", 0) & stat.FILE_ATTRIBUTE_REPARSE_POINT + ): + msg = f"{directory} exists but is not a directory or is a symlink; refusing to use it" + raise TreeReplacedError(msg) + owner = foreign_directory_owner(found, parent) + if owner is not None: # pragma: needs foreign-sticky-tmp + msg = f"{directory} belongs to uid {owner}, not to this process's account or the parent's owner" + raise TreeReplacedError(msg) + self._identities[str(directory)] = found.st_dev, found.st_ino + self._prepared = True + + def _open(self, path: str, flags: int) -> int | None: + # Missing is the one answer a caller acts on without raising. A refused open is real on POSIX, where nothing + # this lock does makes its own records unreadable; on Windows it is usually another process mid-delete or + # holding the file without sharing, so it is retried through a short grace before it counts, and a file still + # refusing after that has been in delete-pending for longer than any deletion takes, so it is on its way out: + # missing. + deadline: float | None = None + while True: + if isinstance(opened := self._attempt_open(path, flags), int): + return opened + if opened.errno in _MISSING_ERRNOS: + return None + if not isinstance(opened, PermissionError) or sys.platform != "win32": + raise opened + if deadline is None: # pragma: win32 cover + deadline = time.monotonic() + _WINDOWS_OPEN_GRACE + elif time.monotonic() >= deadline: # pragma: win32 cover + return None + time.sleep(0.002) # pragma: win32 cover + + def _attempt_open(self, path: str, flags: int) -> int | OSError: + try: + with self._relative(path) as (name, dir_fd): + return os.open(name, flags, dir_fd=dir_fd) + except OSError as error: + return error + + @contextmanager + def _relative(self, path: str, depth: int = 2) -> Generator[tuple[str, int | None]]: + directory, name = os.path.split(path) + with self._directory(directory, depth) as dir_fd: + yield (path if dir_fd is None else name), dir_fd + + @contextmanager + def _directory(self, path: str, depth: int = 2) -> Generator[int | None]: + # O_NOFOLLOW applies to the last path component. The check prepare() made on and /gen holds until a + # process that can write the lock's directory renames the tree aside and puts one with a symlinked gen in its + # place. Each call therefore opens the root and then the record's directory without following symlinks and + # works inside what it opened. No descriptor outlives the call, so a later call finds a log removed and started + # again by name, as before. None means descriptors cannot anchor a call and the call uses the whole path. + if depth and _SUPPORTS_DIR_FD: # pragma: needs link-dir-fd + with self._relative(path, depth - 1) as (name, parent_fd): + try: + fd = os.open(name, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | _O_NOFOLLOW, dir_fd=parent_fd) + except OSError as error: + # NetBSD reports EFTYPE rather than ELOOP when O_NOFOLLOW refuses a symlink. + if error.errno not in {ELOOP, ENOTDIR, getattr(errno, "EFTYPE", ELOOP)}: + raise + msg = f"{path} is not a directory or is a symlink; refusing to use it" + raise TreeReplacedError(msg) from error + try: + self._admit(fd, path, parent_fd) + except BaseException: + os.close(fd) + raise + try: + yield fd + finally: + os.close(fd) + else: + # Without dir_fd, hold each of those directories open instead, so no process can rename or replace one. + try: + pins = _pin_directories(*[Path(path), *Path(path).parents][:depth][::-1]) + except NotADirectoryError as error: # pragma: win32 cover + msg = f"{error.filename} is not a directory or is a symlink; refusing to use it" + raise TreeReplacedError(msg) from error + with pins: + yield None + + def _admit(self, fd: int, path: str, parent_fd: int | None) -> None: # pragma: needs link-dir-fd + # A directory other than the one prepare() checked may belong to another account: the tree can be renamed aside + # and recreated while this object lives, and the ownership check has to hold for what is opened now. + found = os.fstat(fd) + if (found.st_dev, found.st_ino) == self._identities.get(path): + return + parent = os.fstat(parent_fd) if parent_fd is not None else Path(path).parent.stat() + if (owner := foreign_directory_owner(found, parent)) is not None: # pragma: needs foreign-sticky-tmp + msg = f"{path} belongs to uid {owner}, not to this process's account or the parent's owner" + raise TreeReplacedError(msg) + self._identities[path] = found.st_dev, found.st_ino + + +def _identity(path: str, dir_fd: int | None) -> tuple[int, int] | None: + try: + st = os.stat(path, dir_fd=dir_fd, follow_symlinks=False) + except OSError: + return None + return st.st_dev, st.st_ino + + +__all__ = [ + "OsFiles", +] diff --git a/.venv/lib/python3.14/site-packages/filelock/_soft_rw/_sync.py b/.venv/lib/python3.14/site-packages/filelock/_soft_rw/_sync.py new file mode 100644 index 0000000000000000000000000000000000000000..aa2e891516cb38f9027cfd840a4db758abde3390 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/filelock/_soft_rw/_sync.py @@ -0,0 +1,793 @@ +"""Cross-process and cross-host reader/writer lock over a generation log of immutable snapshots.""" + +from __future__ import annotations + +__lazy_modules__ = [ + "filelock._error", + "filelock._lease", + "filelock._soft_rw._protocol", + "filelock._soft_rw._storage", + "math", +] + +import atexit +import os +import threading +import time +from contextlib import contextmanager, suppress +from dataclasses import dataclass +from math import isfinite +from typing import TYPE_CHECKING, ClassVar, Final +from weakref import WeakValueDictionary + +from filelock._api import ( + AcquireReturnProxy, + _acquire_thread_lock, + _canonical, + _check_timeout_max, + _ensure_current_process, + _register_fork_class, + _register_fork_object, + _resolve_read_write_timeout, + _seconds, + _timeout_behavior, +) +from filelock._error import Timeout +from filelock._lease import LeaseCompromise +from filelock._soft_rw._protocol import GenerationLog, Ledger, Mode, Participant +from filelock._soft_rw._storage import OsFiles + +if TYPE_CHECKING: + from collections.abc import Callable, Generator + + from filelock._lease import CompromiseReason + +_ALL_INSTANCES: Final[WeakValueDictionary[int, SoftReadWriteLock]] = WeakValueDictionary() +_ALL_INSTANCES_LOCK: threading.Lock = threading.Lock() + + +class _SoftRWMeta(type): + _instances: WeakValueDictionary[str, SoftReadWriteLock] + _instances_lock: threading.RLock + _instances_under_construction: set[str] + + def __call__( # ruff:ignore[too-many-arguments] # forwards the public constructor's documented parameters + cls, + lock_file: str | os.PathLike[str], + timeout: float = -1, + *, + blocking: bool = True, + is_singleton: bool = True, + heartbeat_interval: float = 30.0, + stale_threshold: float | None = None, + poll_interval: float = 0.25, + on_compromise: Callable[[LeaseCompromise], None] | None = None, + ) -> SoftReadWriteLock: + _ensure_current_process() + timeout = _seconds("timeout", timeout) + # Passed through only when set, so a subclass that declares its own constructor without it keeps working. + extra = {} if on_compromise is None else {"on_compromise": on_compromise} + if not is_singleton: + return super().__call__( + lock_file, + timeout, + blocking=blocking, + is_singleton=is_singleton, + heartbeat_interval=heartbeat_interval, + stale_threshold=stale_threshold, + poll_interval=poll_interval, + **extra, + ) + + # Keyed like the log root, so one instance never spans two logs. + normalized = _canonical(lock_file) + with cls._instances_lock: + instance = cls._instances.get(normalized) + if instance is None: + if normalized in cls._instances_under_construction: + msg = f"Singleton lock construction is already active for {lock_file!s}" + raise RuntimeError(msg) + construction_pid = os.getpid() + construction_registry = cls._instances_under_construction + construction_registry.add(normalized) + try: + instance = super().__call__( + lock_file, + timeout, + blocking=blocking, + is_singleton=is_singleton, + heartbeat_interval=heartbeat_interval, + stale_threshold=stale_threshold, + poll_interval=poll_interval, + **extra, + ) + finally: + construction_registry.discard(normalized) + if os.getpid() != construction_pid: # pragma: needs fork + msg = "Lock construction cannot continue after fork; construct a new lock in the child" + raise RuntimeError(msg) + cls._instances[normalized] = instance + elif ( + _timeout_behavior(instance.timeout, any_negative=False) + != _timeout_behavior(timeout, any_negative=False) + or instance.blocking != blocking + ): + msg = ( + f"Singleton lock created with timeout={instance.timeout}, blocking={instance.blocking}," + f" cannot be changed to timeout={timeout}, blocking={blocking}" + ) + raise ValueError(msg) + elif on_compromise is not None and instance._on_compromise != on_compromise: # ruff: ignore[private-member-access] # the metaclass owns the singleton it compares + # A tuning difference keeps the first caller's values; a different loss callback is a different safety + # contract, and returning the cached instance would silently drop it. + msg = f"Singleton lock created with a different on_compromise callback for {lock_file!s}" + raise ValueError(msg) + else: + _validate_intervals(heartbeat_interval, stale_threshold, poll_interval) + return instance + + +class SoftReadWriteLock(metaclass=_SoftRWMeta): + """ + Cross-process and cross-host reader/writer lock for shared filesystems. + + Use this class instead of :class:`~filelock.ReadWriteLock` when the lock file lives on a network filesystem (NFS, + Lustre, HPC cluster shared storage). ``ReadWriteLock`` is backed by SQLite and cannot run on NFS because SQLite's + ``fcntl`` locking is unreliable there. + + The lock's state is a log of immutable snapshots under ``foo.lock.rw/gen/``, each naming the writer and the + readers holding the lock at generation ``N``. Every acquire, release, and eviction publishes the next snapshot with + one atomic no-replace hard link, so no transition can be left half done by a crash on any host. Each participant + also keeps ``foo.lock.rw/holders/``, a record whose nonce a daemon heartbeat thread rewrites every + ``heartbeat_interval`` seconds. A contender evicts a member whose record has not changed for ``stale_threshold`` + seconds of the contender's own monotonic clock; no clock is ever compared across hosts, so skew between hosts or + against the file server cannot make a live holder look dead. + + Writer acquire is writer-preferring: a writer enters the snapshot as soon as no live writer is named, which blocks + any new reader, then waits for the named readers to leave. Writer starvation is impossible. + + An expired holder is not fenced by the lock: a process paused past ``stale_threshold`` resumes believing it holds + the lock until its next heartbeat reports the loss through ``on_compromise``. :attr:`generation` is a monotonic + fencing token for the protected resource: a store that rejects writes carrying a lower generation than the highest + it has accepted refuses such a holder. + + Reentrancy, upgrade/downgrade rules, thread pinning, and singleton caching by canonical path match + :class:`~filelock.ReadWriteLock`. + + Forking invalidates the inherited instance in the child so the child cannot double-own the lock with its parent; + ``release()`` on that instance is a no-op, and the child must construct a new instance if it needs a lock. + + Trust boundary: protects against same-UID non-cooperating processes (one host or cross-host) and same-host + different-UID users via ``0o600`` / ``0o700`` permissions. Does not protect against root compromise or multi-tenant + mounts where hostile co-tenants share the UID. + + :param lock_file: path to the lock file; the protocol directory lives next to it as ``.rw`` + :param timeout: maximum wait time in seconds; ``-1`` means block indefinitely + :param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately on contention + :param is_singleton: if ``True``, reuse existing instances for the same canonical path + :param heartbeat_interval: seconds between heartbeat refreshes; default 30 s + :param stale_threshold: seconds a holder record may stay unchanged before a contender evicts it; defaults to + ``3 * heartbeat_interval``, matching etcd's ``LeaseKeepAlive`` convention + :param poll_interval: seconds between acquire retries under contention; default 0.25 s + :param on_compromise: called from the heartbeat thread with a :class:`~filelock.LeaseCompromise` when the hold is + lost: a peer evicted it, or refreshes failed for long enough that a peer could have + + .. versionadded:: 3.27.0 + + .. versionchanged:: 3.33.0 + + The on-disk protocol is a generation log; earlier releases cannot share a lock path with this one. + + """ + + _instances: WeakValueDictionary[str, SoftReadWriteLock] = WeakValueDictionary() + _instances_lock = threading.RLock() + _instances_under_construction: ClassVar[set[str]] = set() + + def __init_subclass__(cls, **kwargs: object) -> None: + """Give each lock subclass its own singleton registry and lock.""" + super().__init_subclass__(**kwargs) + cls._instances = WeakValueDictionary() + cls._instances_lock = threading.RLock() + cls._instances_under_construction = set() + _register_fork_class(cls) + + def __init__( # ruff:ignore[too-many-arguments] # public constructor: one parameter per documented lock option + self, + lock_file: str | os.PathLike[str], + timeout: float = -1, + *, + blocking: bool = True, + is_singleton: bool = True, # ruff:ignore[unused-method-argument] # consumed by _SoftRWMeta.__call__ + heartbeat_interval: float = 30.0, + stale_threshold: float | None = None, + poll_interval: float = 0.25, + on_compromise: Callable[[LeaseCompromise], None] | None = None, + ) -> None: + self._creator_pid = os.getpid() + heartbeat_interval, stale_threshold, poll_interval = _validate_intervals( + heartbeat_interval, stale_threshold, poll_interval + ) + + self.lock_file: str = os.fspath(lock_file) + self.timeout: float = timeout + self.blocking: bool = blocking + self.heartbeat_interval: float = heartbeat_interval + self.stale_threshold: float = stale_threshold + self.poll_interval: float = poll_interval + + # Resolved once: a relative lock path must keep naming the same log after the process changes directory. The + # final component stays unresolved, as in 4.0.12, so every release shares one log on a symlinked lock path. + self._root = f"{_canonical(self.lock_file)}.rw" + self._files = OsFiles(self.lock_file) + self._ledger = Ledger(time.monotonic) + self._log = GenerationLog(self._files, self.lock_file, self._root) + self._on_compromise = on_compromise + self._locks = _Locks(internal=threading.Lock(), transaction=threading.Lock()) + self._hold: _Hold | None = None + self._compromise: LeaseCompromise | None = None + self._closed: bool = False + + with _ALL_INSTANCES_LOCK: + _ALL_INSTANCES[id(self)] = self + _register_fork_object(self) + + @classmethod + def _reset_class_after_fork(cls) -> None: # pragma: forked child + global _ALL_INSTANCES_LOCK # ruff:ignore[global-statement] # rebinds the module lock to a fresh one in the fork child + _ALL_INSTANCES_LOCK = threading.Lock() + cls._instances = WeakValueDictionary() + cls._instances_lock = threading.RLock() + cls._instances_under_construction = set() + + @property + def generation(self) -> int | None: + """ + The generation at which the current hold was granted, or ``None`` when no lock is held. + + Generations are monotonic across all participants and hosts, so this is a fencing token: pass it to the + protected resource and have the resource reject any operation carrying a lower generation than the highest it + has accepted. That refuses a holder that paused past ``stale_threshold``, was evicted, and resumed. Removing the + ``.rw`` directory starts a new log whose generations begin again at 1. + + .. versionadded:: 3.33.0 + + """ + with self._locks.internal: + return None if self._hold is None else self._hold.participant.generation + + @property + def compromise(self) -> LeaseCompromise | None: + """ + How the current hold was lost, or ``None`` while it stands. + + Set by the heartbeat thread once a peer has evicted this holder or refreshes have failed for long enough that a + peer could have. The holder should stop using the protected resource when it is set. Until the lost hold is + released, acquiring on this instance raises :class:`RuntimeError` instead of joining it. + + .. versionadded:: 3.33.0 + + """ + with self._locks.internal: + return self._compromise + + @contextmanager + def read_lock(self, timeout: float | None = None, *, blocking: bool | None = None) -> Generator[None]: + """ + Context manager that acquires and releases a shared read lock. + + Falls back to instance defaults for *timeout* and *blocking* when ``None``. + + :param timeout: maximum wait time in seconds, or ``None`` to use the instance default + :param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default + + :raises RuntimeError: if a write lock is already held on this instance + :raises Timeout: if the lock cannot be acquired within *timeout* seconds + :raises TypeError: if *timeout* is a :class:`bool` or not a real number + :raises ValueError: if a blocking call gets a ``nan`` *timeout* or a negative one other than ``-1`` + + """ + self.acquire_read(timeout, blocking=blocking) + try: + yield + finally: + self.release() + + @contextmanager + def write_lock(self, timeout: float | None = None, *, blocking: bool | None = None) -> Generator[None]: + """ + Context manager that acquires and releases an exclusive write lock. + + Falls back to instance defaults for *timeout* and *blocking* when ``None``. + + :param timeout: maximum wait time in seconds, or ``None`` to use the instance default + :param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default + + :raises RuntimeError: if a read lock is already held, or a write lock is held by a different thread + :raises Timeout: if the lock cannot be acquired within *timeout* seconds + :raises TypeError: if *timeout* is a :class:`bool` or not a real number + :raises ValueError: if a blocking call gets a ``nan`` *timeout* or a negative one other than ``-1`` + + """ + self.acquire_write(timeout, blocking=blocking) + try: + yield + finally: + self.release() + + def acquire_read(self, timeout: float | None = None, *, blocking: bool | None = None) -> AcquireReturnProxy: + """ + Acquire a shared read lock. + + If this instance already holds a read lock, the lock level is incremented (reentrant). Attempting to acquire a + read lock while holding a write lock raises :class:`RuntimeError` (downgrade not allowed). On the 0→1 + transition the reader publishes its holder record, enters the next snapshot once no live writer is named, and + starts a daemon heartbeat thread that rewrites the record's nonce every ``heartbeat_interval`` seconds so peers + on other hosts do not evict it. + + :param timeout: maximum wait time in seconds, or ``None`` to use the instance default; ``-1`` means block + indefinitely + :param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately when the lock is unavailable; + ``None`` uses the instance default + + :returns: a proxy that can be used as a context manager to release the lock + + :raises RuntimeError: if a write lock is already held on this instance, if this instance was invalidated by + :func:`os.fork`, or if :meth:`close` was called + :raises Timeout: if the lock cannot be acquired within *timeout* seconds + :raises TypeError: if *timeout* is a :class:`bool` or not a real number + :raises ValueError: if a blocking call gets a ``nan`` *timeout* or a negative one other than ``-1`` + :raises SoftFileLockProtocolError: if a snapshot cannot be read without risking overlap, or the filesystem + refuses the no-replace hard links the protocol commits with + + """ + return self._acquire("read", timeout, blocking=blocking) + + def acquire_write(self, timeout: float | None = None, *, blocking: bool | None = None) -> AcquireReturnProxy: + """ + Acquire an exclusive write lock. + + If this instance already holds a write lock from the same thread, the lock level is incremented (reentrant). + Attempting to acquire a write lock while holding a read lock raises :class:`RuntimeError` (upgrade not + allowed). Write locks are pinned to the acquiring thread: a different thread trying to re-enter also raises + :class:`RuntimeError`. + + A writer enters the next snapshot as its writer as soon as no live writer is named, which blocks every new + reader on every host, then waits for the readers that snapshot names to leave. Writer starvation is impossible: + new readers see the named writer and wait behind it. + + :param timeout: maximum wait time in seconds, or ``None`` to use the instance default; ``-1`` means block + indefinitely + :param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately when the lock is unavailable; + ``None`` uses the instance default + + :returns: a proxy that can be used as a context manager to release the lock + + :raises RuntimeError: if a read lock is already held, if a write lock is held by a different thread, if this + instance was invalidated by :func:`os.fork`, or if :meth:`close` was called + :raises Timeout: if the lock cannot be acquired within *timeout* seconds + :raises TypeError: if *timeout* is a :class:`bool` or not a real number + :raises ValueError: if a blocking call gets a ``nan`` *timeout* or a negative one other than ``-1`` + :raises SoftFileLockProtocolError: if a snapshot cannot be read without risking overlap, or the filesystem + refuses the no-replace hard links the protocol commits with + + """ + return self._acquire("write", timeout, blocking=blocking) + + @classmethod + def get_lock( + cls, + lock_file: str | os.PathLike[str], + timeout: float = -1, + *, + blocking: bool = True, + ) -> SoftReadWriteLock: + """ + Return the singleton :class:`SoftReadWriteLock` for *lock_file*. + + :param lock_file: path to the lock file; the protocol directory lives next to it as ``.rw`` + :param timeout: maximum wait time in seconds; ``-1`` means block indefinitely + :param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately when the lock is unavailable + + :returns: the singleton lock instance + + :raises ValueError: if an instance already exists for this path with different *timeout* or *blocking* values + + """ + return cls(lock_file, timeout, blocking=blocking) + + def close(self) -> None: + """ + Release any held lock and mark the instance closed. + + Idempotent. After calling this method the instance can no longer acquire locks — subsequent acquires raise + :class:`RuntimeError`, and so does an acquire another thread is still waiting in, which gives up its claim + instead of taking the lock. A fork-invalidated instance is closed without raising. + """ + if self._creator_pid != os.getpid(): # pragma: forked child + return + # Closed first, under the lock an acquire publishes its hold under: an acquire still waiting then sees the flag + # and leaves instead of publishing, and one that published before it is released below. + with self._locks.internal: + self._closed = True + self.release(force=True) + + def release(self, *, force: bool = False) -> None: + """ + Release one level of the current lock. + + When the lock level reaches zero the heartbeat thread is stopped, the participant is committed out of the next + snapshot, and its holder record is removed. On a fork-invalidated instance (that is, the child of a + :func:`os.fork` call made while the parent held a lock) this method is a no-op so inherited ``with`` blocks can + unwind cleanly in the child. + + :param force: if ``True``, release the lock completely regardless of the current lock level + + :raises RuntimeError: if no lock is currently held and *force* is ``False``, or if the calling thread holds no + level of the current lock and *force* is ``False`` + + """ + if self._creator_pid != os.getpid(): # pragma: forked child + return + self._release(threading.get_ident(), force=force) + + def _acquire_level_of( + self, holder: int, mode: Mode, timeout: float, *, blocking: bool, cancel_check: Callable[[], bool] + ) -> AcquireReturnProxy: + # Records the level under *holder* instead of the calling thread: an async wrapper's executor thread does not + # identify the tasks it holds for. Gives up with Timeout once *cancel_check* is true. + return self._acquire(mode, timeout, blocking=blocking, holder=holder, cancel_check=cancel_check) + + def _release_level_of(self, holder: int) -> None: + self._release(holder, force=False) + + def _holds_level_of(self, holder: int) -> bool: + # Read without the internal lock: an async wrapper asks from its event loop, which must not wait out a release + # running on another thread. + return (hold := self._hold) is not None and (holder == hold.write_thread_id or holder in hold.read_depths) + + def _release(self, thread_id: int, *, force: bool) -> None: + with self._locks.internal: + hold = self._hold + if hold is None: + if force: + return + msg = f"Cannot release a lock on {self.lock_file} (lock id: {id(self)}) that is not held" + raise RuntimeError(msg) + if force: + hold.level = 0 + else: + self._raise_if_not_holder(hold, thread_id) + hold.leave_one_level(thread_id) + if hold.level > 0: + return + self._hold = None + + # Stop the heartbeat before leaving, so a late tick cannot re-read a snapshot this participant just left and + # report an eviction that was its own release. on_compromise runs on that thread and may be what called here, + # in which case there is nothing to join. + hold.heartbeat_stop.set() + try: + if hold.heartbeat_thread is not threading.current_thread(): + hold.heartbeat_thread.join(timeout=min(self.heartbeat_interval + 1.0, threading.TIMEOUT_MAX)) + finally: + # The hold is already gone from this instance, so a join that raises must not strand its record too. + hold.participant.leave() + + def _raise_if_not_holder(self, hold: _Hold, thread_id: int) -> None: + # The hold's own heartbeat thread may release it, since on_compromise runs there. + if thread_id == hold.heartbeat_thread.ident: + return + # Another thread's release would leave the generation log under the holder and let a peer in. + if hold.mode == "write" and thread_id != hold.write_thread_id: + msg = ( + f"Cannot release write lock on {self.lock_file} (lock id: {id(self)}) " + f"from thread {thread_id} while it is held by thread {hold.write_thread_id}" + ) + raise RuntimeError(msg) + # Readers share one participant, so a thread that holds no read level would end another's hold. + if hold.mode == "read" and thread_id not in hold.read_depths: + msg = ( + f"Cannot release read lock on {self.lock_file} (lock id: {id(self)}) " + f"from thread {thread_id}, which does not hold it" + ) + raise RuntimeError(msg) + + def _acquire( + self, + mode: Mode, + timeout: float | None, + *, + blocking: bool | None, + holder: int | None = None, + cancel_check: Callable[[], bool] | None = None, + ) -> AcquireReturnProxy: + if self._creator_pid != os.getpid(): # pragma: forked child + msg = f"SoftReadWriteLock on {self.lock_file} was invalidated by fork(); construct a new instance" + raise RuntimeError(msg) + timeout = self.timeout if timeout is None else timeout + blocking = self.blocking if blocking is None else blocking + holder = threading.get_ident() if holder is None else holder + + with self._locks.internal: + if self._closed: + raise self._closed_error() + timeout = _resolve_read_write_timeout(timeout, blocking=blocking) + if self._hold is not None: + return self._validate_reentrant(mode, holder) + + deadline = None if timeout == -1 else time.perf_counter() + timeout + if not _acquire_thread_lock( + self._locks.transaction, + blocking=blocking, + timeout=timeout, + poll_interval=self.poll_interval, + cancel_check=cancel_check, + ): + raise Timeout(self.lock_file) from None + try: + return self._do_acquire_inner(mode, deadline, holder, blocking=blocking, cancel_check=cancel_check) + finally: + self._locks.transaction.release() + + def _closed_error(self) -> RuntimeError: + return RuntimeError(f"SoftReadWriteLock on {self.lock_file} has been closed") + + def _do_acquire_inner( + self, + mode: Mode, + deadline: float | None, + holder: int, + *, + blocking: bool, + cancel_check: Callable[[], bool] | None, + ) -> AcquireReturnProxy: + with self._locks.internal: + if self._hold is not None: + return self._validate_reentrant(mode, holder) + participant = Participant( + self._files, + self._root, + mode, + stale_threshold=self.stale_threshold, + clock=time.monotonic, + ledger=self._ledger, + log=self._log, + ) + participant.publish() + try: + self._wait_for(participant.advance, deadline=deadline, blocking=blocking, cancel_check=cancel_check) + except BaseException: + # A contender that gives up must not stay named in the snapshot: a writer left there would block every + # reader until a peer waits out the stale threshold. Its own token is all leave() touches, so it cannot + # undo a peer's claim. + with suppress(OSError): + participant.leave() + raise + stop_event = threading.Event() + hold = _Hold( + level=1, + mode=mode, + write_thread_id=holder if mode == "write" else None, + read_depths={holder: 1} if mode == "read" else {}, + participant=participant, + heartbeat_thread=_HeartbeatThread( + refresh=lambda: self._refresh(participant, hold), + interval=self.heartbeat_interval, + stop_event=stop_event, + name=f"filelock-heartbeat-{id(self):x}", + ), + heartbeat_stop=stop_event, + last_refresh=time.monotonic(), + ) + # Publish the hold and start its heartbeat under one internal-lock section, so a concurrent release() never + # observes a hold whose thread has not started and joins it. If the OS refuses the thread, clear the hold and + # leave: left in place, a peer evicts the unrefreshed record and acquires while this instance still believes it + # holds the lock. A close() that landed while this acquire waited gets the same treatment: published, the hold + # would keep its heartbeat and its slot on an instance nobody can release through. + start_error: BaseException | None = None + with self._locks.internal: + if self._closed: + start_error = self._closed_error() + else: + self._hold = hold + self._compromise = None + try: + hold.heartbeat_thread.start() + except BaseException as error: # ruff:ignore[blind-except] # clear the slot below and re-raise + self._hold = None + start_error = error + if start_error is not None: + participant.leave() + raise start_error + return AcquireReturnProxy(lock=self) + + def _validate_reentrant(self, mode: Mode, holder: int) -> AcquireReturnProxy: + hold = self._hold + assert hold is not None # ruff:ignore[assert] # callers dispatch here only inside the self._hold is not None branch + if hold.mode != mode: + opposite = "write" if mode == "read" else "read" + direction = "downgrade" if mode == "read" else "upgrade" + msg = ( + f"Cannot acquire {mode} lock on {self.lock_file} (lock id: {id(self)}): " + f"already holding a {opposite} lock ({direction} not allowed)" + ) + raise RuntimeError(msg) + if mode == "write" and holder != hold.write_thread_id: + msg = ( + f"Cannot acquire write lock on {self.lock_file} (lock id: {id(self)}) " + f"from thread {holder} while it is held by thread {hold.write_thread_id}" + ) + raise RuntimeError(msg) + # Publishing a hold clears the loss recorded for the one before, so a loss seen here is this hold's. Its + # heartbeat has stopped and a peer may hold the lock, so a new level must not bypass the log. + if (compromise := self._compromise) is not None: + raise _lost_hold_error(self.lock_file, mode, compromise) + if mode == "read": + hold.read_depths[holder] = hold.read_depths.get(holder, 0) + 1 + hold.level += 1 + return AcquireReturnProxy(lock=self) + + def _wait_for( + self, + predicate: Callable[[], bool], + *, + deadline: float | None, + blocking: bool, + cancel_check: Callable[[], bool] | None, + ) -> None: + while not predicate(): + # The flag is read without the lock here only to stop waiting early; publishing the hold re-checks it. + if self._closed: + raise self._closed_error() + now = time.perf_counter() + if not blocking or (deadline is not None and now >= deadline): + raise Timeout(self.lock_file) + # An abandoned acquire gives up like a timed-out one, so the caller's rollback withdraws its claim. + if cancel_check is not None and cancel_check(): + raise Timeout(self.lock_file) + sleep_for = self.poll_interval + if deadline is not None: + sleep_for = min(sleep_for, max(deadline - now, 0.0)) + time.sleep(sleep_for) + + def _refresh(self, participant: Participant, hold: _Hold) -> bool: + # The loop ends at the first loss, so the holder hears about it once. A transient filesystem error (ESTALE / + # EIO on the NFS-style filesystems this lock targets) is not a loss: retry rather than report a false + # compromise. Report the record unrefreshable only once failures have run long enough that a peer could evict + # it before the next success would land, a margin before the record actually ages out. + try: + outcome, error = participant.heartbeat() + except Exception as failure: # ruff:ignore[blind-except] # the thread would end with the hold still published + self._report_compromise(hold, "refresh-failed", failure) + return False + if outcome == "ok": + hold.last_refresh = time.monotonic() + return True + if outcome == "lost": + self._report_compromise(hold, "evicted", None) + return False + if time.monotonic() - hold.last_refresh >= self.stale_threshold - self.heartbeat_interval: + self._report_compromise(hold, "refresh-failed", error) + return False + return True + + def _report_compromise(self, hold: _Hold, reason: CompromiseReason, error: Exception | None) -> None: + # A tick that outlived its release's join must not stamp the old hold's loss onto whatever was acquired since. + with self._locks.internal: + if self._hold is not hold: + return + compromise = LeaseCompromise( + lock_file=self.lock_file, token=hold.participant.token, reason=reason, error=error + ) + self._compromise = compromise + if self._on_compromise is not None: + self._on_compromise(compromise) + + def _reset_after_fork_in_child(self) -> None: # pragma: forked child + self._locks = _Locks(internal=threading.Lock(), transaction=threading.Lock()) + self._hold = None + + +class _HeartbeatThread(threading.Thread): + def __init__( + self, + refresh: Callable[[], bool], + interval: float, + stop_event: threading.Event, + name: str, + ) -> None: + super().__init__(name=name, daemon=True) + self._refresh = refresh + self._interval = interval + self._stop_event = stop_event + + def run(self) -> None: + # Set on every exit, an on_compromise that raises included, so nothing waits on a thread that is gone. + try: + while not self._stop_event.wait(self._interval): + if not self._refresh(): + return + finally: + self._stop_event.set() + + +def _lost_hold_error(lock_file: str, mode: Mode, compromise: LeaseCompromise) -> RuntimeError: + return RuntimeError( + f"Cannot acquire {mode} lock on {lock_file}: the current hold was lost ({compromise.reason}), " + "release it before acquiring again" + ) + + +def _validate_intervals( + heartbeat_interval: float, stale_threshold: float | None, poll_interval: float +) -> tuple[float, float, float]: + heartbeat_interval = _seconds("heartbeat_interval", heartbeat_interval) + if not isfinite(heartbeat_interval) or heartbeat_interval <= 0: + msg = f"heartbeat_interval must be positive and finite, got {heartbeat_interval}" + raise ValueError(msg) + # The heartbeat thread waits this long between refreshes, and Event.wait overflows past TIMEOUT_MAX. + _check_timeout_max("heartbeat_interval", heartbeat_interval) + stale_threshold = ( + heartbeat_interval * 3 if stale_threshold is None else _seconds("stale_threshold", stale_threshold) + ) + if not isfinite(stale_threshold) or stale_threshold <= heartbeat_interval: + msg = ( + f"stale_threshold must exceed heartbeat_interval ({heartbeat_interval}) " + f"and be finite, got {stale_threshold}" + ) + raise ValueError(msg) + if not isfinite(poll_interval := _seconds("poll_interval", poll_interval)) or poll_interval <= 0: + msg = f"poll_interval must be positive and finite, got {poll_interval}" + raise ValueError(msg) + _check_timeout_max("poll_interval", poll_interval) + # A waiting writer refreshes its record once per poll, so a poll slower than the threshold reads as dead. + if poll_interval >= stale_threshold: + msg = f"poll_interval must be below stale_threshold ({stale_threshold}), got {poll_interval}" + raise ValueError(msg) + return heartbeat_interval, stale_threshold, poll_interval + + +@dataclass +class _Locks: + internal: threading.Lock + transaction: threading.Lock + + +@dataclass +class _Hold: + """Everything that exists only while a lock is held; ``None`` when the instance has no lock.""" + + level: int + mode: Mode + write_thread_id: int | None + #: Read levels per thread, so a thread can give back only the levels it took. + read_depths: dict[int, int] + participant: Participant + heartbeat_thread: _HeartbeatThread + heartbeat_stop: threading.Event + last_refresh: float + + def leave_one_level(self, thread_id: int) -> None: + self.level -= 1 + if self.mode == "read": + # The heartbeat thread holds no level of its own, so its release gives back one of a reader's. + owner = thread_id if thread_id in self.read_depths else next(iter(self.read_depths)) + if depth := self.read_depths.pop(owner) - 1: + self.read_depths[owner] = depth + + +def _cleanup_all_instances() -> None: # pragma: no cover - runs from atexit at interpreter shutdown + for instance in list(_ALL_INSTANCES.values()): + with suppress(Exception): + instance.release(force=True) + + +atexit.register(_cleanup_all_instances) +_register_fork_class(SoftReadWriteLock) + + +__all__ = [ + "SoftReadWriteLock", + "_lost_hold_error", +] diff --git a/.venv/lib/python3.14/site-packages/filelock/_strict.py b/.venv/lib/python3.14/site-packages/filelock/_strict.py new file mode 100644 index 0000000000000000000000000000000000000000..14b4a4831a54b8ff1d8efacb7e6954faefdf55d8 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/filelock/_strict.py @@ -0,0 +1,1243 @@ +from __future__ import annotations + +__lazy_modules__ = [ + "contextlib", + "filelock._error", + "filelock._identity", + "filelock._marker", + "filelock._soft", + "filelock._util", + "functools", + "pathlib", + "secrets", + "tempfile", +] + +import contextlib +import errno +import functools +import os +import secrets +import stat +import sys +import tempfile +import time +from dataclasses import dataclass +from errno import EACCES, EEXIST, ENOENT, ENOSYS, EPERM, ESTALE, EXDEV +from pathlib import Path +from typing import TYPE_CHECKING, Final, Literal, cast + +from filelock._api import BaseFileLock, ThreadLocalFileContext, _canonical, _raise_cleanup_errors +from filelock._error import SoftFileLockProtocolError +from filelock._identity import host_name, owner_is_stale, process_start_token +from filelock._marker import parse_marker +from filelock._soft import MALFORMED_LOCK_AGE_THRESHOLD, _parse_lock_holder +from filelock._soft_protocol import STRICT_SOFT_SENTINEL_RECORD +from filelock._util import foreign_directory_owner, write_all + +if sys.platform == "win32": # pragma: win32 cover + from filelock._windows import _is_unlinked, _open_deletable_for_reading, _pin_directory, _unpin_directory + +if TYPE_CHECKING: + from collections.abc import Callable, Iterator + +StrictSoftFileClaimState = Literal["intent", "held"] +_OccupantState = Literal["sentinel", "held", "exited", "damaged", "unparsable"] + +_CLAIM_STATES: Final[frozenset[str]] = frozenset({"intent", "held"}) +_COORDINATION_SUFFIX: Final[str] = ".filelock" +_CLAIM_DIRECTORY_NAME: Final[str] = "claims" +_CLAIM_MAGIC: Final[str] = "filelock-strict-v1" +_CLAIM_RECORD_LIMIT: Final[int] = 1024 +_CLAIM_NAME_PART_COUNT: Final[int] = 3 +_TOKEN_HEX_LENGTH: Final[int] = 32 +_PRIVATE_RECORD_MARKER: Final[str] = ".private-v1-" +_PRIVATE_RECORD_SUFFIX: Final[str] = ".tmp" +_PRIVATE_RECORD_RANDOM_HEX_LENGTH: Final[int] = 32 +_PRIVATE_RECORD_GRACE: Final[float] = 2.0 +_UNLINK_MAX_RETRIES: Final[int] = 10 +#: How long a scan waits out a claim held in Windows' delete-pending state before treating it as unreadable. +_CLAIM_READ_GRACE: Final[float] = 0.5 +_CLAIM_READ_RETRY: Final[float] = 0.002 +#: Rides out a peer replacing the lock path mid-inspection; a foreign node or unreadable record outlasting it is damage. +_SENTINEL_READ_GRACE: Final[float] = 0.5 +_NODE_DESCRIPTIONS: Final[dict[int, str]] = {stat.S_IFDIR: "a directory", stat.S_IFLNK: "a symlink"} +#: Windows opens descriptors in text mode by default, which rewrites newlines and truncates a record at a control byte. +#: The claim and sentinel records are exact binary, so every record descriptor must be binary; POSIX ignores the flag. +_O_BINARY: Final[int] = getattr(os, "O_BINARY", 0) +_LEGACY_SENTINEL: Final[bytes] = STRICT_SOFT_SENTINEL_RECORD.encode() +_WINDOWS_HARD_LINK_UNSUPPORTED: Final[frozenset[int]] = frozenset({1, 17, 50}) + +# Termux/Android CPython ships without os.link (bionic long had only linkat), so the strict backend's whole hard-link +# mechanism is absent there. Probe once and gate every os.link reference on it, so importing filelock still works and +# only an actual StrictSoftFileLock acquire reports the missing capability. +_HAS_LINK: Final[bool] = hasattr(os, "link") + +# Probe dir_fd capability once at import. A per-call ``os.unlink in os.supports_dir_fd`` check flips to False the moment +# a test mocks os.unlink, silently diverting the code to a different branch than the one under test. +_OPEN_SUPPORTS_DIR_FD: Final[bool] = os.open in os.supports_dir_fd +_UNLINK_SUPPORTS_DIR_FD: Final[bool] = os.unlink in os.supports_dir_fd +_STAT_SUPPORTS_DIR_FD: Final[bool] = os.stat in os.supports_dir_fd +_LINK_SUPPORTS_DIR_FD: Final[bool] = _HAS_LINK and os.link in os.supports_dir_fd +_MKDIR_SUPPORTS_DIR_FD: Final[bool] = os.mkdir in os.supports_dir_fd +_SCANDIR_SUPPORTS_FD: Final[bool] = os.scandir in os.supports_fd + + +def _probe_link_follow_symlinks() -> bool: + # os.supports_follow_symlinks lists os.link on PyPy, but its linkat then rejects follow_symlinks=False with EINVAL, + # and Windows raises NotImplementedError for the option outright. Link a throwaway file for real so the answer + # reflects the runtime rather than its advertisement, and treat any failure as "not honored": the option only + # hardens a source this process created with O_EXCL, so skipping it is safe, and a real environment fault surfaces + # when the actual link runs. + if not _HAS_LINK: + return False + try: + with tempfile.TemporaryDirectory() as directory: + source = Path(directory, "probe-source") + source.touch() + os.link(source, Path(directory, "probe-link"), follow_symlinks=False) + except (OSError, NotImplementedError, ValueError): + return False + return True + + +_LINK_HONORS_FOLLOW_SYMLINKS: Final[bool] = _probe_link_follow_symlinks() + + +def _probe_hard_link_unsupported_errnos() -> frozenset[int]: + # GraalPy's errno omits ENOTSUP, so importing the name outright breaks every runtime that ships without it. ENOTSUP + # wins wherever it exists, leaving every runtime that names it unchanged. EOPNOTSUPP only stands in for the ones + # that do not, and it approximates rather than matches: the two codes agree on Linux but differ on macOS/BSD and on + # Windows. Where neither exists a runtime that cannot name "operation not supported" cannot raise it either, and + # ENOSYS/EXDEV still classify the link failures it can raise. + not_supported = getattr(errno, "ENOTSUP", getattr(errno, "EOPNOTSUPP", None)) + return frozenset({ENOSYS, EXDEV} if not_supported is None else {ENOSYS, EXDEV, not_supported}) + + +_HARD_LINK_UNSUPPORTED_ERRNOS: Final[frozenset[int]] = _probe_hard_link_unsupported_errnos() + + +class StrictSoftFileLock(BaseFileLock): + """Portable fail-closed lock based on immutable owner claims.""" + + _preserve_lock_file_supported: bool = True + _on_acquired_supported: bool = False + #: Age cannot clear a strict claim: expiring one on a clock is the overlap the fail-closed contract exists to rule + #: out, so only force_break() removes it. + _lifetime_supported: bool = False + _lifetime_unsupported_reason: str = "a strict claim is never broken by age, only by force_break()" + #: Contending processes each publish and rescan several files, so jitter their retries. Seconds. Only a + #: poll_interval below this cap jitters; the default equals it and keeps a fixed cadence. + _poll_backoff_cap: float = 0.05 + _occupant_seen: tuple[tuple[float, int, int], float] | None = None + #: Set by the finalizer, after which no retry of a failed cleanup can run. + _finalizing: bool = False + + def _acquire(self) -> None: + # Finish a doorway whose claim cleanup failed before publishing, so this owner never contends with leftovers. + self._discard_pending_doorways(exited_only=True) + lock_path = Path(self._lock_path) + coordination_directory = Path(f"{lock_path}{_COORDINATION_SUFFIX}") + claim_directory = coordination_directory / _CLAIM_DIRECTORY_NAME + # A process that can write these directories can remove claims from them, so they take the lock's mode plus + # search permission where the mode grants read. + directory_mode: Final = (mode := self._open_mode()) | (mode & 0o444) >> 2 + _ensure_protocol_directory(self.lock_file, coordination_directory, directory_mode) + _ensure_claim_directory(self.lock_file, claim_directory, directory_mode) + if ( + sentinel_fd := _open_or_create_sentinel( + self.lock_file, lock_path, self._open_mode(), self._occupant_unchanged_for + ) + ) is None: + return + try: + sentinel_identity = _file_identity(os.fstat(sentinel_fd)) + except BaseException as inspection_error: # preserve inspection and descriptor cleanup errors + try: + os.close(sentinel_fd) + except BaseException as close_error: # ruff:ignore[blind-except] # preserve inspection and descriptor cleanup errors + _raise_cleanup_errors("strict sentinel inspection cleanup failed", inspection_error, close_error) + raise + self._mark_descriptor_pending(sentinel_fd, sentinel_identity) + try: + if won := self._attempt_doorway(claim_directory): + self._mark_descriptor_owned(sentinel_fd, sentinel_identity) + except BaseException: + # Once recorded as held, the acquire's rollback releases the claim instead. + if not self.is_locked: + self._discard_doorway() + raise + if not won: + self._discard_doorway() + + def _discard_pending_doorways(self, *, exited_only: bool) -> None: + if self._context.owner_claim_paths and not self.is_locked: + self._discard_doorway() + if not isinstance(context := self._context, ThreadLocalFileContext): + return + # Only this registry reaches an exited thread's thread-local doorway; a live thread finishes its own. + own: Final = context.current + with self._transition_lock: + for pending, thread in self._thread_pending.copy().values(): + if exited_only and thread.is_alive(): + continue + context.current = pending + try: + self._discard_doorway() + finally: + context.current = own + + def _occupant_unchanged_for(self, occupant: os.stat_result) -> float: + # SoftFileLock's marker rule: the mtime is only tested for change, timed on this process's monotonic clock. + key, now = (occupant.st_mtime, occupant.st_ino, occupant.st_size), time.monotonic() + if (seen := self._occupant_seen) is None or seen[0] != key: + self._occupant_seen = key, now + return 0.0 + return now - seen[1] + + def _attempt_doorway(self, claim_directory: Path) -> bool: + if _read_existing_claims(self.lock_file, claim_directory): + return False + + token = secrets.token_hex(_TOKEN_HEX_LENGTH // 2) + intent_name = _claim_name("intent", token) + intent_path = str(claim_directory / intent_name) + # Record each claim before its link: once the link may have committed, any later failure must leave the name for + # _discard_doorway. The token makes the name unique, so removing it when the link never happened is a no-op. + self._context.owner_claim_paths = (intent_path,) + try: + with _pin_claim_directory(claim_directory): + publication_cleanup_error = _publish_record( + intent_path, _claim_record(token), self._open_mode(), _open_claim_directory + ) + except _PrivateRecordReclaimedError: + return False + except (NotImplementedError, OSError) as error: + _raise_if_hard_links_unsupported(self.lock_file, error) + if isinstance(error, OSError) and error.errno == EEXIST: + # Another file holds the name, so it is not this owner's to remove. + self._context.owner_claim_paths = () + return False + raise + if publication_cleanup_error is not None: # pragma: needs dir-fd + raise publication_cleanup_error + + claims = _read_existing_claims(self.lock_file, claim_directory) + if ( + not claims + or any(claim.state == "held" for claim in claims) + or min(claim.name for claim in claims) != intent_name + ): + return False + + held_name = _claim_name("held", token) + held_path = str(claim_directory / held_name) + self._context.owner_claim_paths = (held_path, intent_path) + try: + link_cleanup_error = _link_no_replace(claim_directory, intent_name, held_name) + except (NotImplementedError, OSError) as error: + _raise_if_hard_links_unsupported(self.lock_file, error) + if isinstance(error, OSError) and error.errno == EEXIST: + self._context.owner_claim_paths = (intent_path,) + raise + if link_cleanup_error is not None: # pragma: needs dir-fd + raise link_cleanup_error + + claims = _read_existing_claims(self.lock_file, claim_directory) + # Keep the intent claim for the whole hold rather than unlinking it now. The intent has existed, unchanged, + # since this owner published it, so a contender's os.scandir is guaranteed to return it (POSIX only leaves the + # visibility of entries created or removed *during* a scan unspecified). The freshly linked held claim carries + # no such guarantee: a scan that races its creation can miss it. Were the intent removed here, that scan could + # observe neither claim and let a larger-token contender win over this owner. The stable intent is the witness + # that keeps the phase-five min-token decision computed over the true set. Release unlinks both. + return {intent_name, held_name}.issubset(claim.name for claim in claims) and min( + _claim_token_key(claim.name) for claim in claims + ) == f"v1-{token}.claim" + + @property + def claims(self) -> tuple[StrictSoftFileClaim, ...]: + """Published claims that block acquisition.""" + # An inspection writes nothing, so it trusts no record's age rather than probe the filesystem clock. + return _read_existing_claims(self.lock_file, self._claim_directory, probe_clock=False) + + @property + def lock_path_occupant(self) -> str | None: + """ + What sits at the lock path in place of the strict sentinel, or ``None`` when the sentinel or nothing is there. + + A soft-lock owner's marker blocks acquisition without appearing in :attr:`~filelock.StrictSoftFileLock.claims`; + this describes it, naming the PID and host when the marker records them. A directory, a symlink, a marker whose + owner on this host has exited, or a file it cannot parse that stays unchanged for two seconds makes acquisition + raise :class:`~filelock.SoftFileLockProtocolError` instead and needs an operator to remove it. + """ + try: + fd, _, occupant = _inspect_lock_path(Path(_canonical(self.lock_file))) + except FileNotFoundError: + return None + if fd is None: + return occupant + os.close(fd) + return None + + def force_break(self, claim_name: str) -> None: + """Remove one named claim, allowing overlap if its owner still holds the protected resource.""" + _validate_force_break_name(claim_name) + with _pin_claim_directory(directory := self._claim_directory): + _require_exact_name(directory, claim_name) + cleanup_error = _unlink_in_directory(directory, claim_name) + if cleanup_error is not None: # pragma: needs dir-fd + raise cleanup_error + + def _release_abandoned_holds(self) -> None: + self._finalizing = True + super()._release_abandoned_holds() + # release() skips a doorway whose cleanup failed, as it is no hold, so a dropped instance would strand claims. + with contextlib.suppress(Exception): + # GraalPy defers finalizers to the host collector, so its tests cannot drive this cleanup. + self._discard_pending_doorways(exited_only=False) # pragma: needs collected-finalization + + def _reconcile_failed_acquire(self, canonical: str) -> None: + # The acquisition is over, so the next one resolves the working directory again rather than reuse this one's. + self._context.claim_root = None + super()._reconcile_failed_acquire(canonical) + + def _release(self) -> None: + fd = cast("int", self._context.lock_file_fd) + self._context.claim_root = None + remaining, errors = self._unlink_owner_claims() + if remaining: + _raise_recorded_errors("strict claim release failed", errors) + self._mark_descriptor_released() + try: + self._close_released_fd(fd, default_suppresses=False) + except BaseException as close_error: # ruff:ignore[blind-except] # preserve claim and sentinel cleanup errors + errors.append(close_error) + if errors: + _raise_recorded_errors("strict release cleanup failed", errors) + + def _discard_doorway(self) -> None: + remaining, errors = self._unlink_owner_claims() + # The pending descriptor holds no lock and a retry needs the claim paths alone. A dropped instance does not + # retry, so close it now. + if (fd := self._context.pending_lock_file_fd) is not None: + self._context.pending_lock_file_fd = self._context.pending_lock_file_fd_identity = None + try: + self._close_released_fd(fd, default_suppresses=False) + except BaseException as close_error: # ruff:ignore[blind-except] # preserve claim and sentinel cleanup errors + errors.append(close_error) + if remaining: + # Stay pending, not owned: is_locked reports only a held claim that passed the final rescan. The next + # acquire retries this cleanup. + _raise_recorded_errors("strict doorway claim cleanup failed", errors) + self._mark_descriptor_released() + if errors: + _raise_recorded_errors("strict doorway cleanup failed", errors) + + def _unlink_owner_claims(self) -> tuple[list[str], list[BaseException]]: + remaining, errors = _unlink_owner_paths(self._context.owner_claim_paths) + self._context.owner_claim_paths = tuple(remaining) + if self._finalizing: + # No retry runs after the finalizer and no caller sees its errors, so report success and let it close the + # descriptor even when claims remain. + return [], [] # pragma: needs collected-finalization + return remaining, errors + + @property + def _claim_directory(self) -> Path: + if self._context.owner_claim_paths: + return Path(self._context.owner_claim_paths[0]).parent + return Path(f"{_canonical(self.lock_file)}{_COORDINATION_SUFFIX}") / _CLAIM_DIRECTORY_NAME + + +@dataclass(frozen=True) +class StrictSoftFileClaim: + """One parsed strict soft-lock claim.""" + + name: str + state: StrictSoftFileClaimState + token: str + pid: int + hostname: str + #: The owner's process start token, or ``None`` when the platform exposes no proven start time. A strict lock never + #: reclaims a claim on its own, so this identifies the owner for tooling rather than driving any automatic break. + start: int | None = None + + +class _PrivateRecordReclaimedError(Exception): + pass + + +def _open_or_create_sentinel( + lock_file: str, path: Path, mode: int, unchanged_for: Callable[[os.stat_result], float] +) -> int | None: + try: + return _open_sentinel(lock_file, path, unchanged_for) + except FileNotFoundError: + pass + + _reclaim_sentinel_private_records(lock_file, path) + try: + publication_cleanup_error = _publish_record(os.fspath(path), _LEGACY_SENTINEL, mode) + except _PrivateRecordReclaimedError: + return None + except (NotImplementedError, OSError) as error: + _raise_if_hard_links_unsupported(lock_file, error) + if not isinstance(error, OSError) or error.errno != EEXIST: + raise + else: + if publication_cleanup_error is not None: # pragma: needs dir-fd + raise publication_cleanup_error + try: + return _open_sentinel(lock_file, path, unchanged_for) + except FileNotFoundError: + return None + + +def _open_sentinel(lock_file: str, path: Path, unchanged_for: Callable[[os.stat_result], float]) -> int | None: + # A live soft-lock owner blocks like any holder. Anything else raises: claims cannot show it and force_break() + # cannot remove it, so waiting would wedge every strict contender with no diagnostic. A file no lock parses may be a + # soft lock between its create and its write, so it raises only once unchanged as long as a soft contender waits. + deadline: float | None = None + while True: + fd, state, occupant = _inspect_lock_path(path) + if fd is not None or state == "held": + return fd + reason = f"expected the strict sentinel at the lock path, found {occupant}" + if state == "unparsable": + if unchanged_for(path.lstat()) < MALFORMED_LOCK_AGE_THRESHOLD: + return None + raise SoftFileLockProtocolError( + lock_file, None, f"{reason}, unchanged for {MALFORMED_LOCK_AGE_THRESHOLD:g} seconds" + ) + if deadline is None: + deadline = time.monotonic() + _SENTINEL_READ_GRACE + if state == "exited" or time.monotonic() >= deadline: + raise SoftFileLockProtocolError(lock_file, None, reason) + time.sleep(_CLAIM_READ_RETRY) + + +def _inspect_lock_path(path: Path) -> tuple[int | None, _OccupantState, str]: + # "held" may be a soft-lock owner that still runs, "exited" is one this host proves gone, "unparsable" is content no + # lock parses, and "damaged" is a node no lock writes. + if not stat.S_ISREG(mode := path.lstat().st_mode): + return None, "damaged", _NODE_DESCRIPTIONS.get(stat.S_IFMT(mode), "a special file") + try: + fd, record = _open_record((os.fspath(path.parent), None), path.name, _CLAIM_RECORD_LIMIT) + except FileNotFoundError: + raise + except PermissionError as error: + # Windows denies the read while a legacy holder keeps its marker open, and a holder running as another user + # may leave it unreadable here, so a marker that cannot be read may still be held. + return None, "held", f"a file it cannot read ({error.strerror or error})" + except OSError as error: + return None, "damaged", f"a file it cannot read ({error.strerror or error})" + if record == _LEGACY_SENTINEL: + return fd, "sentinel", "the strict sentinel" + os.close(fd) + return None, *_describe_soft_marker(record.decode("utf-8", errors="replace")) + + +def _describe_soft_marker(content: str) -> tuple[_OccupantState, str]: + # A SoftFileLock, MarkerSoftFileLock or SoftFileLease owner holds the path legitimately until it releases, so its + # marker is contention. One this host proves gone is named, never removed: strict never acts on an inferred death. + if (record := parse_marker(content)) is not None: + pid, hostname, start = record.pid, record.hostname, record.start + elif (holder := _parse_lock_holder(content)) is not None: + pid, hostname, start = holder + else: + return "unparsable", "a file that is neither the sentinel nor a soft-lock marker it can parse" + if owner_is_stale(pid, hostname, start): + return "exited", f"a soft-lock marker left by pid {pid} on {hostname}, which has exited" + return "held", f"a soft-lock marker held by pid {pid} on {hostname}" + + +def _read_claims( + lock_file: str, directory_ref: tuple[str, int | None], *, probe_clock: bool +) -> tuple[StrictSoftFileClaim, ...]: + names: Final = _list_claim_names(lock_file, directory_ref, probe_clock=probe_clock) + name_parts: dict[str, tuple[StrictSoftFileClaimState, str]] = {} + for name in names: + if (parts := _parse_claim_name(name)) is None: + raise SoftFileLockProtocolError(lock_file, name, "unknown claim name or protocol version") + name_parts[name] = parts + records: Final = _read_claim_records(lock_file, directory_ref, names, probe_clock=probe_clock) + return tuple(_parse_claim(lock_file, name, name_parts[name], records[name]) for name in names if name in records) + + +def _list_claim_names(lock_file: str, directory_ref: tuple[str, int | None], *, probe_clock: bool) -> list[str]: + try: + with _scandir_relative(directory_ref) as entries: + return _public_claim_names(directory_ref, entries, probe_clock=probe_clock) + except OSError as error: + reason = f"cannot list claim directory: {error.strerror or type(error).__name__}" + raise SoftFileLockProtocolError(lock_file, None, reason) from error + + +def _read_claim_records( + lock_file: str, directory_ref: tuple[str, int | None], names: list[str], *, probe_clock: bool +) -> dict[str, bytes]: + # A contended scan can list a claim that is not yet cleanly readable, in two ways that both resolve on a brief + # retry. Windows fails an open with EACCES while a peer's unlink holds the claim; once the deletion is pending, + # _attempt_claim_read skips the claim outright, as another process's handle can keep the name listed for seconds. + # On NFS, a peer that unlinks its own claim leaves this client's cached filehandle stale, so the next open returns + # ESTALE rather than a clean ENOENT until the lookup revalidates against the server. Retrying re-runs the path + # lookup, which turns the vanished claim into ENOENT (skip) or reads it if it still exists. A genuinely unreadable + # record (a locked-down file, an EIO fault) still fails closed. The grace starts at the first pending result rather + # than before the first attempt, so a slow first read cannot use up the window and leave no retry. One deadline + # covers the scan: a deadline per claim let claims that clear one after another stretch a scan, and the caller's + # timeout with it, by a grace each. + records: dict[str, bytes] = {} + deadline: float | None = None + while True: + pending: dict[str, OSError] = {} + for name in names: + record, error = _attempt_claim_read(lock_file, directory_ref, name) + if error is not None: + pending[name] = error + elif record is not None: + records[name] = record + if not pending: + return records + if deadline is None: + deadline = time.monotonic() + _CLAIM_READ_GRACE + elif time.monotonic() >= deadline: + break + names = list(pending) + time.sleep(_CLAIM_READ_RETRY) + # A stale handle that outlives revalidation is a claim the server no longer has (RFC 1813 NFS3ERR_STALE): skip it + # like ENOENT. Skipping a vanished claim overcounts contention at worst; it cannot free a held lock. + if denied := [name for name, error in pending.items() if error.errno != ESTALE]: + # Windows can deny opens after a peer removes the claim from the directory. + listed: Final = set(_list_claim_names(lock_file, directory_ref, probe_clock=probe_clock)) + if still_listed := [name for name in denied if name in listed]: + error = pending[still_listed[0]] + reason = f"cannot read claim: {error.strerror or str(error) or type(error).__name__}" + raise SoftFileLockProtocolError(lock_file, still_listed[0], reason) from error + return records + + +def _attempt_claim_read( + lock_file: str, directory_ref: tuple[str, int | None], name: str +) -> tuple[bytes | None, OSError | None]: + # Return the record, or (None, None) when the claim has already gone, or (None, error) for an open the caller may + # retry: a Windows EACCES while a peer's unlink holds the claim, or an NFS ESTALE from a peer unlinking its own + # claim out from under this client's cached filehandle. Any other OSError is a real fault and fails closed. + try: + return _read_record(directory_ref, name, _CLAIM_RECORD_LIMIT), None + except FileNotFoundError: + return None, None + except PermissionError as error: + if sys.platform == "win32" and _is_unlinked(str(Path(directory_ref[0], name))): # pragma: win32 cover + return None, None + return None, error + except OSError as error: + if error.errno == ESTALE: + return None, error + reason = f"cannot read claim: {error.strerror or str(error) or type(error).__name__}" + raise SoftFileLockProtocolError(lock_file, name, reason) from error + + +def _public_claim_names( + directory_ref: tuple[str, int | None], entries: Iterator[os.DirEntry[str]], *, probe_clock: bool +) -> list[str]: + names: list[str] = [] + now: Callable[[], float | None] = ( + functools.cache( + functools.partial( + _filesystem_now, directory_ref, _claim_name("held", secrets.token_hex(_TOKEN_HEX_LENGTH // 2)) + ) + ) + if probe_clock + else lambda: None + ) + for entry in entries: + if not entry.name.startswith("."): + names.append(entry.name) + elif (public_name := _private_public_name(entry.name)) is not None and _parse_claim_name( + public_name + ) is not None: + _reclaim_private_record(directory_ref, entry.name, now) + return sorted(names) + + +def _read_existing_claims( + lock_file: str, directory: Path, *, probe_clock: bool = True +) -> tuple[StrictSoftFileClaim, ...]: + if not _OPEN_SUPPORTS_DIR_FD: # pragma: lacks dir-fd + try: + pinned = _pin_claim_directory(directory) + except FileNotFoundError: + return () + except OSError as error: + reason = f"cannot open claim directory: {error.strerror or type(error).__name__}" + raise SoftFileLockProtocolError(lock_file, None, reason) from error + with pinned: + return _read_claims(lock_file, (os.fspath(directory), None), probe_clock=probe_clock) + return _read_claims_through_descriptor(lock_file, directory, probe_clock=probe_clock) # pragma: needs dir-fd + + +def _read_claims_through_descriptor( # pragma: needs dir-fd + lock_file: str, directory: Path, *, probe_clock: bool +) -> tuple[StrictSoftFileClaim, ...]: + try: + directory_fd = _open_claim_directory(os.fspath(directory)) + except FileNotFoundError: + return () + except OSError as error: + reason = f"cannot open claim directory: {error.strerror or type(error).__name__}" + raise SoftFileLockProtocolError(lock_file, None, reason) from error + try: + return _read_claims(lock_file, (os.fspath(directory), directory_fd), probe_clock=probe_clock) + finally: + # The scan wrote nothing through the descriptor, so a failed close has no result to report. + with contextlib.suppress(OSError): + os.close(directory_fd) + + +def _parse_claim( + lock_file: str, + name: str, + name_parts: tuple[StrictSoftFileClaimState, str], + record: bytes, +) -> StrictSoftFileClaim: + try: + magic, token, pid_text, hostname_hex, start_text, trailing = record.decode("ascii").split("\n") + pid = int(pid_text) + start = int(start_text) if start_text else None + hostname = bytes.fromhex(hostname_hex).decode("utf-8") + except (UnicodeDecodeError, ValueError) as error: + raise SoftFileLockProtocolError(lock_file, name, "malformed claim record") from error + if not all(( + not trailing, + magic == _CLAIM_MAGIC, + token == name_parts[1], + 1 <= pid <= 2**31 - 1, + str(pid) == pid_text, + start is None or (start >= 0 and str(start) == start_text), + hostname.encode().hex() == hostname_hex + and hostname.isprintable() + and not any(character.isspace() for character in hostname), + )): + raise SoftFileLockProtocolError(lock_file, name, "malformed claim record") + return StrictSoftFileClaim(name=name, state=name_parts[0], token=token, pid=pid, hostname=hostname, start=start) + + +def _parse_claim_name(name: str) -> tuple[StrictSoftFileClaimState, str] | None: + if not name.endswith(".claim"): + return None + parts = name.removesuffix(".claim").split("-") + if len(parts) != _CLAIM_NAME_PART_COUNT or parts[0] not in _CLAIM_STATES or parts[1] != "v1": + return None + token = parts[2] + if len(token) != _TOKEN_HEX_LENGTH or any(character not in "0123456789abcdef" for character in token): + return None + return cast("StrictSoftFileClaimState", parts[0]), token + + +def _claim_name(state: StrictSoftFileClaimState, token: str) -> str: + return f"{state}-v1-{token}.claim" + + +def _claim_token_key(name: str) -> str: + return name.removeprefix("held-").removeprefix("intent-") + + +def _claim_record(token: str) -> bytes: + hostname_hex = host_name().encode().hex() + start = process_start_token(os.getpid()) + start_text = "" if start is None else str(start) + return f"{_CLAIM_MAGIC}\n{token}\n{os.getpid()}\n{hostname_hex}\n{start_text}\n".encode("ascii") + + +def _publish_record( + public_path: str, + record: bytes, + mode: int, + open_directory: Callable[[str], int] | None = None, +) -> BaseException | None: + directory, public_name = os.path.split(public_path) + directory = directory or os.curdir + private_name = _private_record_name(public_name) + directory_fd = (open_directory or _open_directory)(directory) if _OPEN_SUPPORTS_DIR_FD else None + directory_ref = directory, directory_fd + try: + _publish_record_in_directory(directory_ref, (private_name, public_name), mode, record) + except BaseException as publication_error: # preserve publication and directory cleanup errors + try: + if directory_fd is not None: # pragma: needs dir-fd + os.close(directory_fd) + except BaseException as close_error: # ruff:ignore[blind-except] # pragma: needs dir-fd # preserve publication and directory cleanup errors + _raise_cleanup_errors("strict publication directory cleanup failed", publication_error, close_error) + raise + if directory_fd is not None: # pragma: needs dir-fd + try: + os.close(directory_fd) + except BaseException as close_error: # ruff:ignore[blind-except] # a claim's caller recorded its path before publishing + return close_error + return None + + +def _publish_record_in_directory( + directory_ref: tuple[str, int | None], + names: tuple[str, str], + mode: int, + record: bytes, +) -> None: + flags = os.O_RDWR | os.O_CREAT | os.O_EXCL | _O_BINARY + if (o_nofollow := getattr(os, "O_NOFOLLOW", None)) is not None: # pragma: needs o-nofollow + flags |= o_nofollow + private_fd = _open_relative(directory_ref, names[0], flags, mode) + private_identity: tuple[int, int] | None = None + try: + private_identity = _file_identity(os.fstat(private_fd)) + write_all(private_fd, record) + _link_private_record(directory_ref, names, private_identity) + except BaseException as publication_error: # preserve publication and cleanup errors + close_error, unlink_error = _close_and_unlink_private_record( + directory_ref, + names[0], + private_fd, + private_identity, + ) + if close_error is not None or unlink_error is not None: + _raise_cleanup_errors( + "strict record publication cleanup failed", + publication_error, + close_error, + unlink_error, + ) + raise + close_error, unlink_error = _close_and_unlink_private_record( + directory_ref, + names[0], + private_fd, + private_identity, + ) + if close_error is not None or unlink_error is not None: + _raise_record_finalization_errors(close_error, unlink_error) + + +def _close_and_unlink_private_record( + directory_ref: tuple[str, int | None], + private_name: str, + private_fd: int, + private_identity: tuple[int, int] | None, +) -> tuple[BaseException | None, BaseException | None]: + close_error: BaseException | None = None + try: + os.close(private_fd) + except BaseException as error: # ruff:ignore[blind-except] # returned for grouping with unlink failures + close_error = error + unlink_error: BaseException | None = None + try: + if private_identity is None: + _unlink_relative(directory_ref, private_name) + else: + _unlink_relative_if_identity(directory_ref, private_name, private_identity) + except FileNotFoundError: + pass + except BaseException as error: # ruff:ignore[blind-except] # returned for grouping with close failures + unlink_error = error + return close_error, unlink_error + + +def _link_private_record( + directory_ref: tuple[str, int | None], + names: tuple[str, str], + private_identity: tuple[int, int], +) -> None: + try: + _link_or_find_committed(directory_ref, *names, private_identity) + except FileNotFoundError as error: + if _relative_identity(directory_ref, names[0]) is not None: + raise + msg = "private publication record was reclaimed" + raise _PrivateRecordReclaimedError(msg) from error + if _relative_identity(directory_ref, names[1]) == private_identity: + return + msg_0 = "private publication record was replaced" + raise _PrivateRecordReclaimedError(msg_0) + + +def _raise_record_finalization_errors( + close_error: BaseException | None, + unlink_error: BaseException | None, +) -> None: + errors = [error for error in (close_error, unlink_error) if error is not None] + if len(errors) > 1: + _raise_cleanup_errors("strict record finalization failed", errors[0], *errors[1:]) + raise errors[0] + + +def _private_record_name(public_name: str) -> str: + return f".{public_name}{_PRIVATE_RECORD_MARKER}{secrets.token_hex(_PRIVATE_RECORD_RANDOM_HEX_LENGTH // 2)}.tmp" + + +def _private_public_name(private_name: str) -> str | None: + if not private_name.startswith(".") or not private_name.endswith(_PRIVATE_RECORD_SUFFIX): + return None + public_name, marker, random_hex = private_name[1 : -len(_PRIVATE_RECORD_SUFFIX)].rpartition(_PRIVATE_RECORD_MARKER) + if ( + marker != _PRIVATE_RECORD_MARKER + or len(random_hex) != _PRIVATE_RECORD_RANDOM_HEX_LENGTH + or any(character not in "0123456789abcdef" for character in random_hex) + ): + return None + return public_name + + +def _reclaim_sentinel_private_records(lock_file: str, path: Path) -> None: + directory_ref = os.fspath(path.parent), None + now = functools.cache(functools.partial(_filesystem_now, directory_ref, path.name)) + try: + with os.scandir(path.parent) as entries: + for entry in entries: + if _private_public_name(entry.name) == path.name: + _reclaim_private_record(directory_ref, entry.name, now) + except OSError as error: + reason = f"cannot reclaim sentinel private records: {error.strerror or error}" + raise SoftFileLockProtocolError(lock_file, None, reason) from error + + +def _reclaim_private_record( + directory_ref: tuple[str, int | None], private_name: str, now: Callable[[], float | None] +) -> None: + directory, directory_fd = directory_ref + try: + private_stat = ( + os.stat(private_name, dir_fd=directory_fd, follow_symlinks=False) + if directory_fd is not None and _STAT_SUPPORTS_DIR_FD + else Path(directory, private_name).lstat() + ) + except FileNotFoundError: + return + if not stat.S_ISREG(private_stat.st_mode): + msg = f"{Path(directory, private_name)} is not a regular private record" + raise OSError(msg) + if private_stat.st_nlink == 1 and ( + (current := now()) is None or current - private_stat.st_mtime < _PRIVATE_RECORD_GRACE + ): + return + try: + _unlink_private_record_once(directory_ref, private_name) + except FileNotFoundError: + pass + except OSError as error: + if error.errno not in {EACCES, EPERM}: + raise + + +def _filesystem_now(directory_ref: tuple[str, int | None], public_name: str) -> float | None: + # A network filesystem stamps mtimes with the server's clock, which can run seconds off this client's, so read "now" + # as the mtime of a probe written like a record (liblockfile uses a read's atime). The probe has a private-record + # name, so a scan reaps one left behind. Without a stamp no age is trusted. + probe = _private_record_name(public_name) + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | _O_BINARY | getattr(os, "O_NOFOLLOW", 0) + try: + fd = _open_relative(directory_ref, probe, flags, 0o600) + except OSError: + return None + try: + write_all(fd, b"\n") + return os.fstat(fd).st_mtime + except OSError: + return None + finally: + with contextlib.suppress(OSError): + os.close(fd) + with contextlib.suppress(OSError): + _unlink_private_record_once(directory_ref, probe) + + +def _unlink_private_record_once(directory_ref: tuple[str, int | None], private_name: str) -> None: + directory, directory_fd = directory_ref + if directory_fd is not None and _UNLINK_SUPPORTS_DIR_FD: # pragma: needs dir-fd + os.unlink(private_name, dir_fd=directory_fd) + else: + Path(directory, private_name).unlink() + + +def _relative_identity(directory_ref: tuple[str, int | None], name: str) -> tuple[int, int] | None: + directory, directory_fd = directory_ref + try: + if directory_fd is not None and _STAT_SUPPORTS_DIR_FD: # pragma: needs dir-fd + path_stat = os.stat(name, dir_fd=directory_fd, follow_symlinks=False) + else: # pragma: win32 cover + path_stat = Path(directory, name).lstat() + except FileNotFoundError: + return None + return _file_identity(path_stat) + + +def _unlink_relative_if_identity( + directory_ref: tuple[str, int | None], + name: str, + identity: tuple[int, int], +) -> None: + if _relative_identity(directory_ref, name) == identity: + with contextlib.suppress(FileNotFoundError): + _unlink_relative(directory_ref, name) + + +def _open_relative(directory_ref: tuple[str, int | None], name: str, flags: int, mode: int) -> int: + directory, directory_fd = directory_ref + return ( + os.open(name, flags, mode, dir_fd=directory_fd) + if directory_fd is not None + else os.open(Path(directory, name), flags, mode) + ) + + +def _link_relative(directory_ref: tuple[str, int | None], source_name: str, destination_name: str) -> None: + directory, directory_fd = directory_ref + if directory_fd is not None and _LINK_SUPPORTS_DIR_FD: # pragma: needs dir-fd + _link_no_follow(source_name, destination_name, src_dir_fd=directory_fd, dst_dir_fd=directory_fd) + return + _link_no_follow(Path(directory, source_name), Path(directory, destination_name)) # pragma: win32 cover + + +def _link_or_find_committed( + directory_ref: tuple[str, int | None], + source_name: str, + destination_name: str, + source_identity: tuple[int, int] | None = None, +) -> None: + try: + _link_relative(directory_ref, source_name, destination_name) + except FileExistsError: + # NFS retransmits a LINK whose reply was lost and reports EEXIST for the committed link; as in open(2) NOTES + # and liblockfile, it succeeded when the destination names the source. Prefer the fstat-ed identity: a peer's + # scan reaps a private record once it has two links, so its name may already resolve to nothing. + if (identity := _relative_identity(directory_ref, destination_name)) is None or identity != ( + source_identity or _relative_identity(directory_ref, source_name) + ): + raise + + +def _link_no_follow( + source: str | Path, + destination: str | Path, + *, + src_dir_fd: int | None = None, + dst_dir_fd: int | None = None, +) -> None: + if not _HAS_LINK: + # No os.link at all (Termux/Android): report it as unsupported like a filesystem that refuses hard links, so + # the acquire path raises SoftFileLockProtocolError instead of a bare AttributeError. + msg = "os.link is unavailable on this platform" + raise NotImplementedError(msg) + # The source is a private record this process created with O_CREAT | O_EXCL, so follow_symlinks guards nothing an + # attacker can reach. Pass the option only when the runtime honors it: PyPy advertises it through + # os.supports_follow_symlinks yet its linkat rejects it with EINVAL, so probe once rather than trust the set. + if _LINK_HONORS_FOLLOW_SYMLINKS: + os.link(source, destination, src_dir_fd=src_dir_fd, dst_dir_fd=dst_dir_fd, follow_symlinks=False) + else: # pragma: lacks link-follow-symlinks + os.link(source, destination, src_dir_fd=src_dir_fd, dst_dir_fd=dst_dir_fd) + + +def _unlink_relative(directory_ref: tuple[str, int | None], name: str) -> None: + directory, directory_fd = directory_ref + if directory_fd is not None and _UNLINK_SUPPORTS_DIR_FD: # pragma: needs dir-fd + os.unlink(name, dir_fd=directory_fd) + elif sys.platform == "win32": # pragma: win32 cover + _unlink_in_directory(Path(directory), name) + else: + Path(directory, name).unlink() # pragma: win32 no cover + + +def _link_no_replace(directory: Path, source_name: str, destination_name: str) -> BaseException | None: + if _LINK_SUPPORTS_DIR_FD: # pragma: needs dir-fd + directory_fd = _open_claim_directory(str(directory)) + try: + _link_or_find_committed((str(directory), directory_fd), source_name, destination_name) + except BaseException as link_error: # preserve link and directory cleanup errors + try: + os.close(directory_fd) + except BaseException as close_error: # ruff:ignore[blind-except] # preserve link and directory cleanup errors + _raise_cleanup_errors("strict link directory cleanup failed", link_error, close_error) + raise + try: + os.close(directory_fd) + except BaseException as close_error: # ruff:ignore[blind-except] # the caller recorded the held path before linking + return close_error + return None + with _pin_claim_directory(directory): # pragma: win32 cover + _link_or_find_committed((str(directory), None), source_name, destination_name) + return None # pragma: win32 cover + + +def _unlink_owner_path(path: str) -> BaseException | None: + directory, name = os.path.split(path) + with _pin_claim_directory(claim_directory := Path(directory or os.curdir)): + return _unlink_in_directory(claim_directory, name) + + +def _unlink_owner_paths(paths: tuple[str, ...]) -> tuple[list[str], list[BaseException]]: + results = [(path, _unlink_owner_path_result(path)) for path in paths] + return [path for path, (removed, _) in results if not removed], [ + error for _, (_, error) in results if error is not None + ] + + +def _unlink_owner_path_result(path: str) -> tuple[bool, BaseException | None]: + try: + cleanup_error = _unlink_owner_path(path) + except FileNotFoundError: + return True, None + except BaseException as error: # ruff:ignore[blind-except] # keep ownership when unlink did not commit + return False, error + return True, cleanup_error + + +def _raise_recorded_errors(message: str, errors: list[BaseException]) -> None: + if len(errors) > 1: + _raise_cleanup_errors(message, errors[0], *errors[1:]) + raise errors[0] + + +def _unlink_in_directory(directory: Path, name: str) -> BaseException | None: + if _UNLINK_SUPPORTS_DIR_FD: # pragma: needs dir-fd + directory_fd = _open_claim_directory(str(directory)) + try: + os.unlink(name, dir_fd=directory_fd) + except BaseException as unlink_error: # preserve unlink and directory cleanup errors + try: + os.close(directory_fd) + except BaseException as close_error: # ruff:ignore[blind-except] # preserve unlink and directory cleanup errors + _raise_cleanup_errors("strict unlink directory cleanup failed", unlink_error, close_error) + raise + try: + os.close(directory_fd) + except BaseException as close_error: # ruff:ignore[blind-except] # caller commits the removed path before raising + return close_error + return None + if sys.platform != "win32": # pragma: win32 no cover + Path(directory / name).unlink() + return None + retry_delay = 0.001 # pragma: win32 cover + for attempt in range(_UNLINK_MAX_RETRIES): # pragma: win32 cover + if (error := _unlink_error(directory / name)) is None: + return None + if error.errno not in {EACCES, EPERM} or attempt == _UNLINK_MAX_RETRIES - 1: + raise error + time.sleep(retry_delay) + retry_delay *= 2 + return None # pragma: no cover # the final retry returns or raises + + +def _unlink_error(path: Path) -> OSError | None: # pragma: win32 cover + try: + path.unlink() + except OSError as error: + return error + return None + + +def _pin_claim_directory(directory: Path) -> contextlib.ExitStack[bool | None]: + return _pin_directories(directory.parent, directory) + + +def _pin_directories(*directories: Path) -> contextlib.ExitStack[bool | None]: + # Windows has no dir_fd, so each operation resolves every path component by name again. An open handle on a + # directory blocks a rename of it, and with it a swap for a junction (a directory that points at another directory), + # until the operation ends. Other platforms work through descriptors and hold nothing. + stack: contextlib.ExitStack[bool | None] = contextlib.ExitStack() + if sys.platform == "win32": # pragma: win32 cover + with stack: + for directory in directories: + stack.callback(_unpin_directory, _pin_directory(str(directory))) + return stack.pop_all() + return stack # pragma: win32 no cover + + +def _open_directory(directory: str, dir_fd: int | None = None) -> int: # pragma: needs dir-fd + return os.open(directory, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0), dir_fd=dir_fd) + + +def _open_claim_directory(directory: str) -> int: # pragma: needs dir-fd + # O_NOFOLLOW applies to the last path component. A process that can rename entries in the lock's directory can + # swap the coordination directory for a symlink after any check of its name, so open the claim directory relative + # to a descriptor of the coordination directory: a symlink at either level then fails the open. + coordination_directory, name = os.path.split(directory) + coordination_fd = _open_directory(coordination_directory) + try: + return _open_directory(name, coordination_fd) + finally: + # The walk wrote nothing through it, and an error here would leak the descriptor this function returns. + with contextlib.suppress(OSError): + os.close(coordination_fd) + + +def _scandir_relative( + directory_ref: tuple[str, int | None], +) -> contextlib.AbstractContextManager[Iterator[os.DirEntry[str]]]: + directory, directory_fd = directory_ref + if directory_fd is not None and _SCANDIR_SUPPORTS_FD: # pragma: needs dir-fd + return os.scandir(directory_fd) + return os.scandir(directory) # pragma: lacks dir-fd + + +def _lstat_relative(directory_ref: tuple[str, int | None], name: str) -> os.stat_result: + directory, directory_fd = directory_ref + if directory_fd is not None and _STAT_SUPPORTS_DIR_FD: # pragma: needs dir-fd + return os.stat(name, dir_fd=directory_fd, follow_symlinks=False) + return Path(directory, name).lstat() + + +def _read_record(directory_ref: tuple[str, int | None], name: str, limit: int) -> bytes: + fd, record = _open_record(directory_ref, name, limit, deletable=True) + os.close(fd) + return record + + +def _open_record( + directory_ref: tuple[str, int | None], name: str, limit: int, *, deletable: bool = False +) -> tuple[int, bytes]: + path = Path(directory_ref[0], name) + path_stat = _lstat_relative(directory_ref, name) + if not stat.S_ISREG(path_stat.st_mode): + msg = f"{path} is not a regular file" + raise OSError(msg) + if deletable and sys.platform == "win32": # pragma: win32 cover + # A claim's owner must be able to unlink it while another process reads it. The sentinel stays for the lock's + # lifetime, so it keeps os.open. + fd = _open_deletable_for_reading(str(path)) + else: + flags = os.O_RDONLY | _O_BINARY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0) + fd = _open_relative(directory_ref, name, flags, 0) + try: + record = _read_opened_record(fd, path, path_stat, limit) + except BaseException as read_error: # preserve read and descriptor cleanup errors + try: + os.close(fd) + except BaseException as close_error: # ruff:ignore[blind-except] # preserve read and descriptor cleanup errors + _raise_cleanup_errors("strict record read cleanup failed", read_error, close_error) + raise + return fd, record + + +def _read_opened_record(fd: int, path: Path, path_stat: os.stat_result, limit: int) -> bytes: + opened_stat = os.fstat(fd) + if not stat.S_ISREG(opened_stat.st_mode): + msg = f"{path} is not a regular file" + raise OSError(msg) + if _file_identity(opened_stat) != _file_identity(path_stat): + msg = f"{path} changed while opening" + raise OSError(msg) + record = os.read(fd, limit + 1) + if len(record) > limit: + msg = f"{path} exceeds {limit} bytes" + raise OSError(msg) + return record + + +def _ensure_claim_directory(lock_file: str, directory: Path, mode: int) -> None: + if not (_OPEN_SUPPORTS_DIR_FD and _MKDIR_SUPPORTS_DIR_FD): # pragma: lacks dir-fd + try: + pinned = _pin_directories(directory.parent) + except OSError as error: + raise SoftFileLockProtocolError(lock_file, None, f"cannot open {directory.parent}") from error + with pinned: + _ensure_protocol_directory(lock_file, directory, mode) + else: # pragma: needs dir-fd + # mkdir follows a symlink in every component but the last, so create against a descriptor of the real parent. + try: + coordination_fd = _open_directory(os.fspath(directory.parent)) + except OSError as error: + raise SoftFileLockProtocolError(lock_file, None, f"cannot open {directory.parent}") from error + try: + _ensure_protocol_directory(lock_file, directory, mode, coordination_fd) + finally: + with contextlib.suppress(OSError): + os.close(coordination_fd) + + +def _ensure_protocol_directory(lock_file: str, directory: Path, mode: int, parent_fd: int | None = None) -> None: + try: + if parent_fd is None: + directory.mkdir(mode) + else: # pragma: needs dir-fd + os.mkdir(directory.name, mode, dir_fd=parent_fd) + except FileExistsError: + try: + found = _lstat_relative((os.fspath(directory.parent), parent_fd), directory.name) + owner = foreign_directory_owner( + found, directory.parent.stat() if parent_fd is None else os.fstat(parent_fd) + ) + except OSError as error: + raise SoftFileLockProtocolError(lock_file, None, f"cannot inspect {directory}") from error + if ( + not stat.S_ISDIR(found.st_mode) + or getattr(found, "st_file_attributes", 0) & stat.FILE_ATTRIBUTE_REPARSE_POINT + ): + raise SoftFileLockProtocolError(lock_file, None, f"{directory} is not a real directory") from None + if owner is not None: # pragma: needs foreign-sticky-tmp + reason = f"{directory} belongs to uid {owner}, not to this process's account or the parent's owner" + raise SoftFileLockProtocolError(lock_file, None, reason) from None + _settle_directory_mode(lock_file, directory, mode, parent_fd) + + +def _settle_directory_mode(lock_file: str, directory: Path, mode: int, parent_fd: int | None) -> None: + # The umask or a default ACL can leave the directory wider than the mode, now or in an earlier release, so narrow a + # directory this account owns to the mode. The sticky bit (a directory flag that lets only a file's owner remove it) + # then protects each claim from other accounts even where the mode admits them. Windows has neither mode bits nor a + # sticky bit. + if sys.platform != "win32": # pragma: win32 no cover + try: + fd = os.open( + directory.name if parent_fd is not None else os.fspath(directory), + os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0), + dir_fd=parent_fd, + ) + except OSError as error: + raise SoftFileLockProtocolError(lock_file, None, f"cannot open {directory}") from error + try: + found = os.fstat(fd) + settled = found.st_mode & 0o7000 | found.st_mode & mode & 0o777 | stat.S_ISVTX + if found.st_uid == os.geteuid() and settled != stat.S_IMODE(found.st_mode): + os.fchmod(fd, settled) + except OSError as error: + raise SoftFileLockProtocolError(lock_file, None, f"cannot set the mode of {directory}") from error + finally: + os.close(fd) + + +def _validate_force_break_name(name: str) -> None: + invalid_component = not name or name.startswith(".") or name in {os.curdir, os.pardir} + if invalid_component or any(separator in name for separator in ("/", "\\", "\x00")): + msg = "claim_name must be one public claim basename" + raise ValueError(msg) + + +def _require_exact_name(directory: Path, name: str) -> None: + directory_fd = _open_claim_directory(os.fspath(directory)) if _OPEN_SUPPORTS_DIR_FD else None + try: + with _scandir_relative((os.fspath(directory), directory_fd)) as entries: + if not any(entry.name == name for entry in entries): + raise FileNotFoundError(ENOENT, os.strerror(ENOENT), name) + finally: + if directory_fd is not None: # pragma: needs dir-fd + with contextlib.suppress(OSError): + os.close(directory_fd) + + +def _raise_if_hard_links_unsupported(lock_file: str, error: NotImplementedError | OSError) -> None: + unsupported = ( + isinstance(error, NotImplementedError) + or error.errno in _HARD_LINK_UNSUPPORTED_ERRNOS + or (getattr(error, "winerror", None) in _WINDOWS_HARD_LINK_UNSUPPORTED) + ) + if unsupported: + reason = "filesystem does not support atomic no-replace hard-link publication" + raise SoftFileLockProtocolError(lock_file, None, reason) from error + + +def _file_identity(st: os.stat_result) -> tuple[int, int]: + return st.st_dev, st.st_ino + + +__all__ = [ + "StrictSoftFileClaim", + "StrictSoftFileClaimState", + "StrictSoftFileLock", +] diff --git a/.venv/lib/python3.14/site-packages/filelock/_unix.py b/.venv/lib/python3.14/site-packages/filelock/_unix.py new file mode 100644 index 0000000000000000000000000000000000000000..30c627638c5c32ead1977bf381342c5836ead72f --- /dev/null +++ b/.venv/lib/python3.14/site-packages/filelock/_unix.py @@ -0,0 +1,218 @@ +from __future__ import annotations + +__lazy_modules__ = ["contextlib", "filelock._util", "pathlib", "typing", "warnings"] + +import os +import sys +import warnings +from contextlib import suppress +from errno import EACCES, EAGAIN, ENOSYS, EWOULDBLOCK +from pathlib import Path +from typing import Final, cast + +from filelock._api import BaseFileLock +from filelock._util import remove_own_file + +has_fcntl = False +if sys.platform == "win32": # pragma: win32 cover + + class UnixFileLock(BaseFileLock): + """Uses the :func:`fcntl.flock` to hard lock the lock file on unix systems.""" + + def _acquire(self) -> None: + raise NotImplementedError + + def _release(self) -> None: + raise NotImplementedError + +else: # pragma: win32 no cover + try: + import fcntl + + _ = (fcntl.flock, fcntl.LOCK_EX, fcntl.LOCK_NB, fcntl.LOCK_UN) + except (ImportError, AttributeError): + _FCNTL_UNAVAILABLE: Final[str] = "fcntl is unavailable" + + def _lock_fd_nonblocking(_fd: int) -> bool: + raise OSError(ENOSYS, _FCNTL_UNAVAILABLE) + + def _unlock_fd(_fd: int) -> None: + raise OSError(ENOSYS, _FCNTL_UNAVAILABLE) + + else: + has_fcntl = True + # Contention errnos for a nonblocking flock. EAGAIN/EWOULDBLOCK are the usual "held elsewhere" codes; some + # filesystems report EACCES instead, so treat it as contention too rather than a permanent error. + _CONTENTION_ERRNOS: Final[frozenset[int]] = frozenset({EACCES, EAGAIN, EWOULDBLOCK}) + + def _lock_fd_nonblocking(fd: int) -> bool: + # One nonblocking exclusive flock attempt shared by UnixFileLock and lock_descriptor, so both contend on + # the same lock and classify errors identically. The caller owns fd; this never closes it. + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError as exception: + if exception.errno in _CONTENTION_ERRNOS: + return False + raise + return True + + def _unlock_fd(fd: int) -> None: + fcntl.flock(fd, fcntl.LOCK_UN) + + def _flock_is_missing(fd: int) -> bool: + try: + _lock_fd_nonblocking(fd) + except OSError as exception: + return exception.errno == ENOSYS + return False # a flock the probe won is released when the caller closes fd + + class UnixFileLock(BaseFileLock): + """ + Uses the :func:`fcntl.flock` to hard lock the lock file on unix systems. + + We leave the lock file in place after release. Unlinking a locked file on Unix splits + waiters across inodes and breaks mutual exclusion for processes that coordinate via the + same path. + """ + + def _acquire(self) -> None: + if (missing_flock := self._acquire_native()) is not None: + fd, exception, created = missing_flock + self._switch_to_soft_lock(fd, exception, created=created) + + def _acquire_native(self) -> tuple[int, OSError, bool] | None: + if (opened := self._open_lock_file()) is None: + return None + fd, created = opened + self._mark_descriptor_pending(fd) + if created: + # Before the flock: a peer may win it first, and only the creator may set the mode. Outside the flock + # handler below: an fchmod that answers ENOSYS says nothing about flock. + try: + self._apply_explicit_mode(fd) + except BaseException: + self._mark_descriptor_released() + os.close(fd) + raise + # One guard covers the flock through the recording of the hold. A signal handler that raises in between + # would otherwise leave an unreferenced descriptor, and its flock, open until the process exits. + locked = False + try: + locked = _lock_fd_nonblocking(fd) + if locked and self._finalize_locked_fd(fd): + return None + except BaseException as exception: + if not locked and isinstance(exception, OSError) and exception.errno == ENOSYS: + return fd, exception, created + # Contention returns False from _lock_fd_nonblocking, so a raise here is a failure. + self._mark_descriptor_released() + os.close(fd) + raise + self._mark_descriptor_released() + os.close(fd) # contention or an unlinked inode; let the retry loop try again + return None + + def _open_lock_file(self) -> tuple[int, bool] | None: + # No O_TRUNC: a contender that loses the flock must not truncate the holder's file (#591), and neither may + # the winner, since a file renamed or hard-linked to the lock path is indistinguishable from a lock file. + # O_EXCL tells whether this attempt created the inode: only then does it get the explicit mode, and the + # ENOSYS fallback removes only its own placeholder. O_NONBLOCK: a lease (Linux F_SETLEASE) on the file would + # otherwise block the open for up to fs.lease-break-time, beyond timeout, blocking=False and cancel_check. + open_flags = os.O_RDWR | os.O_NONBLOCK + if (o_nofollow := getattr(os, "O_NOFOLLOW", None)) is not None: + open_flags |= o_nofollow + open_mode = self._open_mode() + try: + return os.open(self._lock_path, open_flags | os.O_CREAT | os.O_EXCL, open_mode), True + except FileExistsError: + pass + except FileNotFoundError: + # On FUSE/NFS a split LOOKUP + CREATE lets a concurrent unlink() race the create, so ENOENT is transient + # contention. An empty or separator-terminated path (Path() strips the separator) fails this way on + # every attempt on macOS, so re-raise rather than retry forever. + if self._lock_path and not self._lock_path.endswith(os.sep) and Path(self._lock_path).parent.exists(): + return None + raise + except PermissionError: + # Sticky-bit dirs (e.g. /tmp): O_CREAT fails if another user owns the file (#317). + if not Path(self._lock_path).exists(): + raise + return self._open_existing_lock_file(open_flags, open_mode) + + def _open_existing_lock_file(self, open_flags: int, open_mode: int) -> tuple[int, bool] | None: + try: + return os.open(self._lock_path, open_flags, open_mode), False + except FileNotFoundError: + return None # unlinked since the create attempt; let the retry loop start over + except BlockingIOError: + return None # a lease is breaking: contention, so the retry loop keeps the deadline + except PermissionError: + # A marker another account left is readable but not writable. Without flock, waiting on it only needs + # reading it; with flock the lock writes its file, so the denial stands. + try: + fd = os.open(self.lock_file, open_flags & ~os.O_RDWR | os.O_RDONLY) + except (FileNotFoundError, BlockingIOError): + return None # unlinked, or leased: the kernel checks permission before the lease + if _flock_is_missing(fd): + return fd, False + os.close(fd) + raise + + def _switch_to_soft_lock(self, fd: int, missing_flock: OSError, *, created: bool) -> None: + # The filesystem does not implement flock. Fail closed when the caller opted out of existence-lock + # semantics (#603), asked to preserve the pathname (#605), or set an on_acquired hook (#607), none of which + # a soft lock can honor. + falls_back: Final = self._fallback_to_soft and not self._preserve_lock_file and self._on_acquired is None + try: + # A file this attempt did not create may be a peer's live marker, so it stays. Our own placeholder goes + # before the close: the open descriptor keeps its inode number from a peer's marker, and an exchange + # puts back a peer's marker that replaced it. + if created and falls_back: + with suppress(OSError): + remove_own_file(self._lock_path, fd) + finally: + self._mark_descriptor_released() + os.close(fd) + if not falls_back: + raise missing_flock + self._fallback_to_soft_lock() + self._acquire() + + def _finalize_locked_fd(self, fd: int) -> bool: + # Runs with the flock held; the caller closes fd on failure. A concurrent _release() may have unlinked + # the file between our open() and flock(), which st_nlink 0 shows. That lock guards no name, so report + # it and let the retry loop start over. + if (stat_result := os.fstat(fd)).st_nlink == 0: + return False + self._mark_descriptor_owned(fd, (stat_result.st_dev, stat_result.st_ino)) + return True + + def _apply_explicit_mode(self, fd: int) -> None: + if self.has_explicit_mode: + with suppress(PermissionError): + os.fchmod(fd, self._context.mode) + + def _fallback_to_soft_lock(self: BaseFileLock) -> None: + # Import lazily: this runs only on the rare flock fallback, and asyncio imports _unix, so a + # module-level import of it here would cycle. + from filelock._soft import SoftFileLock # ruff:ignore[import-outside-top-level] + + warnings.warn("flock not supported on this filesystem, falling back to SoftFileLock", stacklevel=2) + from filelock.asyncio import AsyncSoftFileLock, BaseAsyncFileLock # ruff:ignore[import-outside-top-level] + + self.__class__ = AsyncSoftFileLock if isinstance(self, BaseAsyncFileLock) else SoftFileLock + + def _release(self) -> None: + fd = cast("int", self._context.lock_file_fd) + # Retain the descriptor until flock succeeds: a failed unlock leaves the kernel lock held, so is_locked + # must keep reporting held for a retry. Once flock commits, clear held state and close as post-unlock + # cleanup; a close failure (EIO on FUSE/Docker bind mounts) does not make the kernel lock held again. + _unlock_fd(fd) + self._mark_descriptor_released() + self._close_released_fd(fd, default_suppresses=True) + + +if sys.platform == "win32": # pragma: win32 cover + __all__ = ["UnixFileLock", "has_fcntl"] +else: # pragma: win32 no cover + __all__ = ["UnixFileLock", "_lock_fd_nonblocking", "_unlock_fd", "has_fcntl"] diff --git a/.venv/lib/python3.14/site-packages/filelock/_util.py b/.venv/lib/python3.14/site-packages/filelock/_util.py new file mode 100644 index 0000000000000000000000000000000000000000..8dbdadd74333d6f20c3ea48ebafbfe175ae98b15 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/filelock/_util.py @@ -0,0 +1,399 @@ +from __future__ import annotations + +__lazy_modules__ = ["pathlib", "platform", "secrets"] + +import errno +import os +import platform +import secrets +import stat +import sys +from contextlib import contextmanager, suppress +from errno import EACCES, EINVAL, EIO, EISDIR, ENOSYS, EXDEV +from functools import cache +from pathlib import Path +from typing import TYPE_CHECKING, Final, cast + +if TYPE_CHECKING: + import ctypes + from collections.abc import Callable, Generator + from contextlib import AbstractContextManager + + +def write_all(fd: int, data: bytes) -> None: + """ + Write the whole buffer to *fd*, looping over the short writes POSIX lets ``os.write`` make. + + A caller that stopped after one short write would leave a truncated marker behind. A peer that reads between two + writes sees a prefix of the record; each caller has to tolerate that. Lock records need not survive a power loss, so + we skip ``fsync``. + + :param fd: file descriptor open for writing. + :param data: bytes to write in full. + + :raises OSError: if a write reports zero progress before the record is complete. + + """ + remaining = memoryview(data) + while remaining: + if (written := os.write(fd, remaining)) == 0: + raise OSError(EIO, "os.write wrote 0 bytes before the record was complete") + remaining = remaining[written:] + + +def raise_on_not_writable_file(filename: str) -> None: + """ + Raise an exception if attempting to open the file for writing would fail. + + Separates files that can never be written from files that are writable but currently locked. + + :param filename: file to check + + :raises OSError: as if the file was opened for writing. + + """ + try: + # lstat, not stat: settles exists-and-writable in one syscall, and a hostile symlink at the lock path would + # make stat inspect the link target, letting an attacker turn a contended acquire into a misleading + # PermissionError / IsADirectoryError and probe that target's attributes. The real open passes O_NOFOLLOW and + # refuses the symlink anyway. + file_stat = os.lstat(filename) + except OSError: + return # does not exist, or an error the caller cannot act on + + if stat.S_ISDIR(file_stat.st_mode): + if sys.platform == "win32": # pragma: win32 cover + raise PermissionError(EACCES, "Permission denied", filename) + raise IsADirectoryError(EISDIR, "Is a directory", filename) # pragma: win32 no cover + + # No mtime guard: the old `if st_mtime != 0` skip covered NFS/Linux quirks where os.lstat returned an all-zero + # struct, which it no longer does. Skipping on mtime 0 let a read-only file or a directory at the lock path pass + # as missing, so acquire() blocked forever on an open that cannot succeed. + # Match open() credentials where supported; group permissions and ACLs can grant access without S_IWUSR. Do not + # follow a symlink, as the lstat above does not: one swapped in since would be judged by its target. + if not (file_stat.st_mode & stat.S_IWUSR) and not os.access( + filename, + os.W_OK, + effective_ids=os.access in os.supports_effective_ids, + follow_symlinks=os.access not in os.supports_follow_symlinks, + ): + try: + os.lstat(filename) + except FileNotFoundError: + # A holder may unlink its marker between lstat() and access(); let the caller attempt creation. + return + raise PermissionError(EACCES, "Permission denied", filename) + + +def ensure_directory_exists(filename: Path | str) -> None: + """ + Ensure the directory containing the file exists (create it if necessary). + + :param filename: file. + + """ + Path(filename).parent.mkdir(parents=True, exist_ok=True) + + +@contextmanager +def pin_inode(lock_file: str) -> Generator[int | None]: + """ + Keep the file at *lock_file* allocated for the length of the block and yield its inode number. + + An inode is the kernel's identity for a file, independent of its name, and its number names a file only while the + inode stays allocated. ext4 hands a freed number to the next file created. Once a stale marker is unlinked, the next + holder's marker can therefore carry the number a contender read a moment ago. A caller that reads the marker inside + the block, gives up unless it read the yielded inode, and breaks the marker before leaving can rely on the number. + + :param lock_file: path to the lock file about to be read and perhaps broken. + + :returns: the pinned inode number, or ``None`` where nothing can be pinned. Windows cannot rename a file this + process holds open, so it needs no pin. Without ``O_PATH`` a symlink or a socket cannot be opened. + + :raises FileNotFoundError: if the path is gone. + + """ + if sys.platform == "win32": # pragma: win32 cover + yield None + else: # pragma: win32 no cover + # O_PATH pins any node, a symlink or socket included, without read permission and without the filesystem seeing + # an open; elsewhere a nonblocking read-only open pins what it can. + flags: Final = getattr(os, "O_PATH", os.O_RDONLY | getattr(os, "O_NONBLOCK", 0)) | getattr(os, "O_NOFOLLOW", 0) + try: + fd = os.open(lock_file, flags) + except FileNotFoundError: + raise + except OSError: + yield None + return + try: + yield os.fstat(fd).st_ino + finally: + os.close(fd) + + +def remove_own_file(lock_file: str, fd: int) -> None: # pragma: win32 no cover + """ + Remove *lock_file* while it still names the file open as *fd*; the caller closes the descriptor afterwards. + + Where the platform has an atomic exchange, a stale break takes the file through it, so a peer's marker found at the + path goes back. Otherwise the path is unlinked by name if it still names this file. The open descriptor keeps the + inode allocated, so a matching number cannot belong to a successor's marker. + + :raises OSError: if the descriptor cannot be read or the unlink fails. + + """ + own: Final = os.fstat(fd) + with suppress(FileNotFoundError): + break_lock_file(lock_file, own.st_mtime, own.st_ino) + if os.path.samestat(os.lstat(lock_file), own): + Path(lock_file).unlink() + + +def foreign_directory_owner(directory: os.stat_result, parent: os.stat_result) -> int | None: + """ + Return the uid that owns *directory* when another account controls it under a sticky parent. + + A lock that keeps its ownership records in a directory relies on them staying in place. The sticky bit (a directory + flag that lets only a file's owner remove it) on a parent such as ``/tmp`` stops other accounts from renaming the + directory away. A directory that this account or the parent's owner owns is therefore safe there. One that another + account owns is outside the lock's control. Without the sticky bit, any process that can create the directory can + also rename this account's own one away, so a refusal there would change nothing. Linux applies the same rule for + ``fs.protected_regular``. + + :param directory: ``lstat`` of the directory found in place. + :param parent: ``stat`` of the directory that holds it. + + :returns: the owning uid when another account controls the directory, otherwise ``None``. + + """ + if sys.platform != "win32" and parent.st_mode & stat.S_ISVTX: # pragma: needs foreign-sticky-tmp + return None if directory.st_uid in {os.geteuid(), parent.st_uid} else directory.st_uid + return None + + +def break_lock_file(lock_file: str, mtime_before: float, ino_before: int) -> None: + """ + Remove the lock file a caller found stale with modification time *mtime_before* and inode *ino_before*. + + The caller keeps the judged file open until we return, through :func:`pin_inode` or its own descriptor, so no + other file can carry *ino_before* meanwhile. Right before breaking we ``lstat`` the lock again and leave it alone + if a peer recreated it since the caller judged it stale, as Mercurial re-reads a lock under its break lock. + + On Linux (``renameat2``) and macOS (``renamex_np``) the platform can exchange two names in one step. There we swap + the marker with a placeholder of our own, so the lock path never stands empty, and a contender that polls meanwhile + finds a file it cannot parse and waits. If the file we took is not the one judged stale, a peer recreated the lock, + and a second exchange puts it back. Otherwise we remove it and then our placeholder. Of several processes racing to + break the same lock, only one takes the stale file. + + Windows cannot exchange names, but it refuses to rename a marker its live holder keeps open. There we rename the + marker aside and, if it was recreated after all, link it back. ``link`` never replaces a name, so a marker a third + process created meanwhile keeps the path. Elsewhere without an exchange (NFS, SMB and other filesystems that refuse + it, platforms without the call) a rename would leave the path empty beside a possibly live holder. We then do not + move the marker, and the caller waits for ``timeout`` or ``break_lock()``. + + We compare inodes as well as modification times, because NFS and FAT store coarse modification times. The times are + compared for equality, since a file's owner can set any time. We ``lstat`` so a symlink a peer swaps in is never + followed. The break name carries a random token so no other process can guess it. + + :param lock_file: path to the lock file to break. + :param mtime_before: modification time the caller saw when it judged the lock stale. + :param ino_before: inode number the caller saw when it judged the lock stale. + + :raises OSError: if taking the marker fails, for example because the file vanished, another user owns it in a + sticky directory, or on Windows its holder keeps it open. + + """ + if (current := os.lstat(lock_file)).st_mtime != mtime_before or current.st_ino != ino_before: + return + break_path: Final[str] = f"{lock_file}.break.{os.getpid()}.{secrets.token_hex(16)}" + if exchange_supported(): # pragma: win32 no cover + _swap_out(lock_file, break_path, mtime_before, ino_before) + elif _MOVES_WITHOUT_EXCHANGE: # pragma: win32 cover + _break_by_rename(lock_file, break_path, mtime_before, ino_before) + + +def _swap_out(lock_file: str, break_path: str, mtime_before: float, ino_before: int) -> None: # pragma: win32 no cover + # An atomic exchange with a placeholder takes the marker without the lock path ever being empty. No contender can + # then create a marker while this one is judged, and a live marker goes back the same way. + fd: Final = os.open(break_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0), 0o600) + try: + try: + # Not a record any lock writes, so a contender that reads it waits, and one left by a crash ages out. The + # PID tells the contender that evicts it whose break names to search for a marker this process left. + write_all(fd, f"filelock/break\n{os.getpid()}\n".encode()) + exchanged = exchange_names(break_path, lock_file) + except OSError: + exchanged = False + if not exchanged: + Path(break_path).unlink() + return + if (grabbed := os.lstat(break_path)).st_mtime == mtime_before and grabbed.st_ino == ino_before: + Path(break_path).unlink() + # Our descriptor keeps the placeholder's inode allocated, so a matching number at the path is still ours. + if os.path.samestat(os.lstat(lock_file), os.fstat(fd)): + Path(lock_file).unlink() + else: + exchange_names(break_path, lock_file) + Path(break_path).unlink() + finally: + os.close(fd) + + +def _break_by_rename( + lock_file: str, break_path: str, mtime_before: float, ino_before: int +) -> None: # pragma: win32 cover + # Renaming the marker aside empties the path until a recreated live marker is put back. link never replaces a name, + # so a marker a third process created meanwhile keeps the path, and the live one waits under the break name. + # Without os.link, or where the filesystem refuses hard links, the live one waits there too. + with _pinned_for_rename(lock_file): + Path(lock_file).rename(break_path) + if (st_after := os.lstat(break_path)).st_mtime == mtime_before and st_after.st_ino == ino_before: + unlink_path(break_path) + elif stat.S_ISREG(st_after.st_mode) and hasattr(os, "link"): # pragma: needs hard-link + with suppress(OSError): + os.link(break_path, lock_file, follow_symlinks=False) + unlink_path(break_path) + + +if sys.platform == "win32": # pragma: win32 cover + + def _pinned_for_rename(path: str) -> AbstractContextManager[object]: + # A rename by path waits on a Level 1 or batch oplock holder until it answers or closes. Opening first with + # FILE_COMPLETE_IF_OPLOCKED turns that wait into BlockingIOError, and the open handle shares delete, so the + # rename then proceeds. Imported here because _windows imports this module. + from filelock._windows import open_for_delete_without_oplock_wait # ruff:ignore[import-outside-top-level] + + return os.fdopen(open_for_delete_without_oplock_wait(path), "rb") + + def unlink_path(path: str) -> None: + from filelock._windows import unlink_without_oplock_wait # ruff:ignore[import-outside-top-level] + + unlink_without_oplock_wait(path) + +else: # pragma: win32 no cover + + def unlink_path(path: str) -> None: + Path(path).unlink() + + +def exchange_supported() -> bool: + """Whether this platform has a call that swaps two names in one step; a filesystem may still refuse it.""" + return _rename_call() is not None + + +def exchange_names(source: str, target: str) -> bool: # pragma: win32 no cover + """ + Swap the names *source* and *target* in one step, so neither ever stands empty, where :func:`exchange_supported`. + + :returns: ``False`` where the filesystem cannot, so the caller falls back + + :raises OSError: for any other failure, such as a missing name or a denied rename + + """ + renamer, exchange_flag = cast("tuple[Callable[[bytes, bytes, int], int], int]", _rename_call()) + if not (error := renamer(os.fsencode(source), os.fsencode(target), exchange_flag)): + return True + # EXDEV: overlayfs and some FUSE filesystems refuse an exchange between layers. + if error in {EINVAL, ENOSYS, EXDEV, getattr(errno, "ENOTSUP", EINVAL), getattr(errno, "EOPNOTSUPP", EINVAL)}: + return False + raise OSError(error, os.strerror(error), source, None, target) + + +@cache +def _rename_call() -> tuple[Callable[[bytes, bytes, int], int], int] | None: + # The call returns 0 or the errno. Looked up on first use, since most processes never break a lock and ctypes is + # slow to import. None where ctypes is missing (GraalPy) or the platform has no exchange. + try: + import ctypes # ruff:ignore[import-outside-top-level] # only a stale break needs it + except ImportError: # pragma: no cover # GraalPy ships without ctypes + return None + if sys.platform in {"linux", "android"}: # pragma: linux cover + if (renameat2 := _renameat2(ctypes.CDLL(None, use_errno=True))) is None: + return None # pragma: no cover # an architecture the CI matrix does not run + return renameat2, 2 # RENAME_EXCHANGE in linux/fs.h + if sys.platform == "darwin": # pragma: darwin cover + renamex_np: Final = ctypes.CDLL(None, use_errno=True).renamex_np + renamex_np.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_uint] + renamex_np.restype = ctypes.c_int + + def call_renamex_np(source: bytes, target: bytes, flags: int) -> int: + return 0 if renamex_np(source, target, flags) == 0 else ctypes.get_errno() + + return call_renamex_np, 0x2 # RENAME_SWAP in sys/stdio.h + return None # pragma: linux no cover # pragma: android no cover # pragma: darwin no cover + + +def _renameat2(libc: ctypes.CDLL) -> Callable[[bytes, bytes, int], int] | None: # pragma: linux cover + # Reached through syscall(2) rather than the C library's wrapper. glibc 2.28, musl 1.2.2 and bionic API 30 added + # the wrapper, and Termux ships an older bionic; the kernel has had the call since 3.15. The number is the + # architecture's own, so an architecture outside the table gets no exchange rather than another call. + import ctypes # ruff:ignore[import-outside-top-level] # only a stale break needs it + + numbers: Final = { + "x86_64": 316, + "i386": 353, + "i686": 353, + "armv7l": 382, + "armv8l": 382, + "aarch64": 276, + "riscv64": 276, + "loongarch64": 276, + "ppc64le": 357, + "s390x": 347, + } + if (number := numbers.get(platform.machine())) is None: + return None # pragma: no cover # an architecture the CI matrix does not run + syscall: Final = libc.syscall + syscall.restype = ctypes.c_long + at_fdcwd: Final = -100 + + def call_renameat2(source: bytes, target: bytes, flags: int) -> int: + arguments: Final = ( + ctypes.c_long(number), + ctypes.c_int(at_fdcwd), + ctypes.c_char_p(source), + ctypes.c_int(at_fdcwd), + ctypes.c_char_p(target), + ctypes.c_uint(flags), + ) + return 0 if syscall(*arguments) == 0 else ctypes.get_errno() + + return call_renameat2 + + +def touch(name: str, *, fd: int) -> None: + # Prefer the already-open, already-verified fd so a peer that swaps a symlink or a different file in at the + # path after our O_NOFOLLOW read cannot redirect the touch: utime then targets the inode behind the fd. + # Where the platform cannot utime an fd, fall back to a path-based touch that still refuses to follow a + # symlink where supported, matching the O_NOFOLLOW reads used elsewhere here. + if _SUPPORTS_UTIME_FD: # pragma: needs utime-fd + os.utime(fd, None) + return + os.utime(name, None, follow_symlinks=not _SUPPORTS_UTIME_NOFOLLOW) # pragma: lacks utime-fd + + +# Windows refuses to rename a marker its live holder keeps open, so renaming aside only moves a marker nobody holds. +# Elsewhere without an exchange a contender could take the emptied path beside a live holder, so the marker stays and +# the lock counts as held. +_MOVES_WITHOUT_EXCHANGE: Final[bool] = sys.platform == "win32" +# Retargeting os.utime to an open fd lets a heartbeat refresh the exact inode it verified instead of whatever the +# pathname now names. +_SUPPORTS_UTIME_FD: Final[bool] = sys.platform != "win32" and os.utime in os.supports_fd +# os.utime follows symlinks unless told not to; not every platform can refuse the follow, so probe support. +_SUPPORTS_UTIME_NOFOLLOW: Final[bool] = os.utime in os.supports_follow_symlinks + + +__all__ = [ + "break_lock_file", + "ensure_directory_exists", + "exchange_names", + "exchange_supported", + "foreign_directory_owner", + "pin_inode", + "raise_on_not_writable_file", + "remove_own_file", + "touch", + "unlink_path", + "write_all", +] diff --git a/.venv/lib/python3.14/site-packages/filelock/_windows.py b/.venv/lib/python3.14/site-packages/filelock/_windows.py new file mode 100644 index 0000000000000000000000000000000000000000..0258961a7ec86ae6fd2938890eab949e9a783a6c --- /dev/null +++ b/.venv/lib/python3.14/site-packages/filelock/_windows.py @@ -0,0 +1,780 @@ +from __future__ import annotations + +__lazy_modules__ = ["filelock._util", "msvcrt", "struct"] + +import os +import struct +import sys +import time +from contextlib import contextmanager, suppress +from errno import ENOENT, ENOTDIR, EWOULDBLOCK +from functools import cache +from typing import TYPE_CHECKING, Final, cast + +from filelock._api import BaseFileLock +from filelock._util import raise_on_not_writable_file + +if TYPE_CHECKING: + from collections.abc import Generator + +if sys.platform == "win32": # pragma: win32 cover + import ctypes + import msvcrt + from ctypes import wintypes + + _GENERIC_READ: Final[int] = 0x80000000 + _GENERIC_WRITE: Final[int] = 0x40000000 + _SYNCHRONIZE: Final[int] = 0x00100000 + _DELETE: Final[int] = 0x00010000 + _DESIRED_ACCESS: Final[int] = _GENERIC_READ | _GENERIC_WRITE | _SYNCHRONIZE + _FILE_SHARE_READ_WRITE: Final[int] = ( + 0x00000001 | 0x00000002 + ) # read | write; matches os.open (_SH_DENYNO), no delete + _FILE_OPEN: Final[int] = 1 # open the file only if it exists; the NtCreateFile OPEN_EXISTING + _FILE_OPEN_IF: Final[int] = 3 # open the file if it exists, create it otherwise; the NtCreateFile OPEN_ALWAYS + # Attribute-only access skips share-mode checks, so the probe sees a name's deletion state rather than a reader. + _FILE_READ_ATTRIBUTES: Final[int] = 0x00000080 + _FILE_SHARE_ALL: Final[int] = 0x00000001 | 0x00000002 | 0x00000004 # read | write | delete + _FILE_ATTRIBUTE_READONLY: Final[int] = 0x00000001 + _FILE_ATTRIBUTE_NORMAL: Final[int] = 0x00000080 + _FILE_ATTRIBUTE_REPARSE_POINT: Final[int] = 0x00000400 + # CreateOptions: keep the handle synchronous (the CRT and msvcrt.locking rely on a maintained file position), + # refuse a directory, return at once instead of waiting for an oplock holder to answer the break, and open a + # reparse point rather than following it so the check below acts on the link itself. + _FILE_SYNCHRONOUS_IO_NONALERT: Final[int] = 0x00000020 + _FILE_NON_DIRECTORY_FILE: Final[int] = 0x00000040 + _FILE_DELETE_ON_CLOSE: Final[int] = 0x00001000 + _FILE_COMPLETE_IF_OPLOCKED: Final[int] = 0x00000100 + _FILE_OPEN_REPARSE_POINT: Final[int] = 0x00200000 + _CREATE_OPTIONS: Final[int] = ( + _FILE_SYNCHRONOUS_IO_NONALERT | _FILE_NON_DIRECTORY_FILE | _FILE_COMPLETE_IF_OPLOCKED | _FILE_OPEN_REPARSE_POINT + ) + # Listing access takes part in share-mode checks, unlike attribute-only access, so a handle that holds it without + # FILE_SHARE_DELETE blocks renames and removals of the directory until it closes. + _FILE_LIST_DIRECTORY: Final[int] = 0x00000001 + _FILE_DIRECTORY_FILE: Final[int] = 0x00000001 + _DIRECTORY_OPTIONS: Final[int] = _FILE_SYNCHRONOUS_IO_NONALERT | _FILE_DIRECTORY_FILE | _FILE_OPEN_REPARSE_POINT + _OBJ_CASE_INSENSITIVE: Final[int] = 0x00000040 # Win32 name lookups are case-insensitive + _OWNER_WRITE: Final[int] = 0o200 + _GROUP_AND_OTHERS: Final[int] = 0o077 + # The DACL os.mkdir(mode=0o700) applies, less the inheritance flags a file has no use for: protected, so the + # directory's inheritable entries stay out, with full access for SYSTEM, Administrators and the owner. + _OWNER_ONLY_SDDL: Final[str] = "D:P(A;;FA;;;SY)(A;;FA;;;BA)(A;;FA;;;OW)" + _SDDL_REVISION_1: Final[int] = 1 + _INVALID_HANDLE_VALUE: Final[int | None] = ctypes.c_void_p(-1).value + + # LockFileEx locks a byte range at an offset carried in OVERLAPPED, independent of the descriptor's file position. + # msvcrt.locking starts at the current position instead, so a metadata write between lock and unlock could shift + # the byte a later unlock targets; the explicit offset removes that hazard for both the path lock and #608's + # descriptor lock. + _LOCKFILE_FAIL_IMMEDIATELY: Final[int] = 0x00000001 + _LOCKFILE_EXCLUSIVE_LOCK: Final[int] = 0x00000002 + _ERROR_ACCESS_DENIED: Final[int] = 5 + _ERROR_LOCK_VIOLATION: Final[int] = 33 # another handle holds the byte range + _ERROR_NOT_LOCKED: Final[int] = 158 # the byte range is not locked + + # NtCreateFile returns the raw NTSTATUS as its value, where CreateFileW collapses several of these into one + # ERROR_ACCESS_DENIED. Telling them apart is the point (#604): a name pending deletion or a share conflict is + # transient and worth a retry, a real access denial is not. + _STATUS_SUCCESS: Final[int] = 0x00000000 + # A success code that comes with a handle: another handle's oplock is breaking and its holder has not answered yet. + _STATUS_OPLOCK_BREAK_IN_PROGRESS: Final[int] = 0x00000108 + _STATUS_ACCESS_DENIED: Final[int] = 0xC0000022 + _STATUS_OBJECT_NAME_NOT_FOUND: Final[int] = 0xC0000034 + _STATUS_OBJECT_PATH_NOT_FOUND: Final[int] = 0xC000003A + _STATUS_SHARING_VIOLATION: Final[int] = 0xC0000043 + _STATUS_DELETE_PENDING: Final[int] = 0xC0000056 + #: Probes, 2 ms apart, for transient STATUS_ACCESS_DENIED while a peer unlinks the name (#604). A count ensures + #: every probe runs even when the process stalls; oplock breaks use the caller's wait instead. + _ACCESS_DENIED_PROBES: Final[int] = 250 + _ACCESS_DENIED_GRACE: Final[float] = 0.5 + _ACCESS_DENIED_RETRY: Final[float] = 0.002 + + _ntdll: Final[ctypes.WinDLL] = ctypes.WinDLL("ntdll") + _kernel32: Final[ctypes.WinDLL] = ctypes.WinDLL("kernel32", use_last_error=True) + _advapi32: Final[ctypes.WinDLL] = ctypes.WinDLL("advapi32", use_last_error=True) + + class _UNICODE_STRING(ctypes.Structure): # ruff:ignore[invalid-class-name] # mirrors the Win32 struct name + _fields_ = ( + ("Length", wintypes.USHORT), # byte length, not character count + ("MaximumLength", wintypes.USHORT), + ("Buffer", wintypes.LPWSTR), + ) + + class _OBJECT_ATTRIBUTES(ctypes.Structure): # ruff:ignore[invalid-class-name] # mirrors the Win32 struct name + _fields_ = ( + ("Length", wintypes.ULONG), + ("RootDirectory", wintypes.HANDLE), + ("ObjectName", ctypes.POINTER(_UNICODE_STRING)), + ("Attributes", wintypes.ULONG), + ("SecurityDescriptor", ctypes.c_void_p), + ("SecurityQualityOfService", ctypes.c_void_p), + ) + + class _IO_STATUS_BLOCK(ctypes.Structure): # ruff:ignore[invalid-class-name] # mirrors the Win32 struct name + _fields_ = ( + ("Status", ctypes.c_void_p), # a union of NTSTATUS and PVOID, so it is pointer-sized + ("Information", ctypes.c_void_p), + ) + + class _SECURITY_ATTRIBUTES(ctypes.Structure): # ruff:ignore[invalid-class-name] # mirrors the Win32 struct name + _fields_ = ( + ("nLength", wintypes.DWORD), + ("lpSecurityDescriptor", ctypes.c_void_p), + ("bInheritHandle", wintypes.BOOL), + ) + + class _OVERLAPPED(ctypes.Structure): # mirrors the Win32 struct name + _fields_ = ( + ("Internal", ctypes.c_void_p), # ULONG_PTR: pointer-sized, not DWORD, or the x64 layout corrupts Offset + ("InternalHigh", ctypes.c_void_p), + ("Offset", wintypes.DWORD), # the DUMMYUNIONNAME struct, flattened: low 32 bits of the byte offset + ("OffsetHigh", wintypes.DWORD), + ("hEvent", wintypes.HANDLE), + ) + + class _BY_HANDLE_FILE_INFORMATION(ctypes.Structure): # ruff:ignore[invalid-class-name] # mirrors the Win32 struct name + _fields_ = ( + ("dwFileAttributes", wintypes.DWORD), + ("ftCreationTime", wintypes.FILETIME), + ("ftLastAccessTime", wintypes.FILETIME), + ("ftLastWriteTime", wintypes.FILETIME), + ("dwVolumeSerialNumber", wintypes.DWORD), + ("nFileSizeHigh", wintypes.DWORD), + ("nFileSizeLow", wintypes.DWORD), + ("nNumberOfLinks", wintypes.DWORD), + ("nFileIndexHigh", wintypes.DWORD), + ("nFileIndexLow", wintypes.DWORD), + ) + + _ntdll.NtCreateFile.restype = wintypes.LONG # NTSTATUS + _ntdll.NtCreateFile.argtypes = [ + ctypes.POINTER(wintypes.HANDLE), + wintypes.DWORD, + ctypes.POINTER(_OBJECT_ATTRIBUTES), + ctypes.POINTER(_IO_STATUS_BLOCK), + ctypes.POINTER(ctypes.c_longlong), # PLARGE_INTEGER AllocationSize + wintypes.ULONG, + wintypes.ULONG, + wintypes.ULONG, + wintypes.ULONG, + ctypes.c_void_p, + wintypes.ULONG, + ] + _ntdll.RtlDosPathNameToNtPathName_U_WithStatus.restype = wintypes.LONG # NTSTATUS + _ntdll.RtlDosPathNameToNtPathName_U_WithStatus.argtypes = [ + wintypes.LPCWSTR, + ctypes.POINTER(_UNICODE_STRING), + ctypes.c_void_p, + ctypes.c_void_p, + ] + _ntdll.RtlFreeUnicodeString.restype = None + _ntdll.RtlFreeUnicodeString.argtypes = [ctypes.POINTER(_UNICODE_STRING)] + _ntdll.RtlNtStatusToDosError.restype = wintypes.ULONG + _ntdll.RtlNtStatusToDosError.argtypes = [wintypes.LONG] + + _kernel32.CloseHandle.argtypes = [wintypes.HANDLE] + _kernel32.CloseHandle.restype = wintypes.BOOL + _kernel32.GetFileInformationByHandle.argtypes = [wintypes.HANDLE, ctypes.POINTER(_BY_HANDLE_FILE_INFORMATION)] + _kernel32.GetFileInformationByHandle.restype = wintypes.BOOL + _kernel32.LockFileEx.argtypes = [ + wintypes.HANDLE, + wintypes.DWORD, + wintypes.DWORD, + wintypes.DWORD, + wintypes.DWORD, + ctypes.POINTER(_OVERLAPPED), + ] + _kernel32.LockFileEx.restype = wintypes.BOOL + _kernel32.UnlockFileEx.argtypes = [ + wintypes.HANDLE, + wintypes.DWORD, + wintypes.DWORD, + wintypes.DWORD, + ctypes.POINTER(_OVERLAPPED), + ] + _kernel32.UnlockFileEx.restype = wintypes.BOOL + _advapi32.ConvertStringSecurityDescriptorToSecurityDescriptorW.argtypes = [ + wintypes.LPCWSTR, + wintypes.DWORD, + ctypes.POINTER(ctypes.c_void_p), + ctypes.POINTER(wintypes.ULONG), + ] + _advapi32.ConvertStringSecurityDescriptorToSecurityDescriptorW.restype = wintypes.BOOL + _advapi32.GetNamedSecurityInfoW.argtypes = [ + wintypes.LPCWSTR, + ctypes.c_int, + wintypes.DWORD, + ctypes.POINTER(ctypes.c_void_p), + ctypes.POINTER(ctypes.c_void_p), + ctypes.POINTER(ctypes.c_void_p), + ctypes.POINTER(ctypes.c_void_p), + ctypes.POINTER(ctypes.c_void_p), + ] + _advapi32.GetNamedSecurityInfoW.restype = wintypes.DWORD + _advapi32.OpenThreadToken.argtypes = [ + wintypes.HANDLE, + wintypes.DWORD, + wintypes.BOOL, + ctypes.POINTER(wintypes.HANDLE), + ] + _advapi32.OpenThreadToken.restype = wintypes.BOOL + _advapi32.OpenProcessToken.argtypes = [wintypes.HANDLE, wintypes.DWORD, ctypes.POINTER(wintypes.HANDLE)] + _advapi32.OpenProcessToken.restype = wintypes.BOOL + _advapi32.GetTokenInformation.argtypes = [ + wintypes.HANDLE, + ctypes.c_int, + ctypes.c_void_p, + wintypes.DWORD, + ctypes.POINTER(wintypes.DWORD), + ] + _advapi32.GetTokenInformation.restype = wintypes.BOOL + _kernel32.GetCurrentProcess.restype = wintypes.HANDLE + _kernel32.GetCurrentThread.restype = wintypes.HANDLE + _kernel32.LocalFree.argtypes = [ctypes.c_void_p] + _kernel32.LocalFree.restype = ctypes.c_void_p + _kernel32.CreateDirectoryW.argtypes = [wintypes.LPCWSTR, ctypes.POINTER(_SECURITY_ATTRIBUTES)] + _kernel32.CreateDirectoryW.restype = wintypes.BOOL + _kernel32.CreateFileW.argtypes = [ + wintypes.LPCWSTR, + wintypes.DWORD, + wintypes.DWORD, + ctypes.c_void_p, + wintypes.DWORD, + wintypes.DWORD, + wintypes.HANDLE, + ] + _kernel32.CreateFileW.restype = ctypes.c_void_p + + def _lock_fd_nonblocking(fd: int) -> bool: + # One nonblocking exclusive LockFileEx attempt shared by WindowsFileLock and lock_descriptor, over the one-byte + # range at offset 0. True on acquisition, False on contention, raise otherwise. The caller owns fd; the handle + # from get_osfhandle belongs to the CRT descriptor and must not be closed here. + overlapped = _OVERLAPPED() # zero-initialized, so Offset/OffsetHigh/hEvent are 0 + flags = _LOCKFILE_EXCLUSIVE_LOCK | _LOCKFILE_FAIL_IMMEDIATELY + if _kernel32.LockFileEx(msvcrt.get_osfhandle(fd), flags, 0, 1, 0, ctypes.byref(overlapped)): + return True + err = ctypes.get_last_error() + if err == _ERROR_LOCK_VIOLATION: + return False + # A non-contention LockFileEx failure is not reproducible in-process. + raise ctypes.WinError(err) # pragma: no cover + + def _unlock_fd(fd: int) -> None: + overlapped = _OVERLAPPED() # the same offset 0 and one-byte length the lock used + if ( # pragma: no cover # only an interrupted earlier release leaves our own range unlocked + not _kernel32.UnlockFileEx(msvcrt.get_osfhandle(fd), 0, 1, 0, ctypes.byref(overlapped)) + and (err := ctypes.get_last_error()) != _ERROR_NOT_LOCKED + ): + raise ctypes.WinError(err) + + class WindowsFileLock(BaseFileLock): + """ + Uses ``LockFileEx`` to hard lock a byte range of the lock file on Windows systems. + + Lock file cleanup: Windows attempts to delete the lock file after release, but deletion is + not guaranteed in multi-threaded scenarios where another thread holds an open handle. The lock + file may persist on disk, which does not affect lock correctness. + """ + + def _acquire(self) -> None: + raise_on_not_writable_file(self._lock_path) + + # The reparse test is bound to the opened handle, so a symlink or junction swapped in cannot defeat it + # through a check-then-open TOCTOU race. + try: + fd = _open_non_reparse_fd(self._lock_path, self._open_mode()) + except BlockingIOError: + return # an oplock holder has not answered its break; let the retry loop keep its deadline + if fd is None: + return # open contention (share conflict or a name pending deletion); let the retry loop try again + try: + locked = _lock_fd_nonblocking(fd) + if locked: + self._mark_descriptor_owned(fd) + except BaseException: + # Closing drops the byte-range lock, so a hold already recorded for this descriptor must go with it. + self._mark_descriptor_released() + os.close(fd) + raise + if not locked: + os.close(fd) # another holder owns the byte-range lock; let the retry loop try again + + def _release(self) -> None: + fd = cast("int", self._context.lock_file_fd) + # Retain the descriptor until the OS unlock succeeds: if UnlockFileEx raises, the byte-range lock is still + # held, so is_locked must keep reporting held rather than losing the fd. Only after the unlock commits do + # close and unlink run as post-unlock cleanup; their failure cannot make the lock held again. + _unlock_fd(fd) + self._mark_descriptor_released() + self._close_released_fd(fd, default_suppresses=False) + if not self._preserve_lock_file: # preserve_lock_file keeps a stable file identity for the caller (#605) + with suppress(OSError): + unlink_without_oplock_wait(self._lock_path) + + def _open_non_reparse_fd(path: str, mode: int, *, oplock_wait: float = _ACCESS_DENIED_GRACE) -> int | None: + """ + Open *path* for locking while refusing reparse points, bound to the handle actually locked. + + The file is opened through ``NtCreateFile`` with ``FILE_OPEN_REPARSE_POINT`` so a symlink or junction planted + at the path is not followed, and the reparse decision is read from *that* handle via + ``GetFileInformationByHandle`` rather than from a prior pathname query. Reading the held handle closes the + check-then-open race: an attacker cannot swap the path between validation and use because both act on the same + handle. Share mode omits delete so a peer cannot unlink or rename the file out from under a live holder, + matching ``os.open``'s ``_SH_DENYNO``. + + ``NtCreateFile`` is used instead of ``CreateFileW`` because its return value carries the raw ``NTSTATUS``. + Windows collapses a transient delete-pending name and a permanent access denial into the same Win32 + ``ERROR_ACCESS_DENIED``; the status keeps them apart, so a real denial fails fast instead of spinning until the + caller's timeout (#604). + + The reparse option only guards the final path component; Windows still follows reparse points in intermediate + directories. This assumes the lock file sits in a lock directory untrusted users cannot modify. A path with + attacker-controlled parent directories would need component-by-component handle validation. + + :param path: the lock file path. + :param mode: the permission mode; as ``os.open`` does on Windows, a cleared owner-write bit creates the file + read-only. A mode without group and other bits creates it with an owner-only DACL (discretionary access + control list, the Windows permission list); any other mode has no Windows counterpart and leaves the file + the entries its directory hands down. Both only take effect when the file is created, not when an existing + one is opened. + + :param oplock_wait: seconds to wait for the holder of an oplock on the file to answer its break. + + :returns: a file descriptor owning the opened handle, or ``None`` on a sharing violation or a delete-pending + name, which the caller should treat as contention and retry. + + :raises BlockingIOError: if an oplock break stays unanswered for *oplock_wait* seconds. + :raises OSError: if the path resolves to a reparse point, or the open fails for any other reason, raised with + the Win32 error the status maps to. An access denial is raised only once it outlasts a run of probes, since + a peer unlinking the file as it releases can make the open answer ``STATUS_ACCESS_DENIED`` for a moment. + + """ + # Emit the audit event os.open would, so consumers watching "open" still see the path-level open and can veto. + sys.audit("open", path, None, os.O_RDWR | os.O_CREAT) + read_only = not mode & _OWNER_WRITE + owner_only = not mode & 0o077 + handle, status = _nt_open(path, read_only=read_only, owner_only=owner_only) + denied_probes = 0 + oplock_deadline = time.monotonic() + oplock_wait + while True: + if status == _STATUS_ACCESS_DENIED: + if denied_probes >= _ACCESS_DENIED_PROBES: + break + denied_probes += 1 + elif status == _STATUS_OPLOCK_BREAK_IN_PROGRESS: + if time.monotonic() >= oplock_deadline: + break + else: + break + time.sleep(_ACCESS_DENIED_RETRY) + handle, status = _nt_open(path, read_only=read_only, owner_only=owner_only) + if status == _STATUS_OPLOCK_BREAK_IN_PROGRESS: + msg = "oplock break in progress" + raise BlockingIOError(EWOULDBLOCK, msg, path) + if status != _STATUS_SUCCESS: + if status in {_STATUS_SHARING_VIOLATION, _STATUS_DELETE_PENDING}: + return None + raise _status_error(status, path) + + return _descriptor_for_non_reparse(handle, path, os.O_RDWR) + + def open_read_without_oplock_wait(path: str) -> int: + """ + Open *path* read-only and return a descriptor, without waiting for an oplock holder to answer its break. + + ``os.open`` waits inside the system call until the holder of an oplock on the file answers or closes its handle, + so no timeout or ``blocking=False`` ends the wait. A reparse point at *path* is refused rather than followed. + + :param path: the file to read. + + :returns: a read-only file descriptor. + + :raises BlockingIOError: if an oplock break is in progress, which callers treat as contention. + :raises FileNotFoundError: if *path* does not exist. + :raises OSError: if the open fails for any other reason, or *path* is a reparse point. + + """ + sys.audit("open", path, None, os.O_RDONLY) + return _open_existing_without_oplock_wait(path, _GENERIC_READ | _SYNCHRONIZE, os.O_RDONLY) + + def open_for_delete_without_oplock_wait(path: str) -> int: + """ + Open *path* with delete access and return a descriptor, without waiting for an oplock holder to answer. + + The open handle shares read, write and delete, so a rename of *path* can proceed while the caller keeps it. + A holder that took an oplock through another handle is asked to answer by this open, which then raises, so the + caller skips the rename instead of waiting for it. + + :param path: the file to rename. + + :returns: a file descriptor owning the handle; the caller closes it. + + :raises BlockingIOError: if an oplock break is in progress, which callers treat as contention. + :raises FileNotFoundError: if *path* does not exist. + :raises OSError: if the open fails for any other reason, or *path* is a reparse point. + + """ + return _open_existing_without_oplock_wait(path, _DELETE | _FILE_READ_ATTRIBUTES | _SYNCHRONIZE, os.O_RDONLY) + + def unlink_without_oplock_wait(path: str) -> None: + """ + Delete *path* through a handle that does not wait for an oplock holder to answer its break. + + ``os.unlink`` waits like ``os.open`` does. Here an unanswered break leaves the file in place, which callers + accept since deleting a lock file is best effort. A symlink at *path* is deleted as a link. + + :param path: the file to delete. + + :raises FileNotFoundError: if *path* does not exist. + :raises OSError: if the open fails for any other reason, such as a sharing violation. + + """ + sys.audit("os.remove", path, None) + handle, status = _nt_create( + path, + _DELETE | _SYNCHRONIZE, + _FILE_ATTRIBUTE_NORMAL, + _FILE_SHARE_ALL, + _FILE_OPEN, + options=_CREATE_OPTIONS | _FILE_DELETE_ON_CLOSE, + ) + if status == _STATUS_SUCCESS: + _kernel32.CloseHandle(handle) # deletes the file + elif status != _STATUS_OPLOCK_BREAK_IN_PROGRESS: + raise _status_error(status, path) + + def _open_existing_without_oplock_wait(path: str, access: int, flags: int) -> int: + handle, status = _nt_create(path, access, _FILE_ATTRIBUTE_NORMAL, _FILE_SHARE_ALL, _FILE_OPEN) + if status != _STATUS_SUCCESS: + raise _status_error(status, path) + return _descriptor_for_non_reparse(handle, path, flags) + + def _status_error(status: int, path: str) -> OSError: + if status == _STATUS_OPLOCK_BREAK_IN_PROGRESS: + return BlockingIOError(EWOULDBLOCK, "oplock break in progress", path) + if status in {_STATUS_OBJECT_NAME_NOT_FOUND, _STATUS_OBJECT_PATH_NOT_FOUND}: + return FileNotFoundError(ENOENT, os.strerror(ENOENT), path) + winerror = _ntdll.RtlNtStatusToDosError(status) + return OSError(None, ctypes.FormatError(winerror).strip(), path, winerror) + + def _descriptor_for_non_reparse(handle: int, path: str, flags: int) -> int: + info = _BY_HANDLE_FILE_INFORMATION() + # Querying an open handle we just created does not fail. + if not _kernel32.GetFileInformationByHandle(handle, ctypes.byref(info)): # pragma: no cover + err = ctypes.get_last_error() + _kernel32.CloseHandle(handle) + raise ctypes.WinError(err) + if info.dwFileAttributes & _FILE_ATTRIBUTE_REPARSE_POINT: + _kernel32.CloseHandle(handle) + msg = f"Lock file is a reparse point (symlink/junction): {path}" + raise OSError(msg) + + try: + # O_NOINHERIT mirrors os.open on Windows: the lock fd must not leak into child processes. + return msvcrt.open_osfhandle(handle, flags | os.O_NOINHERIT) + except BaseException: # pragma: no cover # open_osfhandle audits too; a hook raising must not leak the handle + _kernel32.CloseHandle(handle) + raise + + def _open_deletable_for_reading(path: str) -> int: + """ + Open *path* read-only without blocking a concurrent unlink and without following a reparse point. + + ``os.open`` omits ``FILE_SHARE_DELETE``, so while a process has a file open that way, a delete of it fails with + a sharing violation. Every contender reads a strict claim and its owner unlinks it, so a reader that keeps the + claim open past the owner's retries makes the release fail. + """ + sys.audit("open", path, None, os.O_RDONLY) + handle, status = _nt_create(path, _GENERIC_READ | _SYNCHRONIZE, 0, _FILE_SHARE_ALL, _FILE_OPEN) + if status != _STATUS_SUCCESS: + winerror = _ntdll.RtlNtStatusToDosError(status) + raise OSError(None, ctypes.FormatError(winerror).strip(), path, winerror) + try: + return msvcrt.open_osfhandle(handle, os.O_RDONLY | os.O_NOINHERIT) + except BaseException: # pragma: no cover # open_osfhandle audits too; a hook raising must not leak the handle + _kernel32.CloseHandle(handle) + raise + + def _pin_directory(path: str) -> int: + """ + Open the directory at *path* so that no process can rename or remove it until :func:`_unpin_directory`. + + Windows has no ``dir_fd``, so an operation resolves every path component by name again. A handle on each + directory of a record path, opened without following a reparse point, keeps that name bound to the directory + checked here until the operation ends. + + :raises NotADirectoryError: when *path* is a junction, a symlink or anything other than a directory. + """ + sys.audit("open", path, None, os.O_RDONLY) + # GetFileInformationByHandle needs FILE_READ_ATTRIBUTES; listing access is what takes part in share checks. + access = _FILE_LIST_DIRECTORY | _FILE_READ_ATTRIBUTES | _SYNCHRONIZE + handle, status = _nt_create(path, access, 0, _FILE_SHARE_READ_WRITE, _FILE_OPEN, _DIRECTORY_OPTIONS) + if status != _STATUS_SUCCESS: + winerror = _ntdll.RtlNtStatusToDosError(status) + raise OSError(None, ctypes.FormatError(winerror).strip(), path, winerror) + info = _BY_HANDLE_FILE_INFORMATION() + # Querying an open handle we just created does not fail. + if not _kernel32.GetFileInformationByHandle(handle, ctypes.byref(info)): # pragma: no cover + error = ctypes.get_last_error() + _kernel32.CloseHandle(handle) + raise ctypes.WinError(error) + if info.dwFileAttributes & _FILE_ATTRIBUTE_REPARSE_POINT: + _kernel32.CloseHandle(handle) + raise NotADirectoryError(ENOTDIR, "is a junction or symlink, not a directory", path) + return handle + + def _unpin_directory(handle: int) -> None: + _kernel32.CloseHandle(handle) + + def _is_unlinked(path: str) -> bool: + """ + Report whether *path* is gone or pending deletion, which ``os.open`` reports as the same EACCES as a denial. + + NTFS keeps an unlinked name listed, refusing every open with ``STATUS_DELETE_PENDING``, until the last handle + to the file closes. Another process holding such a handle, such as an antivirus scan, can keep it that way for + longer than any retry grace, yet the name's owner has already removed it. + """ + handle, status = _nt_create(path, _FILE_READ_ATTRIBUTES | _SYNCHRONIZE, 0, _FILE_SHARE_ALL, _FILE_OPEN) + if handle: + _kernel32.CloseHandle(handle) + return status in {_STATUS_OBJECT_NAME_NOT_FOUND, _STATUS_DELETE_PENDING} + + def _nt_open(path: str, *, read_only: bool, owner_only: bool) -> tuple[int, int]: + """ + Open or create *path* for locking through ``NtCreateFile`` and return ``(handle, status)``. + + Without *owner_only* a file this call creates gets no DACL of its own and inherits its directory's ACL. + """ + attributes = _FILE_ATTRIBUTE_READONLY if read_only else _FILE_ATTRIBUTE_NORMAL + descriptor = _owner_only_descriptor() if owner_only else None + return _nt_create( + path, _DESIRED_ACCESS, attributes, _FILE_SHARE_READ_WRITE, _FILE_OPEN_IF, descriptor=descriptor + ) + + @cache + def _owner_only_descriptor() -> ctypes.c_void_p: + # The DACL os.mkdir(mode=0o700) applies, minus the directory-only inheritance flags: protected (P), so the + # directory's inheritable entries stay out, with full access (FA) for SYSTEM, Administrators and the owner. + # Converted once and never freed: the open retries every few milliseconds while it waits. + descriptor = ctypes.c_void_p() + if not _advapi32.ConvertStringSecurityDescriptorToSecurityDescriptorW( + "D:P(A;;FA;;;SY)(A;;FA;;;BA)(A;;FA;;;OW)", 1, ctypes.byref(descriptor), None + ): # pragma: no cover # a constant string converts + raise ctypes.WinError(ctypes.get_last_error()) + return descriptor + + def _nt_create( # ruff:ignore[too-many-arguments, too-many-positional-arguments] # NtCreateFile's own knobs + path: str, + access: int, + attributes: int, + share: int, + disposition: int, + options: int = _CREATE_OPTIONS, + *, + descriptor: ctypes.c_void_p | None = None, + ) -> tuple[int, int]: + """ + Open *path* through ``NtCreateFile`` and return ``(handle, status)``. + + ``RtlDosPathNameToNtPathName_U_WithStatus`` translates the Win32 path to the NT namespace, handling relative, + drive, UNC and extended-length path forms as Win32 itself would, and allocates a buffer that + ``RtlFreeUnicodeString`` releases. The handle is ``0`` unless the status is ``STATUS_SUCCESS``: one that comes + with ``STATUS_OPLOCK_BREAK_IN_PROGRESS`` is closed here, since the file is not ours until the oplock holder + answers. + """ + nt_name = _UNICODE_STRING() + status = _ntdll.RtlDosPathNameToNtPathName_U_WithStatus(path, ctypes.byref(nt_name), None, None) & 0xFFFFFFFF + if status != _STATUS_SUCCESS: + return 0, status + try: + object_attributes = _OBJECT_ATTRIBUTES() + object_attributes.Length = ctypes.sizeof(_OBJECT_ATTRIBUTES) + object_attributes.ObjectName = ctypes.pointer(nt_name) + object_attributes.Attributes = _OBJ_CASE_INSENSITIVE + object_attributes.SecurityDescriptor = descriptor + handle = wintypes.HANDLE() + io_status = _IO_STATUS_BLOCK() + status = ( + _ntdll.NtCreateFile( + ctypes.byref(handle), + access, + ctypes.byref(object_attributes), + ctypes.byref(io_status), + None, + attributes, + share, + disposition, + options, + None, + 0, + ) + & 0xFFFFFFFF + ) + finally: + _ntdll.RtlFreeUnicodeString(ctypes.byref(nt_name)) + if status == _STATUS_OPLOCK_BREAK_IN_PROGRESS: + _kernel32.CloseHandle(handle) + if status != _STATUS_SUCCESS: + return 0, status + return handle.value or 0, status + + def _directory_is_private(path: str) -> bool: + """ + Report whether only the current user, ``SYSTEM`` and ``Administrators`` can create or remove names in *path*. + + A directory whose owner is anyone else can have its ACL rewritten by that owner. Entries that only a child + inherits count too, since the DACL says nothing more precise about who may write. + """ + owner, dacl, descriptor = ctypes.c_void_p(), ctypes.c_void_p(), ctypes.c_void_p() + error = _advapi32.GetNamedSecurityInfoW( + path, + 1, # SE_FILE_OBJECT + 0x1 | 0x4, # OWNER_SECURITY_INFORMATION | DACL_SECURITY_INFORMATION + ctypes.byref(owner), + None, + ctypes.byref(dacl), + None, + ctypes.byref(descriptor), + ) + if error == _ERROR_ACCESS_DENIED: + # C:\Windows\Temp is one. Anything that cannot be proven private is not. + return False + if error: # pragma: no cover # the directory was opened or listed an instant earlier + raise ctypes.WinError(error) + try: + # SYSTEM, Administrators, CREATOR OWNER and OWNER RIGHTS; the last two stand for the owner. + trusted = {_sid_from_text(text) for text in ("S-1-5-18", "S-1-5-32-544", "S-1-3-0", "S-1-3-4")} + trusted.add(_user_sid()) + # A missing DACL grants everyone full access. + return ( + bool(dacl.value) + and _sid_at(cast("int", owner.value)) in trusted + and _name_writers(cast("int", dacl.value)) <= trusted + ) + finally: + _kernel32.LocalFree(descriptor) + + def _name_writers(dacl: int) -> set[bytes]: + (count,) = struct.unpack("<4xH2x", ctypes.string_at(dacl, 8)) + writers = set() + offset = 8 + for _ in range(count): + ace_type, _flags, ace_size, mask = struct.unpack(" bytes: + token = wintypes.HANDLE() + token_query: Final = 0x0008 + open_as_self: Final = True + # An impersonating thread acts as the account it impersonates, which is the one whose names the lock creates. + if not ( + _advapi32.OpenThreadToken(_kernel32.GetCurrentThread(), token_query, open_as_self, ctypes.byref(token)) + or _advapi32.OpenProcessToken(_kernel32.GetCurrentProcess(), token_query, ctypes.byref(token)) + ): # pragma: no cover # every process may query its own token + raise ctypes.WinError(ctypes.get_last_error()) + try: + size = wintypes.DWORD() + _advapi32.GetTokenInformation(token, 1, None, 0, ctypes.byref(size)) + buffer = ctypes.create_string_buffer(size.value) + if not _advapi32.GetTokenInformation( + token, 1, buffer, size, ctypes.byref(size) + ): # pragma: no cover # the buffer has the size the first call asked for + raise ctypes.WinError(ctypes.get_last_error()) + return _sid_at(cast("int", ctypes.c_void_p.from_buffer(buffer).value)) + finally: + _kernel32.CloseHandle(token) + + def _sid_from_text(text: str) -> bytes: + revision, authority, *subauthorities = (int(part) for part in text.removeprefix("S-").split("-")) + return ( + struct.pack(" bytes: + return ctypes.string_at(address, 8 + 4 * ctypes.string_at(address + 1, 1)[0]) + + def _make_private_directory(path: str) -> None: + """Create *path* with the owner-only descriptor, so that nobody else can add a name to it.""" + attributes = _SECURITY_ATTRIBUTES( + nLength=ctypes.sizeof(_SECURITY_ATTRIBUTES), + lpSecurityDescriptor=_owner_only_descriptor(), + bInheritHandle=False, + ) + if not _kernel32.CreateDirectoryW(path, ctypes.byref(attributes)): # pragma: no cover # the name is fresh + raise ctypes.WinError(ctypes.get_last_error()) + + @contextmanager + def _pinned(path: str) -> Generator[tuple[int, int]]: + """Hold *path* open without delete sharing, which keeps its name from being renamed or removed.""" + with _opened(path, _FILE_SHARE_READ_WRITE) as handle: + yield _identity_of(handle) + + def _identity_at(path: str) -> tuple[int, int]: + with _opened(path, _FILE_SHARE_ALL) as handle: + return _identity_of(handle) + + def _identity_of_fd(fd: int) -> tuple[int, int]: + return _identity_of(msvcrt.get_osfhandle(fd)) + + @contextmanager + def _opened(path: str, share: int) -> Generator[int]: + # The final component is opened as itself, never through a reparse point. + handle = _kernel32.CreateFileW( + path, + _FILE_READ_ATTRIBUTES, + share, + None, + 3, # OPEN_EXISTING + 0x02000000 | 0x00200000, # FILE_FLAG_BACKUP_SEMANTICS opens a directory; FILE_FLAG_OPEN_REPARSE_POINT + None, + ) + if handle in {None, _INVALID_HANDLE_VALUE}: # pragma: no cover # callers open a name they just made or listed + raise ctypes.WinError(ctypes.get_last_error()) + try: + yield cast("int", handle) + finally: + _kernel32.CloseHandle(handle) + + def _identity_of(handle: int) -> tuple[int, int]: + info = _BY_HANDLE_FILE_INFORMATION() + if not _kernel32.GetFileInformationByHandle(handle, ctypes.byref(info)): # pragma: no cover + raise ctypes.WinError(ctypes.get_last_error()) + return info.dwVolumeSerialNumber, info.nFileIndexHigh << 32 | info.nFileIndexLow + +else: # pragma: win32 no cover + + class WindowsFileLock(BaseFileLock): + """Uses ``LockFileEx`` to hard lock a byte range of the lock file on Windows systems.""" + + def _acquire(self) -> None: + raise NotImplementedError + + def _release(self) -> None: + raise NotImplementedError + + +__all__ = [ + "WindowsFileLock", + "_directory_is_private", + "_identity_at", + "_identity_of_fd", + "_is_unlinked", + "_make_private_directory", + "_open_deletable_for_reading", + "_open_non_reparse_fd", + "_pin_directory", + "_pinned", + "_unpin_directory", + "open_for_delete_without_oplock_wait", + "open_read_without_oplock_wait", + "unlink_without_oplock_wait", +] diff --git a/.venv/lib/python3.14/site-packages/filelock/asyncio.py b/.venv/lib/python3.14/site-packages/filelock/asyncio.py new file mode 100644 index 0000000000000000000000000000000000000000..8c3703737ec510da51792b19924d066099afaea6 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/filelock/asyncio.py @@ -0,0 +1,925 @@ +"""An asyncio-based implementation of the file lock.""" + +from __future__ import annotations + +__lazy_modules__ = [ + "asyncio", + "concurrent", + "contextlib", + "filelock._async", + "filelock._error", + "filelock._util", + "functools", + "inspect", + "threading", +] + +import asyncio +import contextlib +import functools +import logging +import os +import sys +import time +from concurrent import futures +from dataclasses import dataclass, field, fields +from inspect import iscoroutinefunction +from threading import RLock +from typing import TYPE_CHECKING, Final, NoReturn, TypeVar, cast + +from filelock._api import ( + _UNSET_FILE_MODE, + BaseFileLock, + CloseErrorPolicy, + ContextErrorPolicy, + FileLockContext, + FileLockMeta, + ThreadLocalFileContext, + _append_exception_context, + _canonical, + _delegate_to_current, + _ExtraValue, + _fork_transition, + _grouped_errors, + _is_system_call_failure, + _raise_body_and_release, + _raise_chained_errors, + _raise_cleanup_errors, + _raise_grouped_errors, + _register_fork_object, + _resolve_poll_interval, + _resolve_timeout, +) +from filelock._async import ( + _AsyncTransitionGate, + _AsyncTransitionUnavailableError, + _BackendOutcome, + _capture_awaitable, + _capture_call, + _drain_future, + _future_result, + _run_in_executor, + _wait_until_done, +) +from filelock._error import Timeout +from filelock._lease import SoftFileLease +from filelock._soft import SoftFileLock +from filelock._strict import StrictSoftFileLock +from filelock._unix import UnixFileLock +from filelock._util import ensure_directory_exists +from filelock._windows import WindowsFileLock + +if TYPE_CHECKING: + from collections.abc import Awaitable, Callable, Coroutine, Hashable, Mapping + from types import TracebackType + from typing import Self + + +_LOGGER: Final[logging.Logger] = logging.getLogger("filelock") +_ASYNC_RELEASE_CANCELLATION_ERRORS: Final[str] = "lock release cancellation and backend release both failed" +_ASYNC_CONTEXT_RELEASE_ERRORS: Final[str] = "context body, release cancellation, and backend release failed" +_ASYNC_RELEASE_CANCELLATION_MARKER_ATTR: Final[str] = "_filelock_async_release_cancellation" +_ASYNC_RELEASE_CANCELLATION_MARKER: Final[list[None]] = [] +_FINALIZER_RELEASES: Final[set[asyncio.Task[None] | futures.Future[None]]] = set() + +_AT = TypeVar("_AT", bound="BaseAsyncFileLock") +_T = TypeVar("_T") + + +class AsyncFileLockMeta(FileLockMeta): + def __call__( # ruff:ignore[too-many-arguments] # forwards the public constructor's documented parameters + cls: type[_AT], # ruff:ignore[invalid-first-argument-name-for-method] # metaclass __call__ receives the class being constructed + lock_file: str | os.PathLike[str], + timeout: float = -1, + mode: int = _UNSET_FILE_MODE, + thread_local: bool = False, # ruff:ignore[boolean-type-hint-positional-argument, boolean-default-value-positional-argument] # public API: positional bool kept for backwards compatibility + *, + blocking: bool = True, + is_singleton: bool = False, + poll_interval: float = 0.05, + lifetime: float | None = None, + context_error_policy: ContextErrorPolicy = "chain", + close_error_policy: CloseErrorPolicy = "default", + fallback_to_soft: bool = True, + preserve_lock_file: bool = False, + on_acquired: Callable[[int], None] | None = None, + loop: asyncio.AbstractEventLoop | None = None, + run_in_executor: bool = True, + executor: futures.Executor | None = None, + **kwargs: _ExtraValue, + ) -> _AT: + if thread_local and run_in_executor: + msg = "run_in_executor is not supported when thread_local is True" + raise ValueError(msg) + return super().__call__( # a subclass may add options of its own, as AsyncSoftFileLease does + **kwargs, + lock_file=lock_file, + timeout=timeout, + mode=mode, + thread_local=thread_local, + blocking=blocking, + is_singleton=is_singleton, + poll_interval=poll_interval, + lifetime=lifetime, + context_error_policy=context_error_policy, + close_error_policy=close_error_policy, + fallback_to_soft=fallback_to_soft, + preserve_lock_file=preserve_lock_file, + on_acquired=on_acquired, + loop=loop, + run_in_executor=run_in_executor, + executor=executor, + ) + + +class BaseAsyncFileLock(BaseFileLock, metaclass=AsyncFileLockMeta): + """ + Base class for asynchronous file locks. + + .. versionadded:: 3.15.0 + + """ + + _deadlock_holder_desc: str = "BaseAsyncFileLock instance in this task" + _constructor_lifetime_warning_stacklevel: int = 4 + + @staticmethod + def _deadlock_scope() -> Hashable | None: + # One event loop thread runs every task, so a thread-scoped registry cannot tell a same-task reacquire + # (a real deadlock: the polling task never reaches its own release) from another task queuing behind the + # holder (no deadlock: each poll yields, so the holder runs on and releases). Only the first may fail + # fast, so scope holders to the task. + return asyncio.current_task() + + def __init__( # ruff:ignore[too-many-arguments] # public constructor: one parameter per documented lock option + self, + lock_file: str | os.PathLike[str], + timeout: float = -1, + mode: int = _UNSET_FILE_MODE, + thread_local: bool = False, # ruff:ignore[boolean-type-hint-positional-argument, boolean-default-value-positional-argument] # public API: positional bool kept for backwards compatibility + *, + blocking: bool = True, + is_singleton: bool = False, + poll_interval: float = 0.05, + lifetime: float | None = None, + context_error_policy: ContextErrorPolicy = "chain", + close_error_policy: CloseErrorPolicy = "default", + fallback_to_soft: bool = True, + preserve_lock_file: bool = False, + on_acquired: Callable[[int], None] | None = None, + loop: asyncio.AbstractEventLoop | None = None, + run_in_executor: bool = True, + executor: futures.Executor | None = None, + ) -> None: + """ + Create a new lock object. + + :param lock_file: path to the file + :param timeout: default timeout when acquiring the lock, in seconds. It will be used as fallback value in the + acquire method, if no timeout value (``None``) is given. If you want to disable the timeout, set it to a + negative value. A timeout of 0 means that there is exactly one attempt to acquire the file lock. + :param mode: file permissions for the lockfile. When not specified, the OS controls permissions via umask and + default ACLs, preserving POSIX default ACL inheritance in shared directories. + :param thread_local: Whether this object's internal context should be thread local or not. If this is set to + ``False`` (the default) then the lock will be reentrant across threads. When ``True``, **all fields of the + lock's internal context are per-thread**, including the configuration values ``poll_interval``, ``timeout``, + ``blocking``, ``mode``, and ``lifetime``. Setting one of these properties from one thread does not change + the value seen by another thread; threads that did not perform the write continue to see the value supplied + at construction time. ``mode`` has no setter, so construction is the only place it is ever set. If you need + configuration values to be visible across threads, construct the lock with ``thread_local=False``. + :param blocking: whether the lock should be blocking or not + :param is_singleton: If this is set to ``True`` then only one instance of this class will be created per lock + file. This is useful if you want to use the lock object for reentrant locking without needing to pass the + same object around. + :param poll_interval: default interval for polling the lock file, in seconds. It will be used as fallback value + in the acquire method, if no poll_interval value (``None``) is given. + :param lifetime: for :class:`AsyncSoftFileLock`, the age in seconds after which a waiting process may delete + the marker, even while its holder remains alive. This legacy expiry mode does not provide strict mutual + exclusion. ``None`` (the default) disables age-based expiry. Native OS locks (:class:`AsyncFileLock`) + cannot be revoked by file age and ignore a non-``None`` ``lifetime`` with a warning. + :param context_error_policy: how a context manager reconciles a failure in its body with a failure while + releasing on exit. ``"chain"`` (the default) keeps Python's behavior: the release error propagates with the + body error in its ``__context__``. ``"group"`` raises a :class:`BaseExceptionGroup` holding the body error + first and the release error second, so neither hides the other. + :param close_error_policy: for native locks (:class:`AsyncFileLock`), what to do with an ``os.close`` failure + after the OS unlock has already committed. ``"default"`` keeps each platform's historical behavior, + ``"raise"`` always propagates the ``OSError``, and ``"suppress"`` always ignores it. + :param fallback_to_soft: for :class:`AsyncFileLock`, whether to fall back to soft existence locking when + ``flock`` returns ``ENOSYS``. ``True`` (default) keeps the fallback; ``False`` propagates the error. + :param preserve_lock_file: for native locks (:class:`AsyncFileLock`), whether filelock promises not to unlink + the lock pathname on release. ``False`` (default) keeps each backend's cleanup; ``True`` keeps a stable file + identity (Windows skips its unlink, Unix refuses the ``ENOSYS`` soft fallback). :class:`AsyncSoftFileLock` + rejects ``True``. + :param on_acquired: for native locks (:class:`AsyncFileLock`), a callable invoked with the borrowed lock + descriptor once per physical acquisition, after the lock is held but before + :meth:`~BaseAsyncFileLock.acquire` returns. With ``run_in_executor=True`` (the default) it runs in the + backend executor. It must not close or unlock the descriptor; a raise rolls the acquisition back. + :class:`AsyncSoftFileLock` rejects it. + :param loop: the event loop a garbage-collected lock schedules its release on when no loop runs in the + collecting thread. :meth:`~BaseAsyncFileLock.acquire` and :meth:`~BaseAsyncFileLock.release` always run + on the loop that awaits them. + :param run_in_executor: If this is set to ``True`` then the lock will be acquired in an executor. + :param executor: The executor to use. If not specified, the default executor will be used. + + """ + self._creator_pid = os.getpid() + self._holder_id = id(self) + self._is_thread_local = thread_local + self._is_singleton = is_singleton + self._context_error_policy = context_error_policy # already validated by the metaclass + self._close_error_policy = close_error_policy # already validated by the metaclass + self._fallback_to_soft = fallback_to_soft + self._preserve_lock_file = preserve_lock_file # already validated by the metaclass + self._on_acquired = on_acquired # already validated by the metaclass + # BaseFileLock.__init__ never runs here, and the finalizer releases thread-local holds through these. + self._transition_lock = RLock() + self._thread_holds = {} + self._thread_pending = {} + + self._async_context: Final[AsyncFileLockContext] = ( + AsyncThreadLocalFileContext if thread_local else AsyncFileLockContext + )( + lock_file=os.fspath(lock_file), + timeout=timeout, + mode=mode, + blocking=blocking, + poll_interval=poll_interval, + lifetime=lifetime, + loop=loop, + run_in_executor=run_in_executor, + executor=executor, + ) + # BaseFileLock declares _context as FileLockContext; narrowing that writable attribute here is unsound. + self._context = self._async_context + _register_fork_object(self) + + def _singleton_extra_mismatches(self, kwargs: Mapping[str, _ExtraValue], /) -> dict[str, tuple[str, str]]: + mismatches = super()._singleton_extra_mismatches(kwargs) + # Loops and executors compare by identity: a different one schedules the lock's work elsewhere. + mismatches.update( + (name, (str(kwargs[name]), str(value))) + for name, value in (("loop", self.loop), ("executor", self.executor)) + if kwargs[name] is not value + ) + if kwargs["run_in_executor"] != self.run_in_executor: + mismatches["run_in_executor"] = (str(kwargs["run_in_executor"]), str(self.run_in_executor)) + return mismatches + + @property + def run_in_executor(self) -> bool: + """Whether run in executor.""" + return self._async_context.run_in_executor + + @property + def executor(self) -> futures.Executor | None: + """The executor.""" + return self._async_context.executor + + @executor.setter + def executor(self, value: futures.Executor | None) -> None: # pragma: no cover + """ + Change the executor. + + :param futures.Executor | None value: the new executor or ``None`` + + """ + self._async_context.executor = value + + @property + def loop(self) -> asyncio.AbstractEventLoop | None: + """The event loop.""" + return self._async_context.loop + + async def acquire( # ty: ignore[invalid-method-override] + self, + timeout: float | None = None, + poll_interval: float | None = None, + *, + blocking: bool | None = None, + cancel_check: Callable[[], bool] | None = None, + ) -> AsyncAcquireReturnProxy: + """ + Try to acquire the file lock. + + :param timeout: maximum wait time for acquiring the lock, ``None`` means use the default + :attr:`~BaseFileLock.timeout` is and if ``timeout < 0``, there is no timeout and this method will block + until the lock could be acquired + :param poll_interval: interval of trying to acquire the lock file, ``None`` means use the default + :attr:`~BaseFileLock.poll_interval` + :param blocking: whether to retry until this call holds the lock or the timeout expires, ``None`` means use + the default :attr:`~BaseFileLock.blocking`. ``False`` makes one attempt and raises :class:`~Timeout` on + failure. + :param cancel_check: a callable returning ``True`` when the acquisition should be canceled. Checked on each poll + iteration. When triggered, raises :class:`~Timeout` just like an expired timeout. + + :returns: a context object that will unlock the file when the context is exited + + :raises Timeout: if fails to acquire lock within the timeout period + :raises ValueError: if a blocking call gets a ``nan`` timeout, or *poll_interval* exceeds + :data:`threading.TIMEOUT_MAX` + + .. code-block:: python + + # You can use this method in the context manager (recommended) + async with await lock.acquire(): + pass + + # Or use an equivalent try-finally construct: + await lock.acquire() + try: + pass + finally: + await lock.release() + + """ + self._raise_if_inherited() + if timeout is None: + timeout = self._context.timeout + + if blocking is None: + blocking = self._context.blocking + + if poll_interval is None: + poll_interval = self._context.poll_interval + poll_interval = _resolve_poll_interval(poll_interval) + timeout = _resolve_timeout(timeout, blocking=blocking) + + start_time = time.perf_counter() + try: + return await self._acquire_with_admission( + blocking=blocking, + cancel_check=cancel_check, + timeout=timeout, + poll_interval=poll_interval, + start_time=start_time, + ) + except _AsyncTransitionUnavailableError: + raise Timeout(self.lock_file) from None + + async def _acquire_with_admission( + self, + *, + blocking: bool, + cancel_check: Callable[[], bool] | None, + timeout: float, + poll_interval: float, + start_time: float, + ) -> AsyncAcquireReturnProxy: + async with self._async_context.transition_gate.hold_for_acquire( + blocking=blocking, + cancel_check=cancel_check, + deadline=None if timeout < 0 else start_time + timeout, + poll_interval=poll_interval, + ): + # Re-entry only counts, so it skips the executor round trip that resolving the path costs. + if self.is_locked: + self._context.lock_counter += 1 + return AsyncAcquireReturnProxy(lock=self) + # A canceled provisional acquire must finish rollback before another caller can claim its descriptor. + canonical = await self._off_loop(functools.partial(_canonical, self.lock_file)) + self._context.lock_counter += 1 + self._raise_if_would_deadlock(canonical, timeout=timeout, blocking=blocking) + self._pin_lock_path(canonical) + try: + await self._async_poll_until_acquired( + blocking=blocking, + cancel_check=cancel_check, + timeout=timeout, + poll_interval=poll_interval, + start_time=start_time, + ) + except BaseException: + self._reconcile_failed_acquire(canonical) + raise + self._commit_acquire(canonical) + return AsyncAcquireReturnProxy(lock=self) + + async def _off_loop(self, call: Callable[[], _T]) -> _T: + # A slow mount stalls a filesystem call, and the loop with it; a coroutine backend runs on the loop anyway. + if not self.run_in_executor or iscoroutinefunction(self._acquire): + return call() + future = _run_in_executor(asyncio.get_running_loop(), self.executor, call) + await _wait_until_done(future) + return _future_result(future) + + async def _async_poll_until_acquired( + self, + *, + blocking: bool, + cancel_check: Callable[[], bool] | None, + timeout: float, + poll_interval: float, + start_time: float, + ) -> None: + lock_id = id(self) + lock_filename = self.lock_file + await self._off_loop(functools.partial(ensure_directory_exists, self._lock_path)) + while True: + self._raise_if_inherited() + _LOGGER.debug("Attempting to acquire lock %s on %s", lock_id, lock_filename) + await self._run_acquire_attempt() + self._raise_if_inherited() + if self.is_locked: + _LOGGER.debug("Lock %s acquired on %s", lock_id, lock_filename) + return + if self._check_give_up( + blocking=blocking, + cancel_check=cancel_check, + timeout=timeout, + start_time=start_time, + ): + raise Timeout(lock_filename, await self._off_loop(self._unprobeable_holder)) + delay = poll_interval + if timeout >= 0: + delay = min(delay, max(start_time + timeout - time.perf_counter(), 0.0)) + _LOGGER.debug("Lock %s not acquired on %s, waiting %s seconds ...", lock_id, lock_filename, delay) + await asyncio.sleep(delay) + + async def _run_acquire_attempt(self) -> None: + acquire_future = self._start_internal_method( + self._acquire_with_fork_tracking_async + if iscoroutinefunction(self._acquire) + else self._break_expired_and_acquire + ) + try: + await _wait_until_done(acquire_future) + except asyncio.CancelledError as cancellation: + acquire_error: BaseException | None = None + try: + await _drain_future(acquire_future) + except BaseException as error: # ruff:ignore[blind-except] # reported with the cancellation below + acquire_error = error + + rollback_error: BaseException | None = None + if self.is_locked: + try: + await _drain_future(self._start_tracked_release()) + except BaseException as error: # ruff:ignore[blind-except] # reported with the cancellation below + rollback_error = error + + if acquire_error is not None: + if rollback_error is not None: # pragma: needs fcntl + self._raise_cancelled_errors( + "lock acquisition cancellation, backend attempt, and rollback failed", + cancellation, + acquire_error, + rollback_error, + ) + self._raise_cancelled_errors( + "lock acquisition cancellation and backend attempt both failed", cancellation, acquire_error + ) + if rollback_error is not None: + self._raise_cancelled_errors( + "lock acquisition cancellation and rollback both failed", cancellation, rollback_error + ) + raise + try: + _future_result(acquire_future) + except asyncio.CancelledError as acquire_error: + await self._rollback_backend_cancelled_acquire(acquire_error) + + async def _rollback_backend_cancelled_acquire(self, acquire_error: asyncio.CancelledError) -> NoReturn: + if self.is_locked: + try: + await _drain_future(self._start_tracked_release()) + except BaseException as rollback_error: # ruff:ignore[blind-except] # both backend errors must surface + self._raise_acquire_rollback_errors(acquire_error, rollback_error) + raise acquire_error + + def _raise_acquire_rollback_errors(self, acquire_error: BaseException, rollback_error: BaseException) -> NoReturn: + if self._context_error_policy == "group": + _raise_grouped_errors("lock acquisition backend and rollback both failed", acquire_error, rollback_error) + _raise_chained_errors(acquire_error, rollback_error) + + def _raise_cancelled_errors( + self, + message: str, + cancellation: asyncio.CancelledError, + first_error: BaseException, + second_error: BaseException | None = None, + ) -> NoReturn: + if self._context_error_policy == "group": + marker = ( + (_ASYNC_RELEASE_CANCELLATION_MARKER_ATTR, _ASYNC_RELEASE_CANCELLATION_MARKER) + if message == _ASYNC_RELEASE_CANCELLATION_ERRORS + else None + ) + if second_error is None: + _raise_grouped_errors(message, cancellation, first_error, marker=marker) + _raise_grouped_errors(message, cancellation, first_error, second_error, marker=marker) + if (context := first_error.__context__) is not None and context is not cancellation: + if (cancellation_context := cancellation.__context__) is not None: + _append_exception_context(context, cancellation_context) + cancellation.__context__ = context + first_error.__context__ = cancellation + _raise_chained_errors(first_error, second_error) + + async def release(self, force: bool = False) -> None: # ty: ignore[invalid-method-override] # ruff:ignore[boolean-type-hint-positional-argument, boolean-default-value-positional-argument] # public API: positional bool kept for backwards compatibility + """ + Release the file lock. The lock is only completely released when the lock counter reaches 0. The lock file + itself may be deleted automatically, the behavior is platform-specific. + + :param force: If true, the lock counter is ignored and the lock is released in every case. + + """ + if self._creator_pid != os.getpid() or not self.is_locked: + return + async with self._async_context.transition_gate.hold(): + await self._release_serialized(force=force) + + async def _release_serialized(self, *, force: bool) -> None: + counter_before: Final = self._context.lock_counter + if not self.is_locked: + return + if not force and self._context.lock_counter > 1: + self._context.lock_counter -= 1 + return + + try: + await self._release_through_backend() + except BaseException as release_error: + if self._release_interrupted_before_commit(counter_before, release_error): + with contextlib.suppress(Exception): + self._release_with_fork_tracking() + self._commit_release_if_released() + raise + + def _release_interrupted_before_commit(self, counter_before: int, release_error: BaseException) -> bool: + # Same rule as the sync release: a handler that raised before anything committed leaves the counter as it was, + # and only then does a second attempt release the hold. A worker thread never runs a handler, so a lock that + # runs its backend in the executor has nothing to retry. + return ( + counter_before == self._context.lock_counter + and self.is_locked + and not self.run_in_executor + and not iscoroutinefunction(self._release) + and not isinstance(release_error, asyncio.CancelledError) + and not _is_system_call_failure(release_error) + ) + + async def _release_through_backend(self) -> None: + lock_id, lock_filename = id(self), self.lock_file + _LOGGER.debug("Attempting to release lock %s on %s", lock_id, lock_filename) + release_future = self._start_tracked_release() + try: + await _wait_until_done(release_future) + except asyncio.CancelledError as cancellation: + try: + await _drain_future(release_future) + except BaseException as release_error: # ruff:ignore[blind-except] # cancellation and backend failure must both surface + self._commit_release_if_released() + self._raise_cancelled_errors(_ASYNC_RELEASE_CANCELLATION_ERRORS, cancellation, release_error) + self._commit_release() + raise + try: + _future_result(release_future) + except BaseException: # state follows the backend for control-flow exceptions too + self._commit_release_if_released() + raise + self._commit_release() + _LOGGER.debug("Lock %s released on %s", lock_id, lock_filename) + + def _commit_release_if_released(self) -> None: + # Commit only when the backend actually unlocked (close or unlink failed after the OS unlock). If the lock is + # still held, keep the counter so a later release can retry. + if not self.is_locked: + self._commit_release() + + def _start_internal_method( + self, method: Callable[[], None] | Callable[[], Coroutine[None, None, None]] + ) -> asyncio.Future[_BackendOutcome[None]]: + if iscoroutinefunction(method): + return asyncio.create_task(_capture_awaitable(cast("Callable[[], Coroutine[None, None, None]]", method)())) + loop = asyncio.get_running_loop() + sync_method = cast("Callable[[], None]", method) + if self.run_in_executor: + return _run_in_executor(loop, self.executor, sync_method) + future: asyncio.Future[_BackendOutcome[None]] = loop.create_future() + future.set_result(_capture_call(sync_method)) + return future + + def _start_tracked_release(self) -> asyncio.Future[_BackendOutcome[None]]: + return self._start_internal_method( + self._release_with_fork_tracking_async + if iscoroutinefunction(self._release) + else self._release_with_fork_tracking + ) + + def _break_expired_and_acquire(self) -> None: + # Runs as one backend call so the lifetime check's lstat and unlink stay off the event loop with the acquire. + self._try_break_expired_lock() + self._acquire_with_fork_tracking() + + async def _acquire_with_fork_tracking_async(self) -> None: + self._try_break_expired_lock() + with _fork_transition(self): + try: + await cast("Callable[[], Awaitable[None]]", self._acquire)() + except BaseException as acquisition_error: + await self._rollback_failed_acquire_async(acquisition_error) + raise + try: + self._register_context_descriptor() + except BaseException as registration_error: # cancellation must roll back the descriptor + await self._rollback_failed_registration_async(registration_error) + raise + if self.is_locked: + await self._invoke_on_acquired_async() + + async def _rollback_failed_acquire_async(self, acquisition_error: BaseException) -> None: + if not self.is_locked: + return + registration_error: BaseException | None = None + tracking_error: BaseException | None = None + try: + self._register_context_descriptor() + except BaseException as error: # ruff:ignore[blind-except] # preserve registration and acquisition failures + registration_error = error + try: + # Rollback may fail too; retain the fd so a child can close it without another identity probe. + self._register_unverified_context_descriptor() + except BaseException as error: # ruff:ignore[blind-except] # pragma: no cover - allocation/control-flow during fallback + tracking_error = error + try: + await _drain_future(self._start_tracked_release()) + except BaseException as rollback_error: # ruff:ignore[blind-except] # preserve rollback and acquisition failures + if registration_error is None and tracking_error is None: + self._raise_acquire_rollback_errors(acquisition_error, rollback_error) + _raise_cleanup_errors( + "lock acquisition cleanup failed", + acquisition_error, + registration_error, + tracking_error, + rollback_error, + ) + if registration_error is not None: + _raise_cleanup_errors( + "lock acquisition cleanup failed", acquisition_error, registration_error, tracking_error + ) + + async def _rollback_failed_registration_async(self, registration_error: BaseException) -> None: + tracking_error: BaseException | None = None + try: + # Rollback may fail too; retain the fd so a child can close it without another identity probe. + self._register_unverified_context_descriptor() + except BaseException as error: # ruff:ignore[blind-except] # pragma: no cover - allocation/control-flow during fallback + tracking_error = error + try: + await _drain_future(self._start_tracked_release()) + except BaseException as rollback_error: # ruff:ignore[blind-except] # preserve rollback and registration failures + _raise_cleanup_errors( + "descriptor registration cleanup failed", registration_error, tracking_error, rollback_error + ) + if tracking_error is not None: # pragma: no cover - requires failed in-memory fallback + _raise_cleanup_errors("descriptor registration cleanup failed", registration_error, tracking_error) + + async def _invoke_on_acquired_async(self) -> None: + if self._on_acquired is None or self._context.lock_counter != 1: + return + try: + self._on_acquired(cast("int", self._context.lock_file_fd)) + except BaseException as callback_error: # caller control-flow errors must release the lock + callback_context = callback_error.__context__ + try: + await _drain_future(self._start_tracked_release()) + except BaseException as release_error: # ruff:ignore[blind-except] # both errors surface via the group below + _raise_body_and_release(callback_error, release_error) + callback_error.__context__ = callback_context + raise + + async def _release_with_fork_tracking_async(self) -> None: + with _fork_transition(self): + try: + await cast("Callable[[], Awaitable[None]]", self._release)() + finally: + self._unregister_released_descriptor() + + def __enter__(self) -> NoReturn: + """Sync context manager entry is not supported because lock acquisition is a coroutine.""" + msg = "Use `async with`: acquire/release are coroutines and cannot be awaited in a sync context manager." + raise NotImplementedError(msg) + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: + """Sync context manager exit is not supported because lock release is a coroutine.""" + msg = "Use `async with`: acquire/release are coroutines and cannot be awaited in a sync context manager." + raise NotImplementedError(msg) + + async def __aenter__(self) -> Self: + """ + Acquire the lock. + + :returns: the lock object + + """ + await self.acquire() + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: + """ + Release the lock, reconciling a release failure with any body failure per :attr:`context_error_policy`. + + :param exc_type: the exception type if raised + :param exc_value: the exception value if raised + :param traceback: the exception traceback if raised + + """ + await self._release_in_context(exc_value) + + async def _release_in_context( # ty: ignore[invalid-method-override] + self, body_error: BaseException | None + ) -> None: + # The async counterpart of BaseFileLock._release_in_context: await release, then apply the same policy. + try: + await self.release() + except BaseException as release_error: + if body_error is None: + raise + if self._context_error_policy == "chain": + _append_exception_context(release_error, body_error) + raise + if ( + errors := _grouped_errors( + release_error, + _ASYNC_RELEASE_CANCELLATION_ERRORS, + (_ASYNC_RELEASE_CANCELLATION_MARKER_ATTR, _ASYNC_RELEASE_CANCELLATION_MARKER), + ) + ) is not None: + match errors: + # The marker is only ever set on a (CancelledError, backend_error) pair, so this always matches. + case (asyncio.CancelledError() as cancellation, backend_error): # pragma: no branch + _raise_grouped_errors(_ASYNC_CONTEXT_RELEASE_ERRORS, body_error, cancellation, backend_error) + _raise_body_and_release(body_error, release_error) + + def _release_abandoned_holds(self) -> None: + # Runs from garbage collection, where no event loop may be running. + with contextlib.suppress(Exception): + # A thread-local lock never uses an executor, so an in-place release matches the loop's and also reaches + # threads whose loops it cannot schedule on. + if self._is_thread_local and not iscoroutinefunction(self._release): + self._release_without_loop() + return + try: + loop = asyncio.get_running_loop() + except RuntimeError: + # At interpreter exit daemon threads no longer run, so a loop running in one never runs the release. + if (loop := self._async_context.loop) is None or loop.is_closed() or sys.is_finalizing(): + self._release_without_loop() + return + if not loop.is_running(): # pragma: no cover + loop.run_until_complete(self.release(force=True)) + return + # Off the deadlock registry, a new lock in this task waits for the scheduled release instead of failing. + self._drop_registry_entry() + # The stored loop runs in another thread, and only its own thread may create tasks on it. + pending = asyncio.run_coroutine_threadsafe(self.release(force=True), loop) + else: + self._drop_registry_entry() + pending = loop.create_task(self.release(force=True)) + # The loop keeps only a weak reference to a task, so hold one until the release finishes. + _FINALIZER_RELEASES.add(pending) + pending.add_done_callback(self._settle_finalizer_release) + + def _settle_finalizer_release(self, pending: asyncio.Task[None] | futures.Future[None]) -> None: + _FINALIZER_RELEASES.discard(pending) + # Nothing awaits this release, so a failure or cancellation falls back to the sync release here. + with contextlib.suppress(asyncio.CancelledError, futures.CancelledError): + if pending.exception() is None: + return + with contextlib.suppress(Exception): + self._release_without_loop() + + def _release_without_loop(self) -> None: + # No loop can run a release here, so call a sync backend directly as BaseFileLock.__del__ does; a coroutine + # backend cannot run and keeps its lock. + if iscoroutinefunction(self._release): + return + if self.is_locked: + try: + self._release_with_fork_tracking() + finally: + self._commit_release_if_released() + # A thread-local lock also holds for the threads that ran its loops, and nothing else releases those now. + self._release_thread_holds(exited_only=False) + + +def _new_transition_gate() -> _AsyncTransitionGate: + # Called per context rather than passed as the factory, so the lazy import resolves on first use, not at import. + return _AsyncTransitionGate() + + +@dataclass +class AsyncFileLockContext(FileLockContext): + """A dataclass which holds the context for a ``BaseAsyncFileLock`` object.""" + + #: Whether run in executor + run_in_executor: bool = True + + #: The executor + executor: futures.Executor | None = None + + #: The loop + loop: asyncio.AbstractEventLoop | None = None + + #: Serializes the tasks that transition this context. An acquire keeps it while it polls, so a thread-local lock + #: needs one per thread. A thread waiting for the lock must not keep the thread that holds it from releasing. + transition_gate: _AsyncTransitionGate = field(default_factory=_new_transition_gate) + + +class AsyncThreadLocalFileContext(AsyncFileLockContext, ThreadLocalFileContext): + """A thread local version of the ``FileLockContext`` class.""" + + # threading.local runs __init__ again with these arguments the first time each thread touches the context. + def __init__( # ruff:ignore[too-many-arguments] # mirrors the AsyncFileLockContext fields set at construction + self, + *, + lock_file: str, + timeout: float, + mode: int, + blocking: bool, + poll_interval: float, + lifetime: float | None = None, + run_in_executor: bool = True, + executor: futures.Executor | None = None, + loop: asyncio.AbstractEventLoop | None = None, + ) -> None: + self.current = AsyncFileLockContext( + lock_file=lock_file, + timeout=timeout, + mode=mode, + blocking=blocking, + poll_interval=poll_interval, + lifetime=lifetime, + run_in_executor=run_in_executor, + executor=executor, + loop=loop, + ) + + +for _field in fields(AsyncFileLockContext): + setattr(AsyncThreadLocalFileContext, _field.name, _delegate_to_current(_field.name)) + + +class AsyncAcquireReturnProxy: + """A context-aware object that will release the lock file when exiting.""" + + def __init__(self, lock: BaseAsyncFileLock) -> None: # ruff:ignore[undocumented-public-init] # trivial release-on-exit proxy + self.lock = lock + + async def __aenter__(self) -> BaseAsyncFileLock: # ruff:ignore[undocumented-magic-method] # returns the wrapped lock + return self.lock + + async def __aexit__( # ruff:ignore[undocumented-magic-method] # releases the wrapped lock on exit + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: + await self.lock._release_in_context(exc_value) # ruff:ignore[private-member-access] # releases the wrapped lock's context + + +class AsyncSoftFileLock(SoftFileLock, BaseAsyncFileLock): + """Simply watches the existence of the lock file.""" + + _lifetime_replacements: tuple[str, str] | None = ("AsyncStrictSoftFileLock", "AsyncSoftFileLease") + + +class AsyncStrictSoftFileLock(StrictSoftFileLock, BaseAsyncFileLock): + """Run strict owner-claim locking without blocking the event loop.""" + + +class AsyncSoftFileLease(SoftFileLease, BaseAsyncFileLock): + """Existence lock whose claim expires, so a peer may take it while the previous holder still runs.""" + + +class AsyncUnixFileLock(UnixFileLock, BaseAsyncFileLock): + """Uses the :func:`fcntl.flock` to hard lock the lock file on unix systems.""" + + +class AsyncWindowsFileLock(WindowsFileLock, BaseAsyncFileLock): + """Uses ``LockFileEx`` to hard lock a byte range of the lock file on Windows systems.""" + + +__all__ = [ + "AsyncAcquireReturnProxy", + "AsyncSoftFileLease", + "AsyncSoftFileLock", + "AsyncStrictSoftFileLock", + "AsyncUnixFileLock", + "AsyncWindowsFileLock", + "BaseAsyncFileLock", +] + + +def __dir__() -> list[str]: + # Hide the sync bases and helpers imported at top level. + return list(__all__) diff --git a/.venv/lib/python3.14/site-packages/filelock/py.typed b/.venv/lib/python3.14/site-packages/filelock/py.typed new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.venv/lib/python3.14/site-packages/filelock/version.py b/.venv/lib/python3.14/site-packages/filelock/version.py new file mode 100644 index 0000000000000000000000000000000000000000..e4b292007e51eb25f14c4c0472451059b98ceb9a --- /dev/null +++ b/.venv/lib/python3.14/site-packages/filelock/version.py @@ -0,0 +1,24 @@ +# file generated by vcs-versioning +# don't change, don't track in version control +from __future__ import annotations + +__all__ = [ + "__version__", + "__version_tuple__", + "version", + "version_tuple", + "__commit_id__", + "commit_id", +] + +version: str +__version__: str +__version_tuple__: tuple[int | str, ...] +version_tuple: tuple[int | str, ...] +commit_id: str | None +__commit_id__: str | None + +__version__ = version = '4.2.4' +__version_tuple__ = version_tuple = (4, 2, 4) + +__commit_id__ = commit_id = None diff --git a/.venv/lib/python3.14/site-packages/fsspec/__init__.py b/.venv/lib/python3.14/site-packages/fsspec/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..5de87eda16630e55f6a48eae3e145515c167d9cb --- /dev/null +++ b/.venv/lib/python3.14/site-packages/fsspec/__init__.py @@ -0,0 +1,75 @@ +from . import caching +from .callbacks import Callback +from .compression import available_compressions +from .core import get_fs_token_paths, open, open_files, open_local, url_to_fs +from .exceptions import FSTimeoutError +from .mapping import FSMap, get_mapper +from .registry import ( + available_protocols, + filesystem, + get_filesystem_class, + register_implementation, + registry, +) +from .spec import AbstractFileSystem + +try: + from ._version import __version__ # noqa: F401 +except ImportError: + __version__ = "unknown" + +__all__ = [ + "AbstractFileSystem", + "FSTimeoutError", + "FSMap", + "filesystem", + "register_implementation", + "get_filesystem_class", + "get_fs_token_paths", + "get_mapper", + "open", + "open_files", + "open_local", + "registry", + "caching", + "Callback", + "available_protocols", + "available_compressions", + "url_to_fs", +] + + +def process_entries(): + try: + from importlib.metadata import entry_points + except ImportError: + return + if entry_points is not None: + try: + eps = entry_points() + except TypeError: + pass # importlib-metadata < 0.8 + else: + if hasattr(eps, "select"): # Python 3.10+ / importlib_metadata >= 3.9.0 + specs = eps.select(group="fsspec.specs") + else: + specs = eps.get("fsspec.specs", []) + registered_names = {} + for spec in specs: + err_msg = f"Unable to load filesystem from {spec}" + name = spec.name + if name in registered_names: + continue + registered_names[name] = True + register_implementation( + name, + spec.value.replace(":", "."), + errtxt=err_msg, + # We take our implementations as the ones to overload with if + # for some reason we encounter some, may be the same, already + # registered + clobber=True, + ) + + +process_entries() diff --git a/.venv/lib/python3.14/site-packages/fsspec/__pycache__/__init__.cpython-314.pyc b/.venv/lib/python3.14/site-packages/fsspec/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..47d2e81cc40cb80bbd4ebd7c71335a555d8465e2 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/fsspec/__pycache__/__init__.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/fsspec/__pycache__/_version.cpython-314.pyc b/.venv/lib/python3.14/site-packages/fsspec/__pycache__/_version.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..577b60c5cfa2a38d3c5bbcbc22b98ee65b925f75 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/fsspec/__pycache__/_version.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/fsspec/__pycache__/archive.cpython-314.pyc b/.venv/lib/python3.14/site-packages/fsspec/__pycache__/archive.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..39ace8b6ad1a89b26d1c31611574a5a295249454 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/fsspec/__pycache__/archive.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/fsspec/__pycache__/asyn.cpython-314.pyc b/.venv/lib/python3.14/site-packages/fsspec/__pycache__/asyn.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e6fa3141a7fc658a4632ff66562496995acce24b Binary files /dev/null and b/.venv/lib/python3.14/site-packages/fsspec/__pycache__/asyn.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/fsspec/__pycache__/caching.cpython-314.pyc b/.venv/lib/python3.14/site-packages/fsspec/__pycache__/caching.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b754ecc2e76ad1e5406e0684dcaa4ce10554ca42 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/fsspec/__pycache__/caching.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/fsspec/__pycache__/callbacks.cpython-314.pyc b/.venv/lib/python3.14/site-packages/fsspec/__pycache__/callbacks.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..62f030bb72270701beacba03521c1705ed35b80a Binary files /dev/null and b/.venv/lib/python3.14/site-packages/fsspec/__pycache__/callbacks.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/fsspec/__pycache__/compression.cpython-314.pyc b/.venv/lib/python3.14/site-packages/fsspec/__pycache__/compression.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7dfce486d0804fb9fa1e537ab026082191c48471 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/fsspec/__pycache__/compression.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/fsspec/__pycache__/config.cpython-314.pyc b/.venv/lib/python3.14/site-packages/fsspec/__pycache__/config.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e8eb4858e2bf736b459b3d78bc77218c7f53fdca Binary files /dev/null and b/.venv/lib/python3.14/site-packages/fsspec/__pycache__/config.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/fsspec/__pycache__/conftest.cpython-314.pyc b/.venv/lib/python3.14/site-packages/fsspec/__pycache__/conftest.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9b2fbe2d40bce517745480799fb9d307830ee54b Binary files /dev/null and b/.venv/lib/python3.14/site-packages/fsspec/__pycache__/conftest.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/fsspec/__pycache__/core.cpython-314.pyc b/.venv/lib/python3.14/site-packages/fsspec/__pycache__/core.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..91744da9cab2d146f3473eeb106691ef72eb8ad5 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/fsspec/__pycache__/core.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/fsspec/__pycache__/dircache.cpython-314.pyc b/.venv/lib/python3.14/site-packages/fsspec/__pycache__/dircache.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..dc7b6181b1fc3594f01abd43238c6f007a022b70 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/fsspec/__pycache__/dircache.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/fsspec/__pycache__/exceptions.cpython-314.pyc b/.venv/lib/python3.14/site-packages/fsspec/__pycache__/exceptions.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5320d4bbf55a8da1689562dc49e1cf057f5bbac4 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/fsspec/__pycache__/exceptions.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/fsspec/__pycache__/fuse.cpython-314.pyc b/.venv/lib/python3.14/site-packages/fsspec/__pycache__/fuse.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bf923dea2a61eb728c33d1fa906febd0c0e4e9ad Binary files /dev/null and b/.venv/lib/python3.14/site-packages/fsspec/__pycache__/fuse.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/fsspec/__pycache__/generic.cpython-314.pyc b/.venv/lib/python3.14/site-packages/fsspec/__pycache__/generic.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4cc76c04e10fb030d583cdd5ea463bfa38d84e73 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/fsspec/__pycache__/generic.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/fsspec/__pycache__/gui.cpython-314.pyc b/.venv/lib/python3.14/site-packages/fsspec/__pycache__/gui.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..257c787ac2f5e2590402d2f0683fb7ba2cd4fc94 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/fsspec/__pycache__/gui.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/fsspec/__pycache__/json.cpython-314.pyc b/.venv/lib/python3.14/site-packages/fsspec/__pycache__/json.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cc0490e31f4981a47b5aedf23866956bd4f15919 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/fsspec/__pycache__/json.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/fsspec/__pycache__/mapping.cpython-314.pyc b/.venv/lib/python3.14/site-packages/fsspec/__pycache__/mapping.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a26daed0f776f0e7a49b697d3653c6053061007c Binary files /dev/null and b/.venv/lib/python3.14/site-packages/fsspec/__pycache__/mapping.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/fsspec/__pycache__/parquet.cpython-314.pyc b/.venv/lib/python3.14/site-packages/fsspec/__pycache__/parquet.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..072b91d24931d7d2b816be696ef0249da2d163cc Binary files /dev/null and b/.venv/lib/python3.14/site-packages/fsspec/__pycache__/parquet.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/fsspec/__pycache__/prefetcher.cpython-314.pyc b/.venv/lib/python3.14/site-packages/fsspec/__pycache__/prefetcher.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2e4e9660fb727250d0bbbac25645ddb5e7142f28 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/fsspec/__pycache__/prefetcher.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/fsspec/__pycache__/registry.cpython-314.pyc b/.venv/lib/python3.14/site-packages/fsspec/__pycache__/registry.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..64fa553dd31f634dd2631faadfe99a4f6bace26c Binary files /dev/null and b/.venv/lib/python3.14/site-packages/fsspec/__pycache__/registry.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/fsspec/__pycache__/spec.cpython-314.pyc b/.venv/lib/python3.14/site-packages/fsspec/__pycache__/spec.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9d362752c52fd87dd0a71d8491c253d82b12db2a Binary files /dev/null and b/.venv/lib/python3.14/site-packages/fsspec/__pycache__/spec.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/fsspec/__pycache__/transaction.cpython-314.pyc b/.venv/lib/python3.14/site-packages/fsspec/__pycache__/transaction.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0ceb3f3425781f0ff4c8d81052a17042106ee433 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/fsspec/__pycache__/transaction.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/fsspec/__pycache__/utils.cpython-314.pyc b/.venv/lib/python3.14/site-packages/fsspec/__pycache__/utils.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0cdd357bd59f047936b4ee143894fcf3eb301169 Binary files /dev/null and b/.venv/lib/python3.14/site-packages/fsspec/__pycache__/utils.cpython-314.pyc differ diff --git a/.venv/lib/python3.14/site-packages/fsspec/_version.py b/.venv/lib/python3.14/site-packages/fsspec/_version.py new file mode 100644 index 0000000000000000000000000000000000000000..98754697aa15bf37be0e64540bdbbad03ea1e306 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/fsspec/_version.py @@ -0,0 +1,24 @@ +# file generated by vcs-versioning +# don't change, don't track in version control +from __future__ import annotations + +__all__ = [ + "__version__", + "__version_tuple__", + "version", + "version_tuple", + "__commit_id__", + "commit_id", +] + +version: str +__version__: str +__version_tuple__: tuple[int | str, ...] +version_tuple: tuple[int | str, ...] +commit_id: str | None +__commit_id__: str | None + +__version__ = version = '2026.9.0' +__version_tuple__ = version_tuple = (2026, 9, 0) + +__commit_id__ = commit_id = None diff --git a/.venv/lib/python3.14/site-packages/fsspec/archive.py b/.venv/lib/python3.14/site-packages/fsspec/archive.py new file mode 100644 index 0000000000000000000000000000000000000000..7aa317fc5167edb4e79c9d077569b498bb6cd887 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/fsspec/archive.py @@ -0,0 +1,81 @@ +import operator + +from fsspec import AbstractFileSystem +from fsspec.utils import tokenize + + +class AbstractArchiveFileSystem(AbstractFileSystem): + """ + A generic superclass for implementing Archive-based filesystems. + + Currently, it is shared amongst + :class:`~fsspec.implementations.zip.ZipFileSystem`, + :class:`~fsspec.implementations.libarchive.LibArchiveFileSystem` and + :class:`~fsspec.implementations.tar.TarFileSystem`. + """ + + def __str__(self): + return f"" + + __repr__ = __str__ + + def ukey(self, path): + return tokenize(path, self.fo, self.protocol) + + def _all_dirnames(self, paths): + """Returns *all* directory names for each path in paths, including intermediate + ones. + + Parameters + ---------- + paths: Iterable of path strings + """ + if len(paths) == 0: + return set() + + dirnames = {self._parent(path) for path in paths} - {self.root_marker} + return dirnames | self._all_dirnames(dirnames) + + def info(self, path, **kwargs): + self._get_dirs() + path = self._strip_protocol(path) + if path in {"", "/"} and self.dir_cache: + return {"name": "", "type": "directory", "size": 0} + if path in self.dir_cache: + return self.dir_cache[path] + elif path + "/" in self.dir_cache: + return self.dir_cache[path + "/"] + else: + raise FileNotFoundError(path) + + def ls(self, path, detail=True, **kwargs): + self._get_dirs() + paths = {} + for p, f in self.dir_cache.items(): + p = p.rstrip("/") + if "/" in p: + root = p.rsplit("/", 1)[0] + else: + root = "" + if root == path.rstrip("/"): + paths[p] = f + elif all( + (a == b) + for a, b in zip(path.split("/"), [""] + p.strip("/").split("/")) + ): + # root directory entry + ppath = p.rstrip("/").split("/", 1)[0] + if ppath not in paths: + out = {"name": ppath, "size": 0, "type": "directory"} + paths[ppath] = out + if not paths and path.strip("/"): + # Nothing is listed under ``path``, so it is a file, an empty + # directory or missing; ``info`` raises FileNotFoundError for the last. + info = self.info(path) + if info["type"] != "directory": + paths[info["name"]] = info + if detail: + out = sorted(paths.values(), key=operator.itemgetter("name")) + return out + else: + return sorted(paths) diff --git a/.venv/lib/python3.14/site-packages/fsspec/asyn.py b/.venv/lib/python3.14/site-packages/fsspec/asyn.py new file mode 100644 index 0000000000000000000000000000000000000000..faf194685c308d45b03f403f19a00079a8fe26e1 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/fsspec/asyn.py @@ -0,0 +1,1277 @@ +import asyncio +import asyncio.events +import concurrent.futures +import functools +import inspect +import io +import logging +import numbers +import os +import re +import sys +import threading +from collections.abc import Iterable +from glob import has_magic +from typing import TYPE_CHECKING + +from .callbacks import DEFAULT_CALLBACK +from .exceptions import FSTimeoutError +from .implementations.local import LocalFileSystem, make_path_posix, trailing_sep +from .spec import AbstractBufferedFile, AbstractFileSystem +from .utils import check_contained, glob_translate, is_exception, other_paths + +private = re.compile("_[^_]") +iothread = [None] # dedicated fsspec IO thread +loop = [None] # global event loop for any non-async instance +_lock = None # global lock placeholder +get_running_loop = asyncio.get_running_loop + + +def get_lock(): + """Allocate or return a threading lock. + + The lock is allocated on first use to allow setting one lock per forked process. + """ + global _lock + if not _lock: + _lock = threading.Lock() + return _lock + + +def reset_lock(): + """Reset the global lock. + + This should be called only on the init of a forked process to reset the lock to + None, enabling the new forked process to get a new lock. + """ + global _lock + + iothread[0] = None + loop[0] = None + _lock = None + + +async def _runner(event, coro, result, timeout=None): + timeout = timeout if timeout else None # convert 0 or 0.0 to None + if timeout is not None: + coro = asyncio.wait_for(coro, timeout=timeout) + try: + result[0] = await coro + except Exception as ex: + result[0] = ex + finally: + event.set() + + +def sync(loop, func, *args, timeout=None, **kwargs): + """ + Make loop run coroutine until it returns. Runs in other thread + + Examples + -------- + >>> fsspec.asyn.sync(fsspec.asyn.get_loop(), func, *args, + timeout=timeout, **kwargs) + """ + timeout = timeout if timeout else None # convert 0 or 0.0 to None + # NB: if the loop is not running *yet*, it is OK to submit work + # and we will wait for it + if loop is None or loop.is_closed(): + raise RuntimeError("Loop is not running") + try: + loop0 = asyncio.events.get_running_loop() + if loop0 is loop: + raise NotImplementedError("Calling sync() from within a running loop") + except NotImplementedError: + raise + except RuntimeError: + pass + coro = func(*args, **kwargs) + result = [None] + event = threading.Event() + asyncio.run_coroutine_threadsafe(_runner(event, coro, result, timeout), loop) + while True: + # this loops allows thread to get interrupted + if event.wait(1): + break + if timeout is not None: + timeout -= 1 + if timeout < 0: + raise FSTimeoutError + + return_result = result[0] + if isinstance(return_result, asyncio.TimeoutError): + # suppress asyncio.TimeoutError, raise FSTimeoutError + raise FSTimeoutError from return_result + elif isinstance(return_result, BaseException): + raise return_result + else: + return return_result + + +_deferred_close_tasks = set() +_deferred_close_lock = threading.Lock() + + +def _on_loop_thread(loop): + """Returns True if the current thread is servicing the given event loop.""" + if loop is None: + return False + try: + return asyncio.get_running_loop() is loop + except RuntimeError: + return False + + +def _defer_task( + loop, + coro, + description="deferred task", + logger=None, + log_level=logging.WARNING, +): + """Schedules a coroutine as a tracked background task on ``loop``. + + Retains a strong reference in ``_deferred_close_tasks`` until completion to + prevent asyncio garbage collection from discarding pending tasks mid-flight, + and ensures unhandled task exceptions are retrieved and logged. + """ + task = loop.create_task(coro) + with _deferred_close_lock: + _deferred_close_tasks.add(task) + + def _on_done(t): + with _deferred_close_lock: + _deferred_close_tasks.discard(t) + if not t.cancelled(): + exc = t.exception() + if exc: + log = logger or logging.getLogger("fsspec.asyn") + log.log( + log_level, + "%s failed during asynchronous execution: %s", + description, + exc, + exc_info=exc, + ) + + task.add_done_callback(_on_done) + return task + + +def sync_teardown( + loop, + func_or_coro, + *args, + timeout=None, + description="teardown", + **kwargs, +): + """Safely runs an async teardown coroutine on ``loop`` from synchronous context. + + Schedules via :func:`asyncio.run_coroutine_threadsafe` or defers on the loop + thread to prevent deadlocks. + """ + coro = func_or_coro(*args, **kwargs) if callable(func_or_coro) else func_or_coro + if not asyncio.iscoroutine(coro): + return + + if sys.is_finalizing(): + coro.close() + return + + if loop is None or not loop.is_running() or loop.is_closed(): + coro.close() + raise RuntimeError(f"Skipping {description}: no usable IO loop available.") + + if _on_loop_thread(loop): + _defer_task(loop, coro, description=description, log_level=logging.ERROR) + return + + try: + future = asyncio.run_coroutine_threadsafe(coro, loop) + except RuntimeError: + coro.close() + raise RuntimeError(f"Skipping {description}: event loop is closed.") from None + + if timeout is not None and timeout <= 0: + return + + try: + return future.result(timeout) + except concurrent.futures.TimeoutError: + raise FSTimeoutError( + f"{description} did not complete within {timeout}s." + ) from None + + +def sync_wrapper(func, obj=None): + """Given a function, make so can be called in blocking contexts + + Leave obj=None if defining within a class. Pass the instance if attaching + as an attribute of the instance. + """ + + @functools.wraps(func) + def wrapper(*args, **kwargs): + self = obj or args[0] + return sync(self.loop, func, *args, **kwargs) + + return wrapper + + +def async_gen_wrapper(func, obj=None): + """Given a async generator, make so can be called in blocking contexts""" + + @functools.wraps(func) + def wrapper(*args, **kwargs): + self = obj or args[0] + gen = func(*args, **kwargs) + while True: + try: + yield sync(self.loop, gen.__anext__) + except StopAsyncIteration: + break + + return wrapper + + +def get_loop(): + """Create or return the default fsspec IO loop + + The loop will be running on a separate thread. + """ + if loop[0] is None: + with get_lock(): + # repeat the check just in case the loop got filled between the + # previous two calls from another thread + if loop[0] is None: + loop[0] = asyncio.new_event_loop() + th = threading.Thread(target=loop[0].run_forever, name="fsspecIO") + th.daemon = True + th.start() + iothread[0] = th + return loop[0] + + +def reset_after_fork(): + global lock + loop[0] = None + iothread[0] = None + lock = None + + +if hasattr(os, "register_at_fork"): + # should be posix; this will do nothing for spawn or forkserver subprocesses + os.register_at_fork(after_in_child=reset_after_fork) + + +if TYPE_CHECKING: + import resource + + ResourceError = resource.error +else: + try: + import resource + except ImportError: + resource = None + ResourceError = OSError + else: + ResourceError = getattr(resource, "error", OSError) + +_DEFAULT_BATCH_SIZE = 128 +_NOFILES_DEFAULT_BATCH_SIZE = 1280 + + +def _get_batch_size(nofiles=False): + from fsspec.config import conf + + if nofiles: + if "nofiles_gather_batch_size" in conf: + return conf["nofiles_gather_batch_size"] + else: + if "gather_batch_size" in conf: + return conf["gather_batch_size"] + if nofiles: + return _NOFILES_DEFAULT_BATCH_SIZE + if resource is None: + return _DEFAULT_BATCH_SIZE + + try: + soft_limit, _ = resource.getrlimit(resource.RLIMIT_NOFILE) + except (ImportError, ValueError, ResourceError): + return _DEFAULT_BATCH_SIZE + + if soft_limit == resource.RLIM_INFINITY: + return -1 + else: + return soft_limit // 8 + + +def running_async() -> bool: + """Being executed by an event loop?""" + try: + asyncio.get_running_loop() + return True + except RuntimeError: + return False + + +async def _run_coros_in_chunks( + coros, + batch_size=None, + callback=DEFAULT_CALLBACK, + timeout=None, + return_exceptions=False, + nofiles=False, +): + """Run the given coroutines in chunks. + + Parameters + ---------- + coros: list of coroutines to run + batch_size: int or None + Number of coroutines to submit/wait on simultaneously. + If -1, then it will not be any throttling. If + None, it will be inferred from _get_batch_size() + callback: fsspec.callbacks.Callback instance + Gets a relative_update when each coroutine completes + timeout: number or None + If given, each coroutine times out after this time. Note that, since + there are multiple batches, the total run time of this function will in + general be longer + return_exceptions: bool + Same meaning as in asyncio.gather + nofiles: bool + If inferring the batch_size, does this operation involve local files? + If yes, you normally expect smaller batches. + """ + + if batch_size is None: + batch_size = _get_batch_size(nofiles=nofiles) + + if batch_size == -1: + batch_size = len(coros) + elif batch_size <= 0: + raise ValueError + + async def _run_coro(coro, i): + try: + return await asyncio.wait_for(coro, timeout=timeout), i + except Exception as e: + if not return_exceptions: + raise + return e, i + finally: + callback.relative_update(1) + + i = 0 + n = len(coros) + results = [None] * n + pending = set() + + while pending or i < n: + while len(pending) < batch_size and i < n: + pending.add(asyncio.ensure_future(_run_coro(coros[i], i))) + i += 1 + + if not pending: + break + + done, pending = await asyncio.wait(pending, return_when=asyncio.FIRST_COMPLETED) + first_exc = None + while done: + task = done.pop() + try: + result, k = await task + results[k] = result + except Exception as exc: + if first_exc is None: + first_exc = exc + + if first_exc is not None: + for task in pending: + task.cancel() + if pending: + await asyncio.gather(*pending, return_exceptions=True) + raise first_exc + + return results + + +# these methods should be implemented as async by any async-able backend +async_methods = [ + "_ls", + "_cat_file", + "_get_file", + "_put_file", + "_rm_file", + "_cp_file", + "_pipe_file", + "_expand_path", + "_info", + "_isfile", + "_isdir", + "_exists", + "_walk", + "_glob", + "_find", + "_du", + "_size", + "_mkdir", + "_makedirs", +] + + +class AsyncFileSystem(AbstractFileSystem): + """Async file operations, default implementations + + Passes bulk operations to asyncio.gather for concurrent operation. + + Implementations that have concurrent batch operations and/or async methods + should inherit from this class instead of AbstractFileSystem. Docstrings are + copied from the un-underscored method in AbstractFileSystem, if not given. + """ + + # note that methods do not have docstring here; they will be copied + # for _* methods and inferred for overridden methods. + + async_impl = True + mirror_sync_methods = True + disable_throttling = False + + def __init__(self, *args, asynchronous=False, loop=None, batch_size=None, **kwargs): + self.asynchronous = asynchronous + self._pid = os.getpid() + if not asynchronous: + self._loop = loop or get_loop() + else: + self._loop = None + self.batch_size = batch_size + super().__init__(*args, **kwargs) + + @property + def loop(self): + if self._pid != os.getpid(): + raise RuntimeError("This class is not fork-safe") + return self._loop + + async def _rm_file(self, path, **kwargs): + if ( + inspect.iscoroutinefunction(self._rm) + and type(self)._rm is not AsyncFileSystem._rm + ): + return await self._rm(path, recursive=False, batch_size=1, **kwargs) + raise NotImplementedError + + async def _rm( + self, path, recursive=False, batch_size=None, maxdepth=None, **kwargs + ): + # TODO: implement on_error + batch_size = batch_size or self.batch_size + path = await self._expand_path(path, recursive=recursive, maxdepth=maxdepth) + return await _run_coros_in_chunks( + [self._rm_file(p, **kwargs) for p in reversed(path)], + batch_size=batch_size, + nofiles=True, + ) + + async def _cp_file(self, path1, path2, **kwargs): + raise NotImplementedError + + async def _mv_file(self, path1, path2): + await self._cp_file(path1, path2) + await self._rm_file(path1) + + async def _copy( + self, + path1, + path2, + recursive=False, + on_error=None, + maxdepth=None, + batch_size=None, + **kwargs, + ): + if on_error is None and recursive: + on_error = "ignore" + elif on_error is None: + on_error = "raise" + + if isinstance(path1, list) and isinstance(path2, list): + # No need to expand paths when both source and destination + # are provided as lists + paths1 = path1 + paths2 = path2 + else: + source_is_str = isinstance(path1, str) + paths1 = await self._expand_path( + path1, maxdepth=maxdepth, recursive=recursive + ) + if source_is_str and (not recursive or maxdepth is not None): + # Non-recursive glob does not copy directories + paths1 = [ + p for p in paths1 if not (trailing_sep(p) or await self._isdir(p)) + ] + if not paths1: + return + + source_is_file = len(paths1) == 1 + dest_is_dir = isinstance(path2, str) and ( + trailing_sep(path2) or await self._isdir(path2) + ) + + exists = source_is_str and ( + (has_magic(path1) and source_is_file) + or (not has_magic(path1) and dest_is_dir and not trailing_sep(path1)) + ) + paths2 = other_paths( + paths1, + path2, + exists=exists, + flatten=not source_is_str, + ) + + batch_size = batch_size or self.batch_size + coros = [self._cp_file(p1, p2, **kwargs) for p1, p2 in zip(paths1, paths2)] + result = await _run_coros_in_chunks( + coros, batch_size=batch_size, return_exceptions=True, nofiles=True + ) + + for ex in filter(is_exception, result): + if on_error == "ignore" and isinstance(ex, FileNotFoundError): + continue + raise ex + + async def _pipe_file(self, path, value, mode="overwrite", **kwargs): + raise NotImplementedError + + async def _pipe(self, path, value=None, batch_size=None, **kwargs): + if isinstance(path, str): + path = {path: value} + batch_size = batch_size or self.batch_size + return await _run_coros_in_chunks( + [self._pipe_file(k, v, **kwargs) for k, v in path.items()], + batch_size=batch_size, + nofiles=True, + ) + + async def _process_limits(self, url, start, end): + """Helper for "Range"-based _cat_file""" + size = None + suff = False + if start is not None and start < 0: + # if start is negative and end None, end is the "suffix length" + if end is None: + end = -start + start = "" + suff = True + else: + size = size or (await self._info(url))["size"] + start = size + start + elif start is None: + start = 0 + if not suff: + if end is not None and end < 0: + if start is not None: + size = size or (await self._info(url))["size"] + end = size + end + elif end is None: + end = "" + if isinstance(end, numbers.Integral): + end -= 1 # bytes range is inclusive + return f"bytes={start}-{end}" + + async def _cat_file(self, path, start=None, end=None, **kwargs): + raise NotImplementedError + + async def _cat( + self, path, recursive=False, on_error="raise", batch_size=None, **kwargs + ): + paths = await self._expand_path(path, recursive=recursive) + coros = [self._cat_file(path, **kwargs) for path in paths] + batch_size = batch_size or self.batch_size + out = await _run_coros_in_chunks( + coros, batch_size=batch_size, nofiles=True, return_exceptions=True + ) + if on_error == "raise": + ex = next(filter(is_exception, out), False) + if ex: + raise ex + if ( + len(paths) > 1 + or isinstance(path, list) + or paths[0] != self._strip_protocol(path) + ): + return { + k: v + for k, v in zip(paths, out) + if on_error != "omit" or not is_exception(v) + } + else: + return out[0] + + async def _cat_ranges( + self, + paths, + starts, + ends, + max_gap=None, + batch_size=None, + on_error="return", + **kwargs, + ): + """Get the contents of byte ranges from one or more files + + Parameters + ---------- + paths: list + A list of of filepaths on this filesystems + starts, ends: int or list + Bytes limits of the read. If using a single int, the same value will be + used to read all the specified files. + on_error: "return" or "raise" + If "return" (default), any per-range exception is placed in the output + list at the corresponding position. Otherwise the first such exception + is raised. Matches ``AbstractFileSystem.cat_ranges``. + """ + if max_gap is not None: + # use utils.merge_offset_ranges + raise NotImplementedError + if not isinstance(paths, list): + raise TypeError + if not isinstance(starts, Iterable): + starts = [starts] * len(paths) + if not isinstance(ends, Iterable): + ends = [ends] * len(paths) + if len(starts) != len(paths) or len(ends) != len(paths): + raise ValueError + coros = [ + self._cat_file(p, start=s, end=e, **kwargs) + for p, s, e in zip(paths, starts, ends) + ] + batch_size = batch_size or self.batch_size + out = await _run_coros_in_chunks( + coros, batch_size=batch_size, nofiles=True, return_exceptions=True + ) + if on_error != "return": + ex = next(filter(is_exception, out), None) + if ex is not None: + raise ex + return out + + async def _put_file(self, lpath, rpath, mode="overwrite", **kwargs): + raise NotImplementedError + + async def _put( + self, + lpath, + rpath, + recursive=False, + callback=DEFAULT_CALLBACK, + batch_size=None, + maxdepth=None, + **kwargs, + ): + """Copy file(s) from local. + + Copies a specific file or tree of files (if recursive=True). If rpath + ends with a "/", it will be assumed to be a directory, and target files + will go within. + + The put_file method will be called concurrently on a batch of files. The + batch_size option can configure the amount of futures that can be executed + at the same time. If it is -1, then all the files will be uploaded concurrently. + The default can be set for this instance by passing "batch_size" in the + constructor, or for all instances by setting the "gather_batch_size" key + in ``fsspec.config.conf``, falling back to 1/8th of the system limit . + """ + if isinstance(lpath, list) and isinstance(rpath, list): + # No need to expand paths when both source and destination + # are provided as lists + rpaths = rpath + lpaths = lpath + else: + source_is_str = isinstance(lpath, str) + if source_is_str: + lpath = make_path_posix(lpath) + fs = LocalFileSystem() + lpaths = fs.expand_path(lpath, recursive=recursive, maxdepth=maxdepth) + if source_is_str and (not recursive or maxdepth is not None): + # Non-recursive glob does not copy directories + lpaths = [p for p in lpaths if not (trailing_sep(p) or fs.isdir(p))] + if not lpaths: + return + + source_is_file = len(lpaths) == 1 + dest_is_dir = isinstance(rpath, str) and ( + trailing_sep(rpath) or await self._isdir(rpath) + ) + + rpath = self._strip_protocol(rpath) + exists = source_is_str and ( + (has_magic(lpath) and source_is_file) + or (not has_magic(lpath) and dest_is_dir and not trailing_sep(lpath)) + ) + rpaths = other_paths( + lpaths, + rpath, + exists=exists, + flatten=not source_is_str, + ) + + is_dir = {l: os.path.isdir(l) for l in lpaths} + rdirs = [r for l, r in zip(lpaths, rpaths) if is_dir[l]] + file_pairs = [(l, r) for l, r in zip(lpaths, rpaths) if not is_dir[l]] + + await asyncio.gather(*[self._makedirs(d, exist_ok=True) for d in rdirs]) + batch_size = batch_size or self.batch_size + + coros = [] + callback.set_size(len(file_pairs)) + for lfile, rfile in file_pairs: + put_file = callback.branch_coro(self._put_file) + coros.append(put_file(lfile, rfile, **kwargs)) + + return await _run_coros_in_chunks( + coros, batch_size=batch_size, callback=callback + ) + + async def _get_file(self, rpath, lpath, **kwargs): + raise NotImplementedError + + async def _get( + self, + rpath, + lpath, + recursive=False, + callback=DEFAULT_CALLBACK, + maxdepth=None, + **kwargs, + ): + """Copy file(s) to local. + + Copies a specific file or tree of files (if recursive=True). If lpath + ends with a "/", it will be assumed to be a directory, and target files + will go within. Can submit a list of paths, which may be glob-patterns + and will be expanded. + + The get_file method will be called concurrently on a batch of files. The + batch_size option can configure the amount of futures that can be executed + at the same time. If it is -1, then all the files will be uploaded concurrently. + The default can be set for this instance by passing "batch_size" in the + constructor, or for all instances by setting the "gather_batch_size" key + in ``fsspec.config.conf``, falling back to 1/8th of the system limit . + """ + if isinstance(lpath, list) and isinstance(rpath, list): + # No need to expand paths when both source and destination + # are provided as lists + rpaths = rpath + lpaths = lpath + else: + source_is_str = isinstance(rpath, str) + # First check for rpath trailing slash as _strip_protocol removes it. + source_not_trailing_sep = source_is_str and not trailing_sep(rpath) + rpath = self._strip_protocol(rpath) + rpaths = await self._expand_path( + rpath, recursive=recursive, maxdepth=maxdepth + ) + if source_is_str and (not recursive or maxdepth is not None): + # Non-recursive glob does not copy directories + rpaths = [ + p for p in rpaths if not (trailing_sep(p) or await self._isdir(p)) + ] + if not rpaths: + return + + lpath = make_path_posix(lpath) + source_is_file = len(rpaths) == 1 + dest_is_dir = isinstance(lpath, str) and ( + trailing_sep(lpath) or LocalFileSystem().isdir(lpath) + ) + + exists = source_is_str and ( + (has_magic(rpath) and source_is_file) + or (not has_magic(rpath) and dest_is_dir and source_not_trailing_sep) + ) + lpaths = other_paths( + rpaths, + lpath, + exists=exists, + flatten=not source_is_str, + ) + if isinstance(lpath, str): + # The names came from the source listing; ".." in one of them + # would otherwise place the copy above the destination. When + # lpath is a list the caller named every destination itself. + check_contained(lpath, lpaths) + + [os.makedirs(os.path.dirname(lp), exist_ok=True) for lp in lpaths] + batch_size = kwargs.pop("batch_size", self.batch_size) + + coros = [] + callback.set_size(len(lpaths)) + for lpath, rpath in zip(lpaths, rpaths): + get_file = callback.branch_coro(self._get_file) + coros.append(get_file(rpath, lpath, **kwargs)) + return await _run_coros_in_chunks( + coros, batch_size=batch_size, callback=callback + ) + + async def _isfile(self, path): + try: + return (await self._info(path))["type"] == "file" + except Exception: + return False + + async def _isdir(self, path): + try: + return (await self._info(path))["type"] == "directory" + except OSError: + return False + + async def _size(self, path): + return (await self._info(path)).get("size", None) + + async def _sizes(self, paths, batch_size=None): + batch_size = batch_size or self.batch_size + return await _run_coros_in_chunks( + [self._size(p) for p in paths], batch_size=batch_size + ) + + async def _exists(self, path, **kwargs): + try: + await self._info(path, **kwargs) + return True + except FileNotFoundError: + return False + + async def _info(self, path, **kwargs): + raise NotImplementedError + + async def _ls(self, path, detail=True, **kwargs): + raise NotImplementedError + + async def _walk(self, path, maxdepth=None, topdown=True, on_error="omit", **kwargs): + if maxdepth is not None and maxdepth < 1: + raise ValueError("maxdepth must be at least 1") + + path = self._strip_protocol(path) + full_dirs = {} + dirs = {} + files = {} + + detail = kwargs.pop("detail", False) + try: + listing = await self._ls(path, detail=True, **kwargs) + except (FileNotFoundError, OSError) as e: + if on_error == "raise": + raise + elif callable(on_error): + on_error(e) + if detail: + yield path, {}, {} + else: + yield path, [], [] + return + + for info in listing: + # each info name must be at least [path]/part , but here + # we check also for names like [path]/part/ + pathname = info["name"].rstrip("/") + name = pathname.rsplit("/", 1)[-1] + if info["type"] == "directory" and pathname != path: + # do not include "self" path + full_dirs[name] = pathname + dirs[name] = info + elif pathname == path: + # file-like with same name as give path + files[""] = info + else: + files[name] = info + + if not detail: + dirs = list(dirs) + files = list(files) + + if topdown: + # Yield before recursion if walking top down + yield path, dirs, files + + if maxdepth is not None: + maxdepth -= 1 + if maxdepth < 1: + if not topdown: + yield path, dirs, files + return + + for d in dirs: + async for _ in self._walk( + full_dirs[d], + maxdepth=maxdepth, + detail=detail, + topdown=topdown, + **kwargs, + ): + yield _ + + if not topdown: + # Yield after recursion if walking bottom up + yield path, dirs, files + + async def _glob(self, path, maxdepth=None, **kwargs): + if maxdepth is not None and maxdepth < 1: + raise ValueError("maxdepth must be at least 1") + + import re + + seps = (os.path.sep, os.path.altsep) if os.path.altsep else (os.path.sep,) + ends_with_sep = path.endswith(seps) # _strip_protocol strips trailing slash + path = self._strip_protocol(path) + append_slash_to_dirname = ends_with_sep or path.endswith( + tuple(sep + "**" for sep in seps) + ) + idx_star = path.find("*") if path.find("*") >= 0 else len(path) + idx_qmark = path.find("?") if path.find("?") >= 0 else len(path) + idx_brace = path.find("[") if path.find("[") >= 0 else len(path) + + min_idx = min(idx_star, idx_qmark, idx_brace) + + detail = kwargs.pop("detail", False) + withdirs = kwargs.pop("withdirs", True) + + if not has_magic(path): + if await self._exists(path, **kwargs): + if not detail: + return [path] + else: + return {path: await self._info(path, **kwargs)} + else: + if not detail: + return [] # glob of non-existent returns empty + else: + return {} + elif "/" in path[:min_idx]: + first_wildcard_idx = min_idx + min_idx = path[:min_idx].rindex("/") + root = path[ + : min_idx + 1 + ] # everything up to the last / before the first wildcard + prefix = path[ + min_idx + 1 : first_wildcard_idx + ] # stem between last "/" and first wildcard + depth = path[min_idx + 1 :].count("/") + 1 + else: + root = "" + prefix = path[:min_idx] # stem up to the first wildcard + depth = path[min_idx + 1 :].count("/") + 1 + + if "**" in path: + if maxdepth is not None: + idx_double_stars = path.find("**") + depth_double_stars = path[idx_double_stars:].count("/") + 1 + depth = depth - depth_double_stars + maxdepth + else: + depth = None + + # Pass the filename stem as prefix= so backends that support it such as + # gcsfs, s3fs and adlfs can filter server-side up to the first wildcard. + if prefix: + kwargs["prefix"] = prefix + allpaths = await self._find( + root, maxdepth=depth, withdirs=withdirs, detail=True, **kwargs + ) + + pattern = glob_translate(path + ("/" if ends_with_sep else "")) + pattern = re.compile(pattern) + + out = { + p: info + for p, info in sorted(allpaths.items()) + if pattern.match( + p + "/" + if append_slash_to_dirname and info["type"] == "directory" + else p + ) + } + + if detail: + return out + else: + return list(out) + + async def _du(self, path, total=True, maxdepth=None, **kwargs): + sizes = {} + # async for? + for f in await self._find(path, maxdepth=maxdepth, **kwargs): + info = await self._info(f) + sizes[info["name"]] = info["size"] + if total: + return sum(sizes.values()) + else: + return sizes + + async def _find(self, path, maxdepth=None, withdirs=False, **kwargs): + path = self._strip_protocol(path) + out = {} + detail = kwargs.pop("detail", False) + + # Add the root directory if withdirs is requested + # This is needed for posix glob compliance + if withdirs and path != "" and await self._isdir(path): + out[path] = await self._info(path) + + # async for? + async for _, dirs, files in self._walk(path, maxdepth, detail=True, **kwargs): + if withdirs: + files.update(dirs) + out.update({info["name"]: info for name, info in files.items()}) + if not out and (await self._isfile(path)): + # walk works on directories, but find should also return [path] + # when path happens to be a file + out[path] = {} + names = sorted(out) + if not detail: + return names + else: + return {name: out[name] for name in names} + + async def _expand_path( + self, path, recursive=False, maxdepth=None, assume_literal=False + ): + if maxdepth is not None and maxdepth < 1: + raise ValueError("maxdepth must be at least 1") + + if isinstance(path, str): + out = await self._expand_path([path], recursive, maxdepth) + else: + out = set() + path = [self._strip_protocol(p) for p in path] + for p in path: # can gather here + if not assume_literal and has_magic(p): + bit = set(await self._glob(p, maxdepth=maxdepth)) + out |= bit + if recursive: + # glob call above expanded one depth so if maxdepth is defined + # then decrement it in expand_path call below. If it is zero + # after decrementing then avoid expand_path call. + if maxdepth is not None and maxdepth <= 1: + continue + out |= set( + await self._expand_path( + list(bit), + recursive=recursive, + maxdepth=maxdepth - 1 if maxdepth is not None else None, + assume_literal=True, + ) + ) + continue + elif recursive: + rec = set(await self._find(p, maxdepth=maxdepth, withdirs=True)) + out |= rec + if p not in out and (recursive is False or (await self._exists(p))): + # should only check once, for the root + out.add(p) + if not out: + raise FileNotFoundError(path) + return sorted(out) + + async def _mkdir(self, path, create_parents=True, **kwargs): + pass # not necessary to implement, may not have directories + + async def _makedirs(self, path, exist_ok=False): + pass # not necessary to implement, may not have directories + + async def open_async(self, path, mode="rb", **kwargs): + if "b" not in mode or kwargs.get("compression"): + raise ValueError + raise NotImplementedError + + +def mirror_sync_methods(obj): + """Populate sync and async methods for obj + + For each method will create a sync version if the name refers to an async method + (coroutine) and there is no override in the child class; will create an async + method for the corresponding sync method if there is no implementation. + + Uses the methods specified in + - async_methods: the set that an implementation is expected to provide + - default_async_methods: that can be derived from their sync version in + AbstractFileSystem + - AsyncFileSystem: async-specific default coroutines + """ + from fsspec import AbstractFileSystem + + for method in set(async_methods + dir(AsyncFileSystem)): + if not method.startswith("_"): + continue + smethod = method[1:] + if private.match(method): + isco = inspect.iscoroutinefunction(getattr(obj, method, None)) + unsync = getattr(getattr(obj, smethod, False), "__func__", None) + is_default = unsync is getattr(AbstractFileSystem, smethod, "") + if isco and is_default: + mth = sync_wrapper(getattr(obj, method), obj=obj) + elif inspect.isasyncgenfunction(getattr(obj, method, None)) and is_default: + mth = async_gen_wrapper(getattr(obj, method), obj=obj) + else: + continue + setattr(obj, smethod, mth) + if not mth.__doc__: + mth.__doc__ = getattr( + getattr(AbstractFileSystem, smethod, None), "__doc__", "" + ) + + +class FSSpecCoroutineCancel(Exception): + pass + + +def _dump_running_tasks( + printout=True, cancel=True, exc=FSSpecCoroutineCancel, with_task=False +): + import traceback + + tasks = [t for t in asyncio.tasks.all_tasks(loop[0]) if not t.done()] + if printout: + [task.print_stack() for task in tasks] + out = [ + { + "locals": task._coro.cr_frame.f_locals, + "file": task._coro.cr_frame.f_code.co_filename, + "firstline": task._coro.cr_frame.f_code.co_firstlineno, + "linelo": task._coro.cr_frame.f_lineno, + "stack": traceback.format_stack(task._coro.cr_frame), + "task": task if with_task else None, + } + for task in tasks + ] + if cancel: + for t in tasks: + cbs = t._callbacks + t.cancel() + asyncio.futures.Future.set_exception(t, exc) + asyncio.futures.Future.cancel(t) + [cb[0](t) for cb in cbs] # cancels any dependent concurrent.futures + try: + t._coro.throw(exc) # exits coro, unless explicitly handled + except exc: + pass + return out + + +class AbstractAsyncStreamedFile(AbstractBufferedFile): + # no read buffering, and always auto-commit + # TODO: readahead might still be useful here, but needs async version + + async def read(self, length=-1): + """ + Return data from cache, or fetch pieces as necessary + + Parameters + ---------- + length: int (-1) + Number of bytes to read; if <0, all remaining bytes. + """ + length = -1 if length is None else int(length) + if self.mode != "rb": + raise ValueError("File not in read mode") + if length < 0: + length = self.size - self.loc + if self.closed: + raise ValueError("I/O operation on closed file.") + if length == 0: + # don't even bother calling fetch + return b"" + out = await self._fetch_range(self.loc, self.loc + length) + self.loc += len(out) + return out + + async def write(self, data): + """ + Write data to buffer. + + Buffer only sent on flush() or if buffer is greater than + or equal to blocksize. + + Parameters + ---------- + data: bytes + Set of bytes to be written. + """ + if self.mode not in {"wb", "ab"}: + raise ValueError("File not in write mode") + if self.closed: + raise ValueError("I/O operation on closed file.") + if self.forced: + raise ValueError("This file has been force-flushed, can only close") + out = self.buffer.write(data) + self.loc += out + if self.buffer.tell() >= self.blocksize: + await self.flush() + return out + + async def close(self): + """Close file + + Finalizes writes, discards cache + """ + if getattr(self, "_unclosable", False): + return + if self.closed: + return + if self.mode == "rb": + self.cache = None + else: + if not self.forced: + await self.flush(force=True) + + if self.fs is not None: + self.fs.invalidate_cache(self.path) + self.fs.invalidate_cache(self.fs._parent(self.path)) + + self.closed = True + + async def flush(self, force=False): + if self.closed: + raise ValueError("Flush on closed file") + if force and self.forced: + raise ValueError("Force flush cannot be called more than once") + if force: + self.forced = True + + if self.mode not in {"wb", "ab"}: + # no-op to flush on read-mode + return + + if not force and self.buffer.tell() < self.blocksize: + # Defer write on small block + return + + if self.offset is None: + # Initialize a multipart upload + self.offset = 0 + try: + await self._initiate_upload() + except: + self.closed = True + raise + + if await self._upload_chunk(final=force) is not False: + self.offset += self.buffer.seek(0, 2) + self.buffer = io.BytesIO() + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + await self.close() + + async def _fetch_range(self, start, end): + raise NotImplementedError + + async def _initiate_upload(self): + pass + + async def _upload_chunk(self, final=False): + raise NotImplementedError diff --git a/.venv/lib/python3.14/site-packages/fsspec/caching.py b/.venv/lib/python3.14/site-packages/fsspec/caching.py new file mode 100644 index 0000000000000000000000000000000000000000..5cf220067913029ea17149a76a7d14bbe07695eb --- /dev/null +++ b/.venv/lib/python3.14/site-packages/fsspec/caching.py @@ -0,0 +1,1142 @@ +from __future__ import annotations + +import asyncio +import collections +import functools +import logging +import math +import os +import threading +from collections import OrderedDict +from collections.abc import Callable +from concurrent.futures import Future, ThreadPoolExecutor +from itertools import groupby +from operator import itemgetter +from typing import TYPE_CHECKING, Any, ClassVar, Generic, NamedTuple, TypeVar + +if TYPE_CHECKING: + import mmap + + from typing_extensions import ParamSpec + + P = ParamSpec("P") +else: + P = TypeVar("P") + +T = TypeVar("T") + + +logger = logging.getLogger("fsspec.caching") + +Fetcher = Callable[[int, int], bytes] # Maps (start, end) to bytes +MultiFetcher = Callable[[list[int, int]], bytes] # Maps [(start, end)] to bytes + + +class BaseCache: + """Pass-though cache: doesn't keep anything, calls every time + + Acts as base class for other cachers + + Parameters + ---------- + blocksize: int + How far to read ahead in numbers of bytes + fetcher: func + Function of the form f(start, end) which gets bytes from remote as + specified + size: int + How big this file is + """ + + name: ClassVar[str] = "none" + + def __init__(self, blocksize: int, fetcher: Fetcher, size: int) -> None: + self.blocksize = blocksize + self.nblocks = 0 + self.fetcher = fetcher + self.size = size + self.hit_count = 0 + self.miss_count = 0 + # the bytes that we actually requested + self.total_requested_bytes = 0 + + def _fetch(self, start: int | None, stop: int | None) -> bytes: + if start is None: + start = 0 + if stop is None: + stop = self.size + if start >= self.size or start >= stop: + return b"" + return self.fetcher(start, stop) + + def _reset_stats(self) -> None: + """Reset hit and miss counts for a more ganular report e.g. by file.""" + self.hit_count = 0 + self.miss_count = 0 + self.total_requested_bytes = 0 + + def _log_stats(self) -> str: + """Return a formatted string of the cache statistics.""" + if self.hit_count == 0 and self.miss_count == 0: + # a cache that does nothing, this is for logs only + return "" + return f" , {self.name}: {self.hit_count} hits, {self.miss_count} misses, {self.total_requested_bytes} total requested bytes" + + def __repr__(self) -> str: + # TODO: use rich for better formatting + return f""" + <{self.__class__.__name__}: + block size : {self.blocksize} + block count : {self.nblocks} + file size : {self.size} + cache hits : {self.hit_count} + cache misses: {self.miss_count} + total requested bytes: {self.total_requested_bytes}> + """ + + +class MMapCache(BaseCache): + """memory-mapped sparse file cache + + Opens temporary file, which is filled blocks-wise when data is requested. + Ensure there is enough disc space in the temporary location. + + This cache method might only work on posix + + Parameters + ---------- + blocksize: int + How far to read ahead in numbers of bytes + fetcher: Fetcher + Function of the form f(start, end) which gets bytes from remote as + specified + size: int + How big this file is + location: str + Where to create the temporary file. If None, a temporary file is + created using tempfile.TemporaryFile(). + blocks: set[int] + Set of block numbers that have already been fetched. If None, an empty + set is created. + multi_fetcher: MultiFetcher + Function of the form f([(start, end)]) which gets bytes from remote + as specified. This function is used to fetch multiple blocks at once. + If not specified, the fetcher function is used instead. + """ + + name = "mmap" + + def __init__( + self, + blocksize: int, + fetcher: Fetcher, + size: int, + location: str | None = None, + blocks: set[int] | None = None, + multi_fetcher: MultiFetcher | None = None, + ) -> None: + super().__init__(blocksize, fetcher, size) + self.blocks = set() if blocks is None else blocks + self.location = location + self.multi_fetcher = multi_fetcher + self.cache = self._makefile() + + def _makefile(self) -> mmap.mmap | bytearray: + import mmap + import tempfile + + if self.size == 0: + return bytearray() + + # posix version + if self.location is None or not os.path.exists(self.location): + if self.location is None: + fd = tempfile.TemporaryFile() + self.blocks = set() + else: + fd = open(self.location, "wb+") + fd.seek(self.size - 1) + fd.write(b"1") + fd.flush() + else: + fd = open(self.location, "r+b") + + return mmap.mmap(fd.fileno(), self.size) + + def _fetch(self, start: int | None, end: int | None) -> bytes: + logger.debug(f"MMap cache fetching {start}-{end}") + if start is None: + start = 0 + if end is None: + end = self.size + if start >= self.size or start >= end: + return b"" + start_block = start // self.blocksize + end_block = end // self.blocksize + block_range = range(start_block, end_block + 1) + # Determine which blocks need to be fetched. This sequence is sorted by construction. + need = (i for i in block_range if i not in self.blocks) + # Count the number of blocks already cached + self.hit_count += sum(1 for i in block_range if i in self.blocks) + + ranges = [] + + # Consolidate needed blocks. + # Algorithm adapted from Python 2.x itertools documentation. + # We are grouping an enumerated sequence of blocks. By comparing when the difference + # between an ascending range (provided by enumerate) and the needed block numbers + # we can detect when the block number skips values. The key computes this difference. + # Whenever the difference changes, we know that we have previously cached block(s), + # and a new group is started. In other words, this algorithm neatly groups + # runs of consecutive block numbers so they can be fetched together. + for _, _blocks in groupby(enumerate(need), key=lambda x: x[0] - x[1]): + # Extract the blocks from the enumerated sequence + _blocks = tuple(map(itemgetter(1), _blocks)) + # Compute start of first block + sstart = _blocks[0] * self.blocksize + # Compute the end of the last block. Last block may not be full size. + send = min(_blocks[-1] * self.blocksize + self.blocksize, self.size) + + # Fetch bytes (could be multiple consecutive blocks) + self.total_requested_bytes += send - sstart + logger.debug( + f"MMap get blocks {_blocks[0]}-{_blocks[-1]} ({sstart}-{send})" + ) + ranges.append((sstart, send)) + + # Update set of cached blocks + self.blocks.update(_blocks) + # Update cache statistics with number of blocks we had to cache + self.miss_count += len(_blocks) + + if not ranges: + return self.cache[start:end] + + if self.multi_fetcher: + logger.debug(f"MMap get blocks {ranges}") + for idx, r in enumerate(self.multi_fetcher(ranges)): + sstart, send = ranges[idx] + logger.debug(f"MMap copy block ({sstart}-{send}") + self.cache[sstart:send] = r + else: + for sstart, send in ranges: + logger.debug(f"MMap get block ({sstart}-{send}") + self.cache[sstart:send] = self.fetcher(sstart, send) + + return self.cache[start:end] + + def __getstate__(self) -> dict[str, Any]: + state = self.__dict__.copy() + # Remove the unpicklable entries. + del state["cache"] + return state + + def __setstate__(self, state: dict[str, Any]) -> None: + # Restore instance attributes + self.__dict__.update(state) + self.cache = self._makefile() + + +class ReadAheadCache(BaseCache): + """Cache which reads only when we get beyond a block of data + + This is a much simpler version of BytesCache, and does not attempt to + fill holes in the cache or keep fragments alive. It is best suited to + many small reads in a sequential order (e.g., reading lines from a file). + """ + + name = "readahead" + + def __init__(self, blocksize: int, fetcher: Fetcher, size: int) -> None: + super().__init__(blocksize, fetcher, size) + self.cache = b"" + self.start = 0 + self.end = 0 + + def _fetch(self, start: int | None, end: int | None) -> bytes: + if start is None: + start = 0 + if end is None or end > self.size: + end = self.size + if start >= self.size or start >= end: + return b"" + l = end - start + if start >= self.start and end <= self.end: + # cache hit + self.hit_count += 1 + return self.cache[start - self.start : end - self.start] + elif self.start <= start < self.end: + # partial hit + self.miss_count += 1 + part = self.cache[start - self.start :] + l -= len(part) + start = self.end + else: + # miss + self.miss_count += 1 + part = b"" + end = min(self.size, end + self.blocksize) + self.total_requested_bytes += end - start + self.cache = self.fetcher(start, end) # new block replaces old + self.start = start + self.end = self.start + len(self.cache) + return part + self.cache[:l] + + +class AdaptiveReadaheadCache(BaseCache): + """Cache with adaptive asynchronous prefetching. + + Optimized for sequential and near-sequential reads by dynamically + adjusting the amount of data prefetched in the background. The cache + uses the generic prefetch engine when async loop support is available, + and falls back to ``ReadAheadCache`` when it is not. + + Parameters + ---------- + blocksize: int + Nominal read size used by callers. + fetcher: Fetcher + Function of the form ``f(start, end)`` that returns bytes. + size: int + Total size of the file. + concurrency: int + Maximum number of concurrent background fetch tasks. + max_prefetch_size: int | None + Optional upper bound for adaptive prefetch size in bytes. + """ + + name = "adaptive" + + def __init__( + self, + blocksize: int, + fetcher: Fetcher, + size: int, + concurrency: int = 4, + max_prefetch_size: int | None = None, + ) -> None: + super().__init__(blocksize, fetcher, size) + self._fallback = ReadAheadCache(blocksize, fetcher, size) + self._prefetcher = None + + async def _default_fetcher_async( + start_offset: int, + total_size: int, + split_factor: int = 1, + ) -> bytes: + del split_factor + return await asyncio.to_thread( + self.fetcher, start_offset, start_offset + total_size + ) + + try: + from . import asyn as fsspec_asyn + except ImportError as e: + raise ImportError( + "AdaptiveReadaheadCache requires fsspec.asyn to be available" + ) from e + + try: + from .prefetcher import BackgroundPrefetcher + + self._prefetcher = BackgroundPrefetcher( + fetcher=_default_fetcher_async, + size=size, + concurrency=concurrency, + max_prefetch_size=max_prefetch_size, + loop=fsspec_asyn.get_loop(), + ) + logger.info( + "AdaptiveReadaheadCache enabled (blocksize=%d, size=%d, concurrency=%d, max_prefetch_size=%s)", + blocksize, + size, + concurrency, + max_prefetch_size, + ) + except Exception as e: + logger.info( + "AdaptiveReadaheadCache fallback to ReadAheadCache: %s", + e, + exc_info=True, + ) + self._prefetcher = None + + def _fetch(self, start: int | None, end: int | None) -> bytes: + if self._prefetcher is None: + out = self._fallback._fetch(start, end) + self.hit_count = self._fallback.hit_count + self.miss_count = self._fallback.miss_count + self.total_requested_bytes = self._fallback.total_requested_bytes + return out + + out = self._prefetcher.fetch(start, end) + self.miss_count += 1 + self.total_requested_bytes += len(out) + return out + + def close(self) -> None: + if self._prefetcher is not None: + self._prefetcher.close() + self._prefetcher = None + + def __getstate__(self) -> dict[str, Any]: + # The prefetcher owns asyncio primitives that are not picklable. + self.close() + state = self.__dict__.copy() + state["_prefetcher"] = None + return state + + def __setstate__(self, state: dict[str, Any]) -> None: + self.__dict__.update(state) + self._prefetcher = None + + def __del__(self): + try: + self.close() + except Exception: + # Best-effort cleanup during GC. + pass + + +class FirstChunkCache(BaseCache): + """Caches the first block of a file only + + This may be useful for file types where the metadata is stored in the header, + but is randomly accessed. + """ + + name = "first" + + def __init__(self, blocksize: int, fetcher: Fetcher, size: int) -> None: + if blocksize > size: + # this will buffer the whole thing + blocksize = size + super().__init__(blocksize, fetcher, size) + self.cache: bytes | None = None + + def _fetch(self, start: int | None, end: int | None) -> bytes: + start = start or 0 + if start > self.size: + logger.debug("FirstChunkCache: requested start > file size") + return b"" + + if end is None: + end = self.size + end = min(end, self.size) + + if start < self.blocksize: + if self.cache is None: + self.miss_count += 1 + if end > self.blocksize: + self.total_requested_bytes += end + data = self.fetcher(0, end) + self.cache = data[: self.blocksize] + return data[start:] + self.cache = self.fetcher(0, self.blocksize) + self.total_requested_bytes += self.blocksize + part = self.cache[start:end] + if end > self.blocksize: + self.total_requested_bytes += end - self.blocksize + part += self.fetcher(self.blocksize, end) + self.hit_count += 1 + return part + else: + self.miss_count += 1 + self.total_requested_bytes += end - start + return self.fetcher(start, end) + + +class BlockCache(BaseCache): + """ + Cache holding memory as a set of blocks. + + Requests are only ever made ``blocksize`` at a time, and are + stored in an LRU cache. The least recently accessed block is + discarded when more than ``maxblocks`` are stored. + + Parameters + ---------- + blocksize : int + The number of bytes to store in each block. + Requests are only ever made for ``blocksize``, so this + should balance the overhead of making a request against + the granularity of the blocks. + fetcher : Callable + size : int + The total size of the file being cached. + maxblocks : int + The maximum number of blocks to cache for. The maximum memory + use for this cache is then ``blocksize * maxblocks``. + """ + + name = "blockcache" + + def __init__( + self, blocksize: int, fetcher: Fetcher, size: int, maxblocks: int = 32 + ) -> None: + super().__init__(blocksize, fetcher, size) + self.nblocks = math.ceil(size / blocksize) + self.maxblocks = maxblocks + self._fetch_block_cached = functools.lru_cache(maxblocks)(self._fetch_block) + + def cache_info(self): + """ + The statistics on the block cache. + + Returns + ------- + NamedTuple + Returned directly from the LRU Cache used internally. + """ + return self._fetch_block_cached.cache_info() + + def __getstate__(self) -> dict[str, Any]: + state = self.__dict__.copy() + del state["_fetch_block_cached"] + return state + + def __setstate__(self, state: dict[str, Any]) -> None: + self.__dict__.update(state) + self._fetch_block_cached = functools.lru_cache(state["maxblocks"])( + self._fetch_block + ) + + def _fetch(self, start: int | None, end: int | None) -> bytes: + if start is None: + start = 0 + if end is None or end > self.size: + end = self.size + if start >= self.size or start >= end: + return b"" + + return self._read_cache( + start, end, start // self.blocksize, (end - 1) // self.blocksize + ) + + def _fetch_block(self, block_number: int) -> bytes: + """ + Fetch the block of data for `block_number`. + """ + if block_number > self.nblocks: + raise ValueError( + f"'block_number={block_number}' is greater than " + f"the number of blocks ({self.nblocks})" + ) + + start = block_number * self.blocksize + end = start + self.blocksize + self.total_requested_bytes += end - start + self.miss_count += 1 + logger.info("BlockCache fetching block %d", block_number) + block_contents = super()._fetch(start, end) + return block_contents + + def _read_cache( + self, start: int, end: int, start_block_number: int, end_block_number: int + ) -> bytes: + """ + Read from our block cache. + + Parameters + ---------- + start, end : int + The start and end byte positions. + start_block_number, end_block_number : int + The start and end block numbers. + """ + start_pos = start % self.blocksize + end_pos = end % self.blocksize + if end_pos == 0: + end_pos = self.blocksize + + self.hit_count += 1 + if start_block_number == end_block_number: + block: bytes = self._fetch_block_cached(start_block_number) + return block[start_pos:end_pos] + + else: + # read from the initial + out = [self._fetch_block_cached(start_block_number)[start_pos:]] + + # intermediate blocks + # Note: it'd be nice to combine these into one big request. However + # that doesn't play nicely with our LRU cache. + out.extend( + map( + self._fetch_block_cached, + range(start_block_number + 1, end_block_number), + ) + ) + + # final block + out.append(self._fetch_block_cached(end_block_number)[:end_pos]) + + return b"".join(out) + + +class BytesCache(BaseCache): + """Cache which holds data in a in-memory bytes object + + Implements read-ahead by the block size, for semi-random reads progressing + through the file. + + Parameters + ---------- + trim: bool + As we read more data, whether to discard the start of the buffer when + we are more than a blocksize ahead of it. + """ + + name: ClassVar[str] = "bytes" + + def __init__( + self, blocksize: int, fetcher: Fetcher, size: int, trim: bool = True + ) -> None: + super().__init__(blocksize, fetcher, size) + self.cache = b"" + self.start: int | None = None + self.end: int | None = None + self.trim = trim + + def _fetch(self, start: int | None, end: int | None) -> bytes: + # TODO: only set start/end after fetch, in case it fails? + # is this where retry logic might go? + if start is None: + start = 0 + if end is None: + end = self.size + if start >= self.size or start >= end: + return b"" + if ( + self.start is not None + and start >= self.start + and self.end is not None + and end < self.end + ): + # cache hit: we have all the required data + offset = start - self.start + self.hit_count += 1 + return self.cache[offset : offset + end - start] + + if self.blocksize: + bend = min(self.size, end + self.blocksize) + else: + bend = end + + if bend == start or start > self.size: + return b"" + + if (self.start is None or start < self.start) and ( + self.end is None or end > self.end + ): + # First read, or extending both before and after + self.total_requested_bytes += bend - start + self.miss_count += 1 + self.cache = self.fetcher(start, bend) + self.start = start + else: + assert self.start is not None + assert self.end is not None + self.miss_count += 1 + + if start < self.start: + if self.end is None or self.end - end > self.blocksize: + self.total_requested_bytes += bend - start + self.cache = self.fetcher(start, bend) + self.start = start + else: + self.total_requested_bytes += self.start - start + new = self.fetcher(start, self.start) + self.start = start + self.cache = new + self.cache + elif self.end is not None and bend > self.end: + if self.end > self.size: + pass + elif end - self.end > self.blocksize: + self.total_requested_bytes += bend - start + self.cache = self.fetcher(start, bend) + self.start = start + else: + self.total_requested_bytes += bend - self.end + new = self.fetcher(self.end, bend) + self.cache = self.cache + new + + self.end = self.start + len(self.cache) + offset = start - self.start + out = self.cache[offset : offset + end - start] + if self.trim: + num = (self.end - self.start) // (self.blocksize + 1) + if num > 1: + self.start += self.blocksize * num + self.cache = self.cache[self.blocksize * num :] + return out + + def __len__(self) -> int: + return len(self.cache) + + +class AllBytes(BaseCache): + """Cache entire contents of the file""" + + name: ClassVar[str] = "all" + + def __init__( + self, + blocksize: int | None = None, + fetcher: Fetcher | None = None, + size: int | None = None, + data: bytes | None = None, + ) -> None: + super().__init__(blocksize, fetcher, size) # type: ignore[arg-type] + if data is None: + self.miss_count += 1 + self.total_requested_bytes += self.size + data = self.fetcher(0, self.size) + self.data = data + + def _fetch(self, start: int | None, stop: int | None) -> bytes: + self.hit_count += 1 + return self.data[start:stop] + + +class KnownPartsOfAFile(BaseCache): + """ + Cache holding known file parts. + + Parameters + ---------- + blocksize: int + How far to read ahead in numbers of bytes + fetcher: func + Function of the form f(start, end) which gets bytes from remote as + specified + size: int + How big this file is + data: dict + A dictionary mapping explicit `(start, stop)` file-offset tuples + with known bytes. + strict: bool, default True + Whether to fetch reads that go beyond a known byte-range boundary. + If `False`, any read that ends outside a known part will be zero + padded. Note that zero padding will not be used for reads that + begin outside a known byte-range. + """ + + name: ClassVar[str] = "parts" + + def __init__( + self, + blocksize: int, + fetcher: Fetcher, + size: int, + data: dict[tuple[int, int], bytes] | None = None, + strict: bool = False, + **_: Any, + ): + super().__init__(blocksize, fetcher, size) + self.strict = strict + + # simple consolidation of contiguous blocks + if data: + old_offsets = sorted(data.keys()) + offsets = [old_offsets[0]] + blocks = [data.pop(old_offsets[0])] + for start, stop in old_offsets[1:]: + start0, stop0 = offsets[-1] + if start == stop0: + offsets[-1] = (start0, stop) + blocks[-1] += data.pop((start, stop)) + else: + offsets.append((start, stop)) + blocks.append(data.pop((start, stop))) + + self.data = dict(zip(offsets, blocks)) + else: + self.data = {} + + @property + def size(self): + return sum(_[1] - _[0] for _ in self.data) + + @size.setter + def size(self, value): + pass + + @property + def nblocks(self): + return len(self.data) + + @nblocks.setter + def nblocks(self, value): + pass + + def _fetch(self, start: int | None, stop: int | None) -> bytes: + logger.debug("Known parts request %s %s", start, stop) + if start is None: + start = 0 + if stop is None: + stop = self.size + self.total_requested_bytes += stop - start + out = b"" + started = False + loc_old = 0 + for loc0, loc1 in sorted(self.data): + if (loc0 <= start < loc1) and (loc0 <= stop <= loc1): + # entirely within the block + off = start - loc0 + self.hit_count += 1 + return self.data[(loc0, loc1)][off : off + stop - start] + if stop <= loc0: + break + if started and loc0 > loc_old: + # a gap where we need data + self.miss_count += 1 + if self.strict: + raise ValueError + out += b"\x00" * (loc0 - loc_old) + if loc0 <= start < loc1: + # found the start + self.hit_count += 1 + off = start - loc0 + out = self.data[(loc0, loc1)][off : off + stop - start] + started = True + elif start < loc0 and stop > loc1: + # the whole block + self.hit_count += 1 + out += self.data[(loc0, loc1)] + elif loc0 <= stop <= loc1: + # end block + self.hit_count += 1 + out = out + self.data[(loc0, loc1)][: stop - loc0] + return out + loc_old = loc1 + self.miss_count += 1 + if started and not self.strict: + out = out + b"\x00" * (stop - loc_old) + return out + raise ValueError + + +class UpdatableLRU(Generic[P, T]): + """ + Custom implementation of LRU cache that allows updating keys + + Used by BackgroundBlockCache + """ + + class CacheInfo(NamedTuple): + hits: int + misses: int + maxsize: int + currsize: int + + def __init__(self, func: Callable[P, T], max_size: int = 128) -> None: + self._cache: OrderedDict[Any, T] = collections.OrderedDict() + self._func = func + self._max_size = max_size + self._hits = 0 + self._misses = 0 + self._lock = threading.Lock() + + def __call__(self, *args: P.args, **kwargs: P.kwargs) -> T: + if kwargs: + raise TypeError(f"Got unexpected keyword argument {kwargs.keys()}") + with self._lock: + if args in self._cache: + self._cache.move_to_end(args) + self._hits += 1 + return self._cache[args] + + result = self._func(*args, **kwargs) + + with self._lock: + self._cache[args] = result + self._misses += 1 + if len(self._cache) > self._max_size: + self._cache.popitem(last=False) + + return result + + def is_key_cached(self, *args: Any) -> bool: + with self._lock: + return args in self._cache + + def add_key(self, result: T, *args: Any) -> None: + with self._lock: + self._cache[args] = result + if len(self._cache) > self._max_size: + self._cache.popitem(last=False) + + def cache_info(self) -> UpdatableLRU.CacheInfo: + with self._lock: + return self.CacheInfo( + maxsize=self._max_size, + currsize=len(self._cache), + hits=self._hits, + misses=self._misses, + ) + + +class BackgroundBlockCache(BaseCache): + """ + Cache holding memory as a set of blocks with pre-loading of + the next block in the background. + + Requests are only ever made ``blocksize`` at a time, and are + stored in an LRU cache. The least recently accessed block is + discarded when more than ``maxblocks`` are stored. If the + next block is not in cache, it is loaded in a separate thread + in non-blocking way. + + Parameters + ---------- + blocksize : int + The number of bytes to store in each block. + Requests are only ever made for ``blocksize``, so this + should balance the overhead of making a request against + the granularity of the blocks. + fetcher : Callable + size : int + The total size of the file being cached. + maxblocks : int + The maximum number of blocks to cache for. The maximum memory + use for this cache is then ``blocksize * maxblocks``. + """ + + name: ClassVar[str] = "background" + + def __init__( + self, blocksize: int, fetcher: Fetcher, size: int, maxblocks: int = 32 + ) -> None: + super().__init__(blocksize, fetcher, size) + self.nblocks = math.ceil(size / blocksize) + self.maxblocks = maxblocks + self._fetch_block_cached = UpdatableLRU(self._fetch_block, maxblocks) + + self._thread_executor = ThreadPoolExecutor(max_workers=1) + self._fetch_future_block_number: int | None = None + self._fetch_future: Future[bytes] | None = None + self._fetch_future_lock = threading.Lock() + self._closed = False + + def cache_info(self) -> UpdatableLRU.CacheInfo: + """ + The statistics on the block cache. + + Returns + ------- + NamedTuple + Returned directly from the LRU Cache used internally. + """ + return self._fetch_block_cached.cache_info() + + def close(self) -> None: + """Cancel pending work and shut down the background worker.""" + with self._fetch_future_lock: + if self._closed: + return + self._closed = True + future = self._fetch_future + self._fetch_future = None + self._fetch_future_block_number = None + + if future is not None: + future.cancel() + self._thread_executor.shutdown(wait=True, cancel_futures=True) + + # UpdatableLRU stores a bound method and otherwise forms a reference cycle. + del self._fetch_block_cached + + def __getstate__(self) -> dict[str, Any]: + state = self.__dict__.copy() + del state["_fetch_block_cached"] + del state["_thread_executor"] + del state["_fetch_future_block_number"] + del state["_fetch_future"] + del state["_fetch_future_lock"] + return state + + def __setstate__(self, state) -> None: + self.__dict__.update(state) + self._fetch_block_cached = UpdatableLRU(self._fetch_block, state["maxblocks"]) + self._thread_executor = ThreadPoolExecutor(max_workers=1) + self._fetch_future_block_number = None + self._fetch_future = None + self._fetch_future_lock = threading.Lock() + self._closed = False + + def _fetch(self, start: int | None, end: int | None) -> bytes: + if start is None: + start = 0 + if end is None or end > self.size: + end = self.size + if start >= self.size or start >= end: + return b"" + + # byte position -> block numbers + start_block_number = start // self.blocksize + end_block_number = end // self.blocksize + + fetch_future_block_number = None + fetch_future = None + with self._fetch_future_lock: + # Background thread is running. Check we we can or must join it. + if self._fetch_future is not None: + assert self._fetch_future_block_number is not None + if self._fetch_future.done(): + logger.info("BlockCache joined background fetch without waiting.") + self._fetch_block_cached.add_key( + self._fetch_future.result(), self._fetch_future_block_number + ) + # Cleanup the fetch variables. Done with fetching the block. + self._fetch_future_block_number = None + self._fetch_future = None + else: + # Must join if we need the block for the current fetch + must_join = bool( + start_block_number + <= self._fetch_future_block_number + <= end_block_number + ) + if must_join: + # Copy to the local variables to release lock + # before waiting for result + fetch_future_block_number = self._fetch_future_block_number + fetch_future = self._fetch_future + + # Cleanup the fetch variables. Have a local copy. + self._fetch_future_block_number = None + self._fetch_future = None + + # Need to wait for the future for the current read + if fetch_future is not None: + logger.info("BlockCache waiting for background fetch.") + # Wait until result and put it in cache + self._fetch_block_cached.add_key( + fetch_future.result(), fetch_future_block_number + ) + + # these are cached, so safe to do multiple calls for the same start and end. + for block_number in range(start_block_number, end_block_number + 1): + self._fetch_block_cached(block_number) + + # fetch next block in the background if nothing is running in the background, + # the block is within file and it is not already cached + end_block_plus_1 = end_block_number + 1 + with self._fetch_future_lock: + if ( + self._fetch_future is None + and end_block_plus_1 <= self.nblocks + and not self._fetch_block_cached.is_key_cached(end_block_plus_1) + ): + self._fetch_future_block_number = end_block_plus_1 + self._fetch_future = self._thread_executor.submit( + self._fetch_block, end_block_plus_1, "async" + ) + + return self._read_cache( + start, + end, + start_block_number=start_block_number, + end_block_number=end_block_number, + ) + + def _fetch_block(self, block_number: int, log_info: str = "sync") -> bytes: + """ + Fetch the block of data for `block_number`. + """ + if block_number > self.nblocks: + raise ValueError( + f"'block_number={block_number}' is greater than " + f"the number of blocks ({self.nblocks})" + ) + + start = block_number * self.blocksize + end = start + self.blocksize + logger.info("BlockCache fetching block (%s) %d", log_info, block_number) + self.total_requested_bytes += end - start + self.miss_count += 1 + block_contents = super()._fetch(start, end) + return block_contents + + def _read_cache( + self, start: int, end: int, start_block_number: int, end_block_number: int + ) -> bytes: + """ + Read from our block cache. + + Parameters + ---------- + start, end : int + The start and end byte positions. + start_block_number, end_block_number : int + The start and end block numbers. + """ + start_pos = start % self.blocksize + end_pos = end % self.blocksize + + # kind of pointless to count this as a hit, but it is + self.hit_count += 1 + + if start_block_number == end_block_number: + block = self._fetch_block_cached(start_block_number) + return block[start_pos:end_pos] + + else: + # read from the initial + out = [self._fetch_block_cached(start_block_number)[start_pos:]] + + # intermediate blocks + # Note: it'd be nice to combine these into one big request. However + # that doesn't play nicely with our LRU cache. + out.extend( + map( + self._fetch_block_cached, + range(start_block_number + 1, end_block_number), + ) + ) + + # final block + out.append(self._fetch_block_cached(end_block_number)[:end_pos]) + + return b"".join(out) + + +caches: dict[str | None, type[BaseCache]] = { + # one custom case + None: BaseCache, +} + + +def register_cache(cls: type[BaseCache], clobber: bool = False) -> None: + """'Register' cache implementation. + + Parameters + ---------- + clobber: bool, optional + If set to True (default is False) - allow to overwrite existing + entry. + + Raises + ------ + ValueError + """ + name = cls.name + if not clobber and name in caches: + raise ValueError(f"Cache with name {name!r} is already known: {caches[name]}") + caches[name] = cls + + +for c in ( + BaseCache, + MMapCache, + BytesCache, + ReadAheadCache, + AdaptiveReadaheadCache, + BlockCache, + FirstChunkCache, + AllBytes, + KnownPartsOfAFile, + BackgroundBlockCache, +): + register_cache(c) diff --git a/.venv/lib/python3.14/site-packages/fsspec/callbacks.py b/.venv/lib/python3.14/site-packages/fsspec/callbacks.py new file mode 100644 index 0000000000000000000000000000000000000000..7ca99ca6ac3cd69b28bcd1550f6550e8e648c5fe --- /dev/null +++ b/.venv/lib/python3.14/site-packages/fsspec/callbacks.py @@ -0,0 +1,324 @@ +from functools import wraps + + +class Callback: + """ + Base class and interface for callback mechanism + + This class can be used directly for monitoring file transfers by + providing ``callback=Callback(hooks=...)`` (see the ``hooks`` argument, + below), or subclassed for more specialised behaviour. + + Parameters + ---------- + size: int (optional) + Nominal quantity for the value that corresponds to a complete + transfer, e.g., total number of tiles or total number of + bytes + value: int (0) + Starting internal counter value + hooks: dict or None + A dict of named functions to be called on each update. The signature + of these must be ``f(size, value, **kwargs)`` + """ + + def __init__(self, size=None, value=0, hooks=None, **kwargs): + self.size = size + self.value = value + self.hooks = hooks or {} + self.kw = kwargs + + def __enter__(self): + return self + + def __exit__(self, *exc_args): + self.close() + + def close(self): + """Close callback.""" + + def branched(self, path_1, path_2, **kwargs): + """ + Return callback for child transfers + + If this callback is operating at a higher level, e.g., put, which may + trigger transfers that can also be monitored. The function returns a callback + that has to be passed to the child method, e.g., put_file, + as `callback=` argument. + + The implementation uses `callback.branch` for compatibility. + When implementing callbacks, it is recommended to override this function instead + of `branch` and avoid calling `super().branched(...)`. + + Prefer using this function over `branch`. + + Parameters + ---------- + path_1: str + Child's source path + path_2: str + Child's destination path + **kwargs: + Arbitrary keyword arguments + + Returns + ------- + callback: Callback + A callback instance to be passed to the child method + """ + self.branch(path_1, path_2, kwargs) + # mutate kwargs so that we can force the caller to pass "callback=" explicitly + return kwargs.pop("callback", DEFAULT_CALLBACK) + + def branch_coro(self, fn): + """ + Wraps a coroutine, and pass a new child callback to it. + """ + + @wraps(fn) + async def func(path1, path2: str, **kwargs): + with self.branched(path1, path2, **kwargs) as child: + return await fn(path1, path2, callback=child, **kwargs) + + return func + + def set_size(self, size): + """ + Set the internal maximum size attribute + + Usually called if not initially set at instantiation. Note that this + triggers a ``call()``. + + Parameters + ---------- + size: int + """ + self.size = size + self.call() + + def absolute_update(self, value): + """ + Set the internal value state + + Triggers ``call()`` + + Parameters + ---------- + value: int + """ + self.value = value + self.call() + + def relative_update(self, inc=1): + """ + Delta increment the internal counter + + Triggers ``call()`` + + Parameters + ---------- + inc: int + """ + self.value += inc + self.call() + + def call(self, hook_name=None, **kwargs): + """ + Execute hook(s) with current state + + Each function is passed the internal size and current value + + Parameters + ---------- + hook_name: str or None + If given, execute on this hook + kwargs: passed on to (all) hook(s) + """ + if not self.hooks: + return + kw = self.kw.copy() + kw.update(kwargs) + if hook_name: + if hook_name not in self.hooks: + return + return self.hooks[hook_name](self.size, self.value, **kw) + for hook in self.hooks.values() or []: + hook(self.size, self.value, **kw) + + def wrap(self, iterable): + """ + Wrap an iterable to call ``relative_update`` on each iterations + + Parameters + ---------- + iterable: Iterable + The iterable that is being wrapped + """ + for item in iterable: + self.relative_update() + yield item + + def branch(self, path_1, path_2, kwargs): + """ + Set callbacks for child transfers + + If this callback is operating at a higher level, e.g., put, which may + trigger transfers that can also be monitored. The passed kwargs are + to be *mutated* to add ``callback=``, if this class supports branching + to children. + + Parameters + ---------- + path_1: str + Child's source path + path_2: str + Child's destination path + kwargs: dict + arguments passed to child method, e.g., put_file. + + Returns + ------- + + """ + return None + + def no_op(self, *_, **__): + pass + + def __getattr__(self, item): + """ + If undefined methods are called on this class, nothing happens + """ + return self.no_op + + @classmethod + def as_callback(cls, maybe_callback=None): + """Transform callback=... into Callback instance + + For the special value of ``None``, return the global instance of + ``NoOpCallback``. This is an alternative to including + ``callback=DEFAULT_CALLBACK`` directly in a method signature. + """ + if maybe_callback is None: + return DEFAULT_CALLBACK + return maybe_callback + + +class NoOpCallback(Callback): + """ + This implementation of Callback does exactly nothing + """ + + def call(self, *args, **kwargs): + return None + + +class DotPrinterCallback(Callback): + """ + Simple example Callback implementation + + Almost identical to Callback with a hook that prints a char; here we + demonstrate how the outer layer may print "#" and the inner layer "." + """ + + def __init__(self, chr_to_print="#", **kwargs): + self.chr = chr_to_print + super().__init__(**kwargs) + + def branch(self, path_1, path_2, kwargs): + """Mutate kwargs to add new instance with different print char""" + kwargs["callback"] = DotPrinterCallback(".") + + def call(self, **kwargs): + """Just outputs a character""" + print(self.chr, end="") + + +class TqdmCallback(Callback): + """ + A callback to display a progress bar using tqdm + + Parameters + ---------- + tqdm_kwargs : dict, (optional) + Any argument accepted by the tqdm constructor. + See the `tqdm doc `_. + Will be forwarded to `tqdm_cls`. + tqdm_cls: (optional) + subclass of `tqdm.tqdm`. If not passed, it will default to `tqdm.tqdm`. + + Examples + -------- + >>> import fsspec + >>> from fsspec.callbacks import TqdmCallback + >>> fs = fsspec.filesystem("memory") + >>> path2distant_data = "/your-path" + >>> fs.upload( + ".", + path2distant_data, + recursive=True, + callback=TqdmCallback(), + ) + + You can forward args to tqdm using the ``tqdm_kwargs`` parameter. + + >>> fs.upload( + ".", + path2distant_data, + recursive=True, + callback=TqdmCallback(tqdm_kwargs={"desc": "Your tqdm description"}), + ) + + You can also customize the progress bar by passing a subclass of `tqdm`. + + .. code-block:: python + + class TqdmFormat(tqdm): + '''Provides a `total_time` format parameter''' + @property + def format_dict(self): + d = super().format_dict + total_time = d["elapsed"] * (d["total"] or 0) / max(d["n"], 1) + d.update(total_time=self.format_interval(total_time) + " in total") + return d + + >>> with TqdmCallback( + tqdm_kwargs={ + "desc": "desc", + "bar_format": "{total_time}: {percentage:.0f}%|{bar}{r_bar}", + }, + tqdm_cls=TqdmFormat, + ) as callback: + fs.upload(".", path2distant_data, recursive=True, callback=callback) + """ + + def __init__(self, tqdm_kwargs=None, *args, **kwargs): + try: + from tqdm import tqdm + + except ImportError as exce: + raise ImportError( + "Using TqdmCallback requires tqdm to be installed" + ) from exce + + self._tqdm_cls = kwargs.pop("tqdm_cls", tqdm) + self._tqdm_kwargs = tqdm_kwargs or {} + self.tqdm = None + super().__init__(*args, **kwargs) + + def call(self, *args, **kwargs): + if self.tqdm is None: + self.tqdm = self._tqdm_cls(total=self.size, **self._tqdm_kwargs) + self.tqdm.total = self.size + self.tqdm.update(self.value - self.tqdm.n) + + def close(self): + if self.tqdm is not None: + self.tqdm.close() + self.tqdm = None + + def __del__(self): + return self.close() + + +DEFAULT_CALLBACK = _DEFAULT_CALLBACK = NoOpCallback() diff --git a/.venv/lib/python3.14/site-packages/fsspec/compression.py b/.venv/lib/python3.14/site-packages/fsspec/compression.py new file mode 100644 index 0000000000000000000000000000000000000000..11c2e3d3f142d95186663fa5a747911e66832266 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/fsspec/compression.py @@ -0,0 +1,185 @@ +"""Helper functions for a standard streaming compression API""" + +import sys +from zipfile import ZipFile + +import fsspec.utils +from fsspec.spec import AbstractBufferedFile + + +def noop_file(file, mode, **kwargs): + return file + + +# TODO: files should also be available as contexts +# should be functions of the form func(infile, mode=, **kwargs) -> file-like +compr = {None: noop_file} + + +def register_compression(name, callback, extensions, force=False): + """Register an "inferable" file compression type. + + Registers transparent file compression type for use with fsspec.open. + Compression can be specified by name in open, or "infer"-ed for any files + ending with the given extensions. + + Args: + name: (str) The compression type name. Eg. "gzip". + callback: A callable of form (infile, mode, **kwargs) -> file-like. + Accepts an input file-like object, the target mode and kwargs. + Returns a wrapped file-like object. + extensions: (str, Iterable[str]) A file extension, or list of file + extensions for which to infer this compression scheme. Eg. "gz". + force: (bool) Force re-registration of compression type or extensions. + + Raises: + ValueError: If name or extensions already registered, and not force. + + """ + if isinstance(extensions, str): + extensions = [extensions] + + # Validate registration + if name in compr and not force: + raise ValueError(f"Duplicate compression registration: {name}") + + for ext in extensions: + if ext in fsspec.utils.compressions and not force: + raise ValueError(f"Duplicate compression file extension: {ext} ({name})") + + compr[name] = callback + + for ext in extensions: + fsspec.utils.compressions[ext] = name + + +def unzip(infile, mode="rb", filename=None, **kwargs): + if "r" not in mode: + filename = filename or "file" + z = ZipFile(infile, mode="w", **kwargs) + fo = z.open(filename, mode="w") + fo.close = lambda closer=fo.close: closer() or z.close() + return fo + z = ZipFile(infile) + if filename is None: + filename = z.namelist()[0] + return z.open(filename, mode="r", **kwargs) + + +register_compression("zip", unzip, "zip") + +try: + from bz2 import BZ2File +except ImportError: + pass +else: + register_compression("bz2", BZ2File, "bz2") + +try: # pragma: no cover + from isal import igzip + + def isal(infile, mode="rb", **kwargs): + return igzip.IGzipFile(fileobj=infile, mode=mode, **kwargs) + + register_compression("gzip", isal, "gz") +except ImportError: + from gzip import GzipFile + + register_compression( + "gzip", lambda f, **kwargs: GzipFile(fileobj=f, **kwargs), "gz" + ) + +try: + from lzma import LZMAFile + + register_compression("lzma", LZMAFile, "lzma") + register_compression("xz", LZMAFile, "xz") +except ImportError: + pass + +try: + import lzmaffi + + register_compression("lzma", lzmaffi.LZMAFile, "lzma", force=True) + register_compression("xz", lzmaffi.LZMAFile, "xz", force=True) +except ImportError: + pass + + +class SnappyFile(AbstractBufferedFile): + def __init__(self, infile, mode, **kwargs): + import snappy + + super().__init__( + fs=None, path="snappy", mode=mode.strip("b") + "b", size=999999999, **kwargs + ) + self.infile = infile + if "r" in mode: + self.codec = snappy.StreamDecompressor() + else: + self.codec = snappy.StreamCompressor() + + def _upload_chunk(self, final=False): + self.buffer.seek(0) + out = self.codec.add_chunk(self.buffer.read()) + self.infile.write(out) + return True + + def seek(self, loc, whence=0): + raise NotImplementedError("SnappyFile is not seekable") + + def seekable(self): + return False + + def _fetch_range(self, start, end): + """Get the specified set of bytes from remote""" + data = self.infile.read(end - start) + return self.codec.decompress(data) + + +try: + import snappy + + snappy.compress(b"") + # Snappy may use the .sz file extension, but this is not part of the + # standard implementation. + register_compression("snappy", SnappyFile, []) + +except (ImportError, NameError, AttributeError): + pass + +try: + import lz4.frame + + register_compression("lz4", lz4.frame.open, "lz4") +except ImportError: + pass + +try: + if sys.version_info >= (3, 14): + from compression import zstd + else: + from backports import zstd + + register_compression("zstd", zstd.ZstdFile, "zst") +except ImportError: + try: + import zstandard as zstd + + def zstandard_file(infile, mode="rb"): + if "r" in mode: + cctx = zstd.ZstdDecompressor() + return cctx.stream_reader(infile) + else: + cctx = zstd.ZstdCompressor(level=10) + return cctx.stream_writer(infile) + + register_compression("zstd", zstandard_file, "zst") + except ImportError: + pass + pass + + +def available_compressions(): + """Return a list of the implemented compressions.""" + return list(compr) diff --git a/.venv/lib/python3.14/site-packages/fsspec/config.py b/.venv/lib/python3.14/site-packages/fsspec/config.py new file mode 100644 index 0000000000000000000000000000000000000000..19f68071ca026dc2bb11da70effb0c517de5380b --- /dev/null +++ b/.venv/lib/python3.14/site-packages/fsspec/config.py @@ -0,0 +1,131 @@ +from __future__ import annotations + +import configparser +import json +import os +import warnings +from typing import Any + +conf: dict[str, dict[str, Any]] = {} +default_conf_dir = os.path.join(os.path.expanduser("~"), ".config/fsspec") +conf_dir = os.environ.get("FSSPEC_CONFIG_DIR", default_conf_dir) + + +def set_conf_env(conf_dict, envdict=os.environ): + """Set config values from environment variables + + Looks for variables of the form ``FSSPEC_`` and + ``FSSPEC__``. For ``FSSPEC_`` the value is parsed + as a json dictionary and used to ``update`` the config of the + corresponding protocol. For ``FSSPEC__`` there is no + attempt to convert the string value, but the kwarg keys will be lower-cased. + + The ``FSSPEC__`` variables are applied after the + ``FSSPEC_`` ones. + + Parameters + ---------- + conf_dict : dict(str, dict) + This dict will be mutated + envdict : dict-like(str, str) + Source for the values - usually the real environment + """ + envdict = dict(envdict) + kwarg_keys = [] + for key in envdict: + if key.startswith("FSSPEC_") and len(key) > 7 and key[7] != "_": + try: + value = json.loads(envdict[key]) + envdict[key] = value + except json.decoder.JSONDecodeError: + value = envdict[key] + if key.count("_") > 1: + kwarg_keys.append(key) + continue + else: + if isinstance(value, dict): + _, proto = key.split("_", 1) + conf_dict.setdefault(proto.lower(), {}).update(value) + else: + warnings.warn( + f"Ignoring environment variable {key} due to not being a dict:" + f" {type(value)}" + ) + elif key.startswith("FSSPEC"): + warnings.warn( + f"Ignoring environment variable {key} due to having an unexpected name" + ) + + for key in kwarg_keys: + _, proto, kwarg = key.split("_", 2) + conf_dict.setdefault(proto.lower(), {})[kwarg.lower()] = envdict[key] + + +def set_conf_files(cdir, conf_dict): + """Set config values from files + + Scans for INI and JSON files in the given dictionary, and uses their + contents to set the config. In case of repeated values, later values + win. + + In the case of INI files, all values are strings, and these will not + be converted. + + Parameters + ---------- + cdir : str + Directory to search + conf_dict : dict(str, dict) + This dict will be mutated + """ + if not os.path.isdir(cdir): + return + allfiles = sorted(os.listdir(cdir)) + for fn in allfiles: + if fn.endswith(".ini"): + ini = configparser.ConfigParser() + ini.read(os.path.join(cdir, fn)) + for key in ini: + if key == "DEFAULT": + continue + conf_dict.setdefault(key, {}).update(dict(ini[key])) + if fn.endswith(".json"): + with open(os.path.join(cdir, fn)) as f: + js = json.load(f) + for key in js: + conf_dict.setdefault(key, {}).update(dict(js[key])) + + +def apply_config(cls, kwargs, conf_dict=None): + """Supply default values for kwargs when instantiating class + + Augments the passed kwargs, by finding entries in the config dict + which match the classes ``.protocol`` attribute (one or more str) + + Parameters + ---------- + cls : file system implementation + kwargs : dict + conf_dict : dict of dict + Typically this is the global configuration + + Returns + ------- + dict : the modified set of kwargs + """ + if conf_dict is None: + conf_dict = conf + protos = cls.protocol if isinstance(cls.protocol, (tuple, list)) else [cls.protocol] + kw = {} + for proto in protos: + # default kwargs from the current state of the config + if proto in conf_dict: + kw.update(conf_dict[proto]) + # explicit kwargs always win + kw.update(**kwargs) + kwargs = kw + return kwargs + + +set_conf_files(conf_dir, conf) +set_conf_env(conf) diff --git a/.venv/lib/python3.14/site-packages/fsspec/conftest.py b/.venv/lib/python3.14/site-packages/fsspec/conftest.py new file mode 100644 index 0000000000000000000000000000000000000000..f05eb5c30d42b0c1c5cc432f9c217d8f0e01f412 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/fsspec/conftest.py @@ -0,0 +1,125 @@ +import os +import shutil +import subprocess +import sys +import time +from collections import deque +from collections.abc import Generator, Sequence + +import pytest + +import fsspec + + +@pytest.fixture() +def m(): + """ + Fixture providing a memory filesystem. + """ + m = fsspec.filesystem("memory") + m.store.clear() + m.pseudo_dirs.clear() + m.pseudo_dirs.append("") + try: + yield m + finally: + m.store.clear() + m.pseudo_dirs.clear() + m.pseudo_dirs.append("") + + +class InstanceCacheInspector: + """ + Helper class to inspect instance caches of filesystem classes in tests. + """ + + def clear(self) -> None: + """ + Clear instance caches of all currently imported filesystem classes. + """ + classes = deque([fsspec.spec.AbstractFileSystem]) + while classes: + cls = classes.popleft() + cls.clear_instance_cache() + classes.extend(cls.__subclasses__()) + + def gather_counts(self, *, omit_zero: bool = True) -> dict[str, int]: + """ + Gather counts of filesystem instances in the instance caches + of all currently imported filesystem classes. + + Parameters + ---------- + omit_zero: + Whether to omit instance types with no cached instances. + """ + out: dict[str, int] = {} + classes = deque([fsspec.spec.AbstractFileSystem]) + while classes: + cls = classes.popleft() + count = len(cls._cache) # there is no public interface for the cache + # note: skip intermediate AbstractFileSystem subclasses + # if they proxy the protocol attribute via a property. + if isinstance(cls.protocol, (Sequence, str)): + key = cls.protocol if isinstance(cls.protocol, str) else cls.protocol[0] + if count or not omit_zero: + out[key] = count + classes.extend(cls.__subclasses__()) + return out + + +@pytest.fixture(scope="function", autouse=True) +def instance_caches() -> Generator[InstanceCacheInspector, None, None]: + """ + Fixture to ensure empty filesystem instance caches before and after a test. + + Used by default for all tests. + Clears caches of all imported filesystem classes. + Can be used to write test assertions about instance caches. + + Usage: + + def test_something(instance_caches): + # Test code here + fsspec.open("file://abc") + fsspec.open("memory://foo/bar") + + # Test assertion + assert instance_caches.gather_counts() == {"file": 1, "memory": 1} + + Returns + ------- + instance_caches: An instance cache inspector for clearing and inspecting caches. + """ + ic = InstanceCacheInspector() + + ic.clear() + try: + yield ic + finally: + ic.clear() + + +@pytest.fixture(scope="function") +def ftp_writable(tmpdir): + """ + Fixture providing a writable FTP filesystem. + """ + pytest.importorskip("pyftpdlib") + + d = str(tmpdir) + with open(os.path.join(d, "out"), "wb") as f: + f.write(b"hello" * 10000) + P = subprocess.Popen( + [sys.executable, "-m", "pyftpdlib", "-d", d, "-u", "user", "-P", "pass", "-w"] + ) + try: + time.sleep(1) + yield "localhost", 2121, "user", "pass" + finally: + P.terminate() + P.wait() + try: + shutil.rmtree(tmpdir) + except Exception: + pass diff --git a/.venv/lib/python3.14/site-packages/fsspec/core.py b/.venv/lib/python3.14/site-packages/fsspec/core.py new file mode 100644 index 0000000000000000000000000000000000000000..f1e7e7bbd0639e3607647b1cfe486405ce45c375 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/fsspec/core.py @@ -0,0 +1,765 @@ +from __future__ import annotations + +import io +import logging +import posixpath +import re +from glob import has_magic +from pathlib import Path + +# for backwards compat, we export cache things from here too +from fsspec.caching import ( # noqa: F401 + BaseCache, + BlockCache, + BytesCache, + MMapCache, + ReadAheadCache, + caches, +) +from fsspec.compression import compr +from fsspec.config import conf +from fsspec.registry import available_protocols, filesystem, get_filesystem_class +from fsspec.utils import ( + _unstrip_protocol, + build_name_function, + infer_compression, + stringify_path, +) + +logger = logging.getLogger("fsspec") + + +class OpenFile: + """ + File-like object to be used in a context + + Can layer (buffered) text-mode and compression over any file-system, which + are typically binary-only. + + These instances are safe to serialize, as the low-level file object + is not created until invoked using ``with``. + + Parameters + ---------- + fs: FileSystem + The file system to use for opening the file. Should be a subclass or duck-type + with ``fsspec.spec.AbstractFileSystem`` + path: str + Location to open + mode: str like 'rb', optional + Mode of the opened file + compression: str or None, optional + Compression to apply + encoding: str or None, optional + The encoding to use if opened in text mode. + errors: str or None, optional + How to handle encoding errors if opened in text mode. + newline: None or str + Passed to TextIOWrapper in text mode, how to handle line endings. + autoopen: bool + If True, calls open() immediately. Mostly used by pickle + pos: int + If given and autoopen is True, seek to this location immediately + """ + + def __init__( + self, + fs, + path, + mode="rb", + compression=None, + encoding=None, + errors=None, + newline=None, + ): + self.fs = fs + self.path = path + self.mode = mode + self.compression = get_compression(path, compression) + self.encoding = encoding + self.errors = errors + self.newline = newline + self.fobjects = [] + + def __reduce__(self): + return ( + OpenFile, + ( + self.fs, + self.path, + self.mode, + self.compression, + self.encoding, + self.errors, + self.newline, + ), + ) + + def __repr__(self): + return f"" + + def __enter__(self): + mode = self.mode.replace("t", "").replace("b", "") + "b" + + try: + f = self.fs.open(self.path, mode=mode) + except FileNotFoundError as e: + if has_magic(self.path): + raise FileNotFoundError( + "%s not found. The URL contains glob characters: you maybe needed\n" + "to pass expand=True in fsspec.open() or the storage_options of \n" + "your library. You can also set the config value 'open_expand'\n" + "before import, or fsspec.core.DEFAULT_EXPAND at runtime, to True.", + self.path, + ) from e + raise + + self.fobjects = [f] + + if self.compression is not None: + compress = compr[self.compression] + f = compress(f, mode=mode[0]) + self.fobjects.append(f) + + if "b" not in self.mode: + # assume, for example, that 'r' is equivalent to 'rt' as in builtin + f = PickleableTextIOWrapper( + f, encoding=self.encoding, errors=self.errors, newline=self.newline + ) + self.fobjects.append(f) + + return self.fobjects[-1] + + def __exit__(self, *args): + self.close() + + @property + def full_name(self): + return _unstrip_protocol(self.path, self.fs) + + def open(self): + """Materialise this as a real open file without context + + The OpenFile object should be explicitly closed to avoid enclosed file + instances persisting. You must, therefore, keep a reference to the OpenFile + during the life of the file-like it generates. + """ + return self.__enter__() + + def close(self): + """Close all encapsulated file objects""" + for f in reversed(self.fobjects): + if "r" not in self.mode and not f.closed: + f.flush() + f.close() + self.fobjects.clear() + + +class OpenFiles(list): + """List of OpenFile instances + + Can be used in a single context, which opens and closes all of the + contained files. Normal list access to get the elements works as + normal. + + A special case is made for caching filesystems - the files will + be down/uploaded together at the start or end of the context, and + this may happen concurrently, if the target filesystem supports it. + """ + + def __init__(self, *args, mode="rb", fs=None): + self.mode = mode + self.fs = fs + self.files = [] + super().__init__(*args) + + def __enter__(self): + if self.fs is None: + raise ValueError("Context has already been used") + + fs = self.fs + while True: + if hasattr(fs, "open_many"): + # check for concurrent cache download; or set up for upload + self.files = fs.open_many(self) + return self.files + if hasattr(fs, "fs") and fs.fs is not None: + fs = fs.fs + else: + break + return [s.__enter__() for s in self] + + def __exit__(self, *args): + fs = self.fs + [s.__exit__(*args) for s in self] + if "r" in self.mode: + # open_many() returns files without populating the OpenFile objects. + for f in self.files: + f.close() + else: + while True: + if hasattr(fs, "open_many"): + # check for concurrent cache upload + fs.commit_many(self.files) + return + if hasattr(fs, "fs") and fs.fs is not None: + fs = fs.fs + else: + break + + def __getitem__(self, item): + out = super().__getitem__(item) + if isinstance(item, slice): + return OpenFiles(out, mode=self.mode, fs=self.fs) + return out + + def __repr__(self): + return f"" + + +def open_files( + urlpath, + mode="rb", + compression=None, + encoding="utf8", + errors=None, + name_function=None, + num=1, + protocol=None, + newline=None, + auto_mkdir=True, + expand=True, + **kwargs, +): + """Given a path or paths, return a list of ``OpenFile`` objects. + + For writing, a str path must contain the "*" character, which will be filled + in by increasing numbers, e.g., "part*" -> "part1", "part2" if num=2. + + For either reading or writing, can instead provide explicit list of paths. + + Parameters + ---------- + urlpath: string or list + Absolute or relative filepath(s). Prefix with a protocol like ``s3://`` + to read from alternative filesystems. To read from multiple files you + can pass a globstring or a list of paths, with the caveat that they + must all have the same protocol. + mode: 'rb', 'wt', etc. + compression: string or None + If given, open file using compression codec. Can either be a compression + name (a key in ``fsspec.compression.compr``) or "infer" to guess the + compression from the filename suffix. + encoding: str + For text mode only + errors: None or str + Passed to TextIOWrapper in text mode + name_function: function or None + if opening a set of files for writing, those files do not yet exist, + so we need to generate their names by formatting the urlpath for + each sequence number + num: int [1] + if writing mode, number of files we expect to create (passed to + name+function) + protocol: str or None + If given, overrides the protocol found in the URL. + newline: bytes or None + Used for line terminator in text mode. If None, uses system default; + if blank, uses no translation. + auto_mkdir: bool (True) + If in write mode, this will ensure the target directory exists before + writing, by calling ``fs.mkdirs(exist_ok=True)``. + expand: bool + **kwargs: dict + Extra options that make sense to a particular storage connection, e.g. + host, port, username, password, etc. + + Examples + -------- + >>> files = open_files('2015-*-*.csv') # doctest: +SKIP + >>> files = open_files( + ... 's3://bucket/2015-*-*.csv.gz', compression='gzip' + ... ) # doctest: +SKIP + + Returns + ------- + An ``OpenFiles`` instance, which is a list of ``OpenFile`` objects that can + be used as a single context + + Notes + ----- + For a full list of the available protocols and the implementations that + they map across to see the latest online documentation: + + - For implementations built into ``fsspec`` see + https://filesystem-spec.readthedocs.io/en/latest/api.html#built-in-implementations + - For implementations in separate packages see + https://filesystem-spec.readthedocs.io/en/latest/api.html#other-known-implementations + """ + fs, fs_token, paths = get_fs_token_paths( + urlpath, + mode, + num=num, + name_function=name_function, + storage_options=kwargs, + protocol=protocol, + expand=expand, + ) + if fs.protocol == "file": + fs.auto_mkdir = auto_mkdir + elif "r" not in mode and auto_mkdir: + parents = {fs._parent(path) for path in paths} + for parent in parents: + try: + fs.makedirs(parent, exist_ok=True) + except PermissionError: + pass + return OpenFiles( + [ + OpenFile( + fs, + path, + mode=mode, + compression=compression, + encoding=encoding, + errors=errors, + newline=newline, + ) + for path in paths + ], + mode=mode, + fs=fs, + ) + + +def _un_chain(path, kwargs): + # Avoid a circular import + from fsspec.implementations.chained import ChainedFileSystem + + if "::" in path: + x = re.compile(".*[^a-z]+.*") # test for non protocol-like single word + known_protocols = set(available_protocols()) + bits = [] + + # split on '::', then ensure each bit has a protocol + for p in path.split("::"): + if p in known_protocols: + bits.append(p + "://") + elif "://" in p or x.match(p): + bits.append(p) + else: + bits.append(p + "://") + else: + bits = [path] + + # [[url, protocol, kwargs], ...] + out = [] + previous_bit = None + kwargs = kwargs.copy() + + for bit in reversed(bits): + protocol = kwargs.pop("protocol", None) or split_protocol(bit)[0] or "file" + cls = get_filesystem_class(protocol) + extra_kwargs = cls._get_kwargs_from_urls(bit) + kws = kwargs.pop(protocol, {}) + + if bit is bits[0]: + kws.update(kwargs) + + kw = dict( + **{k: v for k, v in extra_kwargs.items() if k not in kws or v != kws[k]}, + **kws, + ) + bit = cls._strip_protocol(bit) + + if ( + "target_protocol" not in kw + and issubclass(cls, ChainedFileSystem) + and not bit + ): + # replace bit if we are chaining and no path given + bit = previous_bit + + out.append((bit, protocol, kw)) + previous_bit = bit + + out.reverse() + return out + + +def url_to_fs(url, **kwargs): + """ + Turn fully-qualified and potentially chained URL into filesystem instance + + Parameters + ---------- + url : str + The fsspec-compatible URL + **kwargs: dict + Extra options that make sense to a particular storage connection, e.g. + host, port, username, password, etc. + + Returns + ------- + filesystem : FileSystem + The new filesystem discovered from ``url`` and created with + ``**kwargs``. + urlpath : str + The file-systems-specific URL for ``url``. + """ + url = stringify_path(url) + # non-FS arguments that appear in fsspec.open() + # inspect could keep this in sync with open()'s signature + known_kwargs = { + "compression", + "encoding", + "errors", + "expand", + "mode", + "name_function", + "newline", + "num", + } + kwargs = {k: v for k, v in kwargs.items() if k not in known_kwargs} + chain = _un_chain(url, kwargs) + inkwargs = {} + # Reverse iterate the chain, creating a nested target_* structure + for i, ch in enumerate(reversed(chain)): + urls, protocol, kw = ch + if i == len(chain) - 1: + inkwargs = dict(**kw, **inkwargs) + continue + inkwargs["target_options"] = dict(**kw, **inkwargs) + inkwargs["target_protocol"] = protocol + inkwargs["fo"] = urls + urlpath, protocol, _ = chain[0] + fs = filesystem(protocol, **inkwargs) + return fs, urlpath + + +DEFAULT_EXPAND = conf.get("open_expand", False) + + +def open( + urlpath, + mode="rb", + compression=None, + encoding="utf8", + errors=None, + protocol=None, + newline=None, + expand=None, + **kwargs, +): + """Given a path or paths, return one ``OpenFile`` object. + + Parameters + ---------- + urlpath: string or list + Absolute or relative filepath. Prefix with a protocol like ``s3://`` + to read from alternative filesystems. Should not include glob + character(s). + mode: 'rb', 'wt', etc. + compression: string or None + If given, open file using compression codec. Can either be a compression + name (a key in ``fsspec.compression.compr``) or "infer" to guess the + compression from the filename suffix. + encoding: str + For text mode only + errors: None or str + Passed to TextIOWrapper in text mode + protocol: str or None + If given, overrides the protocol found in the URL. + newline: bytes or None + Used for line terminator in text mode. If None, uses system default; + if blank, uses no translation. + expand: bool or None + Whether to regard file paths containing special glob characters as needing + expansion (finding the first match) or absolute. Setting False allows using + paths which do embed such characters. If None (default), this argument + takes its value from the DEFAULT_EXPAND module variable, which takes + its initial value from the "open_expand" config value at startup, which will + be False if not set. + **kwargs: dict + Extra options that make sense to a particular storage connection, e.g. + host, port, username, password, etc. + + Examples + -------- + >>> openfile = open('2015-01-01.csv') # doctest: +SKIP + >>> openfile = open( + ... 's3://bucket/2015-01-01.csv.gz', compression='gzip' + ... ) # doctest: +SKIP + >>> with openfile as f: + ... df = pd.read_csv(f) # doctest: +SKIP + ... + + Returns + ------- + ``OpenFile`` object. + + Notes + ----- + For a full list of the available protocols and the implementations that + they map across to see the latest online documentation: + + - For implementations built into ``fsspec`` see + https://filesystem-spec.readthedocs.io/en/latest/api.html#built-in-implementations + - For implementations in separate packages see + https://filesystem-spec.readthedocs.io/en/latest/api.html#other-known-implementations + """ + expand = DEFAULT_EXPAND if expand is None else expand + out = open_files( + urlpath=[urlpath], + mode=mode, + compression=compression, + encoding=encoding, + errors=errors, + protocol=protocol, + newline=newline, + expand=expand, + **kwargs, + ) + if not out: + raise FileNotFoundError(urlpath) + return out[0] + + +def open_local( + url: str | list[str] | Path | list[Path], + mode: str = "rb", + **storage_options: dict, +) -> str | list[str]: + """Open file(s) which can be resolved to local + + For files which either are local, or get downloaded upon open + (e.g., by file caching) + + Parameters + ---------- + url: str or list(str) + mode: str + Must be read mode + storage_options: + passed on to FS for or used by open_files (e.g., compression) + """ + if "r" not in mode: + raise ValueError("Can only ensure local files when reading") + of = open_files(url, mode=mode, **storage_options) + if not getattr(of[0].fs, "local_file", False): + raise ValueError( + "open_local can only be used on a filesystem which" + " has attribute local_file=True" + ) + with of as files: + paths = [f.name for f in files] + if (isinstance(url, str) and not has_magic(url)) or isinstance(url, Path): + return paths[0] + return paths + + +def get_compression(urlpath, compression): + if compression == "infer": + compression = infer_compression(urlpath) + if compression is not None and compression not in compr: + raise ValueError(f"Compression type {compression} not supported") + return compression + + +def split_protocol(urlpath): + """Return protocol, path pair""" + urlpath = stringify_path(urlpath) + if "://" in urlpath: + protocol, path = urlpath.split("://", 1) + if len(protocol) > 1: + # excludes Windows paths + return protocol, path + if urlpath.startswith("data:"): + return urlpath.split(":", 1) + return None, urlpath + + +def strip_protocol(urlpath): + """Return only path part of full URL, according to appropriate backend""" + protocol, _ = split_protocol(urlpath) + cls = get_filesystem_class(protocol) + return cls._strip_protocol(urlpath) + + +def expand_paths_if_needed(paths, mode, num, fs, name_function): + """Expand paths if they have a ``*`` in them (write mode) or any of ``*?[]`` + in them (read mode). + + :param paths: list of paths + mode: str + Mode in which to open files. + num: int + If opening in writing mode, number of files we expect to create. + fs: filesystem object + name_function: callable + If opening in writing mode, this callable is used to generate path + names. Names are generated for each partition by + ``urlpath.replace('*', name_function(partition_index))``. + :return: list of paths + """ + expanded_paths = [] + paths = list(paths) + + if "w" in mode or "x" in mode: # write mode + if sum(1 for p in paths if "*" in p) > 1: + raise ValueError( + "When writing data, only one filename mask can be specified." + ) + num = max(num, len(paths)) + + for curr_path in paths: + if "*" in curr_path: + # expand using name_function + expanded_paths.extend(_expand_paths(curr_path, name_function, num)) + else: + expanded_paths.append(curr_path) + # if we generated more paths that asked for, trim the list + if len(expanded_paths) > num: + expanded_paths = expanded_paths[:num] + + else: # read mode + for curr_path in paths: + if has_magic(curr_path): + # expand using glob + expanded_paths.extend(fs.glob(curr_path)) + else: + expanded_paths.append(curr_path) + + return expanded_paths + + +def get_fs_token_paths( + urlpath, + mode="rb", + num=1, + name_function=None, + storage_options=None, + protocol=None, + expand=True, +): + """Filesystem, deterministic token, and paths from a urlpath and options. + + Parameters + ---------- + urlpath: string or iterable + Absolute or relative filepath, URL (may include protocols like + ``s3://``), or globstring pointing to data. + mode: str, optional + Mode in which to open files. + num: int, optional + If opening in writing mode, number of files we expect to create. + name_function: callable, optional + If opening in writing mode, this callable is used to generate path + names. Names are generated for each partition by + ``urlpath.replace('*', name_function(partition_index))``. + storage_options: dict, optional + Additional keywords to pass to the filesystem class. + protocol: str or None + To override the protocol specifier in the URL + expand: bool + Expand string paths for writing, assuming the path is a directory + """ + if isinstance(urlpath, (list, tuple, set)): + if not urlpath: + raise ValueError("empty urlpath sequence") + urlpath0 = stringify_path(next(iter(urlpath))) + else: + urlpath0 = stringify_path(urlpath) + storage_options = storage_options or {} + if protocol: + storage_options["protocol"] = protocol + chain = _un_chain(urlpath0, storage_options or {}) + inkwargs = {} + # Reverse iterate the chain, creating a nested target_* structure + for i, ch in enumerate(reversed(chain)): + urls, nested_protocol, kw = ch + if i == len(chain) - 1: + inkwargs = dict(**kw, **inkwargs) + continue + inkwargs["target_options"] = dict(**kw, **inkwargs) + inkwargs["target_protocol"] = nested_protocol + inkwargs["fo"] = urls + paths, protocol, _ = chain[0] + fs = filesystem(protocol, **inkwargs) + if isinstance(urlpath, (list, tuple, set)): + pchains = [ + _un_chain(stringify_path(u), storage_options or {})[0] for u in urlpath + ] + if len({pc[1] for pc in pchains}) > 1: + raise ValueError("Protocol mismatch getting fs from %s", urlpath) + paths = [pc[0] for pc in pchains] + else: + paths = fs._strip_protocol(paths) + if isinstance(paths, (list, tuple, set)): + if expand: + paths = expand_paths_if_needed(paths, mode, num, fs, name_function) + elif not isinstance(paths, list): + paths = list(paths) + else: + if ("w" in mode or "x" in mode) and expand: + paths = _expand_paths(paths, name_function, num) + elif "*" in paths: + paths = [f for f in sorted(fs.glob(paths)) if not fs.isdir(f)] + else: + paths = [paths] + + return fs, fs._fs_token, paths + + +def _expand_paths(path, name_function, num): + if isinstance(path, str): + if path.count("*") > 1: + raise ValueError("Output path spec must contain exactly one '*'.") + elif "*" not in path: + # paths are "/"-separated on every filesystem, including local ones + path = posixpath.join(path, "*.part") + + if name_function is None: + name_function = build_name_function(num - 1) + + paths = [path.replace("*", name_function(i)) for i in range(num)] + if paths != sorted(paths): + logger.warning( + "In order to preserve order between partitions" + " paths created with ``name_function`` should " + "sort to partition order" + ) + elif isinstance(path, (tuple, list)): + assert len(path) == num + paths = list(path) + else: + raise ValueError( + "Path should be either\n" + "1. A list of paths: ['foo.json', 'bar.json', ...]\n" + "2. A directory: 'foo/\n" + "3. A path with a '*' in it: 'foo.*.json'" + ) + return paths + + +class PickleableTextIOWrapper(io.TextIOWrapper): + """TextIOWrapper cannot be pickled. This solves it. + + Requires that ``buffer`` be pickleable, which all instances of + AbstractBufferedFile are. + """ + + def __init__( + self, + buffer, + encoding=None, + errors=None, + newline=None, + line_buffering=False, + write_through=False, + ): + self.args = buffer, encoding, errors, newline, line_buffering, write_through + super().__init__(*self.args) + + def __reduce__(self): + return PickleableTextIOWrapper, self.args diff --git a/.venv/lib/python3.14/site-packages/fsspec/dircache.py b/.venv/lib/python3.14/site-packages/fsspec/dircache.py new file mode 100644 index 0000000000000000000000000000000000000000..eca19566b135e5a7a4f6e7407d56411ec58bfe44 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/fsspec/dircache.py @@ -0,0 +1,98 @@ +import time +from collections.abc import MutableMapping +from functools import lru_cache + + +class DirCache(MutableMapping): + """ + Caching of directory listings, in a structure like:: + + {"path0": [ + {"name": "path0/file0", + "size": 123, + "type": "file", + ... + }, + {"name": "path0/file1", + }, + ... + ], + "path1": [...] + } + + Parameters to this class control listing expiry or indeed turn + caching off + """ + + def __init__( + self, + use_listings_cache=True, + listings_expiry_time=None, + max_paths=None, + **kwargs, + ): + """ + + Parameters + ---------- + use_listings_cache: bool + If False, this cache never returns items, but always reports KeyError, + and setting items has no effect + listings_expiry_time: int or float (optional) + Time in seconds that a listing is considered valid. If None, + listings do not expire. + max_paths: int (optional) + The number of most recent listings that are considered valid; 'recent' + refers to when the entry was set. + """ + self._cache = {} + self._times = {} + if max_paths: + self._q = lru_cache(max_paths + 1)(lambda key: self._cache.pop(key, None)) + self.use_listings_cache = use_listings_cache + self.listings_expiry_time = listings_expiry_time + self.max_paths = max_paths + + def __getitem__(self, item): + if self.listings_expiry_time is not None: + if self._times.get(item, 0) - time.time() < -self.listings_expiry_time: + del self._cache[item] + if self.max_paths: + self._q(item) + return self._cache[item] # maybe raises KeyError + + def clear(self): + self._cache.clear() + + def __len__(self): + return len(self._cache) + + def __contains__(self, item): + try: + self[item] + return True + except KeyError: + return False + + def __setitem__(self, key, value): + if not self.use_listings_cache: + return + if self.max_paths: + self._q(key) + self._cache[key] = value + if self.listings_expiry_time is not None: + self._times[key] = time.time() + + def __delitem__(self, key): + del self._cache[key] + + def __iter__(self): + entries = list(self._cache) + + return (k for k in entries if k in self) + + def __reduce__(self): + return ( + DirCache, + (self.use_listings_cache, self.listings_expiry_time, self.max_paths), + ) diff --git a/.venv/lib/python3.14/site-packages/fsspec/exceptions.py b/.venv/lib/python3.14/site-packages/fsspec/exceptions.py new file mode 100644 index 0000000000000000000000000000000000000000..ae8905475f02655f4fc5863931d99ca9da55db78 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/fsspec/exceptions.py @@ -0,0 +1,18 @@ +""" +fsspec user-defined exception classes +""" + +import asyncio + + +class BlocksizeMismatchError(ValueError): + """ + Raised when a cached file is opened with a different blocksize than it was + written with + """ + + +class FSTimeoutError(asyncio.TimeoutError): + """ + Raised when a fsspec function timed out occurs + """ diff --git a/.venv/lib/python3.14/site-packages/fsspec/fuse.py b/.venv/lib/python3.14/site-packages/fsspec/fuse.py new file mode 100644 index 0000000000000000000000000000000000000000..566d520fce3e94e3bbaee48c3c6acc9f1db315a8 --- /dev/null +++ b/.venv/lib/python3.14/site-packages/fsspec/fuse.py @@ -0,0 +1,324 @@ +import argparse +import logging +import os +import stat +import threading +import time +from errno import EIO, ENOENT + +from fuse import FUSE, FuseOSError, LoggingMixIn, Operations + +from fsspec import __version__ +from fsspec.core import url_to_fs + +logger = logging.getLogger("fsspec.fuse") + + +class FUSEr(Operations): + def __init__(self, fs, path, ready_file=False): + self.fs = fs + self.cache = {} + self.root = path.rstrip("/") + "/" + self.counter = 0 + logger.info("Starting FUSE at %s", path) + self._ready_file = ready_file + + def getattr(self, path, fh=None): + logger.debug("getattr %s", path) + if self._ready_file and path in ["/.fuse_ready", ".fuse_ready"]: + return {"type": "file", "st_size": 5} + + path = "".join([self.root, path.lstrip("/")]).rstrip("/") + try: + info = self.fs.info(path) + except FileNotFoundError as exc: + raise FuseOSError(ENOENT) from exc + + data = {"st_uid": info.get("uid", 1000), "st_gid": info.get("gid", 1000)} + perm = info.get("mode", 0o777) + + if info["type"] != "file": + data["st_mode"] = stat.S_IFDIR | perm + data["st_size"] = 0 + data["st_blksize"] = 0 + else: + data["st_mode"] = stat.S_IFREG | perm + data["st_size"] = info["size"] + data["st_blksize"] = 5 * 2**20 + data["st_nlink"] = 1 + data["st_atime"] = info["atime"] if "atime" in info else time.time() + data["st_ctime"] = info["ctime"] if "ctime" in info else time.time() + data["st_mtime"] = info["mtime"] if "mtime" in info else time.time() + return data + + def readdir(self, path, fh): + logger.debug("readdir %s", path) + path = "".join([self.root, path.lstrip("/")]) + files = self.fs.ls(path, False) + files = [os.path.basename(f.rstrip("/")) for f in files] + return [".", ".."] + files + + def mkdir(self, path, mode): + path = "".join([self.root, path.lstrip("/")]) + self.fs.mkdir(path) + return 0 + + def rmdir(self, path): + path = "".join([self.root, path.lstrip("/")]) + self.fs.rmdir(path) + return 0 + + def read(self, path, size, offset, fh): + logger.debug("read %s", (path, size, offset)) + if self._ready_file and path in ["/.fuse_ready", ".fuse_ready"]: + # status indicator + return b"ready" + + f = self.cache[fh] + f.seek(offset) + out = f.read(size) + return out + + def write(self, path, data, offset, fh): + logger.debug("write %s", (path, offset)) + f = self.cache[fh] + f.seek(offset) + f.write(data) + return len(data) + + def create(self, path, flags, fi=None): + logger.debug("create %s", (path, flags)) + fn = "".join([self.root, path.lstrip("/")]) + self.fs.touch(fn) # OS will want to get attributes immediately + f = self.fs.open(fn, "wb") + self.cache[self.counter] = f + self.counter += 1 + return self.counter - 1 + + def open(self, path, flags): + logger.debug("open %s", (path, flags)) + fn = "".join([self.root, path.lstrip("/")]) + if flags % 2 == 0: + # read + mode = "rb" + else: + # write/create + mode = "wb" + self.cache[self.counter] = self.fs.open(fn, mode) + self.counter += 1 + return self.counter - 1 + + def truncate(self, path, length, fh=None): + fn = "".join([self.root, path.lstrip("/")]) + if length != 0: + raise NotImplementedError + # maybe should be no-op since open with write sets size to zero anyway + self.fs.touch(fn) + + def unlink(self, path): + fn = "".join([self.root, path.lstrip("/")]) + try: + self.fs.rm(fn, False) + except (OSError, FileNotFoundError) as exc: + raise FuseOSError(EIO) from exc + + def release(self, path, fh): + try: + if fh in self.cache: + f = self.cache[fh] + f.close() + self.cache.pop(fh) + except Exception as e: + print(e) + return 0 + + def chmod(self, path, mode): + if hasattr(self.fs, "chmod"): + path = "".join([self.root, path.lstrip("/")]) + return self.fs.chmod(path, mode) + raise NotImplementedError + + +def run( + fs, + path, + mount_point, + foreground=True, + threads=False, + ready_file=False, + ops_class=FUSEr, +): + """Mount stuff in a local directory + + This uses fusepy to make it appear as if a given path on an fsspec + instance is in fact resident within the local file-system. + + This requires that fusepy by installed, and that FUSE be available on + the system (typically requiring a package to be installed with + apt, yum, brew, etc.). + + Parameters + ---------- + fs: file-system instance + From one of the compatible implementations + path: str + Location on that file-system to regard as the root directory to + mount. Note that you typically should include the terminating "/" + character. + mount_point: str + An empty directory on the local file-system where the contents of + the remote path will appear. + foreground: bool + Whether or not calling this function will block. Operation will + typically be more stable if True. + threads: bool + Whether or not to create threads when responding to file operations + within the mounter directory. Operation will typically be more + stable if False. + ready_file: bool + Whether the FUSE process is ready. The ``.fuse_ready`` file will + exist in the ``mount_point`` directory if True. Debugging purpose. + ops_class: FUSEr or Subclass of FUSEr + To override the default behavior of FUSEr. For Example, logging + to file. + + """ + func = lambda: FUSE( + ops_class(fs, path, ready_file=ready_file), + mount_point, + nothreads=not threads, + foreground=foreground, + ) + if not foreground: + th = threading.Thread(target=func) + th.daemon = True + th.start() + return th + else: # pragma: no cover + try: + func() + except KeyboardInterrupt: + pass + + +def main(args): + """Mount filesystem from chained URL to MOUNT_POINT. + + Examples: + + python3 -m fsspec.fuse memory /usr/share /tmp/mem + + python3 -m fsspec.fuse local /tmp/source /tmp/local \\ + -l /tmp/fsspecfuse.log + + You can also mount chained-URLs and use special settings: + + python3 -m fsspec.fuse 'filecache::zip::file://data.zip' \\ + / /tmp/zip \\ + -o 'filecache-cache_storage=/tmp/simplecache' + + You can specify the type of the setting by using `[int]` or `[bool]`, + (`true`, `yes`, `1` represents the Boolean value `True`): + + python3 -m fsspec.fuse 'simplecache::ftp://ftp1.at.proftpd.org' \\ + /historic/packages/RPMS /tmp/ftp \\ + -o 'simplecache-cache_storage=/tmp/simplecache' \\ + -o 'simplecache-check_files=false[bool]' \\ + -o 'ftp-listings_expiry_time=60[int]' \\ + -o 'ftp-username=anonymous' \\ + -o 'ftp-password=xieyanbo' + """ + + class RawDescriptionArgumentParser(argparse.ArgumentParser): + def format_help(self): + usage = super().format_help() + parts = usage.split("\n\n") + parts[1] = self.description.rstrip() + return "\n\n".join(parts) + + parser = RawDescriptionArgumentParser(prog="fsspec.fuse", description=main.__doc__) + parser.add_argument("--version", action="version", version=__version__) + parser.add_argument("url", type=str, help="fs url") + parser.add_argument("source_path", type=str, help="source directory in fs") + parser.add_argument("mount_point", type=str, help="local directory") + parser.add_argument( + "-o", + "--option", + action="append", + help="Any options of protocol included in the chained URL", + ) + parser.add_argument( + "-l", "--log-file", type=str, help="Logging FUSE debug info (Default: '')" + ) + parser.add_argument( + "-f", + "--foreground", + action="store_false", + help="Running in foreground or not (Default: False)", + ) + parser.add_argument( + "-t", + "--threads", + action="store_false", + help="Running with threads support (Default: False)", + ) + parser.add_argument( + "-r", + "--ready-file", + action="store_false", + help="The `.fuse_ready` file will exist after FUSE is ready. " + "(Debugging purpose, Default: False)", + ) + args = parser.parse_args(args) + + kwargs = {} + for item in args.option or []: + key, sep, value = item.partition("=") + if not sep: + parser.error(message=f"Wrong option: {item!r}") + val = value.lower() + if val.endswith("[int]"): + value = int(value[: -len("[int]")]) + elif val.endswith("[bool]"): + value = val[: -len("[bool]")] in ["1", "yes", "true"] + + if "-" in key: + fs_name, setting_name = key.split("-", 1) + if fs_name in kwargs: + kwargs[fs_name][setting_name] = value + else: + kwargs[fs_name] = {setting_name: value} + else: + kwargs[key] = value + + if args.log_file: + logging.basicConfig( + level=logging.DEBUG, + filename=args.log_file, + format="%(asctime)s %(message)s", + ) + + class LoggingFUSEr(FUSEr, LoggingMixIn): + pass + + fuser = LoggingFUSEr + else: + fuser = FUSEr + + fs, url_path = url_to_fs(args.url, **kwargs) + logger.debug("Mounting %s to %s", url_path, str(args.mount_point)) + run( + fs, + args.source_path, + args.mount_point, + foreground=args.foreground, + threads=args.threads, + ready_file=args.ready_file, + ops_class=fuser, + ) + + +if __name__ == "__main__": + import sys + + main(sys.argv[1:]) diff --git a/.venv/lib/python3.14/site-packages/fsspec/generic.py b/.venv/lib/python3.14/site-packages/fsspec/generic.py new file mode 100644 index 0000000000000000000000000000000000000000..0600e942d08e8974203795e28d43cfa0928a60ea --- /dev/null +++ b/.venv/lib/python3.14/site-packages/fsspec/generic.py @@ -0,0 +1,396 @@ +from __future__ import annotations + +import inspect +import logging +import os +import shutil +import uuid + +from .asyn import AsyncFileSystem, _run_coros_in_chunks, sync_wrapper +from .callbacks import DEFAULT_CALLBACK +from .core import filesystem, get_filesystem_class, split_protocol, url_to_fs + +_generic_fs = {} +logger = logging.getLogger("fsspec.generic") + + +def set_generic_fs(protocol, **storage_options): + """Populate the dict used for method=="generic" lookups""" + _generic_fs[protocol] = filesystem(protocol, **storage_options) + + +def _resolve_fs(url, method, protocol=None, storage_options=None): + """Pick instance of backend FS""" + url = url[0] if isinstance(url, (list, tuple)) else url + protocol = protocol or split_protocol(url)[0] + storage_options = storage_options or {} + if method == "default": + return filesystem(protocol) + if method == "generic": + return _generic_fs[protocol] + if method == "current": + cls = get_filesystem_class(protocol) + return cls.current() + if method == "options": + fs, _ = url_to_fs(url, **storage_options.get(protocol, {})) + return fs + raise ValueError(f"Unknown FS resolution method: {method}") + + +def rsync( + source, + destination, + delete_missing=False, + source_field="size", + dest_field="size", + update_cond="different", + inst_kwargs=None, + fs=None, + **kwargs, +): + """Sync files between two directory trees + + (experimental) + + Parameters + ---------- + source: str + Root of the directory tree to take files from. This must be a directory, but + do not include any terminating "/" character + destination: str + Root path to copy into. The contents of this location should be + identical to the contents of ``source`` when done. This will be made a + directory, and the terminal "/" should not be included. + delete_missing: bool + If there are paths in the destination that don't exist in the + source and this is True, delete them. Otherwise, leave them alone. + source_field: str | callable + If ``update_field`` is "different", this is the key in the info + of source files to consider for difference. Maybe a function of the + info dict. + dest_field: str | callable + If ``update_field`` is "different", this is the key in the info + of destination files to consider for difference. May be a function of + the info dict. + update_cond: "different"|"always"|"never" + If "always", every file is copied, regardless of whether it exists in + the destination. If "never", files that exist in the destination are + not copied again. If "different" (default), only copy if the info + fields given by ``source_field`` and ``dest_field`` (usually "size") + are different. Other comparisons may be added in the future. + inst_kwargs: dict|None + If ``fs`` is None, use this set of keyword arguments to make a + GenericFileSystem instance + fs: GenericFileSystem|None + Instance to use if explicitly given. The instance defines how to + to make downstream file system instances from paths. + + Returns + ------- + dict of the copy operations that were performed, {source: destination} + """ + fs = fs or GenericFileSystem(**(inst_kwargs or {})) + source = fs._strip_protocol(source) + destination = fs._strip_protocol(destination) + allfiles = fs.find(source, withdirs=True, detail=True) + if not fs.isdir(source): + raise ValueError("Can only rsync on a directory") + otherfiles = fs.find(destination, withdirs=True, detail=True) + dirs = [ + a + for a, v in allfiles.items() + if v["type"] == "directory" and a.replace(source, destination) not in otherfiles + ] + logger.debug(f"{len(dirs)} directories to create") + if dirs: + fs.make_many_dirs( + [dirn.replace(source, destination) for dirn in dirs], exist_ok=True + ) + allfiles = {a: v for a, v in allfiles.items() if v["type"] == "file"} + logger.debug(f"{len(allfiles)} files to consider for copy") + to_delete = [ + o + for o, v in otherfiles.items() + if o.replace(destination, source) not in allfiles and v["type"] == "file" + ] + for k, v in allfiles.copy().items(): + otherfile = k.replace(source, destination) + if otherfile in otherfiles: + if update_cond == "always": + allfiles[k] = otherfile + elif update_cond == "never": + allfiles.pop(k) + elif update_cond == "different": + inf1 = source_field(v) if callable(source_field) else v[source_field] + v2 = otherfiles[otherfile] + inf2 = dest_field(v2) if callable(dest_field) else v2[dest_field] + if inf1 != inf2: + # details mismatch, make copy + allfiles[k] = otherfile + else: + # details match, don't copy + allfiles.pop(k) + else: + # file not in target yet + allfiles[k] = otherfile + logger.debug(f"{len(allfiles)} files to copy") + if allfiles: + source_files, target_files = zip(*allfiles.items()) + fs.cp(source_files, target_files, **kwargs) + logger.debug(f"{len(to_delete)} files to delete") + if delete_missing and to_delete: + fs.rm(to_delete) + return allfiles + + +class GenericFileSystem(AsyncFileSystem): + """Wrapper over all other FS types + + + + This implementation is a single unified interface to be able to run FS operations + over generic URLs, and dispatch to the specific implementations using the URL + protocol prefix. + + Note: instances of this FS are always async, even if you never use it with any async + backend. + """ + + protocol = "generic" # there is no real reason to ever use a protocol with this FS + + def __init__(self, default_method="default", storage_options=None, **kwargs): + """ + + Parameters + ---------- + default_method: str (optional) + Defines how to configure backend FS instances. Options are: + - "default": instantiate like FSClass(), with no + extra arguments; this is the default instance of that FS, and can be + configured via the config system + - "generic": takes instances from the `_generic_fs` dict in this module, + which you must populate before use. Keys are by protocol + - "options": expects storage_options, a dict mapping protocol to + kwargs to use when constructing the filesystem + - "current": takes the most recently instantiated version of each FS + """ + self.method = default_method + self.st_opts = storage_options + super().__init__(**kwargs) + + def _parent(self, path): + fs = _resolve_fs(path, self.method, storage_options=self.st_opts) + return fs.unstrip_protocol(fs._parent(path)) + + def _strip_protocol(self, path): + # normalization only + fs = _resolve_fs(path, self.method, storage_options=self.st_opts) + return fs.unstrip_protocol(fs._strip_protocol(path)) + + async def _find(self, path, maxdepth=None, withdirs=False, detail=False, **kwargs): + fs = _resolve_fs(path, self.method, storage_options=self.st_opts) + if fs.async_impl: + out = await fs._find( + path, maxdepth=maxdepth, withdirs=withdirs, detail=True, **kwargs + ) + else: + out = fs.find( + path, maxdepth=maxdepth, withdirs=withdirs, detail=True, **kwargs + ) + result = {} + for k, v in out.items(): + v = v.copy() # don't corrupt target FS dircache + name = fs.unstrip_protocol(k) + v["name"] = name + result[name] = v + if detail: + return result + return list(result) + + async def _info(self, url, **kwargs): + fs = _resolve_fs(url, self.method, storage_options=self.st_opts) + if fs.async_impl: + out = await fs._info(url, **kwargs) + else: + out = fs.info(url, **kwargs) + out = out.copy() # don't edit originals + out["name"] = fs.unstrip_protocol(out["name"]) + return out + + async def _ls( + self, + url, + detail=True, + **kwargs, + ): + fs = _resolve_fs(url, self.method, storage_options=self.st_opts) + if fs.async_impl: + out = await fs._ls(url, detail=True, **kwargs) + else: + out = fs.ls(url, detail=True, **kwargs) + out = [o.copy() for o in out] # don't edit originals + for o in out: + o["name"] = fs.unstrip_protocol(o["name"]) + if detail: + return out + else: + return [o["name"] for o in out] + + async def _cat_file( + self, + url, + **kwargs, + ): + fs = _resolve_fs(url, self.method, storage_options=self.st_opts) + if fs.async_impl: + return await fs._cat_file(url, **kwargs) + else: + return fs.cat_file(url, **kwargs) + + async def _pipe_file( + self, + path, + value, + **kwargs, + ): + fs = _resolve_fs(path, self.method, storage_options=self.st_opts) + if fs.async_impl: + return await fs._pipe_file(path, value, **kwargs) + else: + return fs.pipe_file(path, value, **kwargs) + + async def _rm(self, url, **kwargs): + urls = url + if isinstance(urls, str): + urls = [urls] + fs = _resolve_fs(urls[0], self.method, storage_options=self.st_opts) + if fs.async_impl: + await fs._rm(urls, **kwargs) + else: + fs.rm(url, **kwargs) + + async def _makedirs(self, path, exist_ok=False): + logger.debug("Make dir %s", path) + fs = _resolve_fs(path, self.method, storage_options=self.st_opts) + if fs.async_impl: + await fs._makedirs(path, exist_ok=exist_ok) + else: + fs.makedirs(path, exist_ok=exist_ok) + + def rsync(self, source, destination, **kwargs): + """Sync files between two directory trees + + See `func:rsync` for more details. + """ + rsync(source, destination, fs=self, **kwargs) + + async def _cp_file( + self, + url, + url2, + blocksize=2**20, + callback=DEFAULT_CALLBACK, + tempdir: str | None = None, + **kwargs, + ): + fs = _resolve_fs(url, self.method, storage_options=self.st_opts) + fs2 = _resolve_fs(url2, self.method, storage_options=self.st_opts) + if fs is fs2: + # pure remote + if fs.async_impl: + return await fs._copy(url, url2, **kwargs) + else: + return fs.copy(url, url2, **kwargs) + await copy_file_op(fs, [url], fs2, [url2], tempdir, 1, on_error="raise") + + async def _make_many_dirs(self, urls, exist_ok=True): + fs = _resolve_fs(urls[0], self.method, storage_options=self.st_opts) + if fs.async_impl: + coros = [fs._makedirs(u, exist_ok=exist_ok) for u in urls] + await _run_coros_in_chunks(coros) + else: + for u in urls: + fs.makedirs(u, exist_ok=exist_ok) + + make_many_dirs = sync_wrapper(_make_many_dirs) + + async def _copy( + self, + path1: list[str], + path2: list[str], + recursive: bool = False, + on_error: str = "ignore", + maxdepth: int | None = None, + batch_size: int | None = None, + tempdir: str | None = None, + **kwargs, + ): + # TODO: special case for one FS being local, which can use get/put + # TODO: special case for one being memFS, which can use cat/pipe + if recursive: + raise NotImplementedError("Please use fsspec.generic.rsync") + path1 = [path1] if isinstance(path1, str) else path1 + path2 = [path2] if isinstance(path2, str) else path2 + + fs = _resolve_fs(path1, self.method, storage_options=self.st_opts) + fs2 = _resolve_fs(path2, self.method, storage_options=self.st_opts) + + if fs is fs2: + if fs.async_impl: + return await fs._copy(path1, path2, **kwargs) + else: + return fs.copy(path1, path2, **kwargs) + + await copy_file_op( + fs, path1, fs2, path2, tempdir, batch_size, on_error=on_error + ) + + +async def copy_file_op( + fs1, url1, fs2, url2, tempdir=None, batch_size=20, on_error="ignore" +): + import tempfile + + tempdir = tempdir or tempfile.mkdtemp() + try: + coros = [ + _copy_file_op( + fs1, + u1, + fs2, + u2, + os.path.join(tempdir, uuid.uuid4().hex), + ) + for u1, u2 in zip(url1, url2) + ] + out = await _run_coros_in_chunks( + coros, batch_size=batch_size, return_exceptions=True + ) + finally: + shutil.rmtree(tempdir) + if on_error == "return": + return out + elif on_error == "raise": + for o in out: + if isinstance(o, Exception): + raise o + + +async def _copy_file_op(fs1, url1, fs2, url2, local, on_error="ignore"): + if fs1.async_impl: + await fs1._get_file(url1, local) + else: + fs1.get_file(url1, local) + if fs2.async_impl: + await fs2._put_file(local, url2) + else: + fs2.put_file(local, url2) + os.unlink(local) + logger.debug("Copy %s -> %s; done", url1, url2) + + +async def maybe_await(cor): + if inspect.iscoroutine(cor): + return await cor + else: + return cor