diff --git a/lib/python3.12/site-packages/IPython/__init__.py b/lib/python3.12/site-packages/IPython/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..c2c800aa2c94b25bcdcd5e93099683197f4f3c02 --- /dev/null +++ b/lib/python3.12/site-packages/IPython/__init__.py @@ -0,0 +1,144 @@ +# PYTHON_ARGCOMPLETE_OK +""" +IPython: tools for interactive and parallel computing in Python. + +https://ipython.org +""" +#----------------------------------------------------------------------------- +# Copyright (c) 2008-2011, IPython Development Team. +# Copyright (c) 2001-2007, Fernando Perez +# Copyright (c) 2001, Janko Hauser +# Copyright (c) 2001, Nathaniel Gray +# +# Distributed under the terms of the Modified BSD License. +# +# The full license is in the file COPYING.txt, distributed with this software. +#----------------------------------------------------------------------------- + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- + +import sys +import warnings + +#----------------------------------------------------------------------------- +# Setup everything +#----------------------------------------------------------------------------- + +# Don't forget to also update setup.py when this changes! +if sys.version_info < (3, 11): + raise ImportError( + """ +IPython 8.31+ supports Python 3.11 and above, following SPEC0 +IPython 8.19+ supports Python 3.10 and above, following SPEC0. +IPython 8.13+ supports Python 3.9 and above, following NEP 29. +IPython 8.0-8.12 supports Python 3.8 and above, following NEP 29. +When using Python 2.7, please install IPython 5.x LTS Long Term Support version. +Python 3.3 and 3.4 were supported up to IPython 6.x. +Python 3.5 was supported with IPython 7.0 to 7.9. +Python 3.6 was supported with IPython up to 7.16. +Python 3.7 was still supported with the 7.x branch. + +See IPython `README.rst` file for more information: + + https://github.com/ipython/ipython/blob/main/README.rst + +""" + ) + +#----------------------------------------------------------------------------- +# Setup the top level names +#----------------------------------------------------------------------------- + +from .core.getipython import get_ipython +from .core import release +from .core.application import Application +from .terminal.embed import embed + +from .core.interactiveshell import InteractiveShell +from .utils.sysinfo import sys_info +from .utils.frame import extract_module_locals + +__all__ = ["start_ipython", "embed", "embed_kernel"] + +# Release data +__author__ = '%s <%s>' % (release.author, release.author_email) +__license__ = release.license +__version__ = release.version +version_info = release.version_info +# list of CVEs that should have been patched in this release. +# this is informational and should not be relied upon. +__patched_cves__ = {"CVE-2022-21699", "CVE-2023-24816"} + + +def embed_kernel(module=None, local_ns=None, **kwargs): + """Embed and start an IPython kernel in a given scope. + + If you don't want the kernel to initialize the namespace + from the scope of the surrounding function, + and/or you want to load full IPython configuration, + you probably want `IPython.start_kernel()` instead. + + This is a deprecated alias for `ipykernel.embed.embed_kernel()`, + to be removed in the future. + You should import directly from `ipykernel.embed`; this wrapper + fails anyway if you don't have `ipykernel` package installed. + + Parameters + ---------- + module : types.ModuleType, optional + The module to load into IPython globals (default: caller) + local_ns : dict, optional + The namespace to load into IPython user namespace (default: caller) + **kwargs : various, optional + Further keyword args are relayed to the IPKernelApp constructor, + such as `config`, a traitlets :class:`Config` object (see :ref:`configure_start_ipython`), + allowing configuration of the kernel. Will only have an effect + on the first embed_kernel call for a given process. + """ + + warnings.warn( + "import embed_kernel from ipykernel.embed directly (since 2013)." + " Importing from IPython will be removed in the future", + DeprecationWarning, + stacklevel=2, + ) + + (caller_module, caller_locals) = extract_module_locals(1) + if module is None: + module = caller_module + if local_ns is None: + local_ns = dict(**caller_locals) + + # Only import .zmq when we really need it + from ipykernel.embed import embed_kernel as real_embed_kernel + real_embed_kernel(module=module, local_ns=local_ns, **kwargs) + +def start_ipython(argv=None, **kwargs): + """Launch a normal IPython instance (as opposed to embedded) + + `IPython.embed()` puts a shell in a particular calling scope, + such as a function or method for debugging purposes, + which is often not desirable. + + `start_ipython()` does full, regular IPython initialization, + including loading startup files, configuration, etc. + much of which is skipped by `embed()`. + + This is a public API method, and will survive implementation changes. + + Parameters + ---------- + argv : list or None, optional + If unspecified or None, IPython will parse command-line options from sys.argv. + To prevent any command-line parsing, pass an empty list: `argv=[]`. + user_ns : dict, optional + specify this dictionary to initialize the IPython user namespace with particular values. + **kwargs : various, optional + Any other kwargs will be passed to the Application constructor, + such as `config`, a traitlets :class:`Config` object (see :ref:`configure_start_ipython`), + allowing configuration of the instance (see :ref:`terminal_options`). + """ + from IPython.terminal.ipapp import launch_new_instance + return launch_new_instance(argv=argv, **kwargs) diff --git a/lib/python3.12/site-packages/IPython/__main__.py b/lib/python3.12/site-packages/IPython/__main__.py new file mode 100644 index 0000000000000000000000000000000000000000..9eabd50e74ac2c8456710f98219723a5ab66472a --- /dev/null +++ b/lib/python3.12/site-packages/IPython/__main__.py @@ -0,0 +1,14 @@ +# PYTHON_ARGCOMPLETE_OK +# encoding: utf-8 +"""Terminal-based IPython entry point.""" +# ----------------------------------------------------------------------------- +# Copyright (c) 2012, IPython Development Team. +# +# Distributed under the terms of the Modified BSD License. +# +# The full license is in the file COPYING.txt, distributed with this software. +# ----------------------------------------------------------------------------- + +from IPython import start_ipython + +start_ipython() diff --git a/lib/python3.12/site-packages/IPython/core/compilerop.py b/lib/python3.12/site-packages/IPython/core/compilerop.py new file mode 100644 index 0000000000000000000000000000000000000000..baa9069e66d8a3328c3c355ee2a67cd73f21fbec --- /dev/null +++ b/lib/python3.12/site-packages/IPython/core/compilerop.py @@ -0,0 +1,192 @@ +"""Compiler tools with improved interactive support. + +Provides compilation machinery similar to codeop, but with caching support so +we can provide interactive tracebacks. + +Authors +------- +* Robert Kern +* Fernando Perez +* Thomas Kluyver +""" + +# Note: though it might be more natural to name this module 'compiler', that +# name is in the stdlib and name collisions with the stdlib tend to produce +# weird problems (often with third-party tools). + +#----------------------------------------------------------------------------- +# Copyright (C) 2010-2011 The IPython Development Team. +# +# Distributed under the terms of the BSD License. +# +# The full license is in the file COPYING.txt, distributed with this software. +#----------------------------------------------------------------------------- + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- + +# Stdlib imports +import __future__ +from ast import PyCF_ONLY_AST +import codeop +import functools +import hashlib +import linecache +import operator +import time +from contextlib import contextmanager + +#----------------------------------------------------------------------------- +# Constants +#----------------------------------------------------------------------------- + +# Roughly equal to PyCF_MASK | PyCF_MASK_OBSOLETE as defined in pythonrun.h, +# this is used as a bitmask to extract future-related code flags. +PyCF_MASK = functools.reduce(operator.or_, + (getattr(__future__, fname).compiler_flag + for fname in __future__.all_feature_names)) + +#----------------------------------------------------------------------------- +# Local utilities +#----------------------------------------------------------------------------- + +def code_name(code, number=0): + """ Compute a (probably) unique name for code for caching. + + This now expects code to be unicode. + """ + hash_digest = hashlib.sha1(code.encode("utf-8")).hexdigest() + # Include the number and 12 characters of the hash in the name. It's + # pretty much impossible that in a single session we'll have collisions + # even with truncated hashes, and the full one makes tracebacks too long + return ''.format(number, hash_digest[:12]) + +#----------------------------------------------------------------------------- +# Classes and functions +#----------------------------------------------------------------------------- + +class CachingCompiler(codeop.Compile): + """A compiler that caches code compiled from interactive statements. + """ + + def __init__(self): + codeop.Compile.__init__(self) + + # Caching a dictionary { filename: execution_count } for nicely + # rendered tracebacks. The filename corresponds to the filename + # argument used for the builtins.compile function. + self._filename_map = {} + + def ast_parse(self, source, filename='', symbol='exec'): + """Parse code to an AST with the current compiler flags active. + + Arguments are exactly the same as ast.parse (in the standard library), + and are passed to the built-in compile function.""" + return compile(source, filename, symbol, self.flags | PyCF_ONLY_AST, 1) + + def reset_compiler_flags(self): + """Reset compiler flags to default state.""" + # This value is copied from codeop.Compile.__init__, so if that ever + # changes, it will need to be updated. + self.flags = codeop.PyCF_DONT_IMPLY_DEDENT + + @property + def compiler_flags(self): + """Flags currently active in the compilation process. + """ + return self.flags + + def get_code_name(self, raw_code, transformed_code, number): + """Compute filename given the code, and the cell number. + + Parameters + ---------- + raw_code : str + The raw cell code. + transformed_code : str + The executable Python source code to cache and compile. + number : int + A number which forms part of the code's name. Used for the execution + counter. + + Returns + ------- + The computed filename. + """ + return code_name(transformed_code, number) + + def format_code_name(self, name): + """Return a user-friendly label and name for a code block. + + Parameters + ---------- + name : str + The name for the code block returned from get_code_name + + Returns + ------- + A (label, name) pair that can be used in tracebacks, or None if the default formatting should be used. + """ + if name in self._filename_map: + return "Cell", "In[%s]" % self._filename_map[name] + + def cache(self, transformed_code, number=0, raw_code=None): + """Make a name for a block of code, and cache the code. + + Parameters + ---------- + transformed_code : str + The executable Python source code to cache and compile. + number : int + A number which forms part of the code's name. Used for the execution + counter. + raw_code : str + The raw code before transformation, if None, set to `transformed_code`. + + Returns + ------- + The name of the cached code (as a string). Pass this as the filename + argument to compilation, so that tracebacks are correctly hooked up. + """ + if raw_code is None: + raw_code = transformed_code + + name = self.get_code_name(raw_code, transformed_code, number) + + # Save the execution count + self._filename_map[name] = number + + # Since Python 2.5, setting mtime to `None` means the lines will + # never be removed by `linecache.checkcache`. This means all the + # monkeypatching has *never* been necessary, since this code was + # only added in 2010, at which point IPython had already stopped + # supporting Python 2.4. + # + # Note that `linecache.clearcache` and `linecache.updatecache` may + # still remove our code from the cache, but those show explicit + # intent, and we should not try to interfere. Normally the former + # is never called except when out of memory, and the latter is only + # called for lines *not* in the cache. + entry = ( + len(transformed_code), + None, + [line + "\n" for line in transformed_code.splitlines()], + name, + ) + linecache.cache[name] = entry + return name + + @contextmanager + def extra_flags(self, flags): + ## bits that we'll set to 1 + turn_on_bits = ~self.flags & flags + + + self.flags = self.flags | flags + try: + yield + finally: + # turn off only the bits we turned on so that something like + # __future__ that set flags stays. + self.flags &= ~turn_on_bits diff --git a/lib/python3.12/site-packages/IPython/core/crashhandler.py b/lib/python3.12/site-packages/IPython/core/crashhandler.py new file mode 100644 index 0000000000000000000000000000000000000000..9887b8718a27cf9fd42be4fd28153b7d607857f6 --- /dev/null +++ b/lib/python3.12/site-packages/IPython/core/crashhandler.py @@ -0,0 +1,244 @@ +"""sys.excepthook for IPython itself, leaves a detailed report on disk. + +Authors: + +* Fernando Perez +* Brian E. Granger +""" + +#----------------------------------------------------------------------------- +# Copyright (C) 2001-2007 Fernando Perez. +# Copyright (C) 2008-2011 The IPython Development Team +# +# Distributed under the terms of the BSD License. The full license is in +# the file COPYING, distributed as part of this software. +#----------------------------------------------------------------------------- + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- + +import sys +import traceback +from pprint import pformat +from pathlib import Path + +import builtins as builtin_mod + +from IPython.core import ultratb +from IPython.core.application import Application +from IPython.core.release import author_email +from IPython.utils.sysinfo import sys_info + +from IPython.core.release import __version__ as version + +from typing import Optional, Dict +import types + +#----------------------------------------------------------------------------- +# Code +#----------------------------------------------------------------------------- + +# Template for the user message. +_default_message_template = """\ +Oops, {app_name} crashed. We do our best to make it stable, but... + +A crash report was automatically generated with the following information: + - A verbatim copy of the crash traceback. + - A copy of your input history during this session. + - Data on your current {app_name} configuration. + +It was left in the file named: +\t'{crash_report_fname}' +If you can email this file to the developers, the information in it will help +them in understanding and correcting the problem. + +You can mail it to: {contact_name} at {contact_email} +with the subject '{app_name} Crash Report'. + +If you want to do it now, the following command will work (under Unix): +mail -s '{app_name} Crash Report' {contact_email} < {crash_report_fname} + +In your email, please also include information about: +- The operating system under which the crash happened: Linux, macOS, Windows, + other, and which exact version (for example: Ubuntu 16.04.3, macOS 10.13.2, + Windows 10 Pro), and whether it is 32-bit or 64-bit; +- How {app_name} was installed: using pip or conda, from GitHub, as part of + a Docker container, or other, providing more detail if possible; +- How to reproduce the crash: what exact sequence of instructions can one + input to get the same crash? Ideally, find a minimal yet complete sequence + of instructions that yields the crash. + +To ensure accurate tracking of this issue, please file a report about it at: +{bug_tracker} +""" + +_lite_message_template = """ +If you suspect this is an IPython {version} bug, please report it at: + https://github.com/ipython/ipython/issues +or send an email to the mailing list at {email} + +You can print a more detailed traceback right now with "%tb", or use "%debug" +to interactively debug it. + +Extra-detailed tracebacks for bug-reporting purposes can be enabled via: + {config}Application.verbose_crash=True +""" + + +class CrashHandler: + """Customizable crash handlers for IPython applications. + + Instances of this class provide a :meth:`__call__` method which can be + used as a ``sys.excepthook``. The :meth:`__call__` signature is:: + + def __call__(self, etype, evalue, etb) + """ + + message_template = _default_message_template + section_sep = '\n\n'+'*'*75+'\n\n' + info: Dict[str, Optional[str]] + + def __init__( + self, + app: Application, + contact_name: Optional[str] = None, + contact_email: Optional[str] = None, + bug_tracker: Optional[str] = None, + show_crash_traceback: bool = True, + call_pdb: bool = False, + ): + """Create a new crash handler + + Parameters + ---------- + app : Application + A running :class:`Application` instance, which will be queried at + crash time for internal information. + contact_name : str + A string with the name of the person to contact. + contact_email : str + A string with the email address of the contact. + bug_tracker : str + A string with the URL for your project's bug tracker. + show_crash_traceback : bool + If false, don't print the crash traceback on stderr, only generate + the on-disk report + call_pdb + Whether to call pdb on crash + + Attributes + ---------- + These instances contain some non-argument attributes which allow for + further customization of the crash handler's behavior. Please see the + source for further details. + + """ + self.crash_report_fname = "Crash_report_%s.txt" % app.name + self.app = app + self.call_pdb = call_pdb + #self.call_pdb = True # dbg + self.show_crash_traceback = show_crash_traceback + self.info = dict(app_name = app.name, + contact_name = contact_name, + contact_email = contact_email, + bug_tracker = bug_tracker, + crash_report_fname = self.crash_report_fname) + + def __call__( + self, + etype: type[BaseException], + evalue: BaseException, + etb: types.TracebackType, + ) -> None: + """Handle an exception, call for compatible with sys.excepthook""" + + # do not allow the crash handler to be called twice without reinstalling it + # this prevents unlikely errors in the crash handling from entering an + # infinite loop. + sys.excepthook = sys.__excepthook__ + + + # Use this ONLY for developer debugging (keep commented out for release) + ipython_dir = getattr(self.app, "ipython_dir", None) + if ipython_dir is not None: + assert isinstance(ipython_dir, str) + rptdir = Path(ipython_dir) + else: + rptdir = Path.cwd() + if not rptdir.is_dir(): + rptdir = Path.cwd() + report_name = rptdir / self.crash_report_fname + # write the report filename into the instance dict so it can get + # properly expanded out in the user message template + self.crash_report_fname = str(report_name) + self.info["crash_report_fname"] = str(report_name) + TBhandler = ultratb.VerboseTB( + theme_name="nocolor", + long_header=True, + call_pdb=self.call_pdb, + ) + if self.call_pdb: + TBhandler(etype,evalue,etb) + return + else: + traceback = TBhandler.text(etype,evalue,etb,context=31) + + # print traceback to screen + if self.show_crash_traceback: + print(traceback, file=sys.stderr) + + # and generate a complete report on disk + try: + report = open(report_name, "w", encoding="utf-8") + except: + print('Could not create crash report on disk.', file=sys.stderr) + return + + with report: + # Inform user on stderr of what happened + print('\n'+'*'*70+'\n', file=sys.stderr) + print(self.message_template.format(**self.info), file=sys.stderr) + + # Construct report on disk + report.write(self.make_report(str(traceback))) + + builtin_mod.input("Hit to quit (your terminal may close):") + + def make_report(self, traceback: str) -> str: + """Return a string containing a crash report.""" + + sec_sep = self.section_sep + + report = ['*'*75+'\n\n'+'IPython post-mortem report\n\n'] + rpt_add = report.append + rpt_add(sys_info()) + + try: + config = pformat(self.app.config) + rpt_add(sec_sep) + rpt_add("Application name: %s\n\n" % self.app.name) + rpt_add("Current user configuration structure:\n\n") + rpt_add(config) + except: + pass + rpt_add(sec_sep+'Crash traceback:\n\n' + traceback) + + return ''.join(report) + + +def crash_handler_lite( + etype: type[BaseException], evalue: BaseException, tb: types.TracebackType +) -> None: + """a light excepthook, adding a small message to the usual traceback""" + traceback.print_exception(etype, evalue, tb) + + from IPython.core.interactiveshell import InteractiveShell + if InteractiveShell.initialized(): + # we are in a Shell environment, give %magic example + config = "%config " + else: + # we are not in a shell, show generic config + config = "c." + print(_lite_message_template.format(email=author_email, config=config, version=version), file=sys.stderr) + diff --git a/lib/python3.12/site-packages/IPython/core/displaypub.py b/lib/python3.12/site-packages/IPython/core/displaypub.py new file mode 100644 index 0000000000000000000000000000000000000000..45afab854baab2276fd275c40f3c355c754a5e5a --- /dev/null +++ b/lib/python3.12/site-packages/IPython/core/displaypub.py @@ -0,0 +1,205 @@ +"""An interface for publishing rich data to frontends. + +There are two components of the display system: + +* Display formatters, which take a Python object and compute the + representation of the object in various formats (text, HTML, SVG, etc.). +* The display publisher that is used to send the representation data to the + various frontends. + +This module defines the logic display publishing. The display publisher uses +the ``display_data`` message type that is defined in the IPython messaging +spec. +""" + +# Copyright (c) IPython Development Team. +# Distributed under the terms of the Modified BSD License. + +import sys + +from traitlets.config.configurable import Configurable +from traitlets import List + +# This used to be defined here - it is imported for backwards compatibility +from .display_functions import publish_display_data +from .history import HistoryOutput + +import typing as t + +# ----------------------------------------------------------------------------- +# Main payload class +# ----------------------------------------------------------------------------- + +_sentinel = object() + + +class DisplayPublisher(Configurable): + """A traited class that publishes display data to frontends. + + Instances of this class are created by the main IPython object and should + be accessed there. + """ + + def __init__(self, shell=None, *args, **kwargs): + self.shell = shell + self._is_publishing = False + self._in_post_execute = False + if self.shell: + self._setup_execution_tracking() + super().__init__(*args, **kwargs) + + def _validate_data(self, data, metadata=None): + """Validate the display data. + + Parameters + ---------- + data : dict + The formata data dictionary. + metadata : dict + Any metadata for the data. + """ + + if not isinstance(data, dict): + raise TypeError("data must be a dict, got: %r" % data) + if metadata is not None: + if not isinstance(metadata, dict): + raise TypeError("metadata must be a dict, got: %r" % data) + + def _setup_execution_tracking(self): + """Set up hooks to track execution state""" + self.shell.events.register("post_execute", self._on_post_execute) + self.shell.events.register("pre_execute", self._on_pre_execute) + + def _on_post_execute(self): + """Called at start of post_execute phase""" + self._in_post_execute = True + + def _on_pre_execute(self): + """Called at start of pre_execute phase""" + self._in_post_execute = False + + # use * to indicate transient, update are keyword-only + def publish( + self, + data, + metadata=None, + source=_sentinel, + *, + transient=None, + update=False, + **kwargs, + ) -> None: + """Publish data and metadata to all frontends. + + See the ``display_data`` message in the messaging documentation for + more details about this message type. + + The following MIME types are currently implemented: + + * text/plain + * text/html + * text/markdown + * text/latex + * application/json + * application/javascript + * image/png + * image/jpeg + * image/svg+xml + + Parameters + ---------- + data : dict + A dictionary having keys that are valid MIME types (like + 'text/plain' or 'image/svg+xml') and values that are the data for + that MIME type. The data itself must be a JSON'able data + structure. Minimally all data should have the 'text/plain' data, + which can be displayed by all frontends. If more than the plain + text is given, it is up to the frontend to decide which + representation to use. + metadata : dict + A dictionary for metadata related to the data. This can contain + arbitrary key, value pairs that frontends can use to interpret + the data. Metadata specific to each mime-type can be specified + in the metadata dict with the same mime-type keys as + the data itself. + source : str, deprecated + Unused. + transient : dict, keyword-only + A dictionary for transient data. + Data in this dictionary should not be persisted as part of saving this output. + Examples include 'display_id'. + update : bool, keyword-only, default: False + If True, only update existing outputs with the same display_id, + rather than creating a new output. + """ + + if source is not _sentinel: + import warnings + + warnings.warn( + "The 'source' parameter is deprecated since IPython 3.0 and will be ignored " + "(this warning is present since 9.0). `source` parameter will be removed in the future.", + DeprecationWarning, + stacklevel=2, + ) + + handlers: t.Dict = {} + if self.shell is not None: + handlers = getattr(self.shell, "mime_renderers", {}) + + outputs = self.shell.history_manager.outputs + + target_execution_count = self.shell.execution_count - 1 + if self._in_post_execute: + # We're in post_execute, so this is likely a matplotlib flush + # Use execution_count - 1 to associate with the cell that created the plot + target_execution_count = self.shell.execution_count - 1 + + outputs[target_execution_count].append( + HistoryOutput(output_type="display_data", bundle=data) + ) + + for mime, handler in handlers.items(): + if mime in data: + handler(data[mime], metadata.get(mime, None)) + return + + self._is_publishing = True + if "text/plain" in data: + print(data["text/plain"]) + self._is_publishing = False + + @property + def is_publishing(self): + return self._is_publishing + + def clear_output(self, wait=False): + """Clear the output of the cell receiving output.""" + print("\033[2K\r", end="") + sys.stdout.flush() + print("\033[2K\r", end="") + sys.stderr.flush() + + +class CapturingDisplayPublisher(DisplayPublisher): + """A DisplayPublisher that stores""" + + outputs: List = List() + + def publish( + self, data, metadata=None, source=None, *, transient=None, update=False + ): + self.outputs.append( + { + "data": data, + "metadata": metadata, + "transient": transient, + "update": update, + } + ) + + def clear_output(self, wait=False): + super(CapturingDisplayPublisher, self).clear_output(wait) + + # empty the list, *do not* reassign a new list + self.outputs.clear() diff --git a/lib/python3.12/site-packages/IPython/core/getipython.py b/lib/python3.12/site-packages/IPython/core/getipython.py new file mode 100644 index 0000000000000000000000000000000000000000..c7e112ba9f036a3ff46b60efeb40932e39b47547 --- /dev/null +++ b/lib/python3.12/site-packages/IPython/core/getipython.py @@ -0,0 +1,23 @@ +"""Simple function to call to get the current InteractiveShell instance +""" + +#----------------------------------------------------------------------------- +# Copyright (C) 2013 The IPython Development Team +# +# Distributed under the terms of the BSD License. The full license is in +# the file COPYING, distributed as part of this software. +#----------------------------------------------------------------------------- + +#----------------------------------------------------------------------------- +# Classes and functions +#----------------------------------------------------------------------------- + + +def get_ipython(): + """Get the global InteractiveShell instance. + + Returns None if no InteractiveShell instance is registered. + """ + from IPython.core.interactiveshell import InteractiveShell + if InteractiveShell.initialized(): + return InteractiveShell.instance() diff --git a/lib/python3.12/site-packages/IPython/core/guarded_eval.py b/lib/python3.12/site-packages/IPython/core/guarded_eval.py new file mode 100644 index 0000000000000000000000000000000000000000..da5a751a7e338b30713674080bf432806f561f3c --- /dev/null +++ b/lib/python3.12/site-packages/IPython/core/guarded_eval.py @@ -0,0 +1,1656 @@ +from copy import copy +from inspect import isclass, signature, Signature, getmodule +from typing import ( + Annotated, + AnyStr, + Literal, + NamedTuple, + NewType, + Optional, + Protocol, + TypeGuard, + Union, + get_args, + get_origin, + is_typeddict, +) +from collections.abc import Callable, Sequence +import ast +import builtins +import collections +import dataclasses +import operator +import sys +import typing +import warnings +from functools import cached_property +from dataclasses import dataclass, field +from types import MethodDescriptorType, ModuleType, MethodType + +from IPython.utils.decorators import undoc + +import types +from typing import Self, LiteralString, get_type_hints + +if sys.version_info < (3, 12): + from typing_extensions import TypeAliasType +else: + from typing import TypeAliasType + + +@undoc +class HasGetItem(Protocol): + def __getitem__(self, key) -> None: + ... + + +@undoc +class InstancesHaveGetItem(Protocol): + def __call__(self, *args, **kwargs) -> HasGetItem: + ... + + +@undoc +class HasGetAttr(Protocol): + def __getattr__(self, key) -> None: + ... + + +@undoc +class DoesNotHaveGetAttr(Protocol): + pass + + +# By default `__getattr__` is not explicitly implemented on most objects +MayHaveGetattr = Union[HasGetAttr, DoesNotHaveGetAttr] + + +def _unbind_method(func: Callable) -> Union[Callable, None]: + """Get unbound method for given bound method. + + Returns None if cannot get unbound method, or method is already unbound. + """ + owner = getattr(func, "__self__", None) + owner_class = type(owner) + name = getattr(func, "__name__", None) + instance_dict_overrides = getattr(owner, "__dict__", None) + if ( + owner is not None + and name + and ( + not instance_dict_overrides + or (instance_dict_overrides and name not in instance_dict_overrides) + ) + ): + return getattr(owner_class, name) + return None + + +@undoc +@dataclass +class EvaluationPolicy: + """Definition of evaluation policy.""" + + allow_locals_access: bool = False + allow_globals_access: bool = False + allow_item_access: bool = False + allow_attr_access: bool = False + allow_builtins_access: bool = False + allow_all_operations: bool = False + allow_any_calls: bool = False + allow_auto_import: bool = False + allowed_calls: set[Callable] = field(default_factory=set) + + def can_get_item(self, value, item): + return self.allow_item_access + + def can_get_attr(self, value, attr): + return self.allow_attr_access + + def can_operate(self, dunders: tuple[str, ...], a, b=None): + if self.allow_all_operations: + return True + + def can_call(self, func): + if self.allow_any_calls: + return True + + if func in self.allowed_calls: + return True + + owner_method = _unbind_method(func) + + if owner_method and owner_method in self.allowed_calls: + return True + + +def _get_external(module_name: str, access_path: Sequence[str]): + """Get value from external module given a dotted access path. + + Only gets value if the module is already imported. + + Raises: + * `KeyError` if module is removed not found, and + * `AttributeError` if access path does not match an exported object + """ + try: + member_type = sys.modules[module_name] + # standard module + for attr in access_path: + member_type = getattr(member_type, attr) + return member_type + except (KeyError, AttributeError): + # handle modules in namespace packages + module_path = ".".join([module_name, *access_path]) + if module_path in sys.modules: + return sys.modules[module_path] + raise + + +def _has_original_dunder_external( + value, + module_name: str, + access_path: Sequence[str], + method_name: str, +): + if module_name not in sys.modules: + full_module_path = ".".join([module_name, *access_path]) + if full_module_path not in sys.modules: + # LBYLB as it is faster + return False + try: + member_type = _get_external(module_name, access_path) + value_type = type(value) + if type(value) == member_type: + return True + if isinstance(member_type, ModuleType): + value_module = getmodule(value_type) + if not value_module or not value_module.__name__: + return False + if ( + value_module.__name__ == member_type.__name__ + or value_module.__name__.startswith(member_type.__name__ + ".") + ): + return True + if method_name == "__getattribute__": + # we have to short-circuit here due to an unresolved issue in + # `isinstance` implementation: https://bugs.python.org/issue32683 + return False + if not isinstance(member_type, ModuleType) and isinstance(value, member_type): + method = getattr(value_type, method_name, None) + member_method = getattr(member_type, method_name, None) + if member_method == method: + return True + if isinstance(member_type, ModuleType): + method = getattr(value_type, method_name, None) + for base_class in value_type.__mro__[1:]: + base_module = getmodule(base_class) + if base_module and ( + base_module.__name__ == member_type.__name__ + or base_module.__name__.startswith(member_type.__name__ + ".") + ): + # Check if the method comes from this trusted base class + base_method = getattr(base_class, method_name, None) + if base_method is not None and base_method == method: + return True + except (AttributeError, KeyError): + return False + + +def _has_original_dunder( + value, allowed_types, allowed_methods, allowed_external, method_name +): + # note: Python ignores `__getattr__`/`__getitem__` on instances, + # we only need to check at class level + value_type = type(value) + + # strict type check passes → no need to check method + if value_type in allowed_types: + return True + + method = getattr(value_type, method_name, None) + + if method is None: + return None + + if method in allowed_methods: + return True + + for module_name, *access_path in allowed_external: + if _has_original_dunder_external(value, module_name, access_path, method_name): + return True + + return False + + +def _coerce_path_to_tuples( + allow_list: set[tuple[str, ...] | str], +) -> set[tuple[str, ...]]: + """Replace dotted paths on the provided allow-list with tuples.""" + return { + path if isinstance(path, tuple) else tuple(path.split(".")) + for path in allow_list + } + + +@undoc +@dataclass +class SelectivePolicy(EvaluationPolicy): + allowed_getitem: set[InstancesHaveGetItem] = field(default_factory=set) + allowed_getitem_external: set[tuple[str, ...] | str] = field(default_factory=set) + + allowed_getattr: set[MayHaveGetattr] = field(default_factory=set) + allowed_getattr_external: set[tuple[str, ...] | str] = field(default_factory=set) + + allowed_operations: set = field(default_factory=set) + allowed_operations_external: set[tuple[str, ...] | str] = field(default_factory=set) + + allow_getitem_on_types: bool = field(default_factory=bool) + + _operation_methods_cache: dict[str, set[Callable]] = field( + default_factory=dict, init=False + ) + + def can_get_attr(self, value, attr): + allowed_getattr_external = _coerce_path_to_tuples(self.allowed_getattr_external) + + has_original_attribute = _has_original_dunder( + value, + allowed_types=self.allowed_getattr, + allowed_methods=self._getattribute_methods, + allowed_external=allowed_getattr_external, + method_name="__getattribute__", + ) + has_original_attr = _has_original_dunder( + value, + allowed_types=self.allowed_getattr, + allowed_methods=self._getattr_methods, + allowed_external=allowed_getattr_external, + method_name="__getattr__", + ) + + accept = False + + # Many objects do not have `__getattr__`, this is fine. + if has_original_attr is None and has_original_attribute: + accept = True + else: + # Accept objects without modifications to `__getattr__` and `__getattribute__` + accept = has_original_attr and has_original_attribute + + if accept: + # We still need to check for overridden properties. + + value_class = type(value) + if not hasattr(value_class, attr): + return True + + class_attr_val = getattr(value_class, attr) + is_property = isinstance(class_attr_val, property) + + if not is_property: + return True + + # Properties in allowed types are ok (although we do not include any + # properties in our default allow list currently). + if type(value) in self.allowed_getattr: + return True # pragma: no cover + + # Properties in subclasses of allowed types may be ok if not changed + for module_name, *access_path in allowed_getattr_external: + try: + external_class = _get_external(module_name, access_path) + external_class_attr_val = getattr(external_class, attr) + except (KeyError, AttributeError): + return False # pragma: no cover + return class_attr_val == external_class_attr_val + + return False + + def can_get_item(self, value, item): + """Allow accessing `__getiitem__` of allow-listed instances unless it was not modified.""" + allowed_getitem_external = _coerce_path_to_tuples(self.allowed_getitem_external) + if self.allow_getitem_on_types: + # e.g. Union[str, int] or Literal[True, 1] + if isinstance(value, (typing._SpecialForm, typing._BaseGenericAlias)): + return True + # PEP 560 e.g. list[str] + if isinstance(value, type) and hasattr(value, "__class_getitem__"): + return True + return _has_original_dunder( + value, + allowed_types=self.allowed_getitem, + allowed_methods=self._getitem_methods, + allowed_external=allowed_getitem_external, + method_name="__getitem__", + ) + + def can_operate(self, dunders: tuple[str, ...], a, b=None): + allowed_operations_external = _coerce_path_to_tuples( + self.allowed_operations_external + ) + objects = [a] + if b is not None: + objects.append(b) + return all( + [ + _has_original_dunder( + obj, + allowed_types=self.allowed_operations, + allowed_methods=self._operator_dunder_methods(dunder), + allowed_external=allowed_operations_external, + method_name=dunder, + ) + for dunder in dunders + for obj in objects + ] + ) + + def _operator_dunder_methods(self, dunder: str) -> set[Callable]: + if dunder not in self._operation_methods_cache: + self._operation_methods_cache[dunder] = self._safe_get_methods( + self.allowed_operations, dunder + ) + return self._operation_methods_cache[dunder] + + @cached_property + def _getitem_methods(self) -> set[Callable]: + return self._safe_get_methods(self.allowed_getitem, "__getitem__") + + @cached_property + def _getattr_methods(self) -> set[Callable]: + return self._safe_get_methods(self.allowed_getattr, "__getattr__") + + @cached_property + def _getattribute_methods(self) -> set[Callable]: + return self._safe_get_methods(self.allowed_getattr, "__getattribute__") + + def _safe_get_methods(self, classes, name) -> set[Callable]: + return { + method + for class_ in classes + for method in [getattr(class_, name, None)] + if method + } + + +class _DummyNamedTuple(NamedTuple): + """Used internally to retrieve methods of named tuple instance.""" + + +EvaluationPolicyName = Literal["forbidden", "minimal", "limited", "unsafe", "dangerous"] + + +@dataclass +class EvaluationContext: + #: Local namespace + locals: dict + #: Global namespace + globals: dict + #: Evaluation policy identifier + evaluation: EvaluationPolicyName = "forbidden" + #: Whether the evaluation of code takes place inside of a subscript. + #: Useful for evaluating ``:-1, 'col'`` in ``df[:-1, 'col']``. + in_subscript: bool = False + #: Auto import method + auto_import: Callable[[Sequence[str]], ModuleType] | None = None + #: Overrides for evaluation policy + policy_overrides: dict = field(default_factory=dict) + #: Transient local namespace used to store mocks + transient_locals: dict = field(default_factory=dict) + #: Transients of class level + class_transients: dict | None = None + #: Instance variable name used in the method definition + instance_arg_name: str | None = None + #: Currently associated value + #: Useful for adding items to _Duck on annotated assignment + current_value: ast.AST | None = None + + def replace(self, /, **changes): + """Return a new copy of the context, with specified changes""" + return dataclasses.replace(self, **changes) + + +class _IdentitySubscript: + """Returns the key itself when item is requested via subscript.""" + + def __getitem__(self, key): + return key + + +IDENTITY_SUBSCRIPT = _IdentitySubscript() +SUBSCRIPT_MARKER = "__SUBSCRIPT_SENTINEL__" +UNKNOWN_SIGNATURE = Signature() +NOT_EVALUATED = object() + + +class GuardRejection(Exception): + """Exception raised when guard rejects evaluation attempt.""" + + pass + + +def guarded_eval(code: str, context: EvaluationContext): + """Evaluate provided code in the evaluation context. + + If evaluation policy given by context is set to ``forbidden`` + no evaluation will be performed; if it is set to ``dangerous`` + standard :func:`eval` will be used; finally, for any other, + policy :func:`eval_node` will be called on parsed AST. + """ + locals_ = context.locals + + if context.evaluation == "forbidden": + raise GuardRejection("Forbidden mode") + + # note: not using `ast.literal_eval` as it does not implement + # getitem at all, for example it fails on simple `[0][1]` + + if context.in_subscript: + # syntactic sugar for ellipsis (:) is only available in subscripts + # so we need to trick the ast parser into thinking that we have + # a subscript, but we need to be able to later recognise that we did + # it so we can ignore the actual __getitem__ operation + if not code: + return tuple() + locals_ = locals_.copy() + locals_[SUBSCRIPT_MARKER] = IDENTITY_SUBSCRIPT + code = SUBSCRIPT_MARKER + "[" + code + "]" + context = context.replace(locals=locals_) + + if context.evaluation == "dangerous": + return eval(code, context.globals, context.locals) + + node = ast.parse(code, mode="exec") + + return eval_node(node, context) + + +BINARY_OP_DUNDERS: dict[type[ast.operator], tuple[str]] = { + ast.Add: ("__add__",), + ast.Sub: ("__sub__",), + ast.Mult: ("__mul__",), + ast.Div: ("__truediv__",), + ast.FloorDiv: ("__floordiv__",), + ast.Mod: ("__mod__",), + ast.Pow: ("__pow__",), + ast.LShift: ("__lshift__",), + ast.RShift: ("__rshift__",), + ast.BitOr: ("__or__",), + ast.BitXor: ("__xor__",), + ast.BitAnd: ("__and__",), + ast.MatMult: ("__matmul__",), +} + +COMP_OP_DUNDERS: dict[type[ast.cmpop], tuple[str, ...]] = { + ast.Eq: ("__eq__",), + ast.NotEq: ("__ne__", "__eq__"), + ast.Lt: ("__lt__", "__gt__"), + ast.LtE: ("__le__", "__ge__"), + ast.Gt: ("__gt__", "__lt__"), + ast.GtE: ("__ge__", "__le__"), + ast.In: ("__contains__",), + # Note: ast.Is, ast.IsNot, ast.NotIn are handled specially +} + +UNARY_OP_DUNDERS: dict[type[ast.unaryop], tuple[str, ...]] = { + ast.USub: ("__neg__",), + ast.UAdd: ("__pos__",), + # we have to check both __inv__ and __invert__! + ast.Invert: ("__invert__", "__inv__"), + ast.Not: ("__not__",), +} + +GENERIC_CONTAINER_TYPES = (dict, list, set, tuple, frozenset) + + +class ImpersonatingDuck: + """A dummy class used to create objects of other classes without calling their ``__init__``""" + + # no-op: override __class__ to impersonate + + +class _Duck: + """A dummy class used to create objects pretending to have given attributes""" + + def __init__(self, attributes: Optional[dict] = None, items: Optional[dict] = None): + self.attributes = attributes if attributes is not None else {} + self.items = items if items is not None else {} + + def __getattr__(self, attr: str): + return self.attributes[attr] + + def __hasattr__(self, attr: str): + return attr in self.attributes + + def __dir__(self): + return [*dir(super), *self.attributes] + + def __getitem__(self, key: str): + return self.items[key] + + def __hasitem__(self, key: str): + return self.items[key] + + def _ipython_key_completions_(self): + return self.items.keys() + + +def _find_dunder(node_op, dunders) -> Union[tuple[str, ...], None]: + dunder = None + for op, candidate_dunder in dunders.items(): + if isinstance(node_op, op): + dunder = candidate_dunder + return dunder + + +def get_policy(context: EvaluationContext) -> EvaluationPolicy: + policy = copy(EVALUATION_POLICIES[context.evaluation]) + + for key, value in context.policy_overrides.items(): + if hasattr(policy, key): + setattr(policy, key, value) + return policy + + +def _validate_policy_overrides( + policy_name: EvaluationPolicyName, policy_overrides: dict +) -> bool: + policy = EVALUATION_POLICIES[policy_name] + + all_good = True + for key, value in policy_overrides.items(): + if not hasattr(policy, key): + warnings.warn( + f"Override {key!r} is not valid with {policy_name!r} evaluation policy" + ) + all_good = False + return all_good + + +def _is_type_annotation(obj) -> bool: + """ + Returns True if obj is a type annotation, False otherwise. + """ + if isinstance(obj, type): + return True + if isinstance(obj, types.GenericAlias): + return True + if hasattr(types, "UnionType") and isinstance(obj, types.UnionType): + return True + if isinstance(obj, (typing._SpecialForm, typing._BaseGenericAlias)): + return True + if isinstance(obj, typing.TypeVar): + return True + # Types that support __class_getitem__ + if isinstance(obj, type) and hasattr(obj, "__class_getitem__"): + return True + # Fallback: check if get_origin returns something + if hasattr(typing, "get_origin") and get_origin(obj) is not None: + return True + + return False + + +def _handle_assign(node: ast.Assign, context: EvaluationContext): + value = eval_node(node.value, context) + transient_locals = context.transient_locals + policy = get_policy(context) + class_transients = context.class_transients + for target in node.targets: + if isinstance(target, (ast.Tuple, ast.List)): + # Handle unpacking assignment + values = list(value) + targets = target.elts + starred = [i for i, t in enumerate(targets) if isinstance(t, ast.Starred)] + + # Unified handling: treat no starred as starred at end + star_or_last_idx = starred[0] if starred else len(targets) + + # Before starred + for i in range(star_or_last_idx): + # Check for self.x assignment + if _is_instance_attribute_assignment(targets[i], context): + class_transients[targets[i].attr] = values[i] + else: + transient_locals[targets[i].id] = values[i] + + # Starred if exists + if starred: + end = len(values) - (len(targets) - star_or_last_idx - 1) + if _is_instance_attribute_assignment( + targets[star_or_last_idx], context + ): + class_transients[targets[star_or_last_idx].attr] = values[ + star_or_last_idx:end + ] + else: + transient_locals[targets[star_or_last_idx].value.id] = values[ + star_or_last_idx:end + ] + + # After starred + for i in range(star_or_last_idx + 1, len(targets)): + if _is_instance_attribute_assignment(targets[i], context): + class_transients[targets[i].attr] = values[ + len(values) - (len(targets) - i) + ] + else: + transient_locals[targets[i].id] = values[ + len(values) - (len(targets) - i) + ] + elif isinstance(target, ast.Subscript): + if isinstance(target.value, ast.Name): + name = target.value.id + container = transient_locals.get(name) + if container is None: + container = context.locals.get(name) + if container is None: + container = context.globals.get(name) + if container is None: + raise NameError( + f"{name} not found in locals, globals, nor builtins" + ) + storage_dict = transient_locals + storage_key = name + elif isinstance( + target.value, ast.Attribute + ) and _is_instance_attribute_assignment(target.value, context): + attr = target.value.attr + container = class_transients.get(attr, None) + if container is None: + raise NameError(f"{attr} not found in class transients") + storage_dict = class_transients + storage_key = attr + else: + return + + key = eval_node(target.slice, context) + attributes = ( + dict.fromkeys(dir(container)) + if policy.can_call(container.__dir__) + else {} + ) + items = {} + + if policy.can_get_item(container, None): + try: + items = dict(container.items()) + except Exception: + pass + + items[key] = value + duck_container = _Duck(attributes=attributes, items=items) + storage_dict[storage_key] = duck_container + elif _is_instance_attribute_assignment(target, context): + class_transients[target.attr] = value + else: + transient_locals[target.id] = value + return None + + +def _handle_annassign(node, context): + context_with_value = context.replace(current_value=getattr(node, "value", None)) + annotation_result = eval_node(node.annotation, context_with_value) + if _is_type_annotation(annotation_result): + annotation_value = _resolve_annotation(annotation_result, context) + # Use Value for generic types + use_value = ( + isinstance(annotation_value, GENERIC_CONTAINER_TYPES) and node.value is not None + ) + else: + annotation_value = annotation_result + use_value = False + + # LOCAL VARIABLE + if getattr(node, "simple", False) and isinstance(node.target, ast.Name): + name = node.target.id + if use_value: + return _handle_assign( + ast.Assign(targets=[node.target], value=node.value), context + ) + context.transient_locals[name] = annotation_value + return None + + # INSTANCE ATTRIBUTE + if _is_instance_attribute_assignment(node.target, context): + attr = node.target.attr + if use_value: + return _handle_assign( + ast.Assign(targets=[node.target], value=node.value), context + ) + context.class_transients[attr] = annotation_value + return None + + return None + +def _extract_args_and_kwargs(node: ast.Call, context: EvaluationContext): + args = [eval_node(arg, context) for arg in node.args] + kwargs = { + k: v + for kw in node.keywords + for k, v in ( + {kw.arg: eval_node(kw.value, context)} + if kw.arg + else eval_node(kw.value, context) + ).items() + } + return args, kwargs + + +def _is_instance_attribute_assignment( + target: ast.AST, context: EvaluationContext +) -> bool: + """Return True if target is an attribute access on the instance argument.""" + return ( + context.class_transients is not None + and context.instance_arg_name is not None + and isinstance(target, ast.Attribute) + and isinstance(getattr(target, "value", None), ast.Name) + and getattr(target.value, "id", None) == context.instance_arg_name + ) + + +def _get_coroutine_attributes() -> dict[str, Optional[object]]: + async def _dummy(): + return None + + coro = _dummy() + try: + return {attr: getattr(coro, attr, None) for attr in dir(coro)} + finally: + coro.close() + + +def eval_node(node: Union[ast.AST, None], context: EvaluationContext): + """Evaluate AST node in provided context. + + Applies evaluation restrictions defined in the context. Currently does not support evaluation of functions with keyword arguments. + + Does not evaluate actions that always have side effects: + + - class definitions (``class sth: ...``) + - function definitions (``def sth: ...``) + - variable assignments (``x = 1``) + - augmented assignments (``x += 1``) + - deletions (``del x``) + + Does not evaluate operations which do not return values: + + - assertions (``assert x``) + - pass (``pass``) + - imports (``import x``) + - control flow: + + - conditionals (``if x:``) except for ternary IfExp (``a if x else b``) + - loops (``for`` and ``while``) + - exception handling + + The purpose of this function is to guard against unwanted side-effects; + it does not give guarantees on protection from malicious code execution. + """ + policy = get_policy(context) + + if node is None: + return None + if isinstance(node, (ast.Interactive, ast.Module)): + result = None + for child_node in node.body: + result = eval_node(child_node, context) + return result + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + is_async = isinstance(node, ast.AsyncFunctionDef) + func_locals = context.transient_locals.copy() + func_context = context.replace(transient_locals=func_locals) + is_property = False + is_static = False + is_classmethod = False + for decorator_node in node.decorator_list: + try: + decorator = eval_node(decorator_node, context) + except NameError: + # if the decorator is not yet defined this is fine + # especialy because we don't handle imports yet + continue + if decorator is property: + is_property = True + elif decorator is staticmethod: + is_static = True + elif decorator is classmethod: + is_classmethod = True + + if func_context.class_transients is not None: + if not is_static and not is_classmethod: + func_context.instance_arg_name = ( + node.args.args[0].arg if node.args.args else None + ) + + return_type = eval_node(node.returns, context=context) + + for child_node in node.body: + eval_node(child_node, func_context) + + if is_property: + if return_type is not None: + if _is_type_annotation(return_type): + context.transient_locals[node.name] = _resolve_annotation( + return_type, context + ) + else: + context.transient_locals[node.name] = return_type + else: + return_value = _infer_return_value(node, func_context) + context.transient_locals[node.name] = return_value + + return None + + def dummy_function(*args, **kwargs): + pass + + if return_type is not None: + if _is_type_annotation(return_type): + dummy_function.__annotations__["return"] = return_type + else: + dummy_function.__inferred_return__ = return_type + else: + inferred_return = _infer_return_value(node, func_context) + if inferred_return is not None: + dummy_function.__inferred_return__ = inferred_return + + dummy_function.__name__ = node.name + dummy_function.__node__ = node + dummy_function.__is_async__ = is_async + context.transient_locals[node.name] = dummy_function + return None + if isinstance(node, ast.Lambda): + + def dummy_function(*args, **kwargs): + pass + + dummy_function.__inferred_return__ = eval_node(node.body, context) + return dummy_function + if isinstance(node, ast.ClassDef): + # TODO support class decorators? + class_locals = {} + outer_locals = context.locals.copy() + outer_locals.update(context.transient_locals) + class_context = context.replace( + transient_locals=class_locals, locals=outer_locals + ) + class_context.class_transients = class_locals + for child_node in node.body: + eval_node(child_node, class_context) + bases = tuple([eval_node(base, context) for base in node.bases]) + dummy_class = type(node.name, bases, class_locals) + context.transient_locals[node.name] = dummy_class + return None + if isinstance(node, ast.Await): + value = eval_node(node.value, context) + if hasattr(value, "__awaited_type__"): + return value.__awaited_type__ + return value + if isinstance(node, ast.While): + loop_locals = context.transient_locals.copy() + loop_context = context.replace(transient_locals=loop_locals) + + result = None + for stmt in node.body: + result = eval_node(stmt, loop_context) + + policy = get_policy(context) + merged_locals = _merge_dicts_by_key( + [loop_locals, context.transient_locals.copy()], policy + ) + context.transient_locals.update(merged_locals) + + return result + if isinstance(node, ast.For): + try: + iterable = eval_node(node.iter, context) + except Exception: + iterable = None + + sample = None + if iterable is not None: + try: + if policy.can_call(getattr(iterable, "__iter__", None)): + sample = next(iter(iterable)) + except Exception: + sample = None + + loop_locals = context.transient_locals.copy() + loop_context = context.replace(transient_locals=loop_locals) + + if sample is not None: + try: + fake_assign = ast.Assign( + targets=[node.target], value=ast.Constant(value=sample) + ) + _handle_assign(fake_assign, loop_context) + except Exception: + pass + + result = None + for stmt in node.body: + result = eval_node(stmt, loop_context) + + policy = get_policy(context) + merged_locals = _merge_dicts_by_key( + [loop_locals, context.transient_locals.copy()], policy + ) + context.transient_locals.update(merged_locals) + + return result + if isinstance(node, ast.If): + branches = [] + current = node + result = None + while True: + branch_locals = context.transient_locals.copy() + branch_context = context.replace(transient_locals=branch_locals) + for stmt in current.body: + result = eval_node(stmt, branch_context) + branches.append(branch_locals) + if not current.orelse: + break + elif len(current.orelse) == 1 and isinstance(current.orelse[0], ast.If): + # It's an elif - continue loop + current = current.orelse[0] + else: + # It's an else block - process and break + else_locals = context.transient_locals.copy() + else_context = context.replace(transient_locals=else_locals) + for stmt in current.orelse: + result = eval_node(stmt, else_context) + branches.append(else_locals) + break + branches.append(context.transient_locals.copy()) + policy = get_policy(context) + merged_locals = _merge_dicts_by_key(branches, policy) + context.transient_locals.update(merged_locals) + return result + if isinstance(node, ast.Assign): + return _handle_assign(node, context) + if isinstance(node, ast.AnnAssign): + return _handle_annassign(node, context) + if isinstance(node, ast.Expression): + return eval_node(node.body, context) + if isinstance(node, ast.Expr): + return eval_node(node.value, context) + if isinstance(node, ast.Pass): + return None + if isinstance(node, ast.Import): + # TODO: populate transient_locals + return None + if isinstance(node, (ast.AugAssign, ast.Delete)): + return None + if isinstance(node, (ast.Global, ast.Nonlocal)): + return None + if isinstance(node, ast.BinOp): + left = eval_node(node.left, context) + right = eval_node(node.right, context) + if ( + isinstance(node.op, ast.BitOr) + and _is_type_annotation(left) + and _is_type_annotation(right) + ): + left_duck = ( + _Duck(dict.fromkeys(dir(left))) + if policy.can_call(left.__dir__) + else _Duck() + ) + right_duck = ( + _Duck(dict.fromkeys(dir(right))) + if policy.can_call(right.__dir__) + else _Duck() + ) + value_node = context.current_value + if value_node is not None and isinstance(value_node, ast.Dict): + if dict in [left, right]: + return _merge_values( + [left_duck, right_duck, ast.literal_eval(value_node)], + policy=get_policy(context), + ) + return _merge_values([left_duck, right_duck], policy=get_policy(context)) + dunders = _find_dunder(node.op, BINARY_OP_DUNDERS) + if dunders: + if policy.can_operate(dunders, left, right): + return getattr(left, dunders[0])(right) + else: + raise GuardRejection( + f"Operation (`{dunders}`) for", + type(left), + f"not allowed in {context.evaluation} mode", + ) + if isinstance(node, ast.Compare): + left = eval_node(node.left, context) + all_true = True + negate = False + for op, right in zip(node.ops, node.comparators): + right = eval_node(right, context) + dunder = None + dunders = _find_dunder(op, COMP_OP_DUNDERS) + if not dunders: + if isinstance(op, ast.NotIn): + dunders = COMP_OP_DUNDERS[ast.In] + negate = True + if isinstance(op, ast.Is): + dunder = "is_" + if isinstance(op, ast.IsNot): + dunder = "is_" + negate = True + if not dunder and dunders: + dunder = dunders[0] + if dunder: + a, b = (right, left) if dunder == "__contains__" else (left, right) + if dunder == "is_" or dunders and policy.can_operate(dunders, a, b): + result = getattr(operator, dunder)(a, b) + if negate: + result = not result + if not result: + all_true = False + left = right + else: + raise GuardRejection( + f"Comparison (`{dunder}`) for", + type(left), + f"not allowed in {context.evaluation} mode", + ) + else: + raise ValueError( + f"Comparison `{dunder}` not supported" + ) # pragma: no cover + return all_true + if isinstance(node, ast.Constant): + return node.value + if isinstance(node, ast.Tuple): + return tuple(eval_node(e, context) for e in node.elts) + if isinstance(node, ast.List): + return [eval_node(e, context) for e in node.elts] + if isinstance(node, ast.Set): + return {eval_node(e, context) for e in node.elts} + if isinstance(node, ast.Dict): + return dict( + zip( + [eval_node(k, context) for k in node.keys], + [eval_node(v, context) for v in node.values], + ) + ) + if isinstance(node, ast.Slice): + return slice( + eval_node(node.lower, context), + eval_node(node.upper, context), + eval_node(node.step, context), + ) + if isinstance(node, ast.UnaryOp): + value = eval_node(node.operand, context) + dunders = _find_dunder(node.op, UNARY_OP_DUNDERS) + if dunders: + if policy.can_operate(dunders, value): + try: + return getattr(value, dunders[0])() + except AttributeError: + raise TypeError( + f"bad operand type for unary {node.op}: {type(value)}" + ) + else: + raise GuardRejection( + f"Operation (`{dunders}`) for", + type(value), + f"not allowed in {context.evaluation} mode", + ) + if isinstance(node, ast.Subscript): + value = eval_node(node.value, context) + slice_ = eval_node(node.slice, context) + if policy.can_get_item(value, slice_): + return value[slice_] + raise GuardRejection( + "Subscript access (`__getitem__`) for", + type(value), # not joined to avoid calling `repr` + f" not allowed in {context.evaluation} mode", + ) + if isinstance(node, ast.Name): + return _eval_node_name(node.id, context) + if isinstance(node, ast.Attribute): + if ( + context.class_transients is not None + and isinstance(node.value, ast.Name) + and node.value.id == context.instance_arg_name + ): + return context.class_transients.get(node.attr) + value = eval_node(node.value, context) + if policy.can_get_attr(value, node.attr): + return getattr(value, node.attr) + try: + cls = ( + value if isinstance(value, type) else getattr(value, "__class__", None) + ) + if cls is not None: + resolved_hints = get_type_hints( + cls, + globalns=(context.globals or {}), + localns=(context.locals or {}), + ) + if node.attr in resolved_hints: + annotated = resolved_hints[node.attr] + return _resolve_annotation(annotated, context) + except Exception: + # Fall through to the guard rejection + pass + raise GuardRejection( + "Attribute access (`__getattr__`) for", + type(value), # not joined to avoid calling `repr` + f"not allowed in {context.evaluation} mode", + ) + if isinstance(node, ast.IfExp): + test = eval_node(node.test, context) + if test: + return eval_node(node.body, context) + else: + return eval_node(node.orelse, context) + if isinstance(node, ast.Call): + func = eval_node(node.func, context) + if policy.can_call(func): + args, kwargs = _extract_args_and_kwargs(node, context) + return func(*args, **kwargs) + if isclass(func): + # this code path gets entered when calling class e.g. `MyClass()` + # or `my_instance.__class__()` - in both cases `func` is `MyClass`. + # Should return `MyClass` if `__new__` is not overridden, + # otherwise whatever `__new__` return type is. + overridden_return_type = _eval_return_type(func.__new__, node, context) + if overridden_return_type is not NOT_EVALUATED: + return overridden_return_type + return _create_duck_for_heap_type(func) + else: + inferred_return = getattr(func, "__inferred_return__", NOT_EVALUATED) + return_type = _eval_return_type(func, node, context) + if getattr(func, "__is_async__", False): + awaited_type = ( + inferred_return if inferred_return is not None else return_type + ) + coroutine_duck = _Duck(attributes=_get_coroutine_attributes()) + coroutine_duck.__awaited_type__ = awaited_type + return coroutine_duck + if inferred_return is not NOT_EVALUATED: + return inferred_return + if return_type is not NOT_EVALUATED: + return return_type + raise GuardRejection( + "Call for", + func, # not joined to avoid calling `repr` + f"not allowed in {context.evaluation} mode", + ) + if isinstance(node, ast.Assert): + # message is always the second item, so if it is defined user would be completing + # on the message, not on the assertion test + if node.msg: + return eval_node(node.msg, context) + return eval_node(node.test, context) + return None + + +def _merge_dicts_by_key(dicts: list, policy: EvaluationPolicy): + """Merge multiple dictionaries, combining values for each key.""" + if len(dicts) == 1: + return dicts[0] + + all_keys = set() + for d in dicts: + all_keys.update(d.keys()) + + merged = {} + for key in all_keys: + values = [d[key] for d in dicts if key in d] + if values: + merged[key] = _merge_values(values, policy) + + return merged + + +def _merge_values(values, policy: EvaluationPolicy): + """Recursively merge multiple values, combining attributes and dict items.""" + if len(values) == 1: + return values[0] + + types = {type(v) for v in values} + merged_items = None + key_values = {} + attributes = set() + for v in values: + if policy.can_call(v.__dir__): + attributes.update(dir(v)) + try: + if policy.can_call(v.items): + try: + for k, val in v.items(): + key_values.setdefault(k, []).append(val) + except Exception as e: + pass + elif policy.can_call(v.keys): + try: + for k in v.keys(): + key_values.setdefault(k, []).append(None) + except Exception as e: + pass + except Exception as e: + pass + + if key_values: + merged_items = { + k: _merge_values(vals, policy) if vals[0] is not None else None + for k, vals in key_values.items() + } + + if len(types) == 1: + t = next(iter(types)) + if t not in (dict,) and not ( + hasattr(next(iter(values)), "__getitem__") + and ( + hasattr(next(iter(values)), "items") + or hasattr(next(iter(values)), "keys") + ) + ): + if t in (list, set, tuple): + return t + return values[0] + + return _Duck(attributes=dict.fromkeys(attributes), items=merged_items) + + +def _infer_return_value(node: ast.FunctionDef, context: EvaluationContext): + """Infer the return value(s) of a function by evaluating all return statements.""" + return_values = _collect_return_values(node.body, context) + + if not return_values: + return None + if len(return_values) == 1: + return return_values[0] + + policy = get_policy(context) + return _merge_values(return_values, policy) + + +def _collect_return_values(body, context): + """Recursively collect return values from a list of AST statements.""" + return_values = [] + for stmt in body: + if isinstance(stmt, ast.Return): + if stmt.value is None: + continue + try: + value = eval_node(stmt.value, context) + if value is not None and value is not NOT_EVALUATED: + return_values.append(value) + except Exception: + pass + if isinstance( + stmt, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef, ast.Lambda) + ): + continue + elif hasattr(stmt, "body") and isinstance(stmt.body, list): + return_values.extend(_collect_return_values(stmt.body, context)) + if isinstance(stmt, ast.Try): + for h in stmt.handlers: + if hasattr(h, "body"): + return_values.extend(_collect_return_values(h.body, context)) + if hasattr(stmt, "orelse"): + return_values.extend(_collect_return_values(stmt.orelse, context)) + if hasattr(stmt, "finalbody"): + return_values.extend(_collect_return_values(stmt.finalbody, context)) + if hasattr(stmt, "orelse") and isinstance(stmt.orelse, list): + return_values.extend(_collect_return_values(stmt.orelse, context)) + return return_values + + +def _eval_return_type(func: Callable, node: ast.Call, context: EvaluationContext): + """Evaluate return type of a given callable function. + + Returns the built-in type, a duck or NOT_EVALUATED sentinel. + """ + try: + sig = signature(func) + except ValueError: + sig = UNKNOWN_SIGNATURE + # if annotation was not stringized, or it was stringized + # but resolved by signature call we know the return type + not_empty = sig.return_annotation is not Signature.empty + if not_empty: + return _resolve_annotation(sig.return_annotation, context, sig, func, node) + return NOT_EVALUATED + + +def _eval_annotation( + annotation: str, + context: EvaluationContext, +): + return ( + _eval_node_name(annotation, context) + if isinstance(annotation, str) + else annotation + ) + + +class _GetItemDuck(dict): + """A dict subclass that always returns the factory instance and claims to have any item.""" + + def __init__(self, factory, *args, **kwargs): + super().__init__(*args, **kwargs) + self._factory = factory + + def __getitem__(self, key): + return self._factory() + + def __contains__(self, key): + return True + + +def _resolve_annotation( + annotation: object | str, + context: EvaluationContext, + sig: Signature | None = None, + func: Callable | None = None, + node: ast.Call | None = None, +): + """Resolve annotation created by user with `typing` module and custom objects.""" + if annotation is None: + return None + annotation = _eval_annotation(annotation, context) + origin = get_origin(annotation) + if annotation is Self and func and hasattr(func, "__self__"): + return func.__self__ + elif origin is Literal: + type_args = get_args(annotation) + if len(type_args) == 1: + return type_args[0] + elif annotation is LiteralString: + return "" + elif annotation is AnyStr: + index = None + if func and hasattr(func, "__node__"): + def_node = func.__node__ + for i, arg in enumerate(def_node.args.args): + if not arg.annotation: + continue + annotation = _eval_annotation(arg.annotation.id, context) + if annotation is AnyStr: + index = i + break + is_bound_method = ( + isinstance(func, MethodType) and getattr(func, "__self__") is not None + ) + if index and is_bound_method: + index -= 1 + elif sig: + for i, (key, value) in enumerate(sig.parameters.items()): + if value.annotation is AnyStr: + index = i + break + if index is None: + return None + if index < 0 or index >= len(node.args): + return None + return eval_node(node.args[index], context) + elif origin is TypeGuard: + return False + elif origin is set or origin is list: + # only one type argument allowed + attributes = [ + attr + for attr in dir( + _resolve_annotation(get_args(annotation)[0], context, sig, func, node) + ) + ] + duck = _Duck(attributes=dict.fromkeys(attributes)) + return _Duck( + attributes=dict.fromkeys(dir(origin())), + # items are not strrictly needed for set + items=_GetItemDuck(lambda: duck), + ) + elif origin is tuple: + # multiple type arguments + return tuple( + _resolve_annotation(arg, context, sig, func, node) + for arg in get_args(annotation) + ) + elif origin is Union: + # multiple type arguments + attributes = [ + attr + for type_arg in get_args(annotation) + for attr in dir(_resolve_annotation(type_arg, context, sig, func, node)) + ] + return _Duck(attributes=dict.fromkeys(attributes)) + elif is_typeddict(annotation): + return _Duck( + attributes=dict.fromkeys(dir(dict())), + items={ + k: _resolve_annotation(v, context, sig, func, node) + for k, v in annotation.__annotations__.items() + }, + ) + elif hasattr(annotation, "_is_protocol"): + return _Duck(attributes=dict.fromkeys(dir(annotation))) + elif origin is Annotated: + type_arg = get_args(annotation)[0] + return _resolve_annotation(type_arg, context, sig, func, node) + elif isinstance(annotation, NewType): + return _eval_or_create_duck(annotation.__supertype__, context) + elif isinstance(annotation, TypeAliasType): + return _eval_or_create_duck(annotation.__value__, context) + else: + return _eval_or_create_duck(annotation, context) + + +def _eval_node_name(node_id: str, context: EvaluationContext): + policy = get_policy(context) + if node_id in context.transient_locals: + return context.transient_locals[node_id] + if policy.allow_locals_access and node_id in context.locals: + return context.locals[node_id] + if policy.allow_globals_access and node_id in context.globals: + return context.globals[node_id] + if policy.allow_builtins_access and hasattr(builtins, node_id): + # note: do not use __builtins__, it is implementation detail of cPython + return getattr(builtins, node_id) + if policy.allow_auto_import and context.auto_import: + return context.auto_import(node_id) + if not policy.allow_globals_access and not policy.allow_locals_access: + raise GuardRejection( + f"Namespace access not allowed in {context.evaluation} mode" + ) + else: + raise NameError(f"{node_id} not found in locals, globals, nor builtins") + + +def _eval_or_create_duck(duck_type, context: EvaluationContext): + policy = get_policy(context) + # if allow-listed builtin is on type annotation, instantiate it + if policy.can_call(duck_type): + return duck_type() + # if custom class is in type annotation, mock it + return _create_duck_for_heap_type(duck_type) + + +def _create_duck_for_heap_type(duck_type): + """Create an imitation of an object of a given type (a duck). + + Returns the duck or NOT_EVALUATED sentinel if duck could not be created. + """ + duck = ImpersonatingDuck() + try: + # this only works for heap types, not builtins + duck.__class__ = duck_type + return duck + except TypeError: + pass + return NOT_EVALUATED + + +SUPPORTED_EXTERNAL_GETITEM = { + ("pandas", "core", "indexing", "_iLocIndexer"), + ("pandas", "core", "indexing", "_LocIndexer"), + ("pandas", "DataFrame"), + ("pandas", "Series"), + ("numpy", "ndarray"), + ("numpy", "void"), +} + + +BUILTIN_GETITEM: set[InstancesHaveGetItem] = { + dict, + str, # type: ignore[arg-type] + bytes, # type: ignore[arg-type] + list, + tuple, + type, # for type annotations like list[str] + _Duck, + collections.defaultdict, + collections.deque, + collections.OrderedDict, + collections.ChainMap, + collections.UserDict, + collections.UserList, + collections.UserString, # type: ignore[arg-type] + _DummyNamedTuple, + _IdentitySubscript, +} + + +def _list_methods(cls, source=None): + """For use on immutable objects or with methods returning a copy""" + return [getattr(cls, k) for k in (source if source else dir(cls))] + + +dict_non_mutating_methods = ("copy", "keys", "values", "items") +list_non_mutating_methods = ("copy", "index", "count") +set_non_mutating_methods = set(dir(set)) & set(dir(frozenset)) + + +dict_keys: type[collections.abc.KeysView] = type({}.keys()) +dict_values: type = type({}.values()) +dict_items: type = type({}.items()) + +NUMERICS = {int, float, complex} + +ALLOWED_CALLS = { + bytes, + *_list_methods(bytes), + bytes.__iter__, + dict, + *_list_methods(dict, dict_non_mutating_methods), + dict.__iter__, + dict_keys.__iter__, + dict_values.__iter__, + dict_items.__iter__, + dict_keys.isdisjoint, + list, + *_list_methods(list, list_non_mutating_methods), + list.__iter__, + set, + *_list_methods(set, set_non_mutating_methods), + set.__iter__, + frozenset, + *_list_methods(frozenset), + frozenset.__iter__, + range, + range.__iter__, + str, + *_list_methods(str), + str.__iter__, + tuple, + *_list_methods(tuple), + tuple.__iter__, + bool, + *_list_methods(bool), + *NUMERICS, + *[method for numeric_cls in NUMERICS for method in _list_methods(numeric_cls)], + collections.deque, + *_list_methods(collections.deque, list_non_mutating_methods), + collections.deque.__iter__, + collections.defaultdict, + *_list_methods(collections.defaultdict, dict_non_mutating_methods), + collections.defaultdict.__iter__, + collections.OrderedDict, + *_list_methods(collections.OrderedDict, dict_non_mutating_methods), + collections.OrderedDict.__iter__, + collections.UserDict, + *_list_methods(collections.UserDict, dict_non_mutating_methods), + collections.UserDict.__iter__, + collections.UserList, + *_list_methods(collections.UserList, list_non_mutating_methods), + collections.UserList.__iter__, + collections.UserString, + *_list_methods(collections.UserString, dir(str)), + collections.UserString.__iter__, + collections.Counter, + *_list_methods(collections.Counter, dict_non_mutating_methods), + collections.Counter.__iter__, + collections.Counter.elements, + collections.Counter.most_common, + object.__dir__, + type.__dir__, + _Duck.__dir__, +} + +BUILTIN_GETATTR: set[MayHaveGetattr] = { + *BUILTIN_GETITEM, + set, + frozenset, + object, + type, # `type` handles a lot of generic cases, e.g. numbers as in `int.real`. + *NUMERICS, + dict_keys, + MethodDescriptorType, + ModuleType, +} + + +BUILTIN_OPERATIONS = {*BUILTIN_GETATTR} + +EVALUATION_POLICIES = { + "minimal": EvaluationPolicy( + allow_builtins_access=True, + allow_locals_access=False, + allow_globals_access=False, + allow_item_access=False, + allow_attr_access=False, + allowed_calls=set(), + allow_any_calls=False, + allow_all_operations=False, + ), + "limited": SelectivePolicy( + allowed_getitem=BUILTIN_GETITEM, + allowed_getitem_external=SUPPORTED_EXTERNAL_GETITEM, + allowed_getattr=BUILTIN_GETATTR, + allowed_getattr_external={ + # pandas Series/Frame implements custom `__getattr__` + ("pandas", "DataFrame"), + ("pandas", "Series"), + }, + allowed_operations=BUILTIN_OPERATIONS, + allow_builtins_access=True, + allow_locals_access=True, + allow_globals_access=True, + allow_getitem_on_types=True, + allowed_calls=ALLOWED_CALLS, + ), + "unsafe": EvaluationPolicy( + allow_builtins_access=True, + allow_locals_access=True, + allow_globals_access=True, + allow_attr_access=True, + allow_item_access=True, + allow_any_calls=True, + allow_all_operations=True, + ), +} + + +__all__ = [ + "guarded_eval", + "eval_node", + "GuardRejection", + "EvaluationContext", + "_unbind_method", +] diff --git a/lib/python3.12/site-packages/IPython/core/magic.py b/lib/python3.12/site-packages/IPython/core/magic.py new file mode 100644 index 0000000000000000000000000000000000000000..a986c0892fbcbed0d7d0f624ecbd2490b08d2155 --- /dev/null +++ b/lib/python3.12/site-packages/IPython/core/magic.py @@ -0,0 +1,786 @@ +from __future__ import annotations + +"""Magic functions for InteractiveShell.""" + +# ----------------------------------------------------------------------------- +# Copyright (C) 2001 Janko Hauser and +# Copyright (C) 2001 Fernando Perez +# Copyright (C) 2008 The IPython Development Team + +# Distributed under the terms of the BSD License. The full license is in +# the file COPYING, distributed as part of this software. +# ----------------------------------------------------------------------------- + +import os +import re +import sys +from getopt import getopt, GetoptError + +from traitlets.config.configurable import Configurable +from . import oinspect +from .error import UsageError +from .inputtransformer2 import ESC_MAGIC, ESC_MAGIC2 +from ..utils.ipstruct import Struct +from ..utils.process import arg_split +from ..utils.text import dedent +from traitlets import Bool, Dict, Instance, observe +from logging import error + +import typing as t + +if t.TYPE_CHECKING: + from IPython.core.interactiveshell import InteractiveShell + + +# ----------------------------------------------------------------------------- +# Globals +# ----------------------------------------------------------------------------- + +# A dict we'll use for each class that has magics, used as temporary storage to +# pass information between the @line/cell_magic method decorators and the +# @magics_class class decorator, because the method decorators have no +# access to the class when they run. See for more details: +# http://stackoverflow.com/questions/2366713/can-a-python-decorator-of-an-instance-method-access-the-class + +magics: t.Dict = dict(line={}, cell={}) + +magic_kinds = ("line", "cell") +magic_spec = ("line", "cell", "line_cell") +magic_escapes = dict(line=ESC_MAGIC, cell=ESC_MAGIC2) + +# ----------------------------------------------------------------------------- +# Utility classes and functions +# ----------------------------------------------------------------------------- + + +class Bunch: + pass + + +def on_off(tag): + """Return an ON/OFF string for a 1/0 input. Simple utility function.""" + return ["OFF", "ON"][tag] + + +def compress_dhist(dh): + """Compress a directory history into a new one with at most 20 entries. + + Return a new list made from the first and last 10 elements of dhist after + removal of duplicates. + """ + head, tail = dh[:-10], dh[-10:] + + newhead = [] + done = set() + for h in head: + if h in done: + continue + newhead.append(h) + done.add(h) + + return newhead + tail + + +def needs_local_scope(func): + """Decorator to mark magic functions which need to local scope to run.""" + func.needs_local_scope = True + return func + + +# ----------------------------------------------------------------------------- +# Class and method decorators for registering magics +# ----------------------------------------------------------------------------- + + +def magics_class(cls): + """Class decorator for all subclasses of the main Magics class. + + Any class that subclasses Magics *must* also apply this decorator, to + ensure that all the methods that have been decorated as line/cell magics + get correctly registered in the class instance. This is necessary because + when method decorators run, the class does not exist yet, so they + temporarily store their information into a module global. Application of + this class decorator copies that global data to the class instance and + clears the global. + + Obviously, this mechanism is not thread-safe, which means that the + *creation* of subclasses of Magic should only be done in a single-thread + context. Instantiation of the classes has no restrictions. Given that + these classes are typically created at IPython startup time and before user + application code becomes active, in practice this should not pose any + problems. + """ + cls.registered = True + cls.magics = dict(line=magics["line"], cell=magics["cell"]) + magics["line"] = {} + magics["cell"] = {} + return cls + + +def record_magic(dct, magic_kind, magic_name, func): + """Utility function to store a function as a magic of a specific kind. + + Parameters + ---------- + dct : dict + A dictionary with 'line' and 'cell' subdicts. + magic_kind : str + Kind of magic to be stored. + magic_name : str + Key to store the magic as. + func : function + Callable object to store. + """ + if magic_kind == "line_cell": + dct["line"][magic_name] = dct["cell"][magic_name] = func + else: + dct[magic_kind][magic_name] = func + + +def validate_type(magic_kind): + """Ensure that the given magic_kind is valid. + + Check that the given magic_kind is one of the accepted spec types (stored + in the global `magic_spec`), raise ValueError otherwise. + """ + if magic_kind not in magic_spec: + raise ValueError( + "magic_kind must be one of %s, %s given" % magic_kinds, magic_kind + ) + + +# The docstrings for the decorator below will be fairly similar for the two +# types (method and function), so we generate them here once and reuse the +# templates below. +_docstring_template = """Decorate the given {0} as {1} magic. + +The decorator can be used with or without arguments, as follows. + +i) without arguments: it will create a {1} magic named as the {0} being +decorated:: + + @deco + def foo(...) + +will create a {1} magic named `foo`. + +ii) with one string argument: which will be used as the actual name of the +resulting magic:: + + @deco('bar') + def foo(...) + +will create a {1} magic named `bar`. + +To register a class magic use ``Interactiveshell.register_magic(class or instance)``. +""" + +# These two are decorator factories. While they are conceptually very similar, +# there are enough differences in the details that it's simpler to have them +# written as completely standalone functions rather than trying to share code +# and make a single one with convoluted logic. + + +def _method_magic_marker(magic_kind): + """Decorator factory for methods in Magics subclasses.""" + + validate_type(magic_kind) + + # This is a closure to capture the magic_kind. We could also use a class, + # but it's overkill for just that one bit of state. + def magic_deco(arg): + if callable(arg): + # "Naked" decorator call (just @foo, no args) + func = arg + name = func.__name__ + retval = arg + record_magic(magics, magic_kind, name, name) + elif isinstance(arg, str): + # Decorator called with arguments (@foo('bar')) + name = arg + + def mark(func, *a, **kw): + record_magic(magics, magic_kind, name, func.__name__) + return func + + retval = mark + else: + raise TypeError("Decorator can only be called with string or function") + return retval + + # Ensure the resulting decorator has a usable docstring + magic_deco.__doc__ = _docstring_template.format("method", magic_kind) + return magic_deco + + +def _function_magic_marker(magic_kind): + """Decorator factory for standalone functions.""" + validate_type(magic_kind) + + # This is a closure to capture the magic_kind. We could also use a class, + # but it's overkill for just that one bit of state. + def magic_deco(arg): + # Find get_ipython() in the caller's namespace + caller = sys._getframe(1) + for ns in ["f_locals", "f_globals", "f_builtins"]: + get_ipython = getattr(caller, ns).get("get_ipython") + if get_ipython is not None: + break + else: + raise NameError( + "Decorator can only run in context where `get_ipython` exists" + ) + + ip = get_ipython() + + if callable(arg): + # "Naked" decorator call (just @foo, no args) + func = arg + name = func.__name__ + ip.register_magic_function(func, magic_kind, name) + retval = arg + elif isinstance(arg, str): + # Decorator called with arguments (@foo('bar')) + name = arg + + def mark(func, *a, **kw): + ip.register_magic_function(func, magic_kind, name) + return func + + retval = mark + else: + raise TypeError("Decorator can only be called with string or function") + return retval + + # Ensure the resulting decorator has a usable docstring + ds = _docstring_template.format("function", magic_kind) + + ds += dedent( + """ + Note: this decorator can only be used in a context where IPython is already + active, so that the `get_ipython()` call succeeds. You can therefore use + it in your startup files loaded after IPython initializes, but *not* in the + IPython configuration file itself, which is executed before IPython is + fully up and running. Any file located in the `startup` subdirectory of + your configuration profile will be OK in this sense. + """ + ) + + magic_deco.__doc__ = ds + return magic_deco + + +MAGIC_NO_VAR_EXPAND_ATTR = "_ipython_magic_no_var_expand" +MAGIC_OUTPUT_CAN_BE_SILENCED = "_ipython_magic_output_can_be_silenced" + + +def no_var_expand(magic_func): + """Mark a magic function as not needing variable expansion + + By default, IPython interprets `{a}` or `$a` in the line passed to magics + as variables that should be interpolated from the interactive namespace + before passing the line to the magic function. + This is not always desirable, e.g. when the magic executes Python code + (%timeit, %time, etc.). + Decorate magics with `@no_var_expand` to opt-out of variable expansion. + + .. versionadded:: 7.3 + """ + setattr(magic_func, MAGIC_NO_VAR_EXPAND_ATTR, True) + return magic_func + + +def output_can_be_silenced(magic_func): + """Mark a magic function so its output may be silenced. + + The output is silenced if the Python code used as a parameter of + the magic ends in a semicolon, not counting a Python comment that can + follow it. + """ + setattr(magic_func, MAGIC_OUTPUT_CAN_BE_SILENCED, True) + return magic_func + + +# Create the actual decorators for public use + +# These three are used to decorate methods in class definitions +line_magic = _method_magic_marker("line") +cell_magic = _method_magic_marker("cell") +line_cell_magic = _method_magic_marker("line_cell") + +# These three decorate standalone functions and perform the decoration +# immediately. They can only run where get_ipython() works +register_line_magic = _function_magic_marker("line") +register_cell_magic = _function_magic_marker("cell") +register_line_cell_magic = _function_magic_marker("line_cell") + +# ----------------------------------------------------------------------------- +# Core Magic classes +# ----------------------------------------------------------------------------- + + +class MagicsManager(Configurable): + """Object that handles all magic-related functionality for IPython.""" + + # Non-configurable class attributes + + # A two-level dict, first keyed by magic type, then by magic function, and + # holding the actual callable object as value. This is the dict used for + # magic function dispatch + magics = Dict() + lazy_magics = Dict( + help=""" + Mapping from magic names to modules to load. + + This can be used in IPython/IPykernel configuration to declare lazy magics + that will only be imported/registered on first use. + + For example:: + + c.MagicsManager.lazy_magics = { + "my_magic": "slow.to.import", + "my_other_magic": "also.slow", + } + + On first invocation of `%my_magic`, `%%my_magic`, `%%my_other_magic` or + `%%my_other_magic`, the corresponding module will be loaded as an ipython + extensions as if you had previously done `%load_ext ipython`. + + Magics names should be without percent(s) as magics can be both cell + and line magics. + + Lazy loading happen relatively late in execution process, and + complex extensions that manipulate Python/IPython internal state or global state + might not support lazy loading. + """ + ).tag( + config=True, + ) + + # A registry of the original objects that we've been given holding magics. + registry = Dict() + + shell = Instance( + "IPython.core.interactiveshell.InteractiveShellABC", allow_none=True + ) + + auto_magic = Bool( + True, help="Automatically call line magics without requiring explicit % prefix" + ).tag(config=True) + + @observe("auto_magic") + def _auto_magic_changed(self, change): + assert self.shell is not None + self.shell.automagic = change["new"] + + _auto_status = [ + "Automagic is OFF, % prefix IS needed for line magics.", + "Automagic is ON, % prefix IS NOT needed for line magics.", + ] + + user_magics = Instance("IPython.core.magics.UserMagics", allow_none=True) + + def __init__(self, shell=None, config=None, user_magics=None, **traits): + super(MagicsManager, self).__init__( + shell=shell, config=config, user_magics=user_magics, **traits + ) + self.magics = dict(line={}, cell={}) + # Let's add the user_magics to the registry for uniformity, so *all* + # registered magic containers can be found there. + self.registry[user_magics.__class__.__name__] = user_magics + + def auto_status(self): + """Return descriptive string with automagic status.""" + return self._auto_status[self.auto_magic] + + def lsmagic(self): + """Return a dict of currently available magic functions. + + The return dict has the keys 'line' and 'cell', corresponding to the + two types of magics we support. Each value is a list of names. + """ + return self.magics + + def lsmagic_docs(self, brief=False, missing=""): + """Return dict of documentation of magic functions. + + The return dict has the keys 'line' and 'cell', corresponding to the + two types of magics we support. Each value is a dict keyed by magic + name whose value is the function docstring. If a docstring is + unavailable, the value of `missing` is used instead. + + If brief is True, only the first line of each docstring will be returned. + """ + docs = {} + for m_type in self.magics: + m_docs = {} + for m_name, m_func in self.magics[m_type].items(): + if m_func.__doc__: + if brief: + m_docs[m_name] = m_func.__doc__.split("\n", 1)[0] + else: + m_docs[m_name] = m_func.__doc__.rstrip() + else: + m_docs[m_name] = missing + docs[m_type] = m_docs + return docs + + def register_lazy(self, name: str, fully_qualified_name: str) -> None: + """ + Lazily register a magic via an extension. + + + Parameters + ---------- + name : str + Name of the magic you wish to register. + fully_qualified_name : + Fully qualified name of the module/submodule that should be loaded + as an extensions when the magic is first called. + It is assumed that loading this extensions will register the given + magic. + """ + + self.lazy_magics[name] = fully_qualified_name + + def register(self, *magic_objects): + """Register one or more instances of Magics. + + Take one or more classes or instances of classes that subclass the main + `core.Magic` class, and register them with IPython to use the magic + functions they provide. The registration process will then ensure that + any methods that have decorated to provide line and/or cell magics will + be recognized with the `%x`/`%%x` syntax as a line/cell magic + respectively. + + If classes are given, they will be instantiated with the default + constructor. If your classes need a custom constructor, you should + instanitate them first and pass the instance. + + The provided arguments can be an arbitrary mix of classes and instances. + + Parameters + ---------- + *magic_objects : one or more classes or instances + """ + # Start by validating them to ensure they have all had their magic + # methods registered at the instance level + for m in magic_objects: + if not m.registered: + raise ValueError( + "Class of magics %r was constructed without " + "the @register_magics class decorator" + ) + if isinstance(m, type): + # If we're given an uninstantiated class + m = m(shell=self.shell) + + # Now that we have an instance, we can register it and update the + # table of callables + self.registry[m.__class__.__name__] = m + for mtype in magic_kinds: + self.magics[mtype].update(m.magics[mtype]) + + def register_function(self, func, magic_kind="line", magic_name=None): + """Expose a standalone function as magic function for IPython. + + This will create an IPython magic (line, cell or both) from a + standalone function. The functions should have the following + signatures: + + * For line magics: `def f(line)` + * For cell magics: `def f(line, cell)` + * For a function that does both: `def f(line, cell=None)` + + In the latter case, the function will be called with `cell==None` when + invoked as `%f`, and with cell as a string when invoked as `%%f`. + + Parameters + ---------- + func : callable + Function to be registered as a magic. + magic_kind : str + Kind of magic, one of 'line', 'cell' or 'line_cell' + magic_name : optional str + If given, the name the magic will have in the IPython namespace. By + default, the name of the function itself is used. + """ + + # Create the new method in the user_magics and register it in the + # global table + validate_type(magic_kind) + magic_name = func.__name__ if magic_name is None else magic_name + setattr(self.user_magics, magic_name, func) + record_magic(self.magics, magic_kind, magic_name, func) + + def register_alias( + self, alias_name, magic_name, magic_kind="line", magic_params=None + ): + """Register an alias to a magic function. + + The alias is an instance of :class:`MagicAlias`, which holds the + name and kind of the magic it should call. Binding is done at + call time, so if the underlying magic function is changed the alias + will call the new function. + + Parameters + ---------- + alias_name : str + The name of the magic to be registered. + magic_name : str + The name of an existing magic. + magic_kind : str + Kind of magic, one of 'line' or 'cell' + """ + + # `validate_type` is too permissive, as it allows 'line_cell' + # which we do not handle. + if magic_kind not in magic_kinds: + raise ValueError( + "magic_kind must be one of %s, %s given" % magic_kinds, magic_kind + ) + + alias = MagicAlias(self.shell, magic_name, magic_kind, magic_params) + setattr(self.user_magics, alias_name, alias) + record_magic(self.magics, magic_kind, alias_name, alias) + + +# Key base class that provides the central functionality for magics. + + +class Magics(Configurable): + """Base class for implementing magic functions. + + Shell functions which can be reached as %function_name. All magic + functions should accept a string, which they can parse for their own + needs. This can make some functions easier to type, eg `%cd ../` + vs. `%cd("../")` + + Classes providing magic functions need to subclass this class, and they + MUST: + + - Use the method decorators `@line_magic` and `@cell_magic` to decorate + individual methods as magic functions, AND + + - Use the class decorator `@magics_class` to ensure that the magic + methods are properly registered at the instance level upon instance + initialization. + + See :mod:`magic_functions` for examples of actual implementation classes. + """ + + # Dict holding all command-line options for each magic. + options_table: dict[str, t.Any] = {} + # Dict for the mapping of magic names to methods, set by class decorator + magics: dict[str, t.Any] = {} + # Flag to check that the class decorator was properly applied + registered: bool = False + # Instance of IPython shell + shell: None | InteractiveShell = None + + def __init__(self, shell=None, **kwargs): + if not (self.__class__.registered): + raise ValueError( + "Magics subclass without registration - " + "did you forget to apply @magics_class?" + ) + if shell is not None: + if hasattr(shell, "configurables"): + shell.configurables.append(self) + if hasattr(shell, "config"): + kwargs.setdefault("parent", shell) + + self.shell = shell + self.options_table = {} + # The method decorators are run when the instance doesn't exist yet, so + # they can only record the names of the methods they are supposed to + # grab. Only now, that the instance exists, can we create the proper + # mapping to bound methods. So we read the info off the original names + # table and replace each method name by the actual bound method. + # But we mustn't clobber the *class* mapping, in case of multiple instances. + class_magics = self.magics + self.magics = {} + for mtype in magic_kinds: + tab = self.magics[mtype] = {} + cls_tab = class_magics[mtype] + for magic_name, meth_name in cls_tab.items(): + if isinstance(meth_name, str): + # it's a method name, grab it + tab[magic_name] = getattr(self, meth_name) + else: + # it's the real thing + tab[magic_name] = meth_name + # Configurable **needs** to be initiated at the end or the config + # magics get screwed up. + super(Magics, self).__init__(**kwargs) + + def arg_err(self, func): + """Print docstring if incorrect arguments were passed""" + print("Error in arguments:") + print(oinspect.getdoc(func)) + + def format_latex(self, strng): + """Format a string for latex inclusion.""" + + # Characters that need to be escaped for latex: + escape_re = re.compile(r"(%|_|\$|#|&)", re.MULTILINE) + # Magic command names as headers: + cmd_name_re = re.compile(r"^(%s.*?):" % ESC_MAGIC, re.MULTILINE) + # Magic commands + cmd_re = re.compile(r"(?P%s.+?\b)(?!\}\}:)" % ESC_MAGIC, re.MULTILINE) + # Paragraph continue + par_re = re.compile(r"\\$", re.MULTILINE) + + # The "\n" symbol + newline_re = re.compile(r"\\n") + + # Now build the string for output: + # strng = cmd_name_re.sub(r'\n\\texttt{\\textsl{\\large \1}}:',strng) + strng = cmd_name_re.sub(r"\n\\bigskip\n\\texttt{\\textbf{ \1}}:", strng) + strng = cmd_re.sub(r"\\texttt{\g}", strng) + strng = par_re.sub(r"\\\\", strng) + strng = escape_re.sub(r"\\\1", strng) + strng = newline_re.sub(r"\\textbackslash{}n", strng) + return strng + + def parse_options(self, arg_str, opt_str, *long_opts, **kw): + """Parse options passed to an argument string. + + The interface is similar to that of :func:`getopt.getopt`, but it + returns a :class:`~IPython.utils.struct.Struct` with the options as keys + and the stripped argument string still as a string. + + arg_str is quoted as a true sys.argv vector by using shlex.split. + This allows us to easily expand variables, glob files, quote + arguments, etc. + + Parameters + ---------- + arg_str : str + The arguments to parse. + opt_str : str + The options specification. + mode : str, default 'string' + If given as 'list', the argument string is returned as a list (split + on whitespace) instead of a string. + list_all : bool, default False + Put all option values in lists. Normally only options + appearing more than once are put in a list. + posix : bool, default True + Whether to split the input line in POSIX mode or not, as per the + conventions outlined in the :mod:`shlex` module from the standard + library. + """ + + # inject default options at the beginning of the input line + caller = sys._getframe(1).f_code.co_name + arg_str = "%s %s" % (self.options_table.get(caller, ""), arg_str) + + mode = kw.get("mode", "string") + if mode not in ["string", "list"]: + raise ValueError("incorrect mode given: %s" % mode) + # Get options + list_all = kw.get("list_all", 0) + posix = kw.get("posix", os.name == "posix") + strict = kw.get("strict", True) + + preserve_non_opts = kw.get("preserve_non_opts", False) + remainder_arg_str = arg_str + + # Check if we have more than one argument to warrant extra processing: + odict: dict[str, t.Any] = {} # Dictionary with options + args = arg_str.split() + if len(args) >= 1: + # If the list of inputs only has 0 or 1 thing in it, there's no + # need to look for options + argv = arg_split(arg_str, posix, strict) + # Do regular option processing + try: + opts, args = getopt(argv, opt_str, long_opts) + except GetoptError as e: + raise UsageError( + '%s (allowed: "%s"%s)' + % (e.msg, opt_str, " ".join(("",) + long_opts) if long_opts else "") + ) from e + for o, a in opts: + if mode == "string" and preserve_non_opts: + # remove option-parts from the original args-string and preserve remaining-part. + # This relies on the arg_split(...) and getopt(...)'s impl spec, that the parsed options are + # returned in the original order. + remainder_arg_str = remainder_arg_str.replace(o, "", 1).replace( + a, "", 1 + ) + if o.startswith("--"): + o = o[2:] + else: + o = o[1:] + try: + odict[o].append(a) + except AttributeError: + odict[o] = [odict[o], a] + except KeyError: + if list_all: + odict[o] = [a] + else: + odict[o] = a + + # Prepare opts,args for return + opts = Struct(odict) # type: ignore[assignment] + if mode == "string": + if preserve_non_opts: + args = remainder_arg_str.lstrip() + else: + args = " ".join(args) + + return opts, args + + def default_option(self, fn, optstr): + """Make an entry in the options_table for fn, with value optstr""" + assert False, "is this even called?" + if fn not in self.lsmagic(): + error("%s is not a magic function" % fn) + self.options_table[fn] = optstr + + +class MagicAlias: + """An alias to another magic function. + + An alias is determined by its magic name and magic kind. Lookup + is done at call time, so if the underlying magic changes the alias + will call the new function. + + Use the :meth:`MagicsManager.register_alias` method or the + `%alias_magic` magic function to create and register a new alias. + """ + + def __init__(self, shell, magic_name, magic_kind, magic_params=None): + self.shell = shell + self.magic_name = magic_name + self.magic_params = magic_params + self.magic_kind = magic_kind + + self.pretty_target = "%s%s" % (magic_escapes[self.magic_kind], self.magic_name) + self.__doc__ = "Alias for `%s`." % self.pretty_target + + self._in_call = False + + def __call__(self, *args, **kwargs): + """Call the magic alias.""" + fn = self.shell.find_magic(self.magic_name, self.magic_kind) + if fn is None: + raise UsageError("Magic `%s` not found." % self.pretty_target) + + # Protect against infinite recursion. + if self._in_call: + raise UsageError( + "Infinite recursion detected; magic aliases cannot call themselves." + ) + self._in_call = True + try: + if self.magic_params: + args_list = list(args) + args_list[0] = self.magic_params + " " + args[0] + args = tuple(args_list) + return fn(*args, **kwargs) + finally: + self._in_call = False diff --git a/lib/python3.12/site-packages/IPython/core/release.py b/lib/python3.12/site-packages/IPython/core/release.py new file mode 100644 index 0000000000000000000000000000000000000000..e675529476ea1b6faa52bcefe21bf6084c96bca2 --- /dev/null +++ b/lib/python3.12/site-packages/IPython/core/release.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +"""Release data for the IPython project.""" + +#----------------------------------------------------------------------------- +# Copyright (c) 2008, IPython Development Team. +# Copyright (c) 2001, Fernando Perez +# Copyright (c) 2001, Janko Hauser +# Copyright (c) 2001, Nathaniel Gray +# +# Distributed under the terms of the Modified BSD License. +# +# The full license is in the file COPYING.txt, distributed with this software. +#----------------------------------------------------------------------------- + +# IPython version information. An empty _version_extra corresponds to a full +# release. 'dev' as a _version_extra string means this is a development +# version +_version_major = 9 +_version_minor = 10 +_version_patch = 0 +_version_extra = ".dev" +# _version_extra = "b2" +_version_extra = "" # Uncomment this for full releases + +# Construct full version string from these. +_ver = [_version_major, _version_minor, _version_patch] + +__version__ = '.'.join(map(str, _ver)) +if _version_extra: + __version__ = __version__ + _version_extra + +version = __version__ # backwards compatibility name +version_info = (_version_major, _version_minor, _version_patch, _version_extra) + + +license = "BSD-3-Clause" + +authors = { + "Fernando": ("Fernando Perez", "fperez.net@gmail.com"), + "M": ("M Bussonnier", "mbussonnier@gmail.com"), +} + +author = 'The IPython Development Team' + +author_email = 'ipython-dev@python.org' diff --git a/lib/python3.12/site-packages/IPython/display.py b/lib/python3.12/site-packages/IPython/display.py new file mode 100644 index 0000000000000000000000000000000000000000..da8982d6e2d70fdb739f97bf21c9905b74616428 --- /dev/null +++ b/lib/python3.12/site-packages/IPython/display.py @@ -0,0 +1,41 @@ +"""Public API for display tools in IPython.""" + +# ----------------------------------------------------------------------------- +# Copyright (C) 2012 The IPython Development Team +# +# Distributed under the terms of the BSD License. The full license is in +# the file COPYING, distributed as part of this software. +# ----------------------------------------------------------------------------- + +# ----------------------------------------------------------------------------- +# Imports +# ----------------------------------------------------------------------------- + +from IPython.core.display_functions import * +from IPython.core.display import ( + display_pretty as display_pretty, + display_html as display_html, + display_markdown as display_markdown, + display_svg as display_svg, + display_png as display_png, + display_jpeg as display_jpeg, + display_latex as display_latex, + display_json as display_json, + display_javascript as display_javascript, + display_pdf as display_pdf, + DisplayObject as DisplayObject, + TextDisplayObject as TextDisplayObject, + Pretty as Pretty, + HTML as HTML, + Markdown as Markdown, + Math as Math, + Latex as Latex, + SVG as SVG, + ProgressBar as ProgressBar, + JSON as JSON, + GeoJSON as GeoJSON, + Javascript as Javascript, + Image as Image, + Video as Video, +) +from IPython.lib.display import * diff --git a/lib/python3.12/site-packages/IPython/paths.py b/lib/python3.12/site-packages/IPython/paths.py new file mode 100644 index 0000000000000000000000000000000000000000..1afb31e5cd428468a453d0536870033151aaeb62 --- /dev/null +++ b/lib/python3.12/site-packages/IPython/paths.py @@ -0,0 +1,122 @@ +"""Find files and directories which IPython uses. +""" +import os.path +import tempfile +from warnings import warn + +import IPython +from IPython.utils.importstring import import_item +from IPython.utils.path import ( + get_home_dir, + get_xdg_dir, + get_xdg_cache_dir, + compress_user, + _writable_dir, + ensure_dir_exists, +) + + +def get_ipython_dir() -> str: + """Get the IPython directory for this platform and user. + + This uses the logic in `get_home_dir` to find the home directory + and then adds .ipython to the end of the path. + """ + + env = os.environ + pjoin = os.path.join + + + ipdir_def = '.ipython' + + home_dir = get_home_dir() + xdg_dir = get_xdg_dir() + + ipdir = env.get("IPYTHONDIR", None) + if ipdir is None: + # not set explicitly, use ~/.ipython + ipdir = pjoin(home_dir, ipdir_def) + if xdg_dir: + # Several IPython versions (up to 1.x) defaulted to .config/ipython + # on Linux. We have decided to go back to using .ipython everywhere + xdg_ipdir = pjoin(xdg_dir, 'ipython') + + if _writable_dir(xdg_ipdir): + cu = compress_user + if os.path.exists(ipdir): + warn(('Ignoring {0} in favour of {1}. Remove {0} to ' + 'get rid of this message').format(cu(xdg_ipdir), cu(ipdir))) + elif os.path.islink(xdg_ipdir): + warn(('{0} is deprecated. Move link to {1} to ' + 'get rid of this message').format(cu(xdg_ipdir), cu(ipdir))) + else: + ipdir = xdg_ipdir + + ipdir = os.path.normpath(os.path.expanduser(ipdir)) + + if os.path.exists(ipdir) and not _writable_dir(ipdir): + # ipdir exists, but is not writable + warn("IPython dir '{0}' is not a writable location," + " using a temp directory.".format(ipdir)) + ipdir = tempfile.mkdtemp() + elif not os.path.exists(ipdir): + parent = os.path.dirname(ipdir) + if not _writable_dir(parent): + # ipdir does not exist and parent isn't writable + warn("IPython parent '{0}' is not a writable location," + " using a temp directory.".format(parent)) + ipdir = tempfile.mkdtemp() + else: + os.makedirs(ipdir, exist_ok=True) + assert isinstance(ipdir, str), "all path manipulation should be str(unicode), but are not." + return ipdir + + +def get_ipython_cache_dir() -> str: + """Get the cache directory it is created if it does not exist.""" + xdgdir = get_xdg_cache_dir() + if xdgdir is None: + return get_ipython_dir() + ipdir = os.path.join(xdgdir, "ipython") + if not os.path.exists(ipdir) and _writable_dir(xdgdir): + ensure_dir_exists(ipdir) + elif not _writable_dir(xdgdir): + return get_ipython_dir() + + return ipdir + + +def get_ipython_package_dir() -> str: + """Get the base directory where IPython itself is installed.""" + ipdir = os.path.dirname(IPython.__file__) + assert isinstance(ipdir, str) + return ipdir + + +def get_ipython_module_path(module_str): + """Find the path to an IPython module in this version of IPython. + + This will always find the version of the module that is in this importable + IPython package. This will always return the path to the ``.py`` + version of the module. + """ + if module_str == 'IPython': + return os.path.join(get_ipython_package_dir(), '__init__.py') + mod = import_item(module_str) + the_path = mod.__file__.replace('.pyc', '.py') + the_path = the_path.replace('.pyo', '.py') + return the_path + + +def locate_profile(profile='default'): + """Find the path to the folder associated with a given profile. + + I.e. find $IPYTHONDIR/profile_whatever. + """ + from IPython.core.profiledir import ProfileDir, ProfileDirError + try: + pd = ProfileDir.find_profile_dir_by_name(get_ipython_dir(), profile) + except ProfileDirError as e: + # IOError makes more sense when people are expecting a path + raise IOError("Couldn't find profile %r" % profile) from e + return pd.location diff --git a/lib/python3.12/site-packages/IPython/py.typed b/lib/python3.12/site-packages/IPython/py.typed new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.12/site-packages/IPython/sphinxext/__init__.py b/lib/python3.12/site-packages/IPython/sphinxext/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.12/site-packages/IPython/sphinxext/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/IPython/sphinxext/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bccd12063a1bed2dcaffd378750e0969751d7973 Binary files /dev/null and b/lib/python3.12/site-packages/IPython/sphinxext/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/IPython/sphinxext/__pycache__/custom_doctests.cpython-312.pyc b/lib/python3.12/site-packages/IPython/sphinxext/__pycache__/custom_doctests.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..dd912ab4b8c0c3b315671a001b39999906778ac0 Binary files /dev/null and b/lib/python3.12/site-packages/IPython/sphinxext/__pycache__/custom_doctests.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/IPython/sphinxext/__pycache__/ipython_console_highlighting.cpython-312.pyc b/lib/python3.12/site-packages/IPython/sphinxext/__pycache__/ipython_console_highlighting.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..59989aeb247dee3451e3fe5f26826022484d6f27 Binary files /dev/null and b/lib/python3.12/site-packages/IPython/sphinxext/__pycache__/ipython_console_highlighting.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/IPython/sphinxext/__pycache__/ipython_directive.cpython-312.pyc b/lib/python3.12/site-packages/IPython/sphinxext/__pycache__/ipython_directive.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..38435721ea12c256113d93966ab52ec490a11469 Binary files /dev/null and b/lib/python3.12/site-packages/IPython/sphinxext/__pycache__/ipython_directive.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/IPython/sphinxext/custom_doctests.py b/lib/python3.12/site-packages/IPython/sphinxext/custom_doctests.py new file mode 100644 index 0000000000000000000000000000000000000000..f0ea034a6db04d72340b0439dceb15ea82f8f71a --- /dev/null +++ b/lib/python3.12/site-packages/IPython/sphinxext/custom_doctests.py @@ -0,0 +1,155 @@ +""" +Handlers for IPythonDirective's @doctest pseudo-decorator. + +The Sphinx extension that provides support for embedded IPython code provides +a pseudo-decorator @doctest, which treats the input/output block as a +doctest, raising a RuntimeError during doc generation if the actual output +(after running the input) does not match the expected output. + +An example usage is: + +.. code-block:: rst + + .. ipython:: + + In [1]: x = 1 + + @doctest + In [2]: x + 2 + Out[3]: 3 + +One can also provide arguments to the decorator. The first argument should be +the name of a custom handler. The specification of any other arguments is +determined by the handler. For example, + +.. code-block:: rst + + .. ipython:: + + @doctest float + In [154]: 0.1 + 0.2 + Out[154]: 0.3 + +allows the actual output ``0.30000000000000004`` to match the expected output +due to a comparison with `np.allclose`. + +This module contains handlers for the @doctest pseudo-decorator. Handlers +should have the following function signature:: + + handler(sphinx_shell, args, input_lines, found, submitted) + +where `sphinx_shell` is the embedded Sphinx shell, `args` contains the list +of arguments that follow: '@doctest handler_name', `input_lines` contains +a list of the lines relevant to the current doctest, `found` is a string +containing the output from the IPython shell, and `submitted` is a string +containing the expected output from the IPython shell. + +Handlers must be registered in the `doctests` dict at the end of this module. + +""" + +def str_to_array(s): + """ + Simplistic converter of strings from repr to float NumPy arrays. + + If the repr representation has ellipsis in it, then this will fail. + + Parameters + ---------- + s : str + The repr version of a NumPy array. + + Examples + -------- + >>> s = "array([ 0.3, inf, nan])" + >>> a = str_to_array(s) + + """ + import numpy as np + + # Need to make sure eval() knows about inf and nan. + # This also assumes default printoptions for NumPy. + from numpy import inf, nan + + if s.startswith(u'array'): + # Remove array( and ) + s = s[6:-1] + + if s.startswith(u'['): + a = np.array(eval(s), dtype=float) + else: + # Assume its a regular float. Force 1D so we can index into it. + a = np.atleast_1d(float(s)) + return a + +def float_doctest(sphinx_shell, args, input_lines, found, submitted): + """ + Doctest which allow the submitted output to vary slightly from the input. + + Here is how it might appear in an rst file: + + .. code-block:: rst + + .. ipython:: + + @doctest float + In [1]: 0.1 + 0.2 + Out[1]: 0.3 + + """ + import numpy as np + + if len(args) == 2: + rtol = 1e-05 + atol = 1e-08 + else: + # Both must be specified if any are specified. + try: + rtol = float(args[2]) + atol = float(args[3]) + except IndexError: + e = ("Both `rtol` and `atol` must be specified " + "if either are specified: {0}".format(args)) + raise IndexError(e) from e + + try: + submitted = str_to_array(submitted) + found = str_to_array(found) + except: + # For example, if the array is huge and there are ellipsis in it. + error = True + else: + found_isnan = np.isnan(found) + submitted_isnan = np.isnan(submitted) + error = not np.allclose(found_isnan, submitted_isnan) + error |= not np.allclose(found[~found_isnan], + submitted[~submitted_isnan], + rtol=rtol, atol=atol) + + TAB = ' ' * 4 + directive = sphinx_shell.directive + if directive is None: + source = 'Unavailable' + content = 'Unavailable' + else: + source = directive.state.document.current_source + # Add tabs and make into a single string. + content = '\n'.join([TAB + line for line in directive.content]) + + if error: + + e = ('doctest float comparison failure\n\n' + 'Document source: {0}\n\n' + 'Raw content: \n{1}\n\n' + 'On input line(s):\n{TAB}{2}\n\n' + 'we found output:\n{TAB}{3}\n\n' + 'instead of the expected:\n{TAB}{4}\n\n') + e = e.format(source, content, '\n'.join(input_lines), repr(found), + repr(submitted), TAB=TAB) + raise RuntimeError(e) + +# dict of allowable doctest handlers. The key represents the first argument +# that must be given to @doctest in order to activate the handler. +doctests = { + 'float': float_doctest, +} diff --git a/lib/python3.12/site-packages/IPython/sphinxext/ipython_console_highlighting.py b/lib/python3.12/site-packages/IPython/sphinxext/ipython_console_highlighting.py new file mode 100644 index 0000000000000000000000000000000000000000..b046467f5fc9ff588d18da1ac63aab742ae9175d --- /dev/null +++ b/lib/python3.12/site-packages/IPython/sphinxext/ipython_console_highlighting.py @@ -0,0 +1,28 @@ +""" +reST directive for syntax-highlighting ipython interactive sessions. + +""" + +from sphinx import highlighting +from ipython_pygments_lexers import IPyLexer + + +def setup(app): + """Setup as a sphinx extension.""" + + # This is only a lexer, so adding it below to pygments appears sufficient. + # But if somebody knows what the right API usage should be to do that via + # sphinx, by all means fix it here. At least having this setup.py + # suppresses the sphinx warning we'd get without it. + metadata = {"parallel_read_safe": True, "parallel_write_safe": True} + return metadata + + +# Register the extension as a valid pygments lexer. +# Alternatively, we could register the lexer with pygments instead. This would +# require using setuptools entrypoints: http://pygments.org/docs/plugins + +ipy3 = IPyLexer() + +highlighting.lexers["ipython"] = ipy3 +highlighting.lexers["ipython3"] = ipy3 diff --git a/lib/python3.12/site-packages/IPython/sphinxext/ipython_directive.py b/lib/python3.12/site-packages/IPython/sphinxext/ipython_directive.py new file mode 100644 index 0000000000000000000000000000000000000000..0fdc70cab5ca7887322ce50c8d9479e801119c9b --- /dev/null +++ b/lib/python3.12/site-packages/IPython/sphinxext/ipython_directive.py @@ -0,0 +1,1278 @@ +# -*- coding: utf-8 -*- +""" +Sphinx directive to support embedded IPython code. + +IPython provides an extension for `Sphinx `_ to +highlight and run code. + +This directive allows pasting of entire interactive IPython sessions, prompts +and all, and their code will actually get re-executed at doc build time, with +all prompts renumbered sequentially. It also allows you to input code as a pure +python input by giving the argument python to the directive. The output looks +like an interactive ipython section. + +Here is an example of how the IPython directive can +**run** python code, at build time. + +.. ipython:: + + In [1]: 1+1 + + In [1]: import datetime + ...: datetime.date.fromisoformat('2022-02-22') + +It supports IPython construct that plain +Python does not understand (like magics): + +.. ipython:: + + In [0]: import time + + In [0]: %pdoc time.sleep + +This will also support top-level async when using IPython 7.0+ + +.. ipython:: + + In [2]: import asyncio + ...: print('before') + ...: await asyncio.sleep(1) + ...: print('after') + + +The namespace will persist across multiple code chucks, Let's define a variable: + +.. ipython:: + + In [0]: who = "World" + +And now say hello: + +.. ipython:: + + In [0]: print('Hello,', who) + +If the current section raises an exception, you can add the ``:okexcept:`` flag +to the current block, otherwise the build will fail. + +.. ipython:: + :okexcept: + + In [1]: 1/0 + +IPython Sphinx directive module +=============================== + +To enable this directive, simply list it in your Sphinx ``conf.py`` file +(making sure the directory where you placed it is visible to sphinx, as is +needed for all Sphinx directives). For example, to enable syntax highlighting +and the IPython directive:: + + extensions = ['IPython.sphinxext.ipython_console_highlighting', + 'IPython.sphinxext.ipython_directive'] + +The IPython directive outputs code-blocks with the language 'ipython'. So +if you do not have the syntax highlighting extension enabled as well, then +all rendered code-blocks will be uncolored. By default this directive assumes +that your prompts are unchanged IPython ones, but this can be customized. +The configurable options that can be placed in conf.py are: + +ipython_savefig_dir: + The directory in which to save the figures. This is relative to the + Sphinx source directory. The default is `html_static_path`. +ipython_rgxin: + The compiled regular expression to denote the start of IPython input + lines. The default is ``re.compile('In \\[(\\d+)\\]:\\s?(.*)\\s*')``. You + shouldn't need to change this. +ipython_warning_is_error: [default to True] + Fail the build if something unexpected happen, for example if a block raise + an exception but does not have the `:okexcept:` flag. The exact behavior of + what is considered strict, may change between the sphinx directive version. +ipython_rgxout: + The compiled regular expression to denote the start of IPython output + lines. The default is ``re.compile('Out\\[(\\d+)\\]:\\s?(.*)\\s*')``. You + shouldn't need to change this. +ipython_promptin: + The string to represent the IPython input prompt in the generated ReST. + The default is ``'In [%d]:'``. This expects that the line numbers are used + in the prompt. +ipython_promptout: + The string to represent the IPython prompt in the generated ReST. The + default is ``'Out [%d]:'``. This expects that the line numbers are used + in the prompt. +ipython_mplbackend: + The string which specifies if the embedded Sphinx shell should import + Matplotlib and set the backend. The value specifies a backend that is + passed to `matplotlib.use()` before any lines in `ipython_execlines` are + executed. If not specified in conf.py, then the default value of 'agg' is + used. To use the IPython directive without matplotlib as a dependency, set + the value to `None`. It may end up that matplotlib is still imported + if the user specifies so in `ipython_execlines` or makes use of the + @savefig pseudo decorator. +ipython_execlines: + A list of strings to be exec'd in the embedded Sphinx shell. Typical + usage is to make certain packages always available. Set this to an empty + list if you wish to have no imports always available. If specified in + ``conf.py`` as `None`, then it has the effect of making no imports available. + If omitted from conf.py altogether, then the default value of + ['import numpy as np', 'import matplotlib.pyplot as plt'] is used. +ipython_holdcount + When the @suppress pseudo-decorator is used, the execution count can be + incremented or not. The default behavior is to hold the execution count, + corresponding to a value of `True`. Set this to `False` to increment + the execution count after each suppressed command. + +As an example, to use the IPython directive when `matplotlib` is not available, +one sets the backend to `None`:: + + ipython_mplbackend = None + +An example usage of the directive is: + +.. code-block:: rst + + .. ipython:: + + In [1]: x = 1 + + In [2]: y = x**2 + + In [3]: print(y) + +See http://matplotlib.org/sampledoc/ipython_directive.html for additional +documentation. + +Pseudo-Decorators +================= + +Note: Only one decorator is supported per input. If more than one decorator +is specified, then only the last one is used. + +In addition to the Pseudo-Decorators/options described at the above link, +several enhancements have been made. The directive will emit a message to the +console at build-time if code-execution resulted in an exception or warning. +You can suppress these on a per-block basis by specifying the :okexcept: +or :okwarning: options: + +.. code-block:: rst + + .. ipython:: + :okexcept: + :okwarning: + + In [1]: 1/0 + In [2]: # raise warning. + +To Do +===== + +- Turn the ad-hoc test() function into a real test suite. +- Break up ipython-specific functionality from matplotlib stuff into better + separated code. + +""" + +# Authors +# ======= +# +# - John D Hunter: original author. +# - Fernando Perez: refactoring, documentation, cleanups, port to 0.11. +# - VáclavŠmilauer : Prompt generalizations. +# - Skipper Seabold, refactoring, cleanups, pure python addition + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- + +# Stdlib +import atexit +import errno +import os +import pathlib +import re +import sys +import tempfile +import ast +import warnings +import shutil +from io import StringIO +from typing import Any, Dict, Set + +# Third-party +from docutils.parsers.rst import directives +from docutils.parsers.rst import Directive +from sphinx.util import logging + +# Our own +from traitlets.config import Config +from IPython import InteractiveShell +from IPython.core.profiledir import ProfileDir + +use_matplotlib = False +try: + import matplotlib + use_matplotlib = True +except Exception: + pass + +#----------------------------------------------------------------------------- +# Globals +#----------------------------------------------------------------------------- +# for tokenizing blocks +COMMENT, INPUT, OUTPUT = range(3) + +PSEUDO_DECORATORS = ["suppress", "verbatim", "savefig", "doctest"] + +#----------------------------------------------------------------------------- +# Functions and class declarations +#----------------------------------------------------------------------------- + +def block_parser(part, rgxin, rgxout, fmtin, fmtout): + """ + part is a string of ipython text, comprised of at most one + input, one output, comments, and blank lines. The block parser + parses the text into a list of:: + + blocks = [ (TOKEN0, data0), (TOKEN1, data1), ...] + + where TOKEN is one of [COMMENT | INPUT | OUTPUT ] and + data is, depending on the type of token:: + + COMMENT : the comment string + + INPUT: the (DECORATOR, INPUT_LINE, REST) where + DECORATOR: the input decorator (or None) + INPUT_LINE: the input as string (possibly multi-line) + REST : any stdout generated by the input line (not OUTPUT) + + OUTPUT: the output string, possibly multi-line + + """ + block = [] + lines = part.split('\n') + N = len(lines) + i = 0 + decorator = None + while 1: + + if i==N: + # nothing left to parse -- the last line + break + + line = lines[i] + i += 1 + line_stripped = line.strip() + if line_stripped.startswith('#'): + block.append((COMMENT, line)) + continue + + if any( + line_stripped.startswith("@" + pseudo_decorator) + for pseudo_decorator in PSEUDO_DECORATORS + ): + if decorator: + raise RuntimeError( + "Applying multiple pseudo-decorators on one line is not supported" + ) + else: + decorator = line_stripped + continue + + # does this look like an input line? + matchin = rgxin.match(line) + if matchin: + lineno, inputline = int(matchin.group(1)), matchin.group(2) + + # the ....: continuation string + continuation = ' %s:'%''.join(['.']*(len(str(lineno))+2)) + Nc = len(continuation) + # input lines can continue on for more than one line, if + # we have a '\' line continuation char or a function call + # echo line 'print'. The input line can only be + # terminated by the end of the block or an output line, so + # we parse out the rest of the input line if it is + # multiline as well as any echo text + + rest = [] + while i 1: + if input_lines[-1] != "": + input_lines.append('') # make sure there's a blank line + # so splitter buffer gets reset + + continuation = ' %s:'%''.join(['.']*(len(str(lineno))+2)) + + if is_savefig: + image_file, image_directive = self.process_image(decorator) + + ret = [] + is_semicolon = False + + # Hold the execution count, if requested to do so. + if is_suppress and self.hold_count: + store_history = False + else: + store_history = True + + # Note: catch_warnings is not thread safe + with warnings.catch_warnings(record=True) as ws: + if input_lines[0].endswith(';'): + is_semicolon = True + #for i, line in enumerate(input_lines): + + # process the first input line + if is_verbatim: + self.process_input_lines(['']) + self.IP.execution_count += 1 # increment it anyway + else: + # only submit the line in non-verbatim mode + self.process_input_lines(input_lines, store_history=store_history) + + if not is_suppress: + for i, line in enumerate(input_lines): + if i == 0: + formatted_line = '%s %s'%(input_prompt, line) + else: + formatted_line = '%s %s'%(continuation, line) + ret.append(formatted_line) + + if not is_suppress and len(rest.strip()) and is_verbatim: + # The "rest" is the standard output of the input. This needs to be + # added when in verbatim mode. If there is no "rest", then we don't + # add it, as the new line will be added by the processed output. + ret.append(rest) + + # Fetch the processed output. (This is not the submitted output.) + self.cout.seek(0) + processed_output = self.cout.read() + if not is_suppress and not is_semicolon: + # + # In IPythonDirective.run, the elements of `ret` are eventually + # combined such that '' entries correspond to newlines. So if + # `processed_output` is equal to '', then the adding it to `ret` + # ensures that there is a blank line between consecutive inputs + # that have no outputs, as in: + # + # In [1]: x = 4 + # + # In [2]: x = 5 + # + # When there is processed output, it has a '\n' at the tail end. So + # adding the output to `ret` will provide the necessary spacing + # between consecutive input/output blocks, as in: + # + # In [1]: x + # Out[1]: 5 + # + # In [2]: x + # Out[2]: 5 + # + # When there is stdout from the input, it also has a '\n' at the + # tail end, and so this ensures proper spacing as well. E.g.: + # + # In [1]: print(x) + # 5 + # + # In [2]: x = 5 + # + # When in verbatim mode, `processed_output` is empty (because + # nothing was passed to IP. Sometimes the submitted code block has + # an Out[] portion and sometimes it does not. When it does not, we + # need to ensure proper spacing, so we have to add '' to `ret`. + # However, if there is an Out[] in the submitted code, then we do + # not want to add a newline as `process_output` has stuff to add. + # The difficulty is that `process_input` doesn't know if + # `process_output` will be called---so it doesn't know if there is + # Out[] in the code block. The requires that we include a hack in + # `process_block`. See the comments there. + # + ret.append(processed_output) + elif is_semicolon: + # Make sure there is a newline after the semicolon. + ret.append('') + + # context information + filename = "Unknown" + lineno = 0 + if self.directive.state: + filename = self.directive.state.document.current_source + lineno = self.directive.state.document.current_line + + # Use sphinx logger for warnings + logger = logging.getLogger(__name__) + + # output any exceptions raised during execution to stdout + # unless :okexcept: has been specified. + if not is_okexcept and ( + ("Traceback" in processed_output) or ("SyntaxError" in processed_output) + ): + s = "\n>>>" + ("-" * 73) + "\n" + s += "Exception in %s at block ending on line %s\n" % (filename, lineno) + s += "Specify :okexcept: as an option in the ipython:: block to suppress this message\n" + s += processed_output + "\n" + s += "<<<" + ("-" * 73) + logger.warning(s) + if self.warning_is_error: + raise RuntimeError( + "Unexpected exception in `{}` line {}".format(filename, lineno) + ) + + # output any warning raised during execution to stdout + # unless :okwarning: has been specified. + if not is_okwarning: + for w in ws: + s = "\n>>>" + ("-" * 73) + "\n" + s += "Warning in %s at block ending on line %s\n" % (filename, lineno) + s += "Specify :okwarning: as an option in the ipython:: block to suppress this message\n" + s += ("-" * 76) + "\n" + s += warnings.formatwarning( + w.message, w.category, w.filename, w.lineno, w.line + ) + s += "<<<" + ("-" * 73) + logger.warning(s) + if self.warning_is_error: + raise RuntimeError( + "Unexpected warning in `{}` line {}".format(filename, lineno) + ) + + self.clear_cout() + return (ret, input_lines, processed_output, + is_doctest, decorator, image_file, image_directive) + + + def process_output(self, data, output_prompt, input_lines, output, + is_doctest, decorator, image_file): + """ + Process data block for OUTPUT token. + + """ + # Recall: `data` is the submitted output, and `output` is the processed + # output from `input_lines`. + + TAB = ' ' * 4 + + if is_doctest and output is not None: + + found = output # This is the processed output + found = found.strip() + submitted = data.strip() + + if self.directive is None: + source = 'Unavailable' + content = 'Unavailable' + else: + source = self.directive.state.document.current_source + content = self.directive.content + # Add tabs and join into a single string. + content = '\n'.join([TAB + line for line in content]) + + # Make sure the output contains the output prompt. + ind = found.find(output_prompt) + if ind < 0: + e = ('output does not contain output prompt\n\n' + 'Document source: {0}\n\n' + 'Raw content: \n{1}\n\n' + 'Input line(s):\n{TAB}{2}\n\n' + 'Output line(s):\n{TAB}{3}\n\n') + e = e.format(source, content, '\n'.join(input_lines), + repr(found), TAB=TAB) + raise RuntimeError(e) + found = found[len(output_prompt):].strip() + + # Handle the actual doctest comparison. + if decorator.strip() == '@doctest': + # Standard doctest + if found != submitted: + e = ('doctest failure\n\n' + 'Document source: {0}\n\n' + 'Raw content: \n{1}\n\n' + 'On input line(s):\n{TAB}{2}\n\n' + 'we found output:\n{TAB}{3}\n\n' + 'instead of the expected:\n{TAB}{4}\n\n') + e = e.format(source, content, '\n'.join(input_lines), + repr(found), repr(submitted), TAB=TAB) + raise RuntimeError(e) + else: + self.custom_doctest(decorator, input_lines, found, submitted) + + # When in verbatim mode, this holds additional submitted output + # to be written in the final Sphinx output. + # https://github.com/ipython/ipython/issues/5776 + out_data = [] + + is_verbatim = decorator=='@verbatim' or self.is_verbatim + if is_verbatim and data.strip(): + # Note that `ret` in `process_block` has '' as its last element if + # the code block was in verbatim mode. So if there is no submitted + # output, then we will have proper spacing only if we do not add + # an additional '' to `out_data`. This is why we condition on + # `and data.strip()`. + + # The submitted output has no output prompt. If we want the + # prompt and the code to appear, we need to join them now + # instead of adding them separately---as this would create an + # undesired newline. How we do this ultimately depends on the + # format of the output regex. I'll do what works for the default + # prompt for now, and we might have to adjust if it doesn't work + # in other cases. Finally, the submitted output does not have + # a trailing newline, so we must add it manually. + out_data.append("{0} {1}\n".format(output_prompt, data)) + + return out_data + + def process_comment(self, data): + """Process data fPblock for COMMENT token.""" + if not self.is_suppress: + return [data] + + def save_image(self, image_file): + """ + Saves the image file to disk. + """ + self.ensure_pyplot() + command = 'plt.gcf().savefig("%s")'%image_file + # print('SAVEFIG', command) # dbg + self.process_input_line('bookmark ipy_thisdir', store_history=False) + self.process_input_line('cd -b ipy_savedir', store_history=False) + self.process_input_line(command, store_history=False) + self.process_input_line('cd -b ipy_thisdir', store_history=False) + self.process_input_line('bookmark -d ipy_thisdir', store_history=False) + self.clear_cout() + + def process_block(self, block): + """ + process block from the block_parser and return a list of processed lines + """ + ret = [] + output = None + input_lines = None + lineno = self.IP.execution_count + + input_prompt = self.promptin % lineno + output_prompt = self.promptout % lineno + image_file = None + image_directive = None + + found_input = False + for token, data in block: + if token == COMMENT: + out_data = self.process_comment(data) + elif token == INPUT: + found_input = True + (out_data, input_lines, output, is_doctest, + decorator, image_file, image_directive) = \ + self.process_input(data, input_prompt, lineno) + elif token == OUTPUT: + if not found_input: + + TAB = ' ' * 4 + linenumber = 0 + source = 'Unavailable' + content = 'Unavailable' + if self.directive: + linenumber = self.directive.state.document.current_line + source = self.directive.state.document.current_source + content = self.directive.content + # Add tabs and join into a single string. + content = '\n'.join([TAB + line for line in content]) + + e = ('\n\nInvalid block: Block contains an output prompt ' + 'without an input prompt.\n\n' + 'Document source: {0}\n\n' + 'Content begins at line {1}: \n\n{2}\n\n' + 'Problematic block within content: \n\n{TAB}{3}\n\n') + e = e.format(source, linenumber, content, block, TAB=TAB) + + # Write, rather than include in exception, since Sphinx + # will truncate tracebacks. + sys.stdout.write(e) + raise RuntimeError('An invalid block was detected.') + out_data = \ + self.process_output(data, output_prompt, input_lines, + output, is_doctest, decorator, + image_file) + if out_data: + # Then there was user submitted output in verbatim mode. + # We need to remove the last element of `ret` that was + # added in `process_input`, as it is '' and would introduce + # an undesirable newline. + assert(ret[-1] == '') + del ret[-1] + + if out_data: + ret.extend(out_data) + + # save the image files + if image_file is not None: + self.save_image(image_file) + + return ret, image_directive + + def ensure_pyplot(self): + """ + Ensures that pyplot has been imported into the embedded IPython shell. + + Also, makes sure to set the backend appropriately if not set already. + + """ + # We are here if the @figure pseudo decorator was used. Thus, it's + # possible that we could be here even if python_mplbackend were set to + # `None`. That's also strange and perhaps worthy of raising an + # exception, but for now, we just set the backend to 'agg'. + + if not self._pyplot_imported: + if 'matplotlib.backends' not in sys.modules: + # Then ipython_matplotlib was set to None but there was a + # call to the @figure decorator (and ipython_execlines did + # not set a backend). + #raise Exception("No backend was set, but @figure was used!") + import matplotlib + matplotlib.use('agg') + + # Always import pyplot into embedded shell. + self.process_input_line('import matplotlib.pyplot as plt', + store_history=False) + self._pyplot_imported = True + + def process_pure_python(self, content): + """ + content is a list of strings. it is unedited directive content + + This runs it line by line in the InteractiveShell, prepends + prompts as needed capturing stderr and stdout, then returns + the content as a list as if it were ipython code + """ + output = [] + savefig = False # keep up with this to clear figure + multiline = False # to handle line continuation + multiline_start = None + fmtin = self.promptin + + ct = 0 + + for lineno, line in enumerate(content): + + line_stripped = line.strip() + if not len(line): + output.append(line) + continue + + # handle pseudo-decorators, whilst ensuring real python decorators are treated as input + if any( + line_stripped.startswith("@" + pseudo_decorator) + for pseudo_decorator in PSEUDO_DECORATORS + ): + output.extend([line]) + if 'savefig' in line: + savefig = True # and need to clear figure + continue + + # handle comments + if line_stripped.startswith('#'): + output.extend([line]) + continue + + # deal with lines checking for multiline + continuation = u' %s:'% ''.join(['.']*(len(str(ct))+2)) + if not multiline: + modified = u"%s %s" % (fmtin % ct, line_stripped) + output.append(modified) + ct += 1 + try: + ast.parse(line_stripped) + output.append(u'') + except Exception: # on a multiline + multiline = True + multiline_start = lineno + else: # still on a multiline + modified = u'%s %s' % (continuation, line) + output.append(modified) + + # if the next line is indented, it should be part of multiline + if len(content) > lineno + 1: + nextline = content[lineno + 1] + if len(nextline) - len(nextline.lstrip()) > 3: + continue + try: + mod = ast.parse( + '\n'.join(content[multiline_start:lineno+1])) + if isinstance(mod.body[0], ast.FunctionDef): + # check to see if we have the whole function + for element in mod.body[0].body: + if isinstance(element, ast.Return): + multiline = False + else: + output.append(u'') + multiline = False + except Exception: + pass + + if savefig: # clear figure if plotted + self.ensure_pyplot() + self.process_input_line('plt.clf()', store_history=False) + self.clear_cout() + savefig = False + + return output + + def custom_doctest(self, decorator, input_lines, found, submitted): + """ + Perform a specialized doctest. + + """ + from .custom_doctests import doctests + + args = decorator.split() + doctest_type = args[1] + if doctest_type in doctests: + doctests[doctest_type](self, args, input_lines, found, submitted) + else: + e = "Invalid option to @doctest: {0}".format(doctest_type) + raise Exception(e) + + +class IPythonDirective(Directive): + + has_content: bool = True + required_arguments: int = 0 + optional_arguments: int = 4 # python, suppress, verbatim, doctest + final_argumuent_whitespace: bool = True + option_spec: Dict[str, Any] = { + "python": directives.unchanged, + "suppress": directives.flag, + "verbatim": directives.flag, + "doctest": directives.flag, + "okexcept": directives.flag, + "okwarning": directives.flag, + } + + shell = None + + seen_docs: Set = set() + + def get_config_options(self): + # contains sphinx configuration variables + config = self.state.document.settings.env.config + + # get config variables to set figure output directory + savefig_dir = config.ipython_savefig_dir + source_dir = self.state.document.settings.env.srcdir + savefig_dir = os.path.join(source_dir, savefig_dir) + + # get regex and prompt stuff + rgxin = config.ipython_rgxin + rgxout = config.ipython_rgxout + warning_is_error= config.ipython_warning_is_error + promptin = config.ipython_promptin + promptout = config.ipython_promptout + mplbackend = config.ipython_mplbackend + exec_lines = config.ipython_execlines + hold_count = config.ipython_holdcount + + return (savefig_dir, source_dir, rgxin, rgxout, + promptin, promptout, mplbackend, exec_lines, hold_count, warning_is_error) + + def setup(self): + # Get configuration values. + (savefig_dir, source_dir, rgxin, rgxout, promptin, promptout, + mplbackend, exec_lines, hold_count, warning_is_error) = self.get_config_options() + + try: + os.makedirs(savefig_dir) + except OSError as e: + if e.errno != errno.EEXIST: + raise + + if self.shell is None: + # We will be here many times. However, when the + # EmbeddedSphinxShell is created, its interactive shell member + # is the same for each instance. + + if mplbackend and 'matplotlib.backends' not in sys.modules and use_matplotlib: + import matplotlib + matplotlib.use(mplbackend) + + # Must be called after (potentially) importing matplotlib and + # setting its backend since exec_lines might import pylab. + self.shell = EmbeddedSphinxShell(exec_lines) + + # Store IPython directive to enable better error messages + self.shell.directive = self + + # reset the execution count if we haven't processed this doc + #NOTE: this may be borked if there are multiple seen_doc tmp files + #check time stamp? + if self.state.document.current_source not in self.seen_docs: + self.shell.IP.history_manager.reset() + self.shell.IP.execution_count = 1 + self.seen_docs.add(self.state.document.current_source) + + # and attach to shell so we don't have to pass them around + self.shell.rgxin = rgxin + self.shell.rgxout = rgxout + self.shell.promptin = promptin + self.shell.promptout = promptout + self.shell.savefig_dir = savefig_dir + self.shell.source_dir = source_dir + self.shell.hold_count = hold_count + self.shell.warning_is_error = warning_is_error + + # setup bookmark for saving figures directory + self.shell.process_input_line( + 'bookmark ipy_savedir "%s"' % savefig_dir, store_history=False + ) + self.shell.clear_cout() + + return rgxin, rgxout, promptin, promptout + + def teardown(self): + # delete last bookmark + self.shell.process_input_line('bookmark -d ipy_savedir', + store_history=False) + self.shell.clear_cout() + + def run(self): + debug = False + + #TODO, any reason block_parser can't be a method of embeddable shell + # then we wouldn't have to carry these around + rgxin, rgxout, promptin, promptout = self.setup() + + options = self.options + self.shell.is_suppress = 'suppress' in options + self.shell.is_doctest = 'doctest' in options + self.shell.is_verbatim = 'verbatim' in options + self.shell.is_okexcept = 'okexcept' in options + self.shell.is_okwarning = 'okwarning' in options + + # handle pure python code + if 'python' in self.arguments: + content = self.content + self.content = self.shell.process_pure_python(content) + + # parts consists of all text within the ipython-block. + # Each part is an input/output block. + parts = '\n'.join(self.content).split('\n\n') + + lines = ['.. code-block:: ipython', ''] + figures = [] + + # Use sphinx logger for warnings + logger = logging.getLogger(__name__) + + for part in parts: + block = block_parser(part, rgxin, rgxout, promptin, promptout) + if len(block): + rows, figure = self.shell.process_block(block) + for row in rows: + lines.extend([' {0}'.format(line) + for line in row.split('\n')]) + + if figure is not None: + figures.append(figure) + else: + message = 'Code input with no code at {}, line {}'\ + .format( + self.state.document.current_source, + self.state.document.current_line) + if self.shell.warning_is_error: + raise RuntimeError(message) + else: + logger.warning(message) + + for figure in figures: + lines.append('') + lines.extend(figure.split('\n')) + lines.append('') + + if len(lines) > 2: + if debug: + print('\n'.join(lines)) + else: + # This has to do with input, not output. But if we comment + # these lines out, then no IPython code will appear in the + # final output. + self.state_machine.insert_input( + lines, self.state_machine.input_lines.source(0)) + + # cleanup + self.teardown() + + return [] + +# Enable as a proper Sphinx directive +def setup(app): + setup.app = app + + app.add_directive('ipython', IPythonDirective) + app.add_config_value('ipython_savefig_dir', 'savefig', 'env') + app.add_config_value('ipython_warning_is_error', True, 'env') + app.add_config_value('ipython_rgxin', + re.compile(r'In \[(\d+)\]:\s?(.*)\s*'), 'env') + app.add_config_value('ipython_rgxout', + re.compile(r'Out\[(\d+)\]:\s?(.*)\s*'), 'env') + app.add_config_value('ipython_promptin', 'In [%d]:', 'env') + app.add_config_value('ipython_promptout', 'Out[%d]:', 'env') + + # We could just let matplotlib pick whatever is specified as the default + # backend in the matplotlibrc file, but this would cause issues if the + # backend didn't work in headless environments. For this reason, 'agg' + # is a good default backend choice. + app.add_config_value('ipython_mplbackend', 'agg', 'env') + + # If the user sets this config value to `None`, then EmbeddedSphinxShell's + # __init__ method will treat it as []. + execlines = ['import numpy as np'] + if use_matplotlib: + execlines.append('import matplotlib.pyplot as plt') + app.add_config_value('ipython_execlines', execlines, 'env') + + app.add_config_value('ipython_holdcount', True, 'env') + + metadata = {'parallel_read_safe': True, 'parallel_write_safe': True} + return metadata + +# Simple smoke test, needs to be converted to a proper automatic test. +def test(): + + examples = [ + r""" +In [9]: pwd +Out[9]: '/home/jdhunter/py4science/book' + +In [10]: cd bookdata/ +/home/jdhunter/py4science/book/bookdata + +In [2]: from pylab import * + +In [2]: ion() + +In [3]: im = imread('stinkbug.png') + +@savefig mystinkbug.png width=4in +In [4]: imshow(im) +Out[4]: + +""", + r""" + +In [1]: x = 'hello world' + +# string methods can be +# used to alter the string +@doctest +In [2]: x.upper() +Out[2]: 'HELLO WORLD' + +@verbatim +In [3]: x.st +x.startswith x.strip +""", + r""" + +In [130]: url = 'http://ichart.finance.yahoo.com/table.csv?s=CROX\ + .....: &d=9&e=22&f=2009&g=d&a=1&br=8&c=2006&ignore=.csv' + +In [131]: print url.split('&') +['http://ichart.finance.yahoo.com/table.csv?s=CROX', 'd=9', 'e=22', 'f=2009', 'g=d', 'a=1', 'b=8', 'c=2006', 'ignore=.csv'] + +In [60]: import urllib + +""", + r"""\ + +In [133]: import numpy.random + +@suppress +In [134]: numpy.random.seed(2358) + +@doctest +In [135]: numpy.random.rand(10,2) +Out[135]: +array([[ 0.64524308, 0.59943846], + [ 0.47102322, 0.8715456 ], + [ 0.29370834, 0.74776844], + [ 0.99539577, 0.1313423 ], + [ 0.16250302, 0.21103583], + [ 0.81626524, 0.1312433 ], + [ 0.67338089, 0.72302393], + [ 0.7566368 , 0.07033696], + [ 0.22591016, 0.77731835], + [ 0.0072729 , 0.34273127]]) + +""", + + r""" +In [106]: print x +jdh + +In [109]: for i in range(10): + .....: print i + .....: + .....: +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +""", + + r""" + +In [144]: from pylab import * + +In [145]: ion() + +# use a semicolon to suppress the output +@savefig test_hist.png width=4in +In [151]: hist(np.random.randn(10000), 100); + + +@savefig test_plot.png width=4in +In [151]: plot(np.random.randn(10000), 'o'); + """, + + r""" +# use a semicolon to suppress the output +In [151]: plt.clf() + +@savefig plot_simple.png width=4in +In [151]: plot([1,2,3]) + +@savefig hist_simple.png width=4in +In [151]: hist(np.random.randn(10000), 100); + +""", + r""" +# update the current fig +In [151]: ylabel('number') + +In [152]: title('normal distribution') + + +@savefig hist_with_text.png +In [153]: grid(True) + +@doctest float +In [154]: 0.1 + 0.2 +Out[154]: 0.3 + +@doctest float +In [155]: np.arange(16).reshape(4,4) +Out[155]: +array([[ 0, 1, 2, 3], + [ 4, 5, 6, 7], + [ 8, 9, 10, 11], + [12, 13, 14, 15]]) + +In [1]: x = np.arange(16, dtype=float).reshape(4,4) + +In [2]: x[0,0] = np.inf + +In [3]: x[0,1] = np.nan + +@doctest float +In [4]: x +Out[4]: +array([[ inf, nan, 2., 3.], + [ 4., 5., 6., 7.], + [ 8., 9., 10., 11.], + [ 12., 13., 14., 15.]]) + + + """, + ] + # skip local-file depending first example: + examples = examples[1:] + + #ipython_directive.DEBUG = True # dbg + #options = dict(suppress=True) # dbg + options = {} + for example in examples: + content = example.split('\n') + IPythonDirective('debug', arguments=None, options=options, + content=content, lineno=0, + content_offset=None, block_text=None, + state=None, state_machine=None, + ) + +# Run test suite as a script +if __name__=='__main__': + if not os.path.isdir('_static'): + os.mkdir('_static') + test() + print('All OK? Check figures in _static/') diff --git a/lib/python3.12/site-packages/IPython/testing/__init__.py b/lib/python3.12/site-packages/IPython/testing/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8fcd65ea41a9406de5b11064524849b5bb4579f3 --- /dev/null +++ b/lib/python3.12/site-packages/IPython/testing/__init__.py @@ -0,0 +1,20 @@ +"""Testing support (tools to test IPython itself). +""" + +#----------------------------------------------------------------------------- +# Copyright (C) 2009-2011 The IPython Development Team +# +# Distributed under the terms of the BSD License. The full license is in +# the file COPYING, distributed as part of this software. +#----------------------------------------------------------------------------- + + +import os + +#----------------------------------------------------------------------------- +# Constants +#----------------------------------------------------------------------------- + +# We scale all timeouts via this factor, slow machines can increase it +IPYTHON_TESTING_TIMEOUT_SCALE = float(os.getenv( + 'IPYTHON_TESTING_TIMEOUT_SCALE', 1)) diff --git a/lib/python3.12/site-packages/IPython/testing/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/IPython/testing/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0f5c38caa50e45590c9352eddfb315917306c4e0 Binary files /dev/null and b/lib/python3.12/site-packages/IPython/testing/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/IPython/testing/__pycache__/decorators.cpython-312.pyc b/lib/python3.12/site-packages/IPython/testing/__pycache__/decorators.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..992d558cacaa4a26eb19d76bf3f8a2644cd7a098 Binary files /dev/null and b/lib/python3.12/site-packages/IPython/testing/__pycache__/decorators.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/IPython/testing/__pycache__/globalipapp.cpython-312.pyc b/lib/python3.12/site-packages/IPython/testing/__pycache__/globalipapp.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1a012b6ed5241b7d767aa422086fdc824c3a9188 Binary files /dev/null and b/lib/python3.12/site-packages/IPython/testing/__pycache__/globalipapp.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/IPython/testing/__pycache__/ipunittest.cpython-312.pyc b/lib/python3.12/site-packages/IPython/testing/__pycache__/ipunittest.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4d3a99a7befb3cf40df61dd8b13452d98031ad79 Binary files /dev/null and b/lib/python3.12/site-packages/IPython/testing/__pycache__/ipunittest.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/IPython/testing/__pycache__/skipdoctest.cpython-312.pyc b/lib/python3.12/site-packages/IPython/testing/__pycache__/skipdoctest.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5778676b1982d295c1ca414ae25bafb48f09ceb5 Binary files /dev/null and b/lib/python3.12/site-packages/IPython/testing/__pycache__/skipdoctest.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/IPython/testing/__pycache__/tools.cpython-312.pyc b/lib/python3.12/site-packages/IPython/testing/__pycache__/tools.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..546ff152db9ea60e2a7583285e4d140c40da69c6 Binary files /dev/null and b/lib/python3.12/site-packages/IPython/testing/__pycache__/tools.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/IPython/testing/decorators.py b/lib/python3.12/site-packages/IPython/testing/decorators.py new file mode 100644 index 0000000000000000000000000000000000000000..865f045f45a3a6cdaef7377b8d1a5a18b9fcf0e9 --- /dev/null +++ b/lib/python3.12/site-packages/IPython/testing/decorators.py @@ -0,0 +1,147 @@ +import os +import shutil +import sys +import tempfile +from importlib import import_module + +import pytest + +# Expose the unittest-driven decorators +from .ipunittest import ipdoctest, ipdocstring + +def skipif(skip_condition, msg=None): + """Make function raise SkipTest exception if skip_condition is true + + Parameters + ---------- + + skip_condition : bool or callable + Flag to determine whether to skip test. If the condition is a + callable, it is used at runtime to dynamically make the decision. This + is useful for tests that may require costly imports, to delay the cost + until the test suite is actually executed. + msg : string + Message to give on raising a SkipTest exception. + + Returns + ------- + decorator : function + Decorator, which, when applied to a function, causes SkipTest + to be raised when the skip_condition was True, and the function + to be called normally otherwise. + """ + if msg is None: + msg = "Test skipped due to test condition." + + assert isinstance(skip_condition, bool) + return pytest.mark.skipif(skip_condition, reason=msg) + + +# A version with the condition set to true, common case just to attach a message +# to a skip decorator +def skip(msg=None): + """Decorator factory - mark a test function for skipping from test suite. + + Parameters + ---------- + msg : string + Optional message to be added. + + Returns + ------- + decorator : function + Decorator, which, when applied to a function, causes SkipTest + to be raised, with the optional message added. + """ + if msg and not isinstance(msg, str): + raise ValueError( + "invalid object passed to `@skip` decorator, did you " + "meant `@skip()` with brackets ?" + ) + return skipif(True, msg) + + +def onlyif(condition, msg): + """The reverse from skipif, see skipif for details.""" + + return skipif(not condition, msg) + + +# ----------------------------------------------------------------------------- +# Utility functions for decorators +def module_not_available(module): + """Can module be imported? Returns true if module does NOT import. + + This is used to make a decorator to skip tests that require module to be + available, but delay the 'import numpy' to test execution time. + """ + try: + mod = import_module(module) + mod_not_avail = False + except ImportError: + mod_not_avail = True + + return mod_not_avail + + +# ----------------------------------------------------------------------------- +# Decorators for public use + +# Decorators to skip certain tests on specific platforms. +skip_win32 = skipif(sys.platform == "win32", "This test does not run under Windows") + + +# Decorators to skip tests if not on specific platforms. +skip_if_not_win32 = skipif(sys.platform != "win32", "This test only runs under Windows") +skip_if_not_osx = skipif( + not sys.platform.startswith("darwin"), "This test only runs under macOS" +) + +_x11_skip_cond = ( + sys.platform not in ("darwin", "win32") and os.environ.get("DISPLAY", "") == "" +) +_x11_skip_msg = "Skipped under *nix when X11/XOrg not available" + +skip_if_no_x11 = skipif(_x11_skip_cond, _x11_skip_msg) + +# Other skip decorators + +# generic skip without module +skip_without = lambda mod: skipif( + module_not_available(mod), "This test requires %s" % mod +) + +skipif_not_numpy = skip_without("numpy") + +skipif_not_matplotlib = skip_without("matplotlib") + +# A null 'decorator', useful to make more readable code that needs to pick +# between different decorators based on OS or other conditions +null_deco = lambda f: f + +# Some tests only run where we can use unicode paths. Note that we can't just +# check os.path.supports_unicode_filenames, which is always False on Linux. +try: + f = tempfile.NamedTemporaryFile(prefix="tmp€") +except UnicodeEncodeError: + unicode_paths = False +# TODO: should this be finnally ? +else: + unicode_paths = True + f.close() + +onlyif_unicode_paths = onlyif( + unicode_paths, + ("This test is only applicable where we can use unicode in filenames."), +) + + +def onlyif_cmds_exist(*commands): + """ + Decorator to skip test when at least one of `commands` is not found. + """ + for cmd in commands: + reason = f"This test runs only if command '{cmd}' is installed" + if not shutil.which(cmd): + return pytest.mark.skip(reason=reason) + return null_deco diff --git a/lib/python3.12/site-packages/IPython/testing/globalipapp.py b/lib/python3.12/site-packages/IPython/testing/globalipapp.py new file mode 100644 index 0000000000000000000000000000000000000000..3a699e07d619fe22fbdaba89d31e6563e36b25f9 --- /dev/null +++ b/lib/python3.12/site-packages/IPython/testing/globalipapp.py @@ -0,0 +1,114 @@ +"""Global IPython app to support test running. + +We must start our own ipython object and heavily muck with it so that all the +modifications IPython makes to system behavior don't send the doctest machinery +into a fit. This code should be considered a gross hack, but it gets the job +done. +""" + +# Copyright (c) IPython Development Team. +# Distributed under the terms of the Modified BSD License. + +import builtins as builtin_mod +import sys +import types + +from pathlib import Path + +from . import tools + +from IPython.core import page +from IPython.utils import io +from IPython.terminal.interactiveshell import TerminalInteractiveShell + + +def get_ipython(): + # This will get replaced by the real thing once we start IPython below + return start_ipython() + + +# A couple of methods to override those in the running IPython to interact +# better with doctest (doctest captures on raw stdout, so we need to direct +# various types of output there otherwise it will miss them). + +def xsys(self, cmd): + """Replace the default system call with a capturing one for doctest. + """ + # We use getoutput, but we need to strip it because pexpect captures + # the trailing newline differently from commands.getoutput + print(self.getoutput(cmd, split=False, depth=1).rstrip(), end='', file=sys.stdout) + sys.stdout.flush() + + +def _showtraceback(self, etype, evalue, stb): + """Print the traceback purely on stdout for doctest to capture it. + """ + print(self.InteractiveTB.stb2text(stb), file=sys.stdout) + + +def start_ipython(): + """Start a global IPython shell, which we need for IPython-specific syntax. + """ + global get_ipython + + # This function should only ever run once! + if hasattr(start_ipython, 'already_called'): + return + start_ipython.already_called = True + + # Store certain global objects that IPython modifies + _displayhook = sys.displayhook + _excepthook = sys.excepthook + _main = sys.modules.get('__main__') + + # Create custom argv and namespaces for our IPython to be test-friendly + config = tools.default_config() + config.TerminalInteractiveShell.simple_prompt = True + + # Create and initialize our test-friendly IPython instance. + shell = TerminalInteractiveShell.instance(config=config, + ) + + # A few more tweaks needed for playing nicely with doctests... + + # remove history file + shell.tempfiles.append(Path(config.HistoryManager.hist_file)) + + # These traps are normally only active for interactive use, set them + # permanently since we'll be mocking interactive sessions. + shell.builtin_trap.activate() + + # Modify the IPython system call with one that uses getoutput, so that we + # can capture subcommands and print them to Python's stdout, otherwise the + # doctest machinery would miss them. + shell.system = types.MethodType(xsys, shell) + + shell._showtraceback = types.MethodType(_showtraceback, shell) + + # IPython is ready, now clean up some global state... + + # Deactivate the various python system hooks added by ipython for + # interactive convenience so we don't confuse the doctest system + sys.modules['__main__'] = _main + sys.displayhook = _displayhook + sys.excepthook = _excepthook + + # So that ipython magics and aliases can be doctested (they work by making + # a call into a global _ip object). Also make the top-level get_ipython + # now return this without recursively calling here again. + _ip = shell + get_ipython = _ip.get_ipython + builtin_mod._ip = _ip + builtin_mod.ip = _ip + builtin_mod.get_ipython = get_ipython + + # Override paging, so we don't require user interaction during the tests. + def nopage(strng, start=0, screen_lines=0, pager_cmd=None): + if isinstance(strng, dict): + strng = strng.get('text/plain', '') + print(strng) + + page.orig_page = page.pager_page + page.pager_page = nopage + + return _ip diff --git a/lib/python3.12/site-packages/IPython/testing/ipunittest.py b/lib/python3.12/site-packages/IPython/testing/ipunittest.py new file mode 100644 index 0000000000000000000000000000000000000000..2ce77a44edf5cef1f83e73eb48fd78770f184614 --- /dev/null +++ b/lib/python3.12/site-packages/IPython/testing/ipunittest.py @@ -0,0 +1,187 @@ +"""Experimental code for cleaner support of IPython syntax with unittest. + +In IPython up until 0.10, we've used very hacked up nose machinery for running +tests with IPython special syntax, and this has proved to be extremely slow. +This module provides decorators to try a different approach, stemming from a +conversation Brian and I (FP) had about this problem Sept/09. + +The goal is to be able to easily write simple functions that can be seen by +unittest as tests, and ultimately for these to support doctests with full +IPython syntax. Nose already offers this based on naming conventions and our +hackish plugins, but we are seeking to move away from nose dependencies if +possible. + +This module follows a different approach, based on decorators. + +- A decorator called @ipdoctest can mark any function as having a docstring + that should be viewed as a doctest, but after syntax conversion. + +Authors +------- + +- Fernando Perez +""" + + +#----------------------------------------------------------------------------- +# Copyright (C) 2009-2011 The IPython Development Team +# +# Distributed under the terms of the BSD License. The full license is in +# the file COPYING, distributed as part of this software. +#----------------------------------------------------------------------------- + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- + +# Stdlib +import re +import sys +import unittest +import builtins +from doctest import DocTestFinder, DocTestRunner, TestResults +from IPython.terminal.interactiveshell import InteractiveShell + +#----------------------------------------------------------------------------- +# Classes and functions +#----------------------------------------------------------------------------- + +def count_failures(runner): + """Count number of failures in a doctest runner. + + Code modeled after the summarize() method in doctest. + """ + if sys.version_info < (3, 13): + return [TestResults(f, t) for f, t in runner._name2ft.values() if f > 0] + else: + return [ + TestResults(failure, try_) + for failure, try_, skip in runner._stats.values() + if failure > 0 + ] + + +class IPython2PythonConverter: + """Convert IPython 'syntax' to valid Python. + + Eventually this code may grow to be the full IPython syntax conversion + implementation, but for now it only does prompt conversion.""" + + def __init__(self): + self.rps1 = re.compile(r'In\ \[\d+\]: ') + self.rps2 = re.compile(r'\ \ \ \.\.\.+: ') + self.rout = re.compile(r'Out\[\d+\]: \s*?\n?') + self.pyps1 = '>>> ' + self.pyps2 = '... ' + self.rpyps1 = re.compile (r'(\s*%s)(.*)$' % self.pyps1) + self.rpyps2 = re.compile (r'(\s*%s)(.*)$' % self.pyps2) + + def __call__(self, ds): + """Convert IPython prompts to python ones in a string.""" + from . import globalipapp + + pyps1 = '>>> ' + pyps2 = '... ' + pyout = '' + + dnew = ds + dnew = self.rps1.sub(pyps1, dnew) + dnew = self.rps2.sub(pyps2, dnew) + dnew = self.rout.sub(pyout, dnew) + ip = InteractiveShell.instance() + + # Convert input IPython source into valid Python. + out = [] + newline = out.append + for line in dnew.splitlines(): + + mps1 = self.rpyps1.match(line) + if mps1 is not None: + prompt, text = mps1.groups() + newline(prompt+ip.prefilter(text, False)) + continue + + mps2 = self.rpyps2.match(line) + if mps2 is not None: + prompt, text = mps2.groups() + newline(prompt+ip.prefilter(text, True)) + continue + + newline(line) + newline('') # ensure a closing newline, needed by doctest + # print("PYSRC:", '\n'.join(out)) # dbg + return '\n'.join(out) + + #return dnew + + +class Doc2UnitTester: + """Class whose instances act as a decorator for docstring testing. + + In practice we're only likely to need one instance ever, made below (though + no attempt is made at turning it into a singleton, there is no need for + that). + """ + def __init__(self, verbose=False): + """New decorator. + + Parameters + ---------- + + verbose : boolean, optional (False) + Passed to the doctest finder and runner to control verbosity. + """ + self.verbose = verbose + # We can reuse the same finder for all instances + self.finder = DocTestFinder(verbose=verbose, recurse=False) + + def __call__(self, func): + """Use as a decorator: doctest a function's docstring as a unittest. + + This version runs normal doctests, but the idea is to make it later run + ipython syntax instead.""" + + # Capture the enclosing instance with a different name, so the new + # class below can see it without confusion regarding its own 'self' + # that will point to the test instance at runtime + d2u = self + + # Rewrite the function's docstring to have python syntax + if func.__doc__ is not None: + func.__doc__ = ip2py(func.__doc__) + + # Now, create a tester object that is a real unittest instance, so + # normal unittest machinery (or Nose, or Trial) can find it. + class Tester(unittest.TestCase): + def test(self): + # Make a new runner per function to be tested + runner = DocTestRunner(verbose=d2u.verbose) + for the_test in d2u.finder.find(func, func.__name__): + runner.run(the_test) + failed = count_failures(runner) + if failed: + # Since we only looked at a single function's docstring, + # failed should contain at most one item. More than that + # is a case we can't handle and should error out on + if len(failed) > 1: + err = "Invalid number of test results: %s" % failed + raise ValueError(err) + # Report a normal failure. + self.fail('failed doctests: %s' % str(failed[0])) + + # Rename it so test reports have the original signature. + Tester.__name__ = func.__name__ + return Tester + + +def ipdocstring(func): + """Change the function docstring via ip2py. + """ + if func.__doc__ is not None: + func.__doc__ = ip2py(func.__doc__) + return func + + +# Make an instance of the classes for public use +ipdoctest = Doc2UnitTester() +ip2py = IPython2PythonConverter() diff --git a/lib/python3.12/site-packages/IPython/testing/plugin/__init__.py b/lib/python3.12/site-packages/IPython/testing/plugin/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.12/site-packages/IPython/testing/plugin/dtexample.py b/lib/python3.12/site-packages/IPython/testing/plugin/dtexample.py new file mode 100644 index 0000000000000000000000000000000000000000..68f7016e34dc57b9ae7aadfd028c36889fe36088 --- /dev/null +++ b/lib/python3.12/site-packages/IPython/testing/plugin/dtexample.py @@ -0,0 +1,167 @@ +"""Simple example using doctests. + +This file just contains doctests both using plain python and IPython prompts. +All tests should be loaded by nose. +""" + +import os + + +def pyfunc(): + """Some pure python tests... + + >>> pyfunc() + 'pyfunc' + + >>> import os + + >>> 2+3 + 5 + + >>> for i in range(3): + ... print(i, end=' ') + ... print(i+1, end=' ') + ... + 0 1 1 2 2 3 + """ + return 'pyfunc' + +def ipfunc(): + """Some ipython tests... + + In [1]: import os + + In [3]: 2+3 + Out[3]: 5 + + In [26]: for i in range(3): + ....: print(i, end=' ') + ....: print(i+1, end=' ') + ....: + 0 1 1 2 2 3 + + + It's OK to use '_' for the last result, but do NOT try to use IPython's + numbered history of _NN outputs, since those won't exist under the + doctest environment: + + In [7]: 'hi' + Out[7]: 'hi' + + In [8]: print(repr(_)) + 'hi' + + In [7]: 3+4 + Out[7]: 7 + + In [8]: _+3 + Out[8]: 10 + + In [9]: ipfunc() + Out[9]: 'ipfunc' + """ + return "ipfunc" + + +def ipos(): + """Examples that access the operating system work: + + In [1]: !echo hello + hello + + In [2]: !echo hello > /tmp/foo_iptest + + In [3]: !cat /tmp/foo_iptest + hello + + In [4]: rm -f /tmp/foo_iptest + """ + pass + + +ipos.__skip_doctest__ = os.name == "nt" + + +def ranfunc(): + """A function with some random output. + + Normal examples are verified as usual: + >>> 1+3 + 4 + + But if you put '# random' in the output, it is ignored: + >>> 1+3 + junk goes here... # random + + >>> 1+2 + again, anything goes #random + if multiline, the random mark is only needed once. + + >>> 1+2 + You can also put the random marker at the end: + # random + + >>> 1+2 + # random + .. or at the beginning. + + More correct input is properly verified: + >>> ranfunc() + 'ranfunc' + """ + return 'ranfunc' + + +def random_all(): + """A function where we ignore the output of ALL examples. + + Examples: + + # all-random + + This mark tells the testing machinery that all subsequent examples should + be treated as random (ignoring their output). They are still executed, + so if a they raise an error, it will be detected as such, but their + output is completely ignored. + + >>> 1+3 + junk goes here... + + >>> 1+3 + klasdfj; + + >>> 1+2 + again, anything goes + blah... + """ + pass + +def iprand(): + """Some ipython tests with random output. + + In [7]: 3+4 + Out[7]: 7 + + In [8]: print('hello') + world # random + + In [9]: iprand() + Out[9]: 'iprand' + """ + return 'iprand' + +def iprand_all(): + """Some ipython tests with fully random output. + + # all-random + + In [7]: 1 + Out[7]: 99 + + In [8]: print('hello') + world + + In [9]: iprand_all() + Out[9]: 'junk' + """ + return 'iprand_all' diff --git a/lib/python3.12/site-packages/IPython/testing/plugin/ipdoctest.py b/lib/python3.12/site-packages/IPython/testing/plugin/ipdoctest.py new file mode 100644 index 0000000000000000000000000000000000000000..5c23373fba2a9bfa7fe08e8bb03303d57c8587a3 --- /dev/null +++ b/lib/python3.12/site-packages/IPython/testing/plugin/ipdoctest.py @@ -0,0 +1,299 @@ +"""Nose Plugin that supports IPython doctests. + +Limitations: + +- When generating examples for use as doctests, make sure that you have + pretty-printing OFF. This can be done either by setting the + ``PlainTextFormatter.pprint`` option in your configuration file to False, or + by interactively disabling it with %Pprint. This is required so that IPython + output matches that of normal Python, which is used by doctest for internal + execution. + +- Do not rely on specific prompt numbers for results (such as using + '_34==True', for example). For IPython tests run via an external process the + prompt numbers may be different, and IPython tests run as normal python code + won't even have these special _NN variables set at all. +""" + +#----------------------------------------------------------------------------- +# Module imports + +# From the standard library +import doctest +import logging +import re + +from testpath import modified_env + +#----------------------------------------------------------------------------- +# Module globals and other constants +#----------------------------------------------------------------------------- + +log = logging.getLogger(__name__) + + +#----------------------------------------------------------------------------- +# Classes and functions +#----------------------------------------------------------------------------- + + +class DocTestFinder(doctest.DocTestFinder): + def _get_test(self, obj, name, module, globs, source_lines): + test = super()._get_test(obj, name, module, globs, source_lines) + + if bool(getattr(obj, "__skip_doctest__", False)) and test is not None: + for example in test.examples: + example.options[doctest.SKIP] = True + + return test + + +class IPDoctestOutputChecker(doctest.OutputChecker): + """Second-chance checker with support for random tests. + + If the default comparison doesn't pass, this checker looks in the expected + output string for flags that tell us to ignore the output. + """ + + random_re = re.compile(r'#\s*random\s+') + + def check_output(self, want, got, optionflags): + """Check output, accepting special markers embedded in the output. + + If the output didn't pass the default validation but the special string + '#random' is included, we accept it.""" + + # Let the original tester verify first, in case people have valid tests + # that happen to have a comment saying '#random' embedded in. + ret = doctest.OutputChecker.check_output(self, want, got, + optionflags) + if not ret and self.random_re.search(want): + # print('RANDOM OK:',want, file=sys.stderr) # dbg + return True + + return ret + + +# A simple subclassing of the original with a different class name, so we can +# distinguish and treat differently IPython examples from pure python ones. +class IPExample(doctest.Example): pass + + +class IPDocTestParser(doctest.DocTestParser): + """ + A class used to parse strings containing doctest examples. + + Note: This is a version modified to properly recognize IPython input and + convert any IPython examples into valid Python ones. + """ + # This regular expression is used to find doctest examples in a + # string. It defines three groups: `source` is the source code + # (including leading indentation and prompts); `indent` is the + # indentation of the first (PS1) line of the source code; and + # `want` is the expected output (including leading indentation). + + # Classic Python prompts or default IPython ones + _PS1_PY = r'>>>' + _PS2_PY = r'\.\.\.' + + _PS1_IP = r'In\ \[\d+\]:' + _PS2_IP = r'\ \ \ \.\.\.+:' + + _RE_TPL = r''' + # Source consists of a PS1 line followed by zero or more PS2 lines. + (?P + (?:^(?P [ ]*) (?P %s) .*) # PS1 line + (?:\n [ ]* (?P %s) .*)*) # PS2 lines + \n? # a newline + # Want consists of any non-blank lines that do not start with PS1. + (?P (?:(?![ ]*$) # Not a blank line + (?![ ]*%s) # Not a line starting with PS1 + (?![ ]*%s) # Not a line starting with PS2 + .*$\n? # But any other line + )*) + ''' + + _EXAMPLE_RE_PY = re.compile( _RE_TPL % (_PS1_PY,_PS2_PY,_PS1_PY,_PS2_PY), + re.MULTILINE | re.VERBOSE) + + _EXAMPLE_RE_IP = re.compile( _RE_TPL % (_PS1_IP,_PS2_IP,_PS1_IP,_PS2_IP), + re.MULTILINE | re.VERBOSE) + + # Mark a test as being fully random. In this case, we simply append the + # random marker ('#random') to each individual example's output. This way + # we don't need to modify any other code. + _RANDOM_TEST = re.compile(r'#\s*all-random\s+') + + def ip2py(self,source): + """Convert input IPython source into valid Python.""" + block = _ip.input_transformer_manager.transform_cell(source) + if len(block.splitlines()) == 1: + return _ip.prefilter(block) + else: + return block + + def parse(self, string, name=''): + """ + Divide the given string into examples and intervening text, + and return them as a list of alternating Examples and strings. + Line numbers for the Examples are 0-based. The optional + argument `name` is a name identifying this string, and is only + used for error messages. + """ + + # print('Parse string:\n',string) # dbg + + string = string.expandtabs() + # If all lines begin with the same indentation, then strip it. + min_indent = self._min_indent(string) + if min_indent > 0: + string = '\n'.join([l[min_indent:] for l in string.split('\n')]) + + output = [] + charno, lineno = 0, 0 + + # We make 'all random' tests by adding the '# random' mark to every + # block of output in the test. + if self._RANDOM_TEST.search(string): + random_marker = '\n# random' + else: + random_marker = '' + + # Whether to convert the input from ipython to python syntax + ip2py = False + # Find all doctest examples in the string. First, try them as Python + # examples, then as IPython ones + terms = list(self._EXAMPLE_RE_PY.finditer(string)) + if terms: + # Normal Python example + Example = doctest.Example + else: + # It's an ipython example. + terms = list(self._EXAMPLE_RE_IP.finditer(string)) + Example = IPExample + ip2py = True + + for m in terms: + # Add the pre-example text to `output`. + output.append(string[charno:m.start()]) + # Update lineno (lines before this example) + lineno += string.count('\n', charno, m.start()) + # Extract info from the regexp match. + (source, options, want, exc_msg) = \ + self._parse_example(m, name, lineno,ip2py) + + # Append the random-output marker (it defaults to empty in most + # cases, it's only non-empty for 'all-random' tests): + want += random_marker + + # Create an Example, and add it to the list. + if not self._IS_BLANK_OR_COMMENT(source): + output.append(Example(source, want, exc_msg, + lineno=lineno, + indent=min_indent+len(m.group('indent')), + options=options)) + # Update lineno (lines inside this example) + lineno += string.count('\n', m.start(), m.end()) + # Update charno. + charno = m.end() + # Add any remaining post-example text to `output`. + output.append(string[charno:]) + return output + + def _parse_example(self, m, name, lineno,ip2py=False): + """ + Given a regular expression match from `_EXAMPLE_RE` (`m`), + return a pair `(source, want)`, where `source` is the matched + example's source code (with prompts and indentation stripped); + and `want` is the example's expected output (with indentation + stripped). + + `name` is the string's name, and `lineno` is the line number + where the example starts; both are used for error messages. + + Optional: + `ip2py`: if true, filter the input via IPython to convert the syntax + into valid python. + """ + + # Get the example's indentation level. + indent = len(m.group('indent')) + + # Divide source into lines; check that they're properly + # indented; and then strip their indentation & prompts. + source_lines = m.group('source').split('\n') + + # We're using variable-length input prompts + ps1 = m.group('ps1') + ps2 = m.group('ps2') + ps1_len = len(ps1) + + self._check_prompt_blank(source_lines, indent, name, lineno,ps1_len) + if ps2: + self._check_prefix(source_lines[1:], ' '*indent + ps2, name, lineno) + + source = '\n'.join([sl[indent+ps1_len+1:] for sl in source_lines]) + + if ip2py: + # Convert source input from IPython into valid Python syntax + source = self.ip2py(source) + + # Divide want into lines; check that it's properly indented; and + # then strip the indentation. Spaces before the last newline should + # be preserved, so plain rstrip() isn't good enough. + want = m.group('want') + want_lines = want.split('\n') + if len(want_lines) > 1 and re.match(r' *$', want_lines[-1]): + del want_lines[-1] # forget final newline & spaces after it + self._check_prefix(want_lines, ' '*indent, name, + lineno + len(source_lines)) + + # Remove ipython output prompt that might be present in the first line + want_lines[0] = re.sub(r'Out\[\d+\]: \s*?\n?','',want_lines[0]) + + want = '\n'.join([wl[indent:] for wl in want_lines]) + + # If `want` contains a traceback message, then extract it. + m = self._EXCEPTION_RE.match(want) + if m: + exc_msg = m.group('msg') + else: + exc_msg = None + + # Extract options from the source. + options = self._find_options(source, name, lineno) + + return source, options, want, exc_msg + + def _check_prompt_blank(self, lines, indent, name, lineno, ps1_len): + """ + Given the lines of a source string (including prompts and + leading indentation), check to make sure that every prompt is + followed by a space character. If any line is not followed by + a space character, then raise ValueError. + + Note: IPython-modified version which takes the input prompt length as a + parameter, so that prompts of variable length can be dealt with. + """ + space_idx = indent+ps1_len + min_len = space_idx+1 + for i, line in enumerate(lines): + if len(line) >= min_len and line[space_idx] != ' ': + raise ValueError('line %r of the docstring for %s ' + 'lacks blank after %s: %r' % + (lineno+i+1, name, + line[indent:space_idx], line)) + + +SKIP = doctest.register_optionflag('SKIP') + + +class IPDocTestRunner(doctest.DocTestRunner): + """Test runner that synchronizes the IPython namespace with test globals. + """ + + def run(self, test, compileflags=None, out=None, clear_globs=True): + # Override terminal size to standardise traceback format + with modified_env({'COLUMNS': '80', 'LINES': '24'}): + return super(IPDocTestRunner,self).run(test, + compileflags,out,clear_globs) diff --git a/lib/python3.12/site-packages/IPython/testing/plugin/pytest_ipdoctest.py b/lib/python3.12/site-packages/IPython/testing/plugin/pytest_ipdoctest.py new file mode 100644 index 0000000000000000000000000000000000000000..2c35c6fe1085b7609ff4c61b69ffcbc9c94fd1ee --- /dev/null +++ b/lib/python3.12/site-packages/IPython/testing/plugin/pytest_ipdoctest.py @@ -0,0 +1,877 @@ +# Based on Pytest doctest.py +# Original license: +# The MIT License (MIT) +# +# Copyright (c) 2004-2021 Holger Krekel and others +"""Discover and run ipdoctests in modules and test files.""" + +import bdb +import builtins +import inspect +import os +import platform +import sys +import traceback +import types +import warnings +from contextlib import contextmanager +from pathlib import Path +from typing import ( + TYPE_CHECKING, + Any, + Dict, + List, + Optional, + Tuple, + Type, + Union, +) +from re import Pattern +from collections.abc import Callable, Generator, Iterable, Sequence + +import pytest +from _pytest import outcomes +from _pytest._code.code import ExceptionInfo, ReprFileLocation, TerminalRepr +from _pytest._io import TerminalWriter +from _pytest.compat import safe_getattr +from _pytest.config import Config +from _pytest.config.argparsing import Parser + +try: + from _pytest.fixtures import TopRequest as FixtureRequest +except ImportError: + from _pytest.fixtures import FixtureRequest +from _pytest.nodes import Collector +from _pytest.outcomes import OutcomeException +from _pytest.pathlib import fnmatch_ex, import_path +from _pytest.python_api import approx +from _pytest.warning_types import PytestWarning + +if TYPE_CHECKING: + import doctest + + from .ipdoctest import IPDoctestOutputChecker + +DOCTEST_REPORT_CHOICE_NONE = "none" +DOCTEST_REPORT_CHOICE_CDIFF = "cdiff" +DOCTEST_REPORT_CHOICE_NDIFF = "ndiff" +DOCTEST_REPORT_CHOICE_UDIFF = "udiff" +DOCTEST_REPORT_CHOICE_ONLY_FIRST_FAILURE = "only_first_failure" + +DOCTEST_REPORT_CHOICES = ( + DOCTEST_REPORT_CHOICE_NONE, + DOCTEST_REPORT_CHOICE_CDIFF, + DOCTEST_REPORT_CHOICE_NDIFF, + DOCTEST_REPORT_CHOICE_UDIFF, + DOCTEST_REPORT_CHOICE_ONLY_FIRST_FAILURE, +) + +# Lazy definition of runner class +RUNNER_CLASS = None +# Lazy definition of output checker class +CHECKER_CLASS: Optional[Type["IPDoctestOutputChecker"]] = None + +pytest_version = tuple([int(part) for part in pytest.__version__.split(".")]) + + +def pytest_addoption(parser: Parser) -> None: + parser.addini( + "ipdoctest_optionflags", + "option flags for ipdoctests", + type="args", + default=["ELLIPSIS"], + ) + parser.addini( + "ipdoctest_encoding", "encoding used for ipdoctest files", default="utf-8" + ) + group = parser.getgroup("collect") + group.addoption( + "--ipdoctest-modules", + action="store_true", + default=False, + help="run ipdoctests in all .py modules", + dest="ipdoctestmodules", + ) + group.addoption( + "--ipdoctest-report", + type=str.lower, + default="udiff", + help="choose another output format for diffs on ipdoctest failure", + choices=DOCTEST_REPORT_CHOICES, + dest="ipdoctestreport", + ) + group.addoption( + "--ipdoctest-glob", + action="append", + default=[], + metavar="pat", + help="ipdoctests file matching pattern, default: test*.txt", + dest="ipdoctestglob", + ) + group.addoption( + "--ipdoctest-ignore-import-errors", + action="store_true", + default=False, + help="ignore ipdoctest ImportErrors", + dest="ipdoctest_ignore_import_errors", + ) + group.addoption( + "--ipdoctest-continue-on-failure", + action="store_true", + default=False, + help="for a given ipdoctest, continue to run after the first failure", + dest="ipdoctest_continue_on_failure", + ) + + +def pytest_unconfigure() -> None: + global RUNNER_CLASS + + RUNNER_CLASS = None + + +def pytest_collect_file( + file_path: Path, + parent: Collector, +) -> Optional[Union["IPDoctestModule", "IPDoctestTextfile"]]: + config = parent.config + if file_path.suffix == ".py": + if config.option.ipdoctestmodules and not any( + (_is_setup_py(file_path), _is_main_py(file_path)) + ): + mod: IPDoctestModule = IPDoctestModule.from_parent(parent, path=file_path) + return mod + elif _is_ipdoctest(config, file_path, parent): + txt: IPDoctestTextfile = IPDoctestTextfile.from_parent(parent, path=file_path) + return txt + return None + + +if pytest_version[0] < 7: + _collect_file = pytest_collect_file + + def pytest_collect_file( + path, + parent: Collector, + ) -> Optional[Union["IPDoctestModule", "IPDoctestTextfile"]]: + return _collect_file(Path(path), parent) + + _import_path = import_path + + def import_path(path, root): + import py.path + + return _import_path(py.path.local(path)) + + +def _is_setup_py(path: Path) -> bool: + if path.name != "setup.py": + return False + contents = path.read_bytes() + return b"setuptools" in contents or b"distutils" in contents + + +def _is_ipdoctest(config: Config, path: Path, parent: Collector) -> bool: + if path.suffix in (".txt", ".rst") and parent.session.isinitpath(path): + return True + globs = config.getoption("ipdoctestglob") or ["test*.txt"] + return any(fnmatch_ex(glob, path) for glob in globs) + + +def _is_main_py(path: Path) -> bool: + return path.name == "__main__.py" + + +class ReprFailDoctest(TerminalRepr): + def __init__( + self, reprlocation_lines: Sequence[Tuple[ReprFileLocation, Sequence[str]]] + ) -> None: + self.reprlocation_lines = reprlocation_lines + + def toterminal(self, tw: TerminalWriter) -> None: + for reprlocation, lines in self.reprlocation_lines: + for line in lines: + tw.line(line) + reprlocation.toterminal(tw) + + +class MultipleDoctestFailures(Exception): + def __init__(self, failures: Sequence["doctest.DocTestFailure"]) -> None: + super().__init__() + self.failures = failures + + +def _init_runner_class() -> Type["IPDocTestRunner"]: + import doctest + from .ipdoctest import IPDocTestRunner + + class PytestDoctestRunner(IPDocTestRunner): + """Runner to collect failures. + + Note that the out variable in this case is a list instead of a + stdout-like object. + """ + + def __init__( + self, + checker: Optional["IPDoctestOutputChecker"] = None, + verbose: Optional[bool] = None, + optionflags: int = 0, + continue_on_failure: bool = True, + ) -> None: + super().__init__(checker=checker, verbose=verbose, optionflags=optionflags) + self.continue_on_failure = continue_on_failure + + def report_failure( + self, + out, + test: "doctest.DocTest", + example: "doctest.Example", + got: str, + ) -> None: + failure = doctest.DocTestFailure(test, example, got) + if self.continue_on_failure: + out.append(failure) + else: + raise failure + + def report_unexpected_exception( + self, + out, + test: "doctest.DocTest", + example: "doctest.Example", + exc_info: Tuple[Type[BaseException], BaseException, types.TracebackType], + ) -> None: + if isinstance(exc_info[1], OutcomeException): + raise exc_info[1] + if isinstance(exc_info[1], bdb.BdbQuit): + outcomes.exit("Quitting debugger") + failure = doctest.UnexpectedException(test, example, exc_info) + if self.continue_on_failure: + out.append(failure) + else: + raise failure + + return PytestDoctestRunner + + +def _get_runner( + checker: Optional["IPDoctestOutputChecker"] = None, + verbose: Optional[bool] = None, + optionflags: int = 0, + continue_on_failure: bool = True, +) -> "IPDocTestRunner": + # We need this in order to do a lazy import on doctest + global RUNNER_CLASS + if RUNNER_CLASS is None: + RUNNER_CLASS = _init_runner_class() + # Type ignored because the continue_on_failure argument is only defined on + # PytestDoctestRunner, which is lazily defined so can't be used as a type. + return RUNNER_CLASS( # type: ignore + checker=checker, + verbose=verbose, + optionflags=optionflags, + continue_on_failure=continue_on_failure, + ) + + +class IPDoctestItem(pytest.Item): + _user_ns_orig: Dict[str, Any] + + def __init__( + self, + name: str, + parent: "Union[IPDoctestTextfile, IPDoctestModule]", + runner: Optional["IPDocTestRunner"] = None, + dtest: Optional["doctest.DocTest"] = None, + ) -> None: + super().__init__(name, parent) + self.runner = runner + self.dtest = dtest + self.obj = None + self.fixture_request: Optional[FixtureRequest] = None + self._user_ns_orig = {} + + @classmethod + def from_parent( # type: ignore + cls, + parent: "Union[IPDoctestTextfile, IPDoctestModule]", + *, + name: str, + runner: "IPDocTestRunner", + dtest: "doctest.DocTest", + ): + # incompatible signature due to imposed limits on subclass + """The public named constructor.""" + return super().from_parent(name=name, parent=parent, runner=runner, dtest=dtest) + + def setup(self) -> None: + if self.dtest is not None: + self.fixture_request = _setup_fixtures(self) + globs = dict(getfixture=self.fixture_request.getfixturevalue) + for name, value in self.fixture_request.getfixturevalue( + "ipdoctest_namespace" + ).items(): + globs[name] = value + self.dtest.globs.update(globs) + + from .ipdoctest import IPExample + + if isinstance(self.dtest.examples[0], IPExample): + # for IPython examples *only*, we swap the globals with the ipython + # namespace, after updating it with the globals (which doctest + # fills with the necessary info from the module being tested). + self._user_ns_orig = {} + self._user_ns_orig.update(_ip.user_ns) + _ip.user_ns.update(self.dtest.globs) + # We must remove the _ key in the namespace, so that Python's + # doctest code sets it naturally + _ip.user_ns.pop("_", None) + _ip.user_ns["__builtins__"] = builtins + self.dtest.globs = _ip.user_ns + + def teardown(self) -> None: + from .ipdoctest import IPExample + + # Undo the test.globs reassignment we made + if isinstance(self.dtest.examples[0], IPExample): + self.dtest.globs = {} + _ip.user_ns.clear() + _ip.user_ns.update(self._user_ns_orig) + del self._user_ns_orig + + self.dtest.globs.clear() + + def runtest(self) -> None: + assert self.dtest is not None + assert self.runner is not None + _check_all_skipped(self.dtest) + self._disable_output_capturing_for_darwin() + failures: List[doctest.DocTestFailure] = [] + + # exec(compile(..., "single", ...), ...) puts result in builtins._ + had_underscore_value = hasattr(builtins, "_") + underscore_original_value = getattr(builtins, "_", None) + + # Save our current directory and switch out to the one where the + # test was originally created, in case another doctest did a + # directory change. We'll restore this in the finally clause. + curdir = os.getcwd() + os.chdir(self.fspath.dirname) + try: + # Type ignored because we change the type of `out` from what + # ipdoctest expects. + self.runner.run(self.dtest, out=failures, clear_globs=False) # type: ignore[arg-type] + finally: + os.chdir(curdir) + if had_underscore_value: + setattr(builtins, "_", underscore_original_value) + elif hasattr(builtins, "_"): + delattr(builtins, "_") + + if failures: + raise MultipleDoctestFailures(failures) + + def _disable_output_capturing_for_darwin(self) -> None: + """Disable output capturing. Otherwise, stdout is lost to ipdoctest (pytest#985).""" + if platform.system() != "Darwin": + return + capman = self.config.pluginmanager.getplugin("capturemanager") + if capman: + capman.suspend_global_capture(in_=True) + out, err = capman.read_global_capture() + sys.stdout.write(out) + sys.stderr.write(err) + + # TODO: Type ignored -- breaks Liskov Substitution. + def repr_failure( # type: ignore[override] + self, + excinfo: ExceptionInfo[BaseException], + ) -> Union[str, TerminalRepr]: + import doctest + + failures: Optional[ + Sequence[Union[doctest.DocTestFailure, doctest.UnexpectedException]] + ] = None + if isinstance( + excinfo.value, (doctest.DocTestFailure, doctest.UnexpectedException) + ): + failures = [excinfo.value] + elif isinstance(excinfo.value, MultipleDoctestFailures): + failures = excinfo.value.failures + + if failures is None: + return super().repr_failure(excinfo) + + reprlocation_lines = [] + for failure in failures: + example = failure.example + test = failure.test + filename = test.filename + if test.lineno is None: + lineno = None + else: + lineno = test.lineno + example.lineno + 1 + message = type(failure).__name__ + # TODO: ReprFileLocation doesn't expect a None lineno. + reprlocation = ReprFileLocation(filename, lineno, message) # type: ignore[arg-type] + checker = _get_checker() + report_choice = _get_report_choice(self.config.getoption("ipdoctestreport")) + if lineno is not None: + assert failure.test.docstring is not None + lines = failure.test.docstring.splitlines(False) + # add line numbers to the left of the error message + assert test.lineno is not None + lines = [ + "%03d %s" % (i + test.lineno + 1, x) for (i, x) in enumerate(lines) + ] + # trim docstring error lines to 10 + lines = lines[max(example.lineno - 9, 0) : example.lineno + 1] + else: + lines = [ + "EXAMPLE LOCATION UNKNOWN, not showing all tests of that example" + ] + indent = ">>>" + for line in example.source.splitlines(): + lines.append(f"??? {indent} {line}") + indent = "..." + if isinstance(failure, doctest.DocTestFailure): + lines += checker.output_difference( + example, failure.got, report_choice + ).split("\n") + else: + inner_excinfo = ExceptionInfo.from_exc_info(failure.exc_info) + lines += ["UNEXPECTED EXCEPTION: %s" % repr(inner_excinfo.value)] + lines += [ + x.strip("\n") for x in traceback.format_exception(*failure.exc_info) + ] + reprlocation_lines.append((reprlocation, lines)) + return ReprFailDoctest(reprlocation_lines) + + def reportinfo(self) -> Tuple[Union["os.PathLike[str]", str], Optional[int], str]: + assert self.dtest is not None + return self.path, self.dtest.lineno, "[ipdoctest] %s" % self.name + + if pytest_version[0] < 7: + + @property + def path(self) -> Path: + return Path(self.fspath) + + +def _get_flag_lookup() -> Dict[str, int]: + import doctest + + return dict( + DONT_ACCEPT_TRUE_FOR_1=doctest.DONT_ACCEPT_TRUE_FOR_1, + DONT_ACCEPT_BLANKLINE=doctest.DONT_ACCEPT_BLANKLINE, + NORMALIZE_WHITESPACE=doctest.NORMALIZE_WHITESPACE, + ELLIPSIS=doctest.ELLIPSIS, + IGNORE_EXCEPTION_DETAIL=doctest.IGNORE_EXCEPTION_DETAIL, + COMPARISON_FLAGS=doctest.COMPARISON_FLAGS, + ALLOW_UNICODE=_get_allow_unicode_flag(), + ALLOW_BYTES=_get_allow_bytes_flag(), + NUMBER=_get_number_flag(), + ) + + +def get_optionflags(parent): + optionflags_str = parent.config.getini("ipdoctest_optionflags") + flag_lookup_table = _get_flag_lookup() + flag_acc = 0 + for flag in optionflags_str: + flag_acc |= flag_lookup_table[flag] + return flag_acc + + +def _get_continue_on_failure(config): + continue_on_failure = config.getvalue("ipdoctest_continue_on_failure") + if continue_on_failure: + # We need to turn off this if we use pdb since we should stop at + # the first failure. + if config.getvalue("usepdb"): + continue_on_failure = False + return continue_on_failure + + +class IPDoctestTextfile(pytest.Module): + obj = None + + def collect(self) -> Iterable[IPDoctestItem]: + import doctest + from .ipdoctest import IPDocTestParser + + # Inspired by doctest.testfile; ideally we would use it directly, + # but it doesn't support passing a custom checker. + encoding = self.config.getini("ipdoctest_encoding") + text = self.path.read_text(encoding) + filename = str(self.path) + name = self.path.name + globs = {"__name__": "__main__"} + + optionflags = get_optionflags(self) + + runner = _get_runner( + verbose=False, + optionflags=optionflags, + checker=_get_checker(), + continue_on_failure=_get_continue_on_failure(self.config), + ) + + parser = IPDocTestParser() + test = parser.get_doctest(text, globs, name, filename, 0) + if test.examples: + yield IPDoctestItem.from_parent( + self, name=test.name, runner=runner, dtest=test + ) + + if pytest_version[0] < 7: + + @property + def path(self) -> Path: + return Path(self.fspath) + + @classmethod + def from_parent( + cls, + parent, + *, + fspath=None, + path: Optional[Path] = None, + **kw, + ): + if path is not None: + import py.path + + fspath = py.path.local(path) + return super().from_parent(parent=parent, fspath=fspath, **kw) + + +def _check_all_skipped(test: "doctest.DocTest") -> None: + """Raise pytest.skip() if all examples in the given DocTest have the SKIP + option set.""" + import doctest + + all_skipped = all(x.options.get(doctest.SKIP, False) for x in test.examples) + if all_skipped: + pytest.skip("all docstests skipped by +SKIP option") + + +def _is_mocked(obj: object) -> bool: + """Return if an object is possibly a mock object by checking the + existence of a highly improbable attribute.""" + return ( + safe_getattr(obj, "pytest_mock_example_attribute_that_shouldnt_exist", None) + is not None + ) + + +@contextmanager +def _patch_unwrap_mock_aware() -> Generator[None, None, None]: + """Context manager which replaces ``inspect.unwrap`` with a version + that's aware of mock objects and doesn't recurse into them.""" + real_unwrap = inspect.unwrap + + def _mock_aware_unwrap( + func: Callable[..., Any], *, stop: Optional[Callable[[Any], Any]] = None + ) -> Any: + try: + if stop is None or stop is _is_mocked: + return real_unwrap(func, stop=_is_mocked) + _stop = stop + return real_unwrap(func, stop=lambda obj: _is_mocked(obj) or _stop(func)) + except Exception as e: + warnings.warn( + "Got %r when unwrapping %r. This is usually caused " + "by a violation of Python's object protocol; see e.g. " + "https://github.com/pytest-dev/pytest/issues/5080" % (e, func), + PytestWarning, + ) + raise + + inspect.unwrap = _mock_aware_unwrap + try: + yield + finally: + inspect.unwrap = real_unwrap + + +class IPDoctestModule(pytest.Module): + def collect(self) -> Iterable[IPDoctestItem]: + import doctest + from .ipdoctest import DocTestFinder, IPDocTestParser + + class MockAwareDocTestFinder(DocTestFinder): + """A hackish ipdoctest finder that overrides stdlib internals to fix a stdlib bug. + + https://github.com/pytest-dev/pytest/issues/3456 + https://bugs.python.org/issue25532 + """ + + def _find_lineno(self, obj, source_lines): + """Doctest code does not take into account `@property`, this + is a hackish way to fix it. https://bugs.python.org/issue17446 + + Wrapped Doctests will need to be unwrapped so the correct + line number is returned. This will be reported upstream. #8796 + """ + if isinstance(obj, property): + obj = getattr(obj, "fget", obj) + + if hasattr(obj, "__wrapped__"): + # Get the main obj in case of it being wrapped + obj = inspect.unwrap(obj) + + # Type ignored because this is a private function. + return super()._find_lineno( # type:ignore[misc] + obj, + source_lines, + ) + + def _find( + self, tests, obj, name, module, source_lines, globs, seen + ) -> None: + if _is_mocked(obj): + return + with _patch_unwrap_mock_aware(): + # Type ignored because this is a private function. + super()._find( # type:ignore[misc] + tests, obj, name, module, source_lines, globs, seen + ) + + if self.path.name == "conftest.py": + if pytest_version[0] < 7: + module = self.config.pluginmanager._importconftest( + self.path, + self.config.getoption("importmode"), + ) + else: + kwargs = {"rootpath": self.config.rootpath} + if pytest_version >= (8, 1): + kwargs["consider_namespace_packages"] = False + module = self.config.pluginmanager._importconftest( + self.path, + self.config.getoption("importmode"), + **kwargs, + ) + else: + try: + kwargs = {"root": self.config.rootpath} + if pytest_version >= (8, 1): + kwargs["consider_namespace_packages"] = False + module = import_path(self.path, **kwargs) + except ImportError: + if self.config.getvalue("ipdoctest_ignore_import_errors"): + pytest.skip("unable to import module %r" % self.path) + else: + raise + # Uses internal doctest module parsing mechanism. + finder = MockAwareDocTestFinder(parser=IPDocTestParser()) + optionflags = get_optionflags(self) + runner = _get_runner( + verbose=False, + optionflags=optionflags, + checker=_get_checker(), + continue_on_failure=_get_continue_on_failure(self.config), + ) + + for test in finder.find(module, module.__name__): + if test.examples: # skip empty ipdoctests + yield IPDoctestItem.from_parent( + self, name=test.name, runner=runner, dtest=test + ) + + if pytest_version[0] < 7: + + @property + def path(self) -> Path: + return Path(self.fspath) + + @classmethod + def from_parent( + cls, + parent, + *, + fspath=None, + path: Optional[Path] = None, + **kw, + ): + if path is not None: + import py.path + + fspath = py.path.local(path) + return super().from_parent(parent=parent, fspath=fspath, **kw) + + +def _setup_fixtures(doctest_item: IPDoctestItem) -> FixtureRequest: + """Used by IPDoctestTextfile and IPDoctestItem to setup fixture information.""" + + def func() -> None: + pass + + doctest_item.funcargs = {} # type: ignore[attr-defined] + fm = doctest_item.session._fixturemanager + kwargs = {"node": doctest_item, "func": func, "cls": None} + if pytest_version <= (8, 0): + kwargs["funcargs"] = False + doctest_item._fixtureinfo = fm.getfixtureinfo( # type: ignore[attr-defined] + **kwargs + ) + fixture_request = FixtureRequest(doctest_item, _ispytest=True) + if pytest_version <= (8, 0): + fixture_request._fillfixtures() + return fixture_request + + +def _init_checker_class() -> Type["IPDoctestOutputChecker"]: + import doctest + import re + from .ipdoctest import IPDoctestOutputChecker + + class LiteralsOutputChecker(IPDoctestOutputChecker): + # Based on doctest_nose_plugin.py from the nltk project + # (https://github.com/nltk/nltk) and on the "numtest" doctest extension + # by Sebastien Boisgerault (https://github.com/boisgera/numtest). + + _unicode_literal_re = re.compile(r"(\W|^)[uU]([rR]?[\'\"])", re.UNICODE) + _bytes_literal_re = re.compile(r"(\W|^)[bB]([rR]?[\'\"])", re.UNICODE) + _number_re = re.compile( + r""" + (?P + (?P + (?P [+-]?\d*)\.(?P\d+) + | + (?P [+-]?\d+)\. + ) + (?: + [Ee] + (?P [+-]?\d+) + )? + | + (?P [+-]?\d+) + (?: + [Ee] + (?P [+-]?\d+) + ) + ) + """, + re.VERBOSE, + ) + + def check_output(self, want: str, got: str, optionflags: int) -> bool: + if super().check_output(want, got, optionflags): + return True + + allow_unicode = optionflags & _get_allow_unicode_flag() + allow_bytes = optionflags & _get_allow_bytes_flag() + allow_number = optionflags & _get_number_flag() + + if not allow_unicode and not allow_bytes and not allow_number: + return False + + def remove_prefixes(regex: Pattern[str], txt: str) -> str: + return re.sub(regex, r"\1\2", txt) + + if allow_unicode: + want = remove_prefixes(self._unicode_literal_re, want) + got = remove_prefixes(self._unicode_literal_re, got) + + if allow_bytes: + want = remove_prefixes(self._bytes_literal_re, want) + got = remove_prefixes(self._bytes_literal_re, got) + + if allow_number: + got = self._remove_unwanted_precision(want, got) + + return super().check_output(want, got, optionflags) + + def _remove_unwanted_precision(self, want: str, got: str) -> str: + wants = list(self._number_re.finditer(want)) + gots = list(self._number_re.finditer(got)) + if len(wants) != len(gots): + return got + offset = 0 + for w, g in zip(wants, gots): + fraction: Optional[str] = w.group("fraction") + exponent: Optional[str] = w.group("exponent1") + if exponent is None: + exponent = w.group("exponent2") + precision = 0 if fraction is None else len(fraction) + if exponent is not None: + precision -= int(exponent) + if float(w.group()) == approx(float(g.group()), abs=10**-precision): + # They're close enough. Replace the text we actually + # got with the text we want, so that it will match when we + # check the string literally. + got = ( + got[: g.start() + offset] + w.group() + got[g.end() + offset :] + ) + offset += w.end() - w.start() - (g.end() - g.start()) + return got + + return LiteralsOutputChecker + + +def _get_checker() -> "IPDoctestOutputChecker": + """Return a IPDoctestOutputChecker subclass that supports some + additional options: + + * ALLOW_UNICODE and ALLOW_BYTES options to ignore u'' and b'' + prefixes (respectively) in string literals. Useful when the same + ipdoctest should run in Python 2 and Python 3. + + * NUMBER to ignore floating-point differences smaller than the + precision of the literal number in the ipdoctest. + + An inner class is used to avoid importing "ipdoctest" at the module + level. + """ + global CHECKER_CLASS + if CHECKER_CLASS is None: + CHECKER_CLASS = _init_checker_class() + return CHECKER_CLASS() + + +def _get_allow_unicode_flag() -> int: + """Register and return the ALLOW_UNICODE flag.""" + import doctest + + return doctest.register_optionflag("ALLOW_UNICODE") + + +def _get_allow_bytes_flag() -> int: + """Register and return the ALLOW_BYTES flag.""" + import doctest + + return doctest.register_optionflag("ALLOW_BYTES") + + +def _get_number_flag() -> int: + """Register and return the NUMBER flag.""" + import doctest + + return doctest.register_optionflag("NUMBER") + + +def _get_report_choice(key: str) -> int: + """Return the actual `ipdoctest` module flag value. + + We want to do it as late as possible to avoid importing `ipdoctest` and all + its dependencies when parsing options, as it adds overhead and breaks tests. + """ + import doctest + + return { + DOCTEST_REPORT_CHOICE_UDIFF: doctest.REPORT_UDIFF, + DOCTEST_REPORT_CHOICE_CDIFF: doctest.REPORT_CDIFF, + DOCTEST_REPORT_CHOICE_NDIFF: doctest.REPORT_NDIFF, + DOCTEST_REPORT_CHOICE_ONLY_FIRST_FAILURE: doctest.REPORT_ONLY_FIRST_FAILURE, + DOCTEST_REPORT_CHOICE_NONE: 0, + }[key] + + +@pytest.fixture(scope="session") +def ipdoctest_namespace() -> Dict[str, Any]: + """Fixture that returns a :py:class:`dict` that will be injected into the + namespace of ipdoctests.""" + return dict() diff --git a/lib/python3.12/site-packages/IPython/testing/plugin/setup.py b/lib/python3.12/site-packages/IPython/testing/plugin/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..a3281d30c8da509f46859b56ee66658e66156e84 --- /dev/null +++ b/lib/python3.12/site-packages/IPython/testing/plugin/setup.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python +"""A Nose plugin to support IPython doctests. +""" + +from setuptools import setup + +setup(name='IPython doctest plugin', + version='0.1', + author='The IPython Team', + description = 'Nose plugin to load IPython-extended doctests', + license = 'LGPL', + py_modules = ['ipdoctest'], + entry_points = { + 'nose.plugins.0.10': ['ipdoctest = ipdoctest:IPythonDoctest', + 'extdoctest = ipdoctest:ExtensionDoctest', + ], + }, + ) diff --git a/lib/python3.12/site-packages/IPython/testing/plugin/simple.py b/lib/python3.12/site-packages/IPython/testing/plugin/simple.py new file mode 100644 index 0000000000000000000000000000000000000000..79c9e0d21cc5dc446fe0197145d4a5347d34f499 --- /dev/null +++ b/lib/python3.12/site-packages/IPython/testing/plugin/simple.py @@ -0,0 +1,45 @@ +"""Simple example using doctests. + +This file just contains doctests both using plain python and IPython prompts. +All tests should be loaded by Pytest. +""" + + +def pyfunc(): + """Some pure python tests... + + >>> pyfunc() + 'pyfunc' + + >>> import os + + >>> 2+3 + 5 + + >>> for i in range(3): + ... print(i, end=' ') + ... print(i+1, end=' ') + ... + 0 1 1 2 2 3 + """ + return "pyfunc" + + +def ipyfunc(): + """Some IPython tests... + + In [1]: ipyfunc() + Out[1]: 'ipyfunc' + + In [2]: import os + + In [3]: 2+3 + Out[3]: 5 + + In [4]: for i in range(3): + ...: print(i, end=' ') + ...: print(i+1, end=' ') + ...: + Out[4]: 0 1 1 2 2 3 + """ + return "ipyfunc" diff --git a/lib/python3.12/site-packages/IPython/testing/plugin/test_combo.txt b/lib/python3.12/site-packages/IPython/testing/plugin/test_combo.txt new file mode 100644 index 0000000000000000000000000000000000000000..6c8759f3e72ae3f1aea989826f58ca6863337435 --- /dev/null +++ b/lib/python3.12/site-packages/IPython/testing/plugin/test_combo.txt @@ -0,0 +1,36 @@ +======================= + Combo testing example +======================= + +This is a simple example that mixes ipython doctests:: + + In [1]: import code + + In [2]: 2**12 + Out[2]: 4096 + +with command-line example information that does *not* get executed:: + + $ mpirun -n 4 ipengine --controller-port=10000 --controller-ip=host0 + +and with literal examples of Python source code:: + + controller = dict(host='myhost', + engine_port=None, # default is 10105 + control_port=None, + ) + + # keys are hostnames, values are the number of engine on that host + engines = dict(node1=2, + node2=2, + node3=2, + node3=2, + ) + + # Force failure to detect that this test is being run. + 1/0 + +These source code examples are executed but no output is compared at all. An +error or failure is reported only if an exception is raised. + +NOTE: the execution of pure python blocks is not yet working! diff --git a/lib/python3.12/site-packages/IPython/testing/plugin/test_exampleip.txt b/lib/python3.12/site-packages/IPython/testing/plugin/test_exampleip.txt new file mode 100644 index 0000000000000000000000000000000000000000..96b1eae19f09fb62702f5ee58851f4cad87aa9d5 --- /dev/null +++ b/lib/python3.12/site-packages/IPython/testing/plugin/test_exampleip.txt @@ -0,0 +1,30 @@ +================================= + Tests in example form - IPython +================================= + +You can write text files with examples that use IPython prompts (as long as you +use the nose ipython doctest plugin), but you can not mix and match prompt +styles in a single file. That is, you either use all ``>>>`` prompts or all +IPython-style prompts. Your test suite *can* have both types, you just need to +put each type of example in a separate. Using IPython prompts, you can paste +directly from your session:: + + In [5]: s="Hello World" + + In [6]: s.upper() + Out[6]: 'HELLO WORLD' + +Another example:: + + In [8]: 1+3 + Out[8]: 4 + +Just like in IPython docstrings, you can use all IPython syntax and features:: + + In [9]: !echo hello + hello + + In [10]: a='hi' + + In [11]: !echo $a + hi diff --git a/lib/python3.12/site-packages/IPython/testing/plugin/test_ipdoctest.py b/lib/python3.12/site-packages/IPython/testing/plugin/test_ipdoctest.py new file mode 100644 index 0000000000000000000000000000000000000000..2686172bb29e50c9fe824ef9f31b1d3badb1c7b4 --- /dev/null +++ b/lib/python3.12/site-packages/IPython/testing/plugin/test_ipdoctest.py @@ -0,0 +1,92 @@ +"""Tests for the ipdoctest machinery itself. + +Note: in a file named test_X, functions whose only test is their docstring (as +a doctest) and which have no test functionality of their own, should be called +'doctest_foo' instead of 'test_foo', otherwise they get double-counted (the +empty function call is counted as a test, which just inflates tests numbers +artificially). +""" + +def doctest_simple(): + """ipdoctest must handle simple inputs + + In [1]: 1 + Out[1]: 1 + + In [2]: print(1) + 1 + """ + +def doctest_multiline1(): + """The ipdoctest machinery must handle multiline examples gracefully. + + In [2]: for i in range(4): + ...: print(i) + ...: + 0 + 1 + 2 + 3 + """ + +def doctest_multiline2(): + """Multiline examples that define functions and print output. + + In [7]: def f(x): + ...: return x+1 + ...: + + In [8]: f(1) + Out[8]: 2 + + In [9]: def g(x): + ...: print('x is:',x) + ...: + + In [10]: g(1) + x is: 1 + + In [11]: g('hello') + x is: hello + """ + + +def doctest_multiline3(): + """Multiline examples with blank lines. + + In [12]: def h(x): + ....: if x>1: + ....: return x**2 + ....: # To leave a blank line in the input, you must mark it + ....: # with a comment character: + ....: # + ....: # otherwise the doctest parser gets confused. + ....: else: + ....: return -1 + ....: + + In [13]: h(5) + Out[13]: 25 + + In [14]: h(1) + Out[14]: -1 + + In [15]: h(0) + Out[15]: -1 + """ + + +def doctest_builtin_underscore(): + """Defining builtins._ should not break anything outside the doctest + while also should be working as expected inside the doctest. + + In [1]: import builtins + + In [2]: builtins._ = 42 + + In [3]: builtins._ + Out[3]: 42 + + In [4]: _ + Out[4]: 42 + """ diff --git a/lib/python3.12/site-packages/IPython/testing/plugin/test_refs.py b/lib/python3.12/site-packages/IPython/testing/plugin/test_refs.py new file mode 100644 index 0000000000000000000000000000000000000000..b92448be074180e8311fa9820b52bc6fa70d696e --- /dev/null +++ b/lib/python3.12/site-packages/IPython/testing/plugin/test_refs.py @@ -0,0 +1,39 @@ +"""Some simple tests for the plugin while running scripts. +""" +# Module imports +# Std lib +import inspect + +# Our own + +#----------------------------------------------------------------------------- +# Testing functions + +def test_trivial(): + """A trivial passing test.""" + pass + +def doctest_run(): + """Test running a trivial script. + + In [13]: run simplevars.py + x is: 1 + """ + +def doctest_runvars(): + """Test that variables defined in scripts get loaded correctly via %run. + + In [13]: run simplevars.py + x is: 1 + + In [14]: x + Out[14]: 1 + """ + +def doctest_ivars(): + """Test that variables defined interactively are picked up. + In [5]: zz=1 + + In [6]: zz + Out[6]: 1 + """ diff --git a/lib/python3.12/site-packages/IPython/testing/skipdoctest.py b/lib/python3.12/site-packages/IPython/testing/skipdoctest.py new file mode 100644 index 0000000000000000000000000000000000000000..f440ea14b274de05919ce9e77bada084d8725218 --- /dev/null +++ b/lib/python3.12/site-packages/IPython/testing/skipdoctest.py @@ -0,0 +1,19 @@ +"""Decorators marks that a doctest should be skipped. + +The IPython.testing.decorators module triggers various extra imports, including +numpy and sympy if they're present. Since this decorator is used in core parts +of IPython, it's in a separate module so that running IPython doesn't trigger +those imports.""" + +# Copyright (C) IPython Development Team +# Distributed under the terms of the Modified BSD License. + + +def skip_doctest(f): + """Decorator - mark a function or method for skipping its doctest. + + This decorator allows you to mark a function whose docstring you wish to + omit from testing, while preserving the docstring for introspection, help, + etc.""" + f.__skip_doctest__ = True + return f diff --git a/lib/python3.12/site-packages/IPython/testing/tools.py b/lib/python3.12/site-packages/IPython/testing/tools.py new file mode 100644 index 0000000000000000000000000000000000000000..d596a000f8f812bf6328b15fa640c394224db7fc --- /dev/null +++ b/lib/python3.12/site-packages/IPython/testing/tools.py @@ -0,0 +1,434 @@ +"""Generic testing tools. + +Authors +------- +- Fernando Perez +""" + + +# Copyright (c) IPython Development Team. +# Distributed under the terms of the Modified BSD License. + +import os +from pathlib import Path +import re +import sys +import tempfile +import unittest + +from contextlib import contextmanager +from io import StringIO +from subprocess import Popen, PIPE +from unittest.mock import patch + +from traitlets.config.loader import Config +from IPython.utils.process import get_output_error_code +from IPython.utils.text import list_strings +from IPython.utils.io import temp_pyfile, Tee +from IPython.utils import py3compat + +from . import decorators as dec +from . import skipdoctest + + +# The docstring for full_path doctests differently on win32 (different path +# separator) so just skip the doctest there. The example remains informative. +doctest_deco = skipdoctest.skip_doctest if sys.platform == 'win32' else dec.null_deco + +@doctest_deco +def full_path(startPath: str, files: list[str]) -> list[str]: + """Make full paths for all the listed files, based on startPath. + + Only the base part of startPath is kept, since this routine is typically + used with a script's ``__file__`` variable as startPath. The base of startPath + is then prepended to all the listed files, forming the output list. + + Parameters + ---------- + startPath : string + Initial path to use as the base for the results. This path is split + using os.path.split() and only its first component is kept. + + files : list + One or more files. + + Examples + -------- + + >>> full_path('/foo/bar.py',['a.txt','b.txt']) + ['/foo/a.txt', '/foo/b.txt'] + + >>> full_path('/foo',['a.txt','b.txt']) + ['/a.txt', '/b.txt'] + + """ + assert isinstance(files, list) + base = os.path.split(startPath)[0] + return [ os.path.join(base,f) for f in files ] + + +def parse_test_output(txt): + """Parse the output of a test run and return errors, failures. + + Parameters + ---------- + txt : str + Text output of a test run, assumed to contain a line of one of the + following forms:: + + 'FAILED (errors=1)' + 'FAILED (failures=1)' + 'FAILED (errors=1, failures=1)' + + Returns + ------- + nerr, nfail + number of errors and failures. + """ + + err_m = re.search(r'^FAILED \(errors=(\d+)\)', txt, re.MULTILINE) + if err_m: + nerr = int(err_m.group(1)) + nfail = 0 + return nerr, nfail + + fail_m = re.search(r'^FAILED \(failures=(\d+)\)', txt, re.MULTILINE) + if fail_m: + nerr = 0 + nfail = int(fail_m.group(1)) + return nerr, nfail + + both_m = re.search(r'^FAILED \(errors=(\d+), failures=(\d+)\)', txt, + re.MULTILINE) + if both_m: + nerr = int(both_m.group(1)) + nfail = int(both_m.group(2)) + return nerr, nfail + + # If the input didn't match any of these forms, assume no error/failures + return 0, 0 + + +# So nose doesn't think this is a test +parse_test_output.__test__ = False + + +def default_argv(): + """Return a valid default argv for creating testing instances of ipython""" + + return [ + "--quick", # so no config file is loaded + # Other defaults to minimize side effects on stdout + "--colors=nocolor", + "--no-term-title", + "--no-banner", + "--autocall=0", + ] + + +def default_config(): + """Return a config object with good defaults for testing.""" + config = Config() + config.TerminalInteractiveShell.colors = "nocolor" + config.TerminalTerminalInteractiveShell.term_title = (False,) + config.TerminalInteractiveShell.autocall = 0 + f = tempfile.NamedTemporaryFile(suffix="test_hist.sqlite", delete=False) + config.HistoryManager.hist_file = Path(f.name) + f.close() + config.HistoryManager.db_cache_size = 10000 + return config + + +def get_ipython_cmd(as_string=False): + """ + Return appropriate IPython command line name. By default, this will return + a list that can be used with subprocess.Popen, for example, but passing + `as_string=True` allows for returning the IPython command as a string. + + Parameters + ---------- + as_string: bool + Flag to allow to return the command as a string. + """ + ipython_cmd = [sys.executable, "-m", "IPython"] + + if as_string: + ipython_cmd = " ".join(ipython_cmd) + + return ipython_cmd + +def ipexec(fname, options=None, commands=()): + """Utility to call 'ipython filename'. + + Starts IPython with a minimal and safe configuration to make startup as fast + as possible. + + Note that this starts IPython in a subprocess! + + Parameters + ---------- + fname : str, Path + Name of file to be executed (should have .py or .ipy extension). + + options : optional, list + Extra command-line flags to be passed to IPython. + + commands : optional, list + Commands to send in on stdin + + Returns + ------- + ``(stdout, stderr)`` of ipython subprocess. + """ + __tracebackhide__ = True + + if options is None: + options = [] + + cmdargs = default_argv() + options + + test_dir = os.path.dirname(__file__) + + ipython_cmd = get_ipython_cmd() + # Absolute path for filename + full_fname = os.path.join(test_dir, fname) + full_cmd = ipython_cmd + cmdargs + ['--', full_fname] + env = os.environ.copy() + # FIXME: ignore all warnings in ipexec while we have shims + # should we keep suppressing warnings here, even after removing shims? + env['PYTHONWARNINGS'] = 'ignore' + # env.pop('PYTHONWARNINGS', None) # Avoid extraneous warnings appearing on stderr + # Prevent coloring under PyCharm ("\x1b[0m" at the end of the stdout) + env.pop("PYCHARM_HOSTED", None) + for k, v in env.items(): + # Debug a bizarre failure we've seen on Windows: + # TypeError: environment can only contain strings + if not isinstance(v, str): + print(k, v) + p = Popen(full_cmd, stdout=PIPE, stderr=PIPE, stdin=PIPE, env=env) + out, err = p.communicate(input=py3compat.encode('\n'.join(commands)) or None) + out, err = py3compat.decode(out), py3compat.decode(err) + # `import readline` causes 'ESC[?1034h' to be output sometimes, + # so strip that out before doing comparisons + if out: + out = re.sub(r'\x1b\[[^h]+h', '', out) + return out, err + + +def ipexec_validate(fname, expected_out, expected_err='', + options=None, commands=()): + """Utility to call 'ipython filename' and validate output/error. + + This function raises an AssertionError if the validation fails. + + Note that this starts IPython in a subprocess! + + Parameters + ---------- + fname : str, Path + Name of the file to be executed (should have .py or .ipy extension). + + expected_out : str + Expected stdout of the process. + + expected_err : optional, str + Expected stderr of the process. + + options : optional, list + Extra command-line flags to be passed to IPython. + + Returns + ------- + None + """ + __tracebackhide__ = True + + out, err = ipexec(fname, options, commands) + # print('OUT', out) # dbg + # print('ERR', err) # dbg + # If there are any errors, we must check those before stdout, as they may be + # more informative than simply having an empty stdout. + if err: + if expected_err: + assert "\n".join(err.strip().splitlines()) == "\n".join( + expected_err.strip().splitlines() + ) + else: + raise ValueError('Running file %r produced error: %r' % + (fname, err)) + # If no errors or output on stderr was expected, match stdout + assert "\n".join(out.strip().splitlines()) == "\n".join( + expected_out.strip().splitlines() + ) + + +class TempFileMixin(unittest.TestCase): + """Utility class to create temporary Python/IPython files. + + Meant as a mixin class for test cases.""" + + def mktmp(self, src, ext='.py'): + """Make a valid python temp file.""" + fname = temp_pyfile(src, ext) + if not hasattr(self, 'tmps'): + self.tmps=[] + self.tmps.append(fname) + self.fname = fname + + def tearDown(self): + # If the tmpfile wasn't made because of skipped tests, like in + # win32, there's nothing to cleanup. + if hasattr(self, 'tmps'): + for fname in self.tmps: + # If the tmpfile wasn't made because of skipped tests, like in + # win32, there's nothing to cleanup. + try: + os.unlink(fname) + except: + # On Windows, even though we close the file, we still can't + # delete it. I have no clue why + pass + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.tearDown() + + +MyStringIO = StringIO + +_re_type = type(re.compile(r'')) + +notprinted_msg = """Did not find {0!r} in printed output (on {1}): +------- +{2!s} +------- +""" + +class AssertPrints: + """Context manager for testing that code prints certain text. + + Examples + -------- + >>> with AssertPrints("abc", suppress=False): + ... print("abcd") + ... print("def") + ... + abcd + def + """ + def __init__(self, s, channel='stdout', suppress=True): + self.s = s + if isinstance(self.s, (str, _re_type)): + self.s = [self.s] + self.channel = channel + self.suppress = suppress + + def __enter__(self): + self.orig_stream = getattr(sys, self.channel) + self.buffer = MyStringIO() + self.tee = Tee(self.buffer, channel=self.channel) + setattr(sys, self.channel, self.buffer if self.suppress else self.tee) + + def __exit__(self, etype, value, traceback): + __tracebackhide__ = True + + try: + if value is not None: + # If an error was raised, don't check anything else + return False + self.tee.flush() + setattr(sys, self.channel, self.orig_stream) + printed = self.buffer.getvalue() + for s in self.s: + if isinstance(s, _re_type): + assert s.search(printed), notprinted_msg.format(s.pattern, self.channel, printed) + else: + assert s in printed, notprinted_msg.format(s, self.channel, printed) + return False + finally: + self.tee.close() + +printed_msg = """Found {0!r} in printed output (on {1}): +------- +{2!s} +------- +""" + +class AssertNotPrints(AssertPrints): + """Context manager for checking that certain output *isn't* produced. + + Counterpart of AssertPrints""" + def __exit__(self, etype, value, traceback): + __tracebackhide__ = True + + try: + if value is not None: + # If an error was raised, don't check anything else + self.tee.close() + return False + self.tee.flush() + setattr(sys, self.channel, self.orig_stream) + printed = self.buffer.getvalue() + for s in self.s: + if isinstance(s, _re_type): + assert not s.search(printed),printed_msg.format( + s.pattern, self.channel, printed) + else: + assert s not in printed, printed_msg.format( + s, self.channel, printed) + return False + finally: + self.tee.close() + +@contextmanager +def make_tempfile(name): + """Create an empty, named, temporary file for the duration of the context.""" + open(name, "w", encoding="utf-8").close() + try: + yield + finally: + os.unlink(name) + +def fake_input(inputs): + """Temporarily replace the input() function to return the given values + + Use as a context manager: + + with fake_input(['result1', 'result2']): + ... + + Values are returned in order. If input() is called again after the last value + was used, EOFError is raised. + """ + it = iter(inputs) + def mock_input(prompt=''): + try: + return next(it) + except StopIteration as e: + raise EOFError('No more inputs given') from e + + return patch('builtins.input', mock_input) + +def help_output_test(subcommand=''): + """test that `ipython [subcommand] -h` works""" + cmd = get_ipython_cmd() + [subcommand, '-h'] + out, err, rc = get_output_error_code(cmd) + assert rc == 0, err + assert "Traceback" not in err + assert "Options" in out + assert "--help-all" in out + return out, err + + +def help_all_output_test(subcommand=''): + """test that `ipython [subcommand] --help-all` works""" + cmd = get_ipython_cmd() + [subcommand, '--help-all'] + out, err, rc = get_output_error_code(cmd) + assert rc == 0, err + assert "Traceback" not in err + assert "Options" in out + assert "Class" in out + return out, err + diff --git a/lib/python3.12/site-packages/IPython/utils/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/IPython/utils/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..dc7d04f51dcf0277b96bafb6d467a22efe19c2a1 Binary files /dev/null and b/lib/python3.12/site-packages/IPython/utils/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/IPython/utils/__pycache__/capture.cpython-312.pyc b/lib/python3.12/site-packages/IPython/utils/__pycache__/capture.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..dd863a026edf62d3a5d9d968a38c1cbd847075e8 Binary files /dev/null and b/lib/python3.12/site-packages/IPython/utils/__pycache__/capture.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/IPython/utils/__pycache__/decorators.cpython-312.pyc b/lib/python3.12/site-packages/IPython/utils/__pycache__/decorators.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e773b58d47c4f540d6db65431647b4705cd2cdc6 Binary files /dev/null and b/lib/python3.12/site-packages/IPython/utils/__pycache__/decorators.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/IPython/utils/__pycache__/dir2.cpython-312.pyc b/lib/python3.12/site-packages/IPython/utils/__pycache__/dir2.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9b9a437fc017ccc018496aa6ecf8277749ce21ce Binary files /dev/null and b/lib/python3.12/site-packages/IPython/utils/__pycache__/dir2.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/IPython/utils/__pycache__/encoding.cpython-312.pyc b/lib/python3.12/site-packages/IPython/utils/__pycache__/encoding.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..77b9d8bdbb803dfb9040586d368dcff2bc1850d4 Binary files /dev/null and b/lib/python3.12/site-packages/IPython/utils/__pycache__/encoding.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/IPython/utils/__pycache__/frame.cpython-312.pyc b/lib/python3.12/site-packages/IPython/utils/__pycache__/frame.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..06209d3d047cb858b595ef71bece7c81c4bf26f8 Binary files /dev/null and b/lib/python3.12/site-packages/IPython/utils/__pycache__/frame.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/IPython/utils/__pycache__/generics.cpython-312.pyc b/lib/python3.12/site-packages/IPython/utils/__pycache__/generics.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9bac0cbc08087bdf85825892f77639f864202aa4 Binary files /dev/null and b/lib/python3.12/site-packages/IPython/utils/__pycache__/generics.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/IPython/utils/__pycache__/ipstruct.cpython-312.pyc b/lib/python3.12/site-packages/IPython/utils/__pycache__/ipstruct.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..34ebee636cdc92b4f0a44e089e7e1f48fdd932b9 Binary files /dev/null and b/lib/python3.12/site-packages/IPython/utils/__pycache__/ipstruct.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/IPython/utils/__pycache__/log.cpython-312.pyc b/lib/python3.12/site-packages/IPython/utils/__pycache__/log.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4cd0f1e149dca3ece659035a2bdf4798f6355c80 Binary files /dev/null and b/lib/python3.12/site-packages/IPython/utils/__pycache__/log.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/IPython/utils/__pycache__/module_paths.cpython-312.pyc b/lib/python3.12/site-packages/IPython/utils/__pycache__/module_paths.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0c1c79bf0e9dcd96b2138598d7eda6219b866bac Binary files /dev/null and b/lib/python3.12/site-packages/IPython/utils/__pycache__/module_paths.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/IPython/utils/__pycache__/openpy.cpython-312.pyc b/lib/python3.12/site-packages/IPython/utils/__pycache__/openpy.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..721d6105bc870c64d8778ed0d695f512c1096af8 Binary files /dev/null and b/lib/python3.12/site-packages/IPython/utils/__pycache__/openpy.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/IPython/utils/__pycache__/process.cpython-312.pyc b/lib/python3.12/site-packages/IPython/utils/__pycache__/process.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ee162ef420d71616f2ca6f234bc52487e5fb8b55 Binary files /dev/null and b/lib/python3.12/site-packages/IPython/utils/__pycache__/process.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/IPython/utils/__pycache__/strdispatch.cpython-312.pyc b/lib/python3.12/site-packages/IPython/utils/__pycache__/strdispatch.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..49a16b5aa6d257d42ed3ce0d77f044eac0a5cb2d Binary files /dev/null and b/lib/python3.12/site-packages/IPython/utils/__pycache__/strdispatch.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/IPython/utils/__pycache__/tempdir.cpython-312.pyc b/lib/python3.12/site-packages/IPython/utils/__pycache__/tempdir.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..86b39afc90d2260345f719beb81c437fa8a6a819 Binary files /dev/null and b/lib/python3.12/site-packages/IPython/utils/__pycache__/tempdir.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/IPython/utils/__pycache__/wildcard.cpython-312.pyc b/lib/python3.12/site-packages/IPython/utils/__pycache__/wildcard.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c63390f827c3a350a1fe7c4a2d6ce95a9c848381 Binary files /dev/null and b/lib/python3.12/site-packages/IPython/utils/__pycache__/wildcard.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/hopper/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/hopper/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4f30a549ec9097e2e84b0723c9fde2bcbc1d905c Binary files /dev/null and b/lib/python3.12/site-packages/hopper/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/hopper/__pycache__/benchmark_attn.cpython-312.pyc b/lib/python3.12/site-packages/hopper/__pycache__/benchmark_attn.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..961fcb594f4843f08a6a2bd56235396de24321f1 Binary files /dev/null and b/lib/python3.12/site-packages/hopper/__pycache__/benchmark_attn.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/hopper/__pycache__/benchmark_flash_attention_fp8.cpython-312.pyc b/lib/python3.12/site-packages/hopper/__pycache__/benchmark_flash_attention_fp8.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8583d9a290c7b162736023c27fbe4f3fd2f27c52 Binary files /dev/null and b/lib/python3.12/site-packages/hopper/__pycache__/benchmark_flash_attention_fp8.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/hopper/__pycache__/benchmark_split_kv.cpython-312.pyc b/lib/python3.12/site-packages/hopper/__pycache__/benchmark_split_kv.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e87c0f1fc42d193277f3fa3ac868849009300f7d Binary files /dev/null and b/lib/python3.12/site-packages/hopper/__pycache__/benchmark_split_kv.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/hopper/__pycache__/flash_attn_interface.cpython-312.pyc b/lib/python3.12/site-packages/hopper/__pycache__/flash_attn_interface.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4797d25e05fc58b708b3d29b8ff6d8fccd4eb67a Binary files /dev/null and b/lib/python3.12/site-packages/hopper/__pycache__/flash_attn_interface.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/hopper/__pycache__/generate_kernels.cpython-312.pyc b/lib/python3.12/site-packages/hopper/__pycache__/generate_kernels.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b5170c1b92e07837ddbcc86ba026cde7476ced42 Binary files /dev/null and b/lib/python3.12/site-packages/hopper/__pycache__/generate_kernels.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/hopper/__pycache__/padding.cpython-312.pyc b/lib/python3.12/site-packages/hopper/__pycache__/padding.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8cc10565c66de20adeed025ca14c08aa722530e3 Binary files /dev/null and b/lib/python3.12/site-packages/hopper/__pycache__/padding.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/hopper/__pycache__/setup.cpython-312.pyc b/lib/python3.12/site-packages/hopper/__pycache__/setup.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..054cc4bb3967e2929e7b8925214d07aa6547f22f Binary files /dev/null and b/lib/python3.12/site-packages/hopper/__pycache__/setup.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/hopper/__pycache__/test_attn_kvcache.cpython-312.pyc b/lib/python3.12/site-packages/hopper/__pycache__/test_attn_kvcache.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..12727c3ca82f8408dbe50b24508450dfd9f0c6ac Binary files /dev/null and b/lib/python3.12/site-packages/hopper/__pycache__/test_attn_kvcache.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/hopper/__pycache__/test_flash_attn.cpython-312.pyc b/lib/python3.12/site-packages/hopper/__pycache__/test_flash_attn.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..30f0edd6e72e08637326deaa90b6ea0ecd679c59 Binary files /dev/null and b/lib/python3.12/site-packages/hopper/__pycache__/test_flash_attn.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/hopper/__pycache__/test_kvcache.cpython-312.pyc b/lib/python3.12/site-packages/hopper/__pycache__/test_kvcache.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f0d8aa39889d39b0caef2c927e36a55d3754a4ec Binary files /dev/null and b/lib/python3.12/site-packages/hopper/__pycache__/test_kvcache.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/hopper/__pycache__/test_util.cpython-312.pyc b/lib/python3.12/site-packages/hopper/__pycache__/test_util.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a94cc8a7de6db3e9e4c6154e51443ad541484ebc Binary files /dev/null and b/lib/python3.12/site-packages/hopper/__pycache__/test_util.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/hopper/benchmark_attn.py b/lib/python3.12/site-packages/hopper/benchmark_attn.py new file mode 100644 index 0000000000000000000000000000000000000000..5f7522a8ac365b79b37ebe166e602ac55193afe2 --- /dev/null +++ b/lib/python3.12/site-packages/hopper/benchmark_attn.py @@ -0,0 +1,425 @@ +from collections import namedtuple +from functools import partial +import math +from typing import NamedTuple +import torch +import torch.nn as nn +import torch.nn.functional as F + +import time + +try: + import cudnn +except ImportError: + cudnn = None +# cudnn = None + +Timing = NamedTuple('timing', [('mean', float)]) + + +from einops import rearrange, repeat + +# from flash_attn.utils.benchmark import benchmark_forward, benchmark_backward, benchmark_combined, benchmark_all, benchmark_fwd_bwd, pytorch_profiler +from flash_attn.utils.benchmark import benchmark_forward, benchmark_backward, benchmark_combined, benchmark_all, benchmark_fwd_bwd, pytorch_profiler +from flash_attn.flash_attn_interface import flash_attn_func, flash_attn_varlen_func +from flash_attn_interface import flash_attn_func as flash_attn_func_v3 +# from flash_attn_interface import flash_attn_with_kvcache as flash_attn_func_v3 +from flash_attn_interface import flash_attn_varlen_func as flash_attn_varlen_func_v3 + +from triton.testing import do_bench + +try: + from triton_fused_attention import attention as triton_attention +except ImportError: + triton_attention = None +triton_attention = None + + +def time_fwd(func, *args, repeats=30, verbose=True, desc="", **kwargs): + # # Warmup + # for _ in range(5): + # func(*args, **kwargs) + # time.sleep(1) + # return benchmark_forward(func, *args, **kwargs, repeats=repeats, verbose=verbose, desc=desc)[1] + # s = torch.cuda.Stream() + # s.wait_stream(torch.cuda.current_stream()) + # with torch.cuda.stream(s): + # for _ in range(2): + # out = func(*args, **kwargs) + # torch.cuda.current_stream().wait_stream(s) + # graph = torch.cuda.CUDAGraph() + # with torch.cuda.graph(graph): + # out = func(*args, **kwargs) + # time_f = benchmark_forward(lambda: graph.replay(), repeats=repeats, verbose=verbose, desc=desc) + # # return time_f[1].mean + # return time_f[1] + return Timing(do_bench(lambda: func(*args, **kwargs), warmup=5, rep=repeats) * 1e-3) + + +def flops(batch, nheads, seqlen_q, seqlen_k, headdim, causal=False, window_size=(-1, -1)): + if causal: + avg_seqlen = (max(0, seqlen_k - seqlen_q) + seqlen_k) / 2 + else: + if window_size == (-1, -1): + avg_seqlen = seqlen_k + else: + row_idx = torch.arange(seqlen_q, device='cuda') + col_left = torch.maximum(row_idx + seqlen_k - seqlen_q - window_size[0], torch.tensor(0)) + col_right = torch.minimum(row_idx + seqlen_k - seqlen_q - window_size[1], torch.tensor(seqlen_k - 1)) + avg_seqlen = (col_right - col_left + 1).float().mean().item() + return batch * nheads * 2 * seqlen_q * avg_seqlen * headdim * 2 + + +def convert_to_cudnn_type(torch_type): + if torch_type == torch.float16: + return cudnn.data_type.HALF + elif torch_type == torch.bfloat16: + return cudnn.data_type.BFLOAT16 + elif torch_type == torch.float32: + return cudnn.data_type.FLOAT + elif torch_type == torch.int32: + return cudnn.data_type.INT32 + elif torch_type == torch.int64: + return cudnn.data_type.INT64 + else: + raise ValueError("Unsupported tensor data type.") + + +def cudnn_spda_setup(q, k, v, causal=False, window_size_left=-1): + b, nheads, seqlen_q, headdim = q.shape + _, nheads_k, seqlen_k, _ = k.shape + assert v.shape == (b, nheads_k, seqlen_k, headdim) + assert cudnn is not None, 'CUDNN is not available' + q_gpu, k_gpu, v_gpu = q, k, v + o_gpu = torch.empty_like(q_gpu) + stats_gpu = torch.empty(b, nheads, seqlen_q, 1, dtype=torch.float32, device=q.device) + graph = cudnn.pygraph( + io_data_type=convert_to_cudnn_type(q.dtype), + intermediate_data_type=cudnn.data_type.FLOAT, + compute_data_type=cudnn.data_type.FLOAT, + ) + q = graph.tensor_like(q_gpu.detach()) + k = graph.tensor_like(k_gpu.detach()) + v = graph.tensor_like(v_gpu.detach()) + + o, stats = graph.sdpa( + name="sdpa", + q=q, + k=k, + v=v, + is_inference=False, + attn_scale=1.0 / math.sqrt(headdim), + # use_causal_mask_bottom_right=causal or window_size_left >= 0, + use_causal_mask=causal or window_size_left >= 0, + sliding_window_length=window_size_left if window_size_left >= 0 and not causal else None, + ) + + o.set_output(True).set_dim(o_gpu.shape).set_stride(o_gpu.stride()) + stats.set_output(True).set_data_type(cudnn.data_type.FLOAT) + + graph.validate() + graph.build_operation_graph() + graph.create_execution_plans([cudnn.heur_mode.A, cudnn.heur_mode.FALLBACK]) + graph.check_support() + graph.build_plans() + + variant_pack = { + q: q_gpu, + k: k_gpu, + v: v_gpu, + o: o_gpu, + stats: stats_gpu, + } + + workspace = torch.empty(graph.get_workspace_size(), device="cuda", dtype=torch.uint8) + + def run(*args, **kwargs): + graph.execute(variant_pack, workspace) + return o_gpu + + return run + + +def cudnn_spda_bwd_setup(q, k, v, o, g, lse, causal=False, window_size_left=-1): + b, nheads, seqlen_q, headdim = q.shape + _, nheads_k, seqlen_k, _ = k.shape + assert v.shape == (b, nheads_k, seqlen_k, headdim) + assert g.shape == (b, nheads, seqlen_q, headdim) + assert o.shape == (b, nheads, seqlen_q, headdim) + assert lse.shape == (b, nheads, seqlen_q, 1) + assert cudnn is not None, 'CUDNN is not available' + q_gpu, k_gpu, v_gpu, o_gpu, g_gpu = q, k, v, o, g + dq_gpu = torch.empty_like(q_gpu) + dk_gpu = torch.empty_like(k_gpu) + dv_gpu = torch.empty_like(v_gpu) + graph = cudnn.pygraph( + io_data_type=convert_to_cudnn_type(q.dtype), + intermediate_data_type=cudnn.data_type.FLOAT, + compute_data_type=cudnn.data_type.FLOAT, + ) + q = graph.tensor_like(q_gpu.detach()) + k = graph.tensor_like(k_gpu.detach()) + v = graph.tensor_like(v_gpu.detach()) + o = graph.tensor_like(o_gpu.detach()) + g = graph.tensor_like(g_gpu.detach()) + stats = graph.tensor_like(lse.detach()) + + dq, dk, dv = graph.sdpa_backward( + name="sdpa_backward", + q=q, + k=k, + v=v, + o=o, + dO=g, + stats=stats, + attn_scale=1.0 / math.sqrt(headdim), + # use_causal_mask_bottom_right=causal or window_size_left >= 0, + use_causal_mask=causal or window_size_left >= 0, + sliding_window_length=window_size_left if window_size_left >= 0 and not causal else None, + ) + + dq.set_output(True).set_dim(dq_gpu.shape).set_stride(dq_gpu.stride()) + dk.set_output(True).set_dim(dk_gpu.shape).set_stride(dk_gpu.stride()) + dv.set_output(True).set_dim(dv_gpu.shape).set_stride(dv_gpu.stride()) + + graph.validate() + graph.build_operation_graph() + graph.create_execution_plans([cudnn.heur_mode.A, cudnn.heur_mode.FALLBACK]) + graph.check_support() + graph.build_plans() + + variant_pack = { + q: q_gpu, + k: k_gpu, + v: v_gpu, + o: o_gpu, + g: g_gpu, + stats: lse, + dq: dq_gpu, + dk: dk_gpu, + dv: dv_gpu, + } + + workspace = torch.empty(graph.get_workspace_size(), device="cuda", dtype=torch.uint8) + + def run(*args, **kwargs): + graph.execute(variant_pack, workspace) + return dq_gpu, dk_gpu, dv_gpu + + return run + + +torch.manual_seed(0) +repeats = 10 +dropout_p = 0.0 +causal = False +dtype = torch.bfloat16 +# dtype = torch.float8_e4m3fn +dtype_gen = torch.bfloat16 if dtype == torch.float8_e4m3fn else dtype +device = 'cuda' +verbose = True +varlen = False +page_size = None +softcap = 0.0 +V_colmajor = False +deterministic = False +batch_size = 2 +# seqlen = 2048 +seqlen = 8192 +# seqlen = 4096 +# seqlen = 2047 +dim = 2048 +# headdim = 128 +# headdim = 64 +headdim = 256 +# for headdim in [64, 128, 256]: +# bs_seqlen_vals = [(32, 512), (16, 1024), (8, 2048), (4, 4096), (2, 8192), (1, 16384)] +# bs_seqlen_vals = [(16, 1024), (8, 2048), (4, 4096), (2, 8192), (1, 16384)] +# bs_seqlen_vals = [(32, 512), (16, 1024)] +# bs_seqlen_vals = [(2, 64 * 132)] +bs_seqlen_vals = [(2, 8192)] +# bs_seqlen_vals = [(1, 16 * 1024)] +time_f = {} +time_b = {} + +# tflops_matmul = {} +# m, n = 8192, 8192 +# for k in [512, 1024, 1536, 2048, 2560, 3072, 3584, 4096, 4608, 5120, 5632, 6144, 6656, 7168, 7680, 8192]: +# a = torch.randn(m, k, device=device, dtype=dtype) +# b = torch.randn(n, k, device=device, dtype=dtype).transpose(-1, -2) +# nFLOPS_matmul = 2 * m * n * k +# m5 = time_fwd(torch.matmul, a, b, desc='cuBLAS') +# print(f'cuBLAS: {m5.mean * 1e3:.3f}ms, {(nFLOPS_matmul / m5.mean * 1e-12):.1f} TFLOPS') +# tflops_matmul[k] = nFLOPS_matmul / m5.mean * 1e-12 +# # import pickle +# # # with open(f'flash3_attn_time_h100_hdim{headdim}_causal.plk', 'wb') as fp: +# # with open(f'flash3_matmul_tflops_h100.plk', 'wb') as fp: +# # pickle.dump(tflops_matmul, fp, protocol=pickle.HIGHEST_PROTOCOL) +# exit(0) + +# for headdim in [64, 128, 256]: +# for headdim in [64, 96, 128, 192]: +# for headdim in [64, 96, 128, 192, 256]: +# for headdim in [64, 96, 128]: +# for headdim in [64, 128, 256]: +# for headdim in [64, 96, 128, 192, 256]: +for headdim in [128]: + nheads = dim // headdim + # headdim = 64 + # batch_size = 64 + # seqlen = 512 + # nheads = 8 + # headdim = 128 + nheads_kv = nheads + # nheads_kv = nheads // 4 + + for batch_size, seqlen in bs_seqlen_vals: + num_splits = 1 + window_size = (-1, -1) + # window_size = (seqlen // 2 - 1, 0) + sink_token_length = 0 + pack_gqa = None + # seqlen_q = 64 + seqlen_q = seqlen + leftpad_k = None + # leftpad_k = torch.full((batch_size,), 0, device=device, dtype=torch.int32) + q = torch.randn(batch_size, seqlen_q, nheads, headdim, device=device, dtype=dtype_gen, requires_grad=True) + k = torch.randn(batch_size, seqlen, nheads_kv, headdim, device=device, dtype=dtype_gen, requires_grad=True) + v = torch.randn(batch_size, seqlen, nheads_kv, headdim, device=device, dtype=dtype_gen, requires_grad=True) + q, k, v = [x.detach().to(dtype).requires_grad_() for x in [q, k, v]] + v_colmajor = v.detach().transpose(-1, -3).contiguous().transpose(-1, -3).requires_grad_() + v_fa3 = v if not V_colmajor else v_colmajor + # q = torch.randint(-2, 3, (batch_size, seqlen, nheads, headdim), device=device, dtype=torch.int32).to(dtype) + # k = torch.randint(-2, 3, (batch_size, seqlen, nheads, headdim), device=device, dtype=torch.int32).to(dtype) + # v = torch.randint(-2, 3, (batch_size, seqlen, nheads, headdim), device=device, dtype=torch.int32).to(dtype) + g = torch.randn(batch_size, seqlen_q, nheads, headdim, device=device, dtype=dtype_gen, requires_grad=True) + o = torch.randn(batch_size, seqlen_q, nheads, headdim, device=device, dtype=dtype_gen, requires_grad=True) + stats = torch.randn(batch_size, seqlen_q, nheads, 1, device=device, dtype=torch.float32) + a = torch.randn(batch_size, seqlen, seqlen, device=device, dtype=dtype_gen) + b = torch.randn(batch_size, dim * 2, seqlen, device=device, dtype=dtype_gen).transpose(-1, -2) + # x = torch.randn(batch_size * seqlen, 4096, device=device, dtype=dtype) + # w = torch.randn(4096 * 2, 4096, device=device, dtype=dtype).transpose(-1, -2) + if varlen: + q_unpad, k_unpad, v_unpad = [rearrange(x.detach(), "b s h d -> (b s) h d").requires_grad_() for x in [q, k, v]] + cu_seqlens_q = torch.arange(batch_size + 1, device=device, dtype=torch.int32) * seqlen_q + cu_seqlens_k = torch.arange(batch_size + 1, device=device, dtype=torch.int32) * seqlen + # cu_seqlens_q = torch.tensor([0, 248, 249, 250, 251, 252, 253, 254, 255, 256], device=device, dtype=torch.int32) + # q_unpad = q_unpad[:256] + # seqlen_q = 256 + # cu_seqlens_q = torch.tensor([0, 376, 377, 378, 379, 380, 381, 382, 383, 384], device=device, dtype=torch.int32) + # q_unpad = q_unpad[:384] + # seqlen_q = 384 + if page_size is not None: + assert seqlen % page_size == 0 + k_paged, v_paged = [rearrange(x, "b (n p) h d -> (b n) p h d", p=page_size) for x in [k, v]] + page_table = rearrange(torch.arange(batch_size * seqlen // page_size, device=device, dtype=torch.int32), + "(b s) -> b s", s=seqlen // page_size) + else: + page_table = None + + for causal in [False, True]: + # for causal in [False]: + print(f"\n### {headdim = }, {causal = }, {seqlen = } ###") + nFLOPS = flops(batch_size, nheads, seqlen_q, seqlen, headdim, causal=causal, window_size=window_size) + if cudnn is not None: + # if False: + if headdim <= 256 and dtype != torch.float8_e4m3fn: + cudnn_spda = cudnn_spda_setup(q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2), causal=causal, window_size_left=window_size[0]) + cudnn_spda_bwd = cudnn_spda_bwd_setup(q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2), o.transpose(1, 2), g.transpose(1, 2), stats.transpose(1, 2), causal=causal, window_size_left=window_size[0]) + # _, m0 = benchmark_forward(flash_attn_func, q, k, v, dropout_p, causal=causal, repeats=repeats, verbose=verbose, desc='Fav2') + if dtype != torch.float8_e4m3fn: + # if False: + if not varlen: + m0 = time_fwd(flash_attn_func, q, k, v, dropout_p, causal=causal, window_size=window_size, softcap=softcap, repeats=repeats, verbose=verbose, desc='Fav2') + else: + m0 = time_fwd(flash_attn_varlen_func, q_unpad, k_unpad, v_unpad, cu_seqlens_q, cu_seqlens_k, seqlen_q, seqlen, dropout_p, causal=causal, window_size=window_size, softcap=softcap, repeats=repeats, verbose=verbose, desc='Fav2') + time_f[(causal, headdim, batch_size, seqlen), "Flash2"] = m0.mean + time.sleep(1) + if not varlen: + _, m0b = benchmark_backward(flash_attn_func, q, k, v, dropout_p, causal=causal, window_size=window_size, softcap=softcap, deterministic=deterministic, + repeats=repeats, verbose=False, desc='Fav2') + else: + _, m0b = benchmark_backward(flash_attn_varlen_func, q_unpad, k_unpad, v_unpad, cu_seqlens_q, cu_seqlens_k, seqlen_q, seqlen, dropout_p, causal=causal, window_size=window_size, softcap=softcap, deterministic=deterministic, + repeats=repeats, verbose=False, desc='Fav2') + time_b[(causal, headdim, batch_size, seqlen), "Flash2"] = m0b.mean + # pytorch_profiler(flash_attn_func, q, k, v, dropout_p, causal=causal, backward=True) + if headdim <= 256 and dtype != torch.float8_e4m3fn: + if triton_attention is not None: + qt, kt, vt = [x.detach().transpose(1, 2).contiguous().requires_grad_() for x in [q, k, v]] + time.sleep(1) # Sleep to avoid residual power throttling from the previous benchmark + m3 = time_fwd(triton_attention, qt, kt, vt, causal, 1 / math.sqrt(headdim), repeats=repeats, verbose=verbose, desc='Triton') + time_f[(causal, headdim, batch_size, seqlen), "Triton"] = m3.mean + # if causal: # triton bwd only works w causal for now + # time.sleep(1) + # _, m3b = benchmark_backward(triton_attention, qt, kt, vt, causal, 1 / math.sqrt(headdim), repeats=repeats, verbose=verbose, desc='Triton') + # time_b[(causal, headdim, batch_size, seqlen), "Triton"] = m3b.mean + # # pytorch_profiler(triton_attention, q.transpose(1, 2).contiguous(), k.transpose(1, 2).contiguous(), v.transpose(1, 2).contiguous(), causal, 1 / math.sqrt(headdim), backward=True) + if cudnn is not None: + # if False: + if headdim <= 256 and dtype != torch.float8_e4m3fn: + time.sleep(1) # Sleep to avoid residual power throttling from the previous benchmark + m2 = time_fwd(cudnn_spda, repeats=repeats, verbose=verbose, desc='CuDNN') + time_f[(causal, headdim, batch_size, seqlen), "cuDNN"] = m2.mean + time.sleep(1) + m2b = time_fwd(cudnn_spda_bwd, repeats=repeats, verbose=verbose, desc='CuDNN') + time_b[(causal, headdim, batch_size, seqlen), "cuDNN"] = m2b.mean + # pytorch_profiler(cudnn_spda, backward=False) + # pytorch_profiler(cudnn_spda_bwd, backward=False) + + time.sleep(1) + if not varlen: + # m1 = time_fwd(flash_attn_func_v3, q, k if page_size is None else k_paged, v_fa3 if page_size is None else v_paged, cache_leftpad = leftpad_k, page_table=page_table, causal=causal, window_size=window_size, sink_token_length=sink_token_length, softcap=softcap, num_splits=num_splits, pack_gqa=pack_gqa, repeats=repeats, verbose=verbose, desc='Fav3') + m1 = time_fwd(flash_attn_func_v3, q, k if page_size is None else k_paged, v_fa3 if page_size is None else v_paged, causal=causal, window_size=window_size, sink_token_length=sink_token_length, softcap=softcap, num_splits=num_splits, pack_gqa=pack_gqa, repeats=repeats, verbose=verbose, desc='Fav3') + # pytorch_profiler(flash_attn_func_v3, q, k if page_size is None else k_paged, v_fa3 if page_size is None else v_paged, page_table=page_table, causal=causal, window_size=window_size, softcap=softcap, num_splits=num_splits, pack_gqa=pack_gqa) + else: + m1 = time_fwd(flash_attn_varlen_func_v3, q_unpad, k_unpad, v_unpad, cu_seqlens_q, cu_seqlens_k, None, None, seqlen_q, seqlen, causal=causal, window_size=window_size, softcap=softcap, num_splits=num_splits, pack_gqa=pack_gqa, repeats=repeats, verbose=verbose, desc='Fav3') + # pytorch_profiler(flash_attn_varlen_func_v3, q_unpad, k_unpad, v_unpad, cu_seqlens_q, cu_seqlens_k, seqlen_q, seqlen, causal=causal, window_size=window_size, softcap=softcap, num_splits=num_splits) + time_f[(causal, headdim, batch_size, seqlen), "Flash3"] = m1.mean + # time.sleep(1) + # m5 = time_fwd(torch.bmm, a, b, desc='cuBLAS', repeats=repeats, verbose=False) + # nFLOPS_matmul = nFLOPS + # nFLOPS_matmul = 2 * x.shape[0] * x.shape[1] * w.shape[1] + # m5 = time_fwd(torch.matmul, x, w, desc='cuBLAS') + if dtype != torch.float8_e4m3fn: + time.sleep(1) + if not varlen: + _, m1b = benchmark_backward(flash_attn_func_v3, q, k, v, causal=causal, window_size=window_size, sink_token_length=sink_token_length, softcap=softcap, deterministic=deterministic, + repeats=repeats, verbose=False, desc='Fav3') + else: + _, m1b = benchmark_backward(flash_attn_varlen_func_v3, q_unpad, k_unpad, v_unpad, cu_seqlens_q, cu_seqlens_k, None, None, seqlen_q, seqlen, causal=causal, window_size=window_size, softcap=softcap, deterministic=deterministic, + repeats=repeats, verbose=False, desc='Fav3') + time_b[(causal, headdim, batch_size, seqlen), "Flash3"] = m1b.mean + # time.sleep(1) + # if not varlen: + # pytorch_profiler(flash_attn_func_v3, q, k, v, causal=causal, deterministic=deterministic, backward=True) + # else: + # pytorch_profiler(flash_attn_varlen_func_v3, q_unpad, k_unpad, v_unpad, cu_seqlens_q, cu_seqlens_k, seqlen_q, seqlen, causal=causal, deterministic=deterministic, backward=True) + # benchmark_forward(torch.clone, k, repeats=repeats, verbose=verbose, desc='Memcpy') + + if dtype != torch.float8_e4m3fn: + # if False: + print(f'Fav2 fwd: {m0.mean * 1e3:.3f}ms, {(nFLOPS / m0.mean * 1e-12):.1f} TFLOPS') + print(f'Fav2 bwd: {m0b.mean * 1e3:.3f}ms, {(2.5 * nFLOPS / m0b.mean * 1e-12):.1f} TFLOPS') + if headdim <= 256 and dtype != torch.float8_e4m3fn: + if triton_attention is not None: + print(f'Triton fwd: {m3.mean * 1e3:.3f}ms, {(nFLOPS / m3.mean * 1e-12):.1f} TFLOPS') + # if causal: + # print(f'Triton bwd: {m3b.mean * 1e3:.3f}ms, {(2.5 * nFLOPS / m3b.mean * 1e-12):.1f} TFLOPS') + if cudnn is not None: + print(f'CuDNN fwd: {m2.mean * 1e3:.3f}ms, {(nFLOPS / m2.mean * 1e-12):.1f} TFLOPS') + print(f'CuDNN bwd: {m2b.mean * 1e3:.3f}ms, {(2.5 * nFLOPS / m2b.mean * 1e-12):.1f} TFLOPS') + print(f'Fav3 fwd: {m1.mean * 1e3:.3f}ms, {(nFLOPS / m1.mean * 1e-12):.1f} TFLOPS') + if dtype != torch.float8_e4m3fn: + print(f'Fav3 bwd: {m1b.mean * 1e3:.3f}ms, {(2.5 * nFLOPS / m1b.mean * 1e-12):.1f} TFLOPS') + # benchmark_forward(torch.square, k) + # print(f'cuBLAS: {m5.mean * 1e3:.3f}ms, {(nFLOPS_matmul / m5.mean * 1e-12):.1f} TFLOPS') + # print(time_f) + # print(time_b) + + # import pickle + # # with open(f'flash3_attn_time_h100_hdim{headdim}_causal.plk', 'wb') as fp: + # # with open(f'flash3_attn_time_h100_cudnn_triton_20241208.plk', 'wb') as fp: + # with open(f'flash3_attn_time_h100_fa3_20241208.plk', 'wb') as fp: + # # with open(f'flash3_attn_time_h100_fp8_hdim{headdim}.plk', 'wb') as fp: + # # with open(f'flash3_attn_time_h100_hdim{headdim}_1031.plk', 'wb') as fp: + # pickle.dump((time_f, time_b), fp, protocol=pickle.HIGHEST_PROTOCOL) diff --git a/lib/python3.12/site-packages/hopper/benchmark_split_kv.py b/lib/python3.12/site-packages/hopper/benchmark_split_kv.py new file mode 100644 index 0000000000000000000000000000000000000000..d3d83590a9631981e45db88c598f000e95b319dc --- /dev/null +++ b/lib/python3.12/site-packages/hopper/benchmark_split_kv.py @@ -0,0 +1,325 @@ +import torch +import flash_attn +import flash_attn_interface +import itertools +import time +import math + +import torch.utils.benchmark as benchmark + +def round_up_to_power_of_2(x): + if x <= 1: + return 1 + return 1 << (x - 1).bit_length() + +def timeit(fn, *args, **kwargs): + torch.cuda.synchronize() + + # Warmup + for _ in range(5): + fn(*args, **kwargs) + + # Benchmark using PyTorch Timer + t = benchmark.Timer( + stmt='fn(*args, **kwargs)', + globals={'fn': fn, 'args': args, 'kwargs': kwargs} + ) + + # Measure execution time + measurement = t.timeit(20) # Runs the function 20 times + # measurement = t.blocked_autorange(min_run_time=1) + avg_time = measurement.mean # Average time in seconds + + return avg_time + +def main(): + num_sms = torch.cuda.get_device_properties( + torch.cuda.current_device() + ).multi_processor_count + + max_splits = 129 + check_all_splits = False + + causal = True + # causal = False + # dtype=torch.float16 + dtype=torch.bfloat16 + + torch.manual_seed(42) + + model_configs = [ + # ("Gemma-2-2B", 8, 4, 256), + # ("Gemma-2-9B", 16, 8, 256), + # ("Gemma-2-27B", 32, 16, 128), + # ("Qwen-2.5-0.5B", 14, 2, 64), + # ("Qwen-2.5-1.5B", 12, 2, 128), + # ("Qwen-2.5-7B", 28, 4, 128), + # ("Llama-3.1-8B", 32, 8, 128), + ("Llama-3.1-70B", 64, 8, 128), + # ("Llama-3.1-405B", 128, 8, 128), + # ("Llama-3.2-1B", 32, 8, 64), + # ("Llama-3.2-3B", 24, 8, 128), + # ("Nemotron-4-15B", 48, 8, 128), + ] + + all_batch_configs = [] + + all_batch_configs.extend(itertools.product( + # [1024, 2048, 4096, 8192, 16384, 32768, 131072], # context_seqlen + [4096, 16384, 65536], # context_seqlen + # [131072], # context_seqlen + # [i for i in range(1, (num_sms) + 1)], # num_requests + [1, 4, 8, 16], # num_requests + # [1], # num_requests + [1, 4, 8, 16], # query_seqlen + # [1], # query_seqlen + )) + + num_caches = max(reqs for _, reqs, _ in all_batch_configs) + cache_seqlen = max(seqlen for seqlen, _, _ in all_batch_configs) + + for model_name, nheads_q, nheads_kv, headdim in model_configs: + k_cache = torch.randn( + (num_caches, cache_seqlen, nheads_kv, headdim), device="cuda", dtype=dtype + ) + v_cache = torch.randn( + (num_caches, cache_seqlen, nheads_kv, headdim), device="cuda", dtype=dtype + ) + print(f"***{model_name}***") + print(f"QHEADS:{nheads_q}, KVHEADS:{nheads_kv}, HEADDIM:{headdim}") + + if check_all_splits is False: + print(f"{'CONTEXT':<9}{'BSZ':<5}{'QLEN':<6}{'FA2':<10}{'FA3':<9}{'RATIO':<7}{'GB/s':<10}") + + for context_seqlen, num_requests, query_seqlen in all_batch_configs: + bytes_kv = (context_seqlen * num_requests * nheads_kv * headdim * 4) + bytes_q = (query_seqlen * num_requests * nheads_q * headdim * 4) + blockH = round_up_to_power_of_2(nheads_q//nheads_kv) + blockM = 128 # true for hdim 128 causal and hdim 64 + blockM_div_H = blockM//blockH + num_work_tiles = nheads_kv * num_requests * math.ceil(query_seqlen/blockM_div_H) + + q = torch.randn((num_requests, query_seqlen, nheads_q, headdim), device="cuda", dtype=dtype) + cache_idxs = torch.randperm(num_caches, dtype=torch.int32, device="cuda")[:num_requests] + cache_seqlens = torch.tensor( + [context_seqlen] * num_requests, dtype=torch.int32, device="cuda" + ) + + fa2_time_heuristic = timeit( + flash_attn.flash_attn_with_kvcache, + q=q, + k_cache=k_cache, + v_cache=v_cache, + cache_seqlens=cache_seqlens, + cache_batch_idx=cache_idxs, + causal=causal, + ) * 1000. * 1000. + # fastest_splitk_time = float("inf") + # fastest_splitk = 0 + # for i in range(1, max_splits): + # t = timeit( + # flash_attn.flash_attn_with_kvcache, + # q=q, + # k_cache=k_cache, + # v_cache=v_cache, + # cache_seqlens=cache_seqlens, + # cache_batch_idx=cache_idxs, + # causal=causal, + # num_splits=i, + # ) * 1000. * 1000. + # if t < fastest_splitk_time: + # fastest_splitk_time = t + # fastest_splitk = i + + fa3_time_one_split = timeit( + flash_attn_interface.flash_attn_with_kvcache, + q=q, + k_cache=k_cache, + v_cache=v_cache, + cache_seqlens=cache_seqlens, + cache_batch_idx=cache_idxs, + causal=causal, + gqa_parallel=False, + num_splits=1, + ) * 1000. * 1000. + + fa3_time_gqa_heuristic = timeit( + flash_attn_interface.flash_attn_with_kvcache, + q=q, + k_cache=k_cache, + v_cache=v_cache, + cache_seqlens=cache_seqlens, + cache_batch_idx=cache_idxs, + causal=causal, + gqa_parallel=True, + num_splits=0, + max_seqlen_k_hint=context_seqlen + ) * 1000. * 1000. + + if check_all_splits: + + fa3_fastest_num_splits = 0 + fa3_fastest_splitk_time = float("inf") + + for num_splits in range(1, max_splits): + t = timeit( + flash_attn_interface.flash_attn_with_kvcache, + q=q, + k_cache=k_cache, + v_cache=v_cache, + cache_seqlens=cache_seqlens, + cache_batch_idx=cache_idxs, + causal=causal, + gqa_parallel=False, + num_splits=num_splits + ) * 1000. * 1000. + + out0 = flash_attn_interface.flash_attn_with_kvcache( + q=q, + k_cache=k_cache, + v_cache=v_cache, + cache_seqlens=cache_seqlens, + cache_batch_idx=cache_idxs, + causal=causal, + gqa_parallel=False, + num_splits=num_splits + ) + + out1 = flash_attn_interface.flash_attn_with_kvcache( + q=q, + k_cache=k_cache, + v_cache=v_cache, + cache_seqlens=cache_seqlens, + cache_batch_idx=cache_idxs, + causal=causal, + gqa_parallel=False, + num_splits=1 + ) + + max_diff = (out0 - out1).abs().max().item() + mean_diff = (out0 - out1).abs().mean().item() + # print (f"splits {num_splits}, out diff-max, {max_diff}, out diff-mean, {mean_diff}, time {t:.2f}") + # print (f"splits {num_splits}, time {t:.2f}") + + if math.isnan(max_diff) or math.isnan(mean_diff) or max_diff > 2e-3 or mean_diff > 1e-4: + print(f"Numerical error too high: Splits: {num_splits}, Max: {max_diff}, Mean: {mean_diff}") + + if t < fa3_fastest_splitk_time: + fa3_fastest_splitk_time = t + fa3_fastest_num_splits = num_splits + + fa3_fastest_num_splits_gqa = 0 + fa3_fastest_splitk_time_gqa = float("inf") + for num_splits in range(1, max_splits): + + t = timeit( + flash_attn_interface.flash_attn_with_kvcache, + q=q, + k_cache=k_cache, + v_cache=v_cache, + cache_seqlens=cache_seqlens, + cache_batch_idx=cache_idxs, + causal=causal, + gqa_parallel=True, + num_splits=num_splits + ) * 1000. * 1000. + + out0 = flash_attn_interface.flash_attn_with_kvcache( + q=q, + k_cache=k_cache, + v_cache=v_cache, + cache_seqlens=cache_seqlens, + cache_batch_idx=cache_idxs, + causal=causal, + gqa_parallel=True, + num_splits=num_splits + ) + + out1 = flash_attn_interface.flash_attn_with_kvcache( + q=q, + k_cache=k_cache, + v_cache=v_cache, + cache_seqlens=cache_seqlens, + cache_batch_idx=cache_idxs, + causal=causal, + gqa_parallel=True, + num_splits=1 + ) + + max_diff = (out0 - out1).abs().max().item() + mean_diff = (out0 - out1).abs().mean().item() + # print (f"gqa splits {num_splits}, out gqa diff-max {max_diff}, out gqa diff-mean {mean_diff}, time {t:.2f}") + # print (f"gqa splits {num_splits}, time {t:.2f}") + + if math.isnan(max_diff) or math.isnan(mean_diff) or max_diff > 2e-3 or mean_diff > 1e-4: + print(f"Numerical error too high (gqa): Splits: {num_splits}, Max: {max_diff}, Mean: {mean_diff}") + + if t < fa3_fastest_splitk_time_gqa: + fa3_fastest_splitk_time_gqa = t + fa3_fastest_num_splits_gqa = num_splits + + efficiency = (num_work_tiles * fa3_fastest_num_splits_gqa)/num_sms + heuristic_ratio = fa3_time_gqa_heuristic/fa3_fastest_splitk_time_gqa + # remeasure to smooth anomalies + if heuristic_ratio > 1.1: + + fa3_time_gqa_heuristic = timeit( + flash_attn_interface.flash_attn_with_kvcache, + q=q, + k_cache=k_cache, + v_cache=v_cache, + cache_seqlens=cache_seqlens, + cache_batch_idx=cache_idxs, + causal=causal, + gqa_parallel=True, + # num_splits=num_splits_select, + # num_splits=1, + num_splits=0, + max_seqlen_k_hint=context_seqlen + ) * 1000. * 1000. + + fa3_fastest_splitk_time_gqa = timeit( + flash_attn_interface.flash_attn_with_kvcache, + q=q, + k_cache=k_cache, + v_cache=v_cache, + cache_seqlens=cache_seqlens, + cache_batch_idx=cache_idxs, + causal=causal, + gqa_parallel=True, + num_splits=fa3_fastest_num_splits_gqa + ) * 1000. * 1000. + + if check_all_splits is True: + print( + f"CONTEXT:{context_seqlen}, BSZ:{num_requests}, QLEN:{query_seqlen}, " + f"FA2:{fa2_time_heuristic:.2f}, " + # f"FA2 MANUAL:{fastest_splitk_time:.2f}, " + # f"FA2 NUM SPLITS:{fastest_splitk}, " + # f"FA3 NOGQA NOSPLIT:{fa3_time_one_split:.2f}, " + # f"FA3 NOGQA SPLIT MANUAL:{fa3_fastest_splitk_time:.2f}, " + # f"FA3 NOSPLIT:{fa3_time_one_split_gqa:.2f}, " + f"FA3 SPLIT MANUAL:{fa3_fastest_splitk_time_gqa:.2f}, " + f"FA3:{fa3_time_gqa_heuristic:.2f}, " + # f"FA3 RATIO (NONSPLIT/SPLIT):{fa3_time_one_split_gqa/fa3_time_gqa_heuristic:.2f}, " + # f"FA2 NUM SPLITS:{fastest_splitk}, " + # f"FA3 NOGQA NUM SPLITS:{fa3_fastest_num_splits}, " + f"FA3 NUM SPLITS:{fa3_fastest_num_splits_gqa}, " + # f"RATIO (FA2/3):{fa2_time_heuristic/fa3_time_gqa_heuristic:.2f}, " + f"RATIO:{fa3_time_gqa_heuristic/fa3_fastest_splitk_time_gqa:.2f}, " + f"EFF:{efficiency:.2f}, " + f"GB/s:{bytes_kv/fa3_time_gqa_heuristic * 1e-3:.2f}" + ) + + if check_all_splits is False: + print( + f"{context_seqlen:<9}{num_requests:<5}{query_seqlen:<6}" + f"{fa2_time_heuristic:<10.2f}{fa3_time_gqa_heuristic:<9.2f}" + f"{fa2_time_heuristic/fa3_time_gqa_heuristic:<7.2f}" + f"{bytes_kv/fa3_time_gqa_heuristic * 1e-3:<10.2f}" + ) + + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/lib/python3.12/site-packages/hopper/generate_kernels.py b/lib/python3.12/site-packages/hopper/generate_kernels.py new file mode 100644 index 0000000000000000000000000000000000000000..e741c13826f196f7b03463b1972673e32f1efec4 --- /dev/null +++ b/lib/python3.12/site-packages/hopper/generate_kernels.py @@ -0,0 +1,208 @@ +# Copied from Driss Guessous's PR in PyTorch: https://github.com/pytorch/pytorch/pull/105602 + +# This file is run to generate the kernel instantiations for the flash_attn kernels +# They are written to several files in order to speed up compilation + +import argparse +import itertools +from collections import namedtuple +from dataclasses import dataclass +from pathlib import Path +from typing import List, Optional + +KERNEL_BATCH = namedtuple("Kernel", ["template", "filename"]) + +DTYPE_MAP = { + "fp16": "cutlass::half_t", + "bf16": "cutlass::bfloat16_t", + "e4m3": "cutlass::float_e4m3_t", +} + +DTYPE_MAP_FWD_SM8x = { + "fp16": "cutlass::half_t", + "bf16": "cutlass::bfloat16_t", +} + +DTYPE_MAP_BWD = { + "fp16": "cutlass::half_t", + "bf16": "cutlass::bfloat16_t", +} + +SM = [80, 90] # Sm kernels support up to +HEAD_DIMENSIONS = [64, 96, 128, 192, 256] +PAGEDKV = [False, True] +SPLIT = [False, True] +SOFTCAP = [False, True] +PACKGQA = [False, True] + +KERNEL_IMPL_TEMPLATE_FWD_SM90 = """#include "flash_fwd_launch_template.h" + +#ifndef FLASHATTENTION_DISABLE_HDIM{HEAD_DIM} +template void run_mha_fwd_<{ARCH}, {DTYPE}, {HEAD_DIM}, {SPLIT}, {PAGEDKV}, {SOFTCAP}, {PACKGQA}>(Flash_fwd_params ¶ms, cudaStream_t stream); +#endif +""" + +KERNEL_IMPL_TEMPLATE_FWD_SM8x = """#include "flash_fwd_launch_template.h" + +#ifndef FLASHATTENTION_DISABLE_SM8x +#ifndef FLASHATTENTION_DISABLE_HDIM{HEAD_DIM} +template void run_mha_fwd_<80, {DTYPE}, {HEAD_DIM}, {SPLIT}, {PAGEDKV}, {SOFTCAP}, {PACKGQA}>(Flash_fwd_params ¶ms, cudaStream_t stream); +template void run_mha_fwd_<86, {DTYPE}, {HEAD_DIM}, {SPLIT}, {PAGEDKV}, {SOFTCAP}, {PACKGQA}>(Flash_fwd_params ¶ms, cudaStream_t stream); +#endif +#endif +""" + +KERNEL_IMPL_TEMPLATE_BWD_SM90 = """#include "flash_bwd_launch_template.h" + +#ifndef FLASHATTENTION_DISABLE_HDIM{HEAD_DIM} +template<> +void run_mha_bwd_<{ARCH}, {DTYPE}, {HEAD_DIM}, {SOFTCAP}>(Flash_bwd_params ¶ms, cudaStream_t stream) {{ + run_mha_bwd_hdim{HEAD_DIM}<{ARCH}, {DTYPE}, {SOFTCAP}>(params, stream); +}} +#endif +""" + +KERNEL_IMPL_TEMPLATE_BWD_SM8x = """#include "flash_bwd_launch_template.h" + +#ifndef FLASHATTENTION_DISABLE_SM8x +#ifndef FLASHATTENTION_DISABLE_HDIM{HEAD_DIM} +template<> +void run_mha_bwd_<80, {DTYPE}, {HEAD_DIM}, {SOFTCAP}>(Flash_bwd_params ¶ms, cudaStream_t stream) {{ + run_mha_bwd_hdim{HEAD_DIM}<80, {DTYPE}, {SOFTCAP}>(params, stream); +}} +template<> +void run_mha_bwd_<86, {DTYPE}, {HEAD_DIM}, {SOFTCAP}>(Flash_bwd_params ¶ms, cudaStream_t stream) {{ + run_mha_bwd_hdim{HEAD_DIM}<86, {DTYPE}, {SOFTCAP}>(params, stream); +}} +#endif +#endif +""" + + + +@dataclass +class Kernel: + sm: int + dtype: str + head_dim: int + split: bool + paged_kv: bool + softcap: bool + packgqa: bool + direction: str + + @property + def template(self) -> str: + if self.direction == "fwd": + if self.sm == 90: + # Always enable PackGQA for PagedKV or Split to reduce compilation + packgqa = self.packgqa or self.paged_kv or self.split + return KERNEL_IMPL_TEMPLATE_FWD_SM90.format( + ARCH=str(self.sm), DTYPE=DTYPE_MAP[self.dtype], HEAD_DIM=self.head_dim, + SPLIT=str(self.split).lower(), PAGEDKV=str(self.paged_kv).lower(), + SOFTCAP=str(self.softcap).lower(), PACKGQA=str(packgqa).lower() + ) + else: + # Always enable PackGQA for Sm8x to reduce compilation + return KERNEL_IMPL_TEMPLATE_FWD_SM8x.format( + DTYPE=DTYPE_MAP[self.dtype], HEAD_DIM=self.head_dim, + SPLIT=str(self.split).lower(), PAGEDKV=str(self.paged_kv).lower(), + SOFTCAP=str(self.softcap).lower(), PACKGQA=str(True).lower() + ) + elif self.direction == "bwd": + if self.sm == 90: + return KERNEL_IMPL_TEMPLATE_BWD_SM90.format( + ARCH=str(self.sm), DTYPE=DTYPE_MAP[self.dtype], HEAD_DIM=self.head_dim, + SOFTCAP=str(self.softcap).lower() + ) + else: + return KERNEL_IMPL_TEMPLATE_BWD_SM8x.format( + DTYPE=DTYPE_MAP[self.dtype], HEAD_DIM=self.head_dim, + SOFTCAP=str(self.softcap).lower() + ) + + @property + def filename(self) -> str: + return f"flash_{self.direction}_hdim{self.head_dim}_{self.dtype}{'_paged' if self.paged_kv else ''}{'_split' if self.split else ''}{'_softcap' if self.softcap else ''}{'_packgqa' if self.packgqa else ''}_sm{self.sm}.cu" + + +def get_all_kernels() -> List[Kernel]: + for dtype, head_dim, split, paged_kv, softcap, packgqa, sm in itertools.product(DTYPE_MAP.keys(), HEAD_DIMENSIONS, SPLIT, PAGEDKV, SOFTCAP, PACKGQA, SM): + # We always enable PackGQA for Sm8x or PagedKV or Split + # so we should just pass in packgqa=False to avoid the `_packgqa` in the filename. + if packgqa and (sm < 90 or (sm >= 90 and (paged_kv or split))): + continue + if sm >= 90 or dtype in DTYPE_MAP_FWD_SM8x: + yield Kernel(sm=sm, dtype=dtype, head_dim=head_dim, split=split, paged_kv=paged_kv, softcap=softcap, packgqa=packgqa, direction="fwd") + for dtype, head_dim, softcap, sm in itertools.product(DTYPE_MAP_BWD.keys(), HEAD_DIMENSIONS, SOFTCAP, SM): + yield Kernel(sm=sm, dtype=dtype, head_dim=head_dim, split=False, paged_kv=False, softcap=softcap, packgqa=False, direction="bwd") + + +def batch_hdim(kernels_all) -> List[KERNEL_BATCH]: + for dtype, split, paged_kv, softcap, packgqa, sm in itertools.product(DTYPE_MAP.keys(), SPLIT, PAGEDKV, SOFTCAP, PACKGQA, SM): + if sm < 90: + continue + kernels = [k for k in kernels_all if k.direction == "fwd" and k.dtype == dtype and k.split == split and k.paged_kv == paged_kv and k.softcap == softcap and k.packgqa == packgqa and k.sm == sm] + if len(kernels) > 0: + filename = f"flash_fwd_hdimall_{dtype}{'_paged' if paged_kv else ''}{'_split' if split else ''}{'_softcap' if softcap else ''}{'_packgqa' if packgqa else ''}_sm{sm}.cu" + template = "\n".join([f"#include \"{k.filename}\"" for k in kernels]) + yield KERNEL_BATCH(template, filename) + + +def batch_softcap(kernels_all) -> List[KERNEL_BATCH]: + for dtype, head_dim, split, paged_kv, packgqa, sm in itertools.product(DTYPE_MAP.keys(), HEAD_DIMENSIONS, SPLIT, PAGEDKV, PACKGQA, SM): + if sm >= 90: + continue + kernels = [k for k in kernels_all if k.direction == "fwd" and k.dtype == dtype and k.head_dim == head_dim and k.split == split and k.paged_kv == paged_kv and k.packgqa == packgqa and k.sm == sm] + if len(kernels) > 0: + filename = f"flash_fwd_hdim{head_dim}_{dtype}{'_paged' if paged_kv else ''}{'_split' if split else ''}_softcapall{'_packgqa' if packgqa else ''}_sm{sm}.cu" + template = "\n".join([f"#include \"{k.filename}\"" for k in kernels]) + yield KERNEL_BATCH(template, filename) + + # Bwd + for dtype, head_dim, sm in itertools.product(DTYPE_MAP.keys(), HEAD_DIMENSIONS, SM): + if sm < 90: + continue + kernels = [k for k in kernels_all if k.direction == "bwd" and k.dtype == dtype and k.head_dim == head_dim and k.sm == sm] + if len(kernels) > 0: + filename = f"flash_bwd_hdim{head_dim}_{dtype}_softcapall_sm{sm}.cu" + template = "\n".join([f"#include \"{k.filename}\"" for k in kernels]) + yield KERNEL_BATCH(template, filename) + + +def write_kernel(kernel: Kernel, autogen_dir: Path) -> None: + prelude = """// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, Pradeep Ramani, Tri Dao. +// Splitting the different template instantiations to different files to speed up compilation. +// This file is auto-generated. See "generate_kernels.py"\n +""" + (autogen_dir / kernel.filename).write_text(prelude + kernel.template) + + +def main(output_dir: Optional[str]) -> None: + output_dir = Path(output_dir) if output_dir is not None else Path(__file__).parent + output_dir.mkdir(parents=True, exist_ok=True) + kernels_all = list(get_all_kernels()) + for kernel in kernels_all: + write_kernel(kernel, output_dir) + for kernel in batch_hdim(kernels_all): + write_kernel(kernel, output_dir) + for kernel in batch_softcap(kernels_all): + write_kernel(kernel, output_dir) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + prog="generate_kernels", + description="Generate the flash_attention kernels template instantiations", + ) + # Set an optional output directory + parser.add_argument( + "-o", + "--output_dir", + default="instantiations", + required=False, + help="Where to generate the kernels " + " will default to the current directory ", + ) + args = parser.parse_args() + main(args.output_dir) diff --git a/lib/python3.12/site-packages/hopper/test_attn_kvcache.py b/lib/python3.12/site-packages/hopper/test_attn_kvcache.py new file mode 100644 index 0000000000000000000000000000000000000000..726d44ce31906fcf506d0423cd04c84060cd9d94 --- /dev/null +++ b/lib/python3.12/site-packages/hopper/test_attn_kvcache.py @@ -0,0 +1,486 @@ +import pytest +from einops import rearrange, repeat +import torch +import flash_attn +import flash_attn_interface +import itertools +import math +import time + +def construct_local_mask( + seqlen_q, + seqlen_k, + window_size=(-1, -1), # -1 means infinite window size + query_padding_mask=None, + key_padding_mask=None, + device=None, + key_leftpad=None, +): + row_idx = rearrange(torch.arange(seqlen_q, device=device, dtype=torch.long), "s -> s 1") + col_idx = torch.arange(seqlen_k, device=device, dtype=torch.long) + if key_leftpad is not None: + key_leftpad = rearrange(key_leftpad, "b -> b 1 1 1") + col_idx = repeat(col_idx, "s -> b 1 1 s", b=key_leftpad.shape[0]) + col_idx = torch.where(col_idx >= key_leftpad, col_idx - key_leftpad, 2**32) + sk = ( + seqlen_k + if key_padding_mask is None + else rearrange(key_padding_mask.sum(-1), "b -> b 1 1 1") + ) + sq = ( + seqlen_q + if query_padding_mask is None + else rearrange(query_padding_mask.sum(-1), "b -> b 1 1 1") + ) + if window_size[0] < 0: + return col_idx > row_idx + sk - sq + window_size[1] + else: + sk = torch.full_like(col_idx, seqlen_k) if key_padding_mask is None else sk + return torch.logical_or( + col_idx > torch.minimum(row_idx + sk - sq + window_size[1], sk), + col_idx < row_idx + sk - sq - window_size[0], + ) + + +def attention_ref( + q, + k, + v, + query_padding_mask=None, + key_padding_mask=None, + attn_bias=None, + dropout_p=0.0, + dropout_mask=None, + causal=False, + window_size=(-1, -1), # -1 means infinite window size + softcap=0.0, + upcast=True, + reorder_ops=False, + key_leftpad=None, +): + """ + Arguments: + q: (batch_size, seqlen_q, nheads, head_dim) + k: (batch_size, seqlen_k, nheads_k, head_dim) + v: (batch_size, seqlen_k, nheads_k, head_dim) + query_padding_mask: (batch_size, seqlen_q) + key_padding_mask: (batch_size, seqlen_k) + attn_bias: broadcastable to (batch_size, nheads, seqlen_q, seqlen_k) + dropout_p: float + dropout_mask: (batch_size, nheads, seqlen_q, seqlen_k) + causal: whether to apply causal masking + window_size: (int, int), left and right window size + upcast: whether to cast all inputs to fp32, do all computation in fp32, then cast + output back to fp16/bf16. + reorder_ops: whether to change the order of operations (scaling k instead of scaling q, etc.) + without changing the math. This is to estimate the numerical error from operation + reordering. + Output: + output: (batch_size, seqlen_q, nheads, head_dim) + attention: (batch_size, nheads, seqlen_q, seqlen_k), softmax after dropout + """ + if causal: + window_size = (window_size[0], 0) + dtype_og = q.dtype + if upcast: + q, k, v = q.float(), k.float(), v.float() + seqlen_q, seqlen_k = q.shape[1], k.shape[1] + k = repeat(k, "b s h d -> b s (h g) d", g=q.shape[2] // k.shape[2]) + v = repeat(v, "b s h d -> b s (h g) d", g=q.shape[2] // v.shape[2]) + d = q.shape[-1] + if not reorder_ops: + scores = torch.einsum("bthd,bshd->bhts", q / math.sqrt(d), k) + else: + scores = torch.einsum("bthd,bshd->bhts", q, k / math.sqrt(d)) + if softcap > 0: + scores = scores / softcap + scores = scores.tanh() + scores = scores * softcap + if key_padding_mask is not None: + scores.masked_fill_(rearrange(~key_padding_mask, "b s -> b 1 1 s"), float("-inf")) + if window_size[0] >= 0 or window_size[1] >= 0: + local_mask = construct_local_mask( + seqlen_q, + seqlen_k, + window_size, + query_padding_mask, + key_padding_mask, + q.device, + key_leftpad=key_leftpad, + ) + scores.masked_fill_(local_mask, float("-inf")) + if attn_bias is not None: + scores = scores + attn_bias + attention = torch.softmax(scores, dim=-1).to(v.dtype) + # Some rows might be completely masked out so we fill them with zero instead of NaN + if window_size[0] >= 0 or window_size[1] >= 0: + attention = attention.masked_fill(torch.all(local_mask, dim=-1, keepdim=True), 0.0) + # We want to mask here so that the attention matrix doesn't have any NaNs + # Otherwise we'll get NaN in dV + if query_padding_mask is not None: + attention = attention.masked_fill(rearrange(~query_padding_mask, "b s -> b 1 s 1"), 0.0) + dropout_scaling = 1.0 / (1 - dropout_p) + # attention_drop = attention.masked_fill(~dropout_mask, 0.0) * dropout_scaling + # output = torch.einsum('bhts,bshd->bthd', attention_drop , v) + if dropout_mask is not None: + attention_drop = attention.masked_fill(~dropout_mask, 0.0) + else: + attention_drop = attention + output = torch.einsum("bhts,bshd->bthd", attention_drop, v * dropout_scaling) + if query_padding_mask is not None: + output.masked_fill_(rearrange(~query_padding_mask, "b s -> b s 1 1"), 0.0) + return output.to(dtype=dtype_og), attention.to(dtype=dtype_og) + + +@pytest.mark.parametrize("causal", [True, False]) +@pytest.mark.parametrize("num_requests", [1, 4]) +@pytest.mark.parametrize("query_seqlen", [1, 8, 120]) +@pytest.mark.parametrize("context_seqlen", [1024, 3131, 4224]) +@pytest.mark.parametrize("headdim", [64, 128, 256]) +@pytest.mark.parametrize("gqa_parallel", [False, True]) +@pytest.mark.parametrize( + "nheads_kv, gqa_ratio", + [ + (1, 1), + (2, 5), + (3, 3), + (1, 32), + (5, 7), + (8, 1), + (1, 16), + (12, 4), + (8, 2), + ], +) +def test_flash_attn_kvcache_nosplit(nheads_kv, gqa_ratio, num_requests, query_seqlen, context_seqlen, headdim, causal, gqa_parallel): + device = "cuda" + num_caches = num_requests + cache_seqlen = context_seqlen + nheads_q = nheads_kv * gqa_ratio + + k_cache = torch.randn( + (num_caches, cache_seqlen, nheads_kv, headdim), device="cuda", dtype=torch.bfloat16 + ) + v_cache = torch.randn( + (num_caches, cache_seqlen, nheads_kv, headdim), device="cuda", dtype=torch.bfloat16 + ) + q = torch.randn((num_requests, query_seqlen, nheads_q, headdim), device="cuda", dtype=torch.bfloat16) + # cache_idxs = torch.randperm(num_caches, dtype=torch.int32, device="cuda")[:num_requests] + cache_seqlens = torch.tensor([context_seqlen] * num_requests, dtype=torch.int32, device="cuda") + torch.cuda.synchronize() + + out_ref, _ = attention_ref( + q, + k_cache, + v_cache, + causal=causal, + ) + + out_fa3, lse_fa3 = flash_attn_interface.flash_attn_with_kvcache( + q=q, + k_cache=k_cache, + v_cache=v_cache, + cache_seqlens=cache_seqlens, + # cache_batch_idx=cache_idxs, + causal=causal, + num_splits=1, + return_softmax_lse=True, + gqa_parallel=gqa_parallel + ) + + + torch.cuda.synchronize() + assert ((out_ref - out_fa3).abs().max().item() <= 4e-3) + assert ((out_ref - out_fa3).abs().mean().item() <= 2e-4) + + +@pytest.mark.parametrize("causal", [True, False]) +@pytest.mark.parametrize("num_requests", [1, 3]) +@pytest.mark.parametrize("query_seqlen", [1, 8, 120]) +@pytest.mark.parametrize("context_seqlen", [1600, 4000, 5555]) +@pytest.mark.parametrize("headdim", [64, 128, 256]) +@pytest.mark.parametrize("gqa_parallel", [True, False]) +@pytest.mark.parametrize( + "nheads_kv, gqa_ratio", + [ + (1, 1), + (2, 5), + (3, 3), + (1, 32), + (5, 7), + (8, 1), + (1, 16), + (12, 4), + (8, 2), + ], +) +def test_flash_attn_kvcache_nosplit_fp8(nheads_kv, gqa_ratio, num_requests, query_seqlen, context_seqlen, headdim, causal, gqa_parallel): + device = "cuda" + num_caches = num_requests + cache_seqlen = context_seqlen + nheads_q = nheads_kv * gqa_ratio + + k_cache = torch.randn( + (num_caches, cache_seqlen, nheads_kv, headdim), device="cuda", dtype=torch.bfloat16 + ) + v_cache = torch.randn( + (num_caches, cache_seqlen, nheads_kv, headdim), device="cuda", dtype=torch.bfloat16 + ) + q = torch.randn((num_requests, query_seqlen, nheads_q, headdim), device="cuda", dtype=torch.bfloat16) + q = q.to(torch.float8_e4m3fn) + k_cache = k_cache.to(torch.float8_e4m3fn) + v_cache = v_cache.to(torch.float8_e4m3fn) + # cache_idxs = torch.randperm(num_caches, dtype=torch.int32, device="cuda")[:num_requests] + cache_seqlens = torch.tensor([context_seqlen] * num_requests, dtype=torch.int32, device="cuda") + torch.cuda.synchronize() + + out_ref, _ = attention_ref( + q, + k_cache, + v_cache, + causal=causal, + ) + + descale_q = torch.tensor([1.0], dtype=torch.float32, device='cuda') + descale_k = torch.tensor([1.0], dtype=torch.float32, device='cuda') + descale_v = torch.tensor([1.0], dtype=torch.float32, device='cuda') + out_fa3, lse_fa3 = flash_attn_interface.flash_attn_with_kvcache( + q=q, + k_cache=k_cache, + v_cache=v_cache, + cache_seqlens=cache_seqlens, + # cache_batch_idx=cache_idxs, + causal=causal, + num_splits=1, + return_softmax_lse=True, + gqa_parallel=gqa_parallel, + descale_q=descale_q, descale_k=descale_k, descale_v=descale_v + ) + + + torch.cuda.synchronize() + assert ((out_ref - out_fa3).abs().max().item() <= 4e-2) + assert ((out_ref - out_fa3).abs().mean().item() <= 2e-3) + +@pytest.mark.parametrize("dtype", [torch.bfloat16]) +@pytest.mark.parametrize("use_heuristic_only", [True]) +# @pytest.mark.parametrize("use_heuristic_only", [False]) +@pytest.mark.parametrize("causal", [True, False]) +# @pytest.mark.parametrize("num_requests", [1, 4, 16]) +@pytest.mark.parametrize("num_requests", [1, 3]) +# @pytest.mark.parametrize("query_seqlen", [1, 16, 32, 128]) +@pytest.mark.parametrize("query_seqlen", [1, 8, 25]) +# @pytest.mark.parametrize("context_seqlen", [4096, 16384, 65536]) +@pytest.mark.parametrize("context_seqlen", [1600, 4000, 5555]) +@pytest.mark.parametrize("headdim", [64, 128, 256]) +@pytest.mark.parametrize("cache_seqlen_rand", [True, False]) +@pytest.mark.parametrize("gqa_parallel", [True, False]) +@pytest.mark.parametrize( + "nheads_kv, gqa_ratio", + [ + (1, 1), + (4, 1), + (2, 2), + (3, 3), + (4, 4), + (2, 5), + (3, 9), + (1, 16), + (1, 32), + ], +) +def test_flash_attn_kvcache_output(nheads_kv, gqa_ratio, num_requests, query_seqlen, context_seqlen, headdim, causal, use_heuristic_only, cache_seqlen_rand, gqa_parallel, dtype): + device = "cuda" + num_caches = 16 + if context_seqlen <= 65536: + cache_seqlen = 65536 + else: + cache_seqlen = context_seqlen + nheads_q = nheads_kv * gqa_ratio + if use_heuristic_only: + max_splits = 1 + else: + max_splits = 128 + + k_cache = torch.randn( + (num_caches, cache_seqlen, nheads_kv, headdim), device="cuda", dtype=torch.bfloat16 + ) + v_cache = torch.randn( + (num_caches, cache_seqlen, nheads_kv, headdim), device="cuda", dtype=torch.bfloat16 + ) + q = torch.randn((num_requests, query_seqlen, nheads_q, headdim), device="cuda", dtype=torch.bfloat16) + + q = q.to(dtype) + k_cache = k_cache.to(dtype) + v_cache = v_cache.to(dtype) + cache_idxs = torch.randperm(num_caches, dtype=torch.int32, device="cuda")[:num_requests] + cache_seqlens = torch.randint(1, context_seqlen-1, (num_requests,), dtype=torch.int32).to(device) if cache_seqlen_rand else torch.tensor([context_seqlen] * num_requests, dtype=torch.int32, device="cuda") + torch.cuda.synchronize() + + out_ref, lse_ref = flash_attn_interface.flash_attn_with_kvcache( + q=q, + k_cache=k_cache, + v_cache=v_cache, + cache_seqlens=cache_seqlens, + cache_batch_idx=cache_idxs, + causal=causal, + num_splits=1, + return_softmax_lse=True, + gqa_parallel=False + ) + + # i=0 case is with num splits heuristic + for i in range(0, max_splits+1): + out_fa3, lse_fa3 = flash_attn_interface.flash_attn_with_kvcache( + q=q, + k_cache=k_cache, + v_cache=v_cache, + cache_seqlens=cache_seqlens, + cache_batch_idx=cache_idxs, + causal=causal, + num_splits=i, + return_softmax_lse=True, + gqa_parallel=gqa_parallel, + max_seqlen_k_hint=context_seqlen + ) + + torch.cuda.synchronize() + print ('output-ref', i, out_ref) + print ('output-fa3',i, out_fa3) + print ('output-max-diff', i, context_seqlen, (out_ref - out_fa3).abs().max().item()) + print ('output-mean-diff',i, context_seqlen, (out_ref - out_fa3).abs().mean().item()) + print ('lse-max-diff',i, context_seqlen, (lse_ref - lse_fa3).abs().max().item()) + print ('lse-mean-diff',i, context_seqlen, (lse_ref - lse_fa3).abs().mean().item()) + + if cache_seqlen_rand: + assert ((out_ref - out_fa3).abs().max().item() <= 1e-2) + assert ((out_ref - out_fa3).abs().mean().item() <= 1e-3) + else: + assert ((out_ref - out_fa3).abs().max().item() <= 2e-3) + assert ((out_ref - out_fa3).abs().mean().item() <= 1e-4) + lse_max_ref = lse_ref.abs().max().item() + lse_mean_ref = lse_ref.abs().mean().item() + lse_max_fa3 = lse_fa3.abs().max().item() + lse_mean_fa3 = lse_fa3.abs().mean().item() + lse_max_diff = (lse_ref - lse_fa3).abs().max().item() + lse_mean_diff = (lse_ref - lse_fa3).abs().mean().item() + assert ((lse_max_ref == math.inf and lse_max_fa3 == math.inf) or lse_max_diff <= 1e-3) + assert ((lse_mean_ref == math.inf and lse_mean_fa3 == math.inf) or lse_mean_diff <= 1e-4) + + + +@pytest.mark.parametrize("dtype", [torch.float8_e4m3fn]) +@pytest.mark.parametrize("use_heuristic_only", [True]) +# @pytest.mark.parametrize("use_heuristic_only", [False]) +@pytest.mark.parametrize("causal", [True, False]) +# @pytest.mark.parametrize("num_requests", [1, 4, 16]) +@pytest.mark.parametrize("num_requests", [1, 3]) +# @pytest.mark.parametrize("query_seqlen", [1, 16, 32, 128]) +@pytest.mark.parametrize("query_seqlen", [1, 8, 25]) +# @pytest.mark.parametrize("context_seqlen", [4096, 16384, 65536]) +@pytest.mark.parametrize("context_seqlen", [1600, 4000, 5555]) +@pytest.mark.parametrize("headdim", [64, 128, 256]) +@pytest.mark.parametrize("cache_seqlen_rand", [True, False]) +@pytest.mark.parametrize("gqa_parallel", [True, False]) +@pytest.mark.parametrize( + "nheads_kv, gqa_ratio", + [ + (1, 1), + (4, 1), + (2, 2), + (3, 3), + (4, 4), + (2, 5), + (3, 9), + (1, 16), + (1, 32), + ], +) +def test_flash_attn_kvcache_output_fp8(nheads_kv, gqa_ratio, num_requests, query_seqlen, context_seqlen, headdim, causal, use_heuristic_only, cache_seqlen_rand, gqa_parallel, dtype): + device = "cuda" + num_caches = 16 + if context_seqlen <= 65536: + cache_seqlen = 65536 + else: + cache_seqlen = context_seqlen + nheads_q = nheads_kv * gqa_ratio + if use_heuristic_only: + max_splits = 1 + else: + max_splits = 128 + + k_cache = torch.randn( + (num_caches, cache_seqlen, nheads_kv, headdim), device="cuda", dtype=torch.bfloat16 + ) + v_cache = torch.randn( + (num_caches, cache_seqlen, nheads_kv, headdim), device="cuda", dtype=torch.bfloat16 + ) + q = torch.randn((num_requests, query_seqlen, nheads_q, headdim), device="cuda", dtype=torch.bfloat16) + + q = q.to(dtype) + k_cache = k_cache.to(dtype) + v_cache = v_cache.to(dtype) + cache_idxs = torch.randperm(num_caches, dtype=torch.int32, device="cuda")[:num_requests] + cache_seqlens = torch.randint(1, context_seqlen-1, (num_requests,), dtype=torch.int32).to(device) if cache_seqlen_rand else torch.tensor([context_seqlen] * num_requests, dtype=torch.int32, device="cuda") + torch.cuda.synchronize() + + + descale_q = torch.tensor([1.0], dtype=torch.float32, device='cuda') + descale_k = torch.tensor([1.0], dtype=torch.float32, device='cuda') + descale_v = torch.tensor([1.0], dtype=torch.float32, device='cuda') + + out_ref, lse_ref = flash_attn_interface.flash_attn_with_kvcache( + q=q, + k_cache=k_cache, + v_cache=v_cache, + cache_seqlens=cache_seqlens, + cache_batch_idx=cache_idxs, + causal=causal, + num_splits=1, + return_softmax_lse=True, + gqa_parallel=False, + descale_q=descale_q, descale_k=descale_k, descale_v=descale_v + ) + + # i=0 case is with num splits heuristic + for i in range(0, max_splits+1): + out_fa3, lse_fa3 = flash_attn_interface.flash_attn_with_kvcache( + q=q, + k_cache=k_cache, + v_cache=v_cache, + cache_seqlens=cache_seqlens, + cache_batch_idx=cache_idxs, + causal=causal, + num_splits=i, + return_softmax_lse=True, + gqa_parallel=gqa_parallel, + max_seqlen_k_hint=context_seqlen, + descale_q=descale_q, descale_k=descale_k, descale_v=descale_v + ) + + torch.cuda.synchronize() + print ('output-ref', i, out_ref) + print ('output-fa3',i, out_fa3) + print ('output-max-diff', i, context_seqlen, (out_ref - out_fa3).abs().max().item()) + print ('output-mean-diff',i, context_seqlen, (out_ref - out_fa3).abs().mean().item()) + print ('lse-max-diff',i, context_seqlen, (lse_ref - lse_fa3).abs().max().item()) + print ('lse-mean-diff',i, context_seqlen, (lse_ref - lse_fa3).abs().mean().item()) + + if cache_seqlen_rand: + assert ((out_ref - out_fa3).abs().max().item() <= 1e-1) + assert ((out_ref - out_fa3).abs().mean().item() <= 1e-2) + else: + assert ((out_ref - out_fa3).abs().max().item() <= 2e-2) + assert ((out_ref - out_fa3).abs().mean().item() <= 2e-3) + lse_max_ref = lse_ref.abs().max().item() + lse_mean_ref = lse_ref.abs().mean().item() + lse_max_fa3 = lse_fa3.abs().max().item() + lse_mean_fa3 = lse_fa3.abs().mean().item() + lse_max_diff = (lse_ref - lse_fa3).abs().max().item() + lse_mean_diff = (lse_ref - lse_fa3).abs().mean().item() + assert ((lse_max_ref == math.inf and lse_max_fa3 == math.inf) or lse_max_diff <= 1e-3) + assert ((lse_mean_ref == math.inf and lse_mean_fa3 == math.inf) or lse_mean_diff <= 1e-4) + + +if __name__ == "__main__": + main() diff --git a/lib/python3.12/site-packages/hopper/test_flash_attn.py b/lib/python3.12/site-packages/hopper/test_flash_attn.py new file mode 100644 index 0000000000000000000000000000000000000000..1fe43e21fa2ca9de780fb929b00c22a6f7f9994d --- /dev/null +++ b/lib/python3.12/site-packages/hopper/test_flash_attn.py @@ -0,0 +1,1034 @@ +import os +import math +import itertools + +import pytest +import torch +import torch.nn.functional as F + +from einops import rearrange, repeat +from flash_attn.layers.rotary import apply_rotary_emb + +from padding import pad_input, unpad_input +from test_util import ( + attention_ref, + generate_qkv, + generate_random_padding_mask, +) + +from flash_attn_interface import flash_attn_func, flash_attn_varlen_func, flash_attn_combine, flash_attn_with_kvcache + + +DISABLE_BACKWARD = os.getenv("FLASH_ATTENTION_DISABLE_BACKWARD", "FALSE") == "TRUE" +DISABLE_SPLIT = os.getenv("FLASH_ATTENTION_DISABLE_SPLIT", "FALSE") == "TRUE" +DISABLE_PAGEDKV = os.getenv("FLASH_ATTENTION_DISABLE_PAGEDKV", "FALSE") == "TRUE" +DISABLE_APPENDKV = os.getenv("FLASH_ATTENTION_DISABLE_APPENDKV", "FALSE") == "TRUE" +DISABLE_LOCAL = os.getenv("FLASH_ATTENTION_DISABLE_LOCAL", "FALSE") == "TRUE" +DISABLE_SOFTCAP = os.getenv("FLASH_ATTENTION_DISABLE_SOFTCAP", "FALSE") == "TRUE" +DISABLE_PACKGQA = os.getenv("FLASH_ATTENTION_DISABLE_PACKGQA", "FALSE") == "TRUE" +DISABLE_FP16 = os.getenv("FLASH_ATTENTION_DISABLE_FP16", "FALSE") == "TRUE" +DISABLE_FP8 = os.getenv("FLASH_ATTENTION_DISABLE_FP8", "FALSE") == "TRUE" or torch.cuda.get_device_capability("cuda")[0] < 9 +DISABLE_HDIM64 = os.getenv("FLASH_ATTENTION_DISABLE_HDIM64", "FALSE") == "TRUE" +DISABLE_HDIM96 = os.getenv("FLASH_ATTENTION_DISABLE_HDIM96", "FALSE") == "TRUE" +DISABLE_HDIM128 = os.getenv("FLASH_ATTENTION_DISABLE_HDIM128", "FALSE") == "TRUE" +DISABLE_HDIM192 = os.getenv("FLASH_ATTENTION_DISABLE_HDIM192", "FALSE") == "TRUE" +DISABLE_HDIM256 = os.getenv("FLASH_ATTENTION_DISABLE_HDIM256", "FALSE") == "TRUE" + +COMPILED_HDIMS = ( + [] + + ([64] if not DISABLE_HDIM64 else []) + + ([96] if not DISABLE_HDIM96 else []) + + ([128] if not DISABLE_HDIM128 else []) + + ([192] if not DISABLE_HDIM192 else []) + + ([256] if not DISABLE_HDIM256 else []) +) + + +# @pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float8_e4m3fn]) +@pytest.mark.parametrize("dtype", [torch.bfloat16] + ([torch.float16] if not DISABLE_FP16 else []) + ([torch.float8_e4m3fn] if not DISABLE_FP8 else [])) +# @pytest.mark.parametrize("dtype", [torch.bfloat16]) +# @pytest.mark.parametrize("dtype", [torch.float8_e4m3fn]) +@pytest.mark.parametrize("mha_type", ["mha", "mqa", "gqa"]) +# @pytest.mark.parametrize("mha_type", ["mha"]) +# @pytest.mark.parametrize("deterministic", [False, True]) +@pytest.mark.parametrize("deterministic", [False]) +@pytest.mark.parametrize("softcap", [0.0] + ([15.0] if not DISABLE_SOFTCAP else [])) +# @pytest.mark.parametrize("softcap", [0.0]) +@pytest.mark.parametrize("local", [False] + ([True] if not DISABLE_LOCAL else [])) +# @pytest.mark.parametrize("local", [False]) +@pytest.mark.parametrize("causal", [False, True]) +# @pytest.mark.parametrize("causal", [False]) +# @pytest.mark.parametrize("V_colmajor", [False, True]) +@pytest.mark.parametrize("V_colmajor", [False]) +# @pytest.mark.parametrize("d", [32, 64, 96, 128, 160, 192, 224, 256]) +# @pytest.mark.parametrize('d', [32, 40, 64, 80, 96, 128, 160, 192, 256]) +# @pytest.mark.parametrize('d', [32, 64, 96, 128, 160, 192]) +# @pytest.mark.parametrize('d', [56, 80]) +# @pytest.mark.parametrize("d", [64, 128, 256]) +# @pytest.mark.parametrize('d', [32, 40, 64, 80, 96, 128]) +# @pytest.mark.parametrize("d", [64, 96, 128, 192]) +@pytest.mark.parametrize("d", COMPILED_HDIMS) +# @pytest.mark.parametrize("d", [128]) +@pytest.mark.parametrize( + "seqlen_q,seqlen_k", + [ + (1, 1), + (64, 128), + (128, 192), + (256, 256), + (239, 1), + (799, 3), + (113, 203), + (113, 128), + (128, 217), + (113, 211), + (108, 256), + (256, 512), + (384, 256), + (640, 128), + (512, 256), + (1024, 1024), + (1023, 1024), + (1024, 1023), + (4096, 4096), + (4224, 4224), + ], +) +# @pytest.mark.parametrize('seqlen_q,seqlen_k', [(128, 128)]) +def test_flash_attn_output( + seqlen_q, seqlen_k, d, causal, local, softcap, V_colmajor, deterministic, mha_type, dtype +): + # sink_token_length = 0 if not local else 4 + sink_token_length = 0 if not local else 0 + if V_colmajor and (seqlen_k % 16 != 0 or dtype != torch.float8_e4m3fn): + pytest.skip("V_colmajor requires seqlen_k to be a multiple of 16 and dtype to be float8_e4m3fn") + device = "cuda" + # set seed + torch.random.manual_seed(0) + # batch_size = 40 + # nheads = 16 + batch_size = 9 if seqlen_k <= 2048 else 2 + # batch_size = 1 + nheads = 6 + # nheads = 1 + nheads_kv = nheads if mha_type == "mha" else (2 if mha_type == "gqa" else 1) + dtype_ref = torch.bfloat16 if dtype == torch.float8_e4m3fn else dtype + q_ref = torch.randn(batch_size, seqlen_q, nheads, d, device=device, dtype=dtype_ref) + if softcap > 0.0: + # Ensure the values of qk are at least within softcap range. + q_ref = (q_ref * softcap / 4) + q_ref = q_ref.to(dtype).to(dtype_ref).requires_grad_() + k_ref = torch.randn(batch_size, seqlen_k, nheads_kv, d, device=device, dtype=dtype_ref).to(dtype).to(dtype_ref).requires_grad_() + v_ref = torch.randn(batch_size, seqlen_k, nheads_kv, d, device=device, dtype=dtype_ref).to(dtype).to(dtype_ref).requires_grad_() + # Put window_size after QKV randn so that window_size changes from test to test + window_size = (-1, -1) if not local else torch.randint(0, seqlen_k, (2,)) + # window_size = (-1, -1) if not local else (16, 0) + if dtype == torch.float8_e4m3fn: + q_descale, k_descale, v_descale = [torch.rand(batch_size, nheads_kv, device=device, dtype=torch.float32) * 2 for _ in range(3)] + else: + q_descale, k_descale, v_descale = None, None, None + q, k, v = [x.detach().to(dtype).requires_grad_() for x in (q_ref, k_ref, v_ref)] + if V_colmajor: + v = rearrange(rearrange(v.detach(), "b s h d -> b h d s").contiguous(), "b h d s -> b s h d").requires_grad_() + out_ref, attn_ref = attention_ref( + q_ref, + k_ref, + v_ref, + None, + None, + causal=causal, + q_descale=q_descale, k_descale=k_descale, v_descale=v_descale, + window_size=window_size, + sink_token_length=sink_token_length, + softcap=softcap + ) + out_pt, attn_pt = attention_ref( + q_ref, + k_ref, + v_ref, + None, + None, + causal=causal, + q_descale=q_descale, k_descale=k_descale, v_descale=v_descale, + window_size=window_size, + sink_token_length=sink_token_length, + softcap=softcap, + upcast=False, + reorder_ops=True, + intermediate_dtype=dtype if dtype == torch.float8_e4m3fn else None, + ) + + # qk = torch.einsum('bshd,bthd->bhst', q_ref, k_ref).float() + # m = qk.amax(-1, keepdim=True) + # s_tmp = torch.exp((qk - m) / math.sqrt(d)) + # exp_sum = s_tmp.sum(-1) + # qk = torch.einsum('bthd,bshd->bhts', q_ref.float() / math.sqrt(d), k_ref.float()) + # lse_ref = torch.logsumexp(qk, dim=-1) + + # Numerical error if we just do any arithmetic on out_ref + fwd_atol = 2 * (out_ref + 0.3 - 0.3 - out_ref).abs().max().item() + rtol = 2 if softcap == 0.0 else 3 + + print(f"Pytorch max diff: {(out_pt - out_ref).abs().max().item()}") + print(f"Pytorch mean diff: {(out_pt - out_ref).abs().mean().item()}") + pack_gqa_vals = [False, True] if not DISABLE_PACKGQA else [False] + num_splits_vals = [1, 3] if not DISABLE_SPLIT else [1] + for pack_gqa, num_splits in itertools.product(pack_gqa_vals, num_splits_vals): + out, lse = flash_attn_func( + q, + k, + v, + causal=causal, + q_descale=q_descale, k_descale=k_descale, v_descale=v_descale, + window_size=window_size, + sink_token_length=sink_token_length, + softcap=softcap, + pack_gqa=pack_gqa, + num_splits=num_splits + ) + print(f"Output max diff: {(out - out_ref).abs().max().item()}") + print(f"Output mean diff: {(out - out_ref).abs().mean().item()}") + # if not causal: + # print(f"LSE max diff: {(lse - lse_ref).abs().max().item()}") + # breakpoint() + + # Check that FlashAttention's numerical error is at most twice the numerical error + # of a Pytorch implementation. + assert (out - out_ref).abs().max().item() <= rtol * (out_pt - out_ref).abs().max().item() + fwd_atol + + if not DISABLE_BACKWARD and dtype != torch.float8_e4m3fn and not V_colmajor: + g = torch.randn_like(out) + do_o = ((g.float() * out.float()).sum(-1)).transpose(1, 2) + # import flash_attn_3_cuda + # dq, dk, dv, softmax_d, dq_accum, dk_accum, dv_accum = flash_attn_3_cuda.bwd( + # g, + # q, + # k, + # v, + # out, + # lse, + # None, + # None, + # None, + # d ** (-0.5), + # causal, + # window_size[0], window_size[1], + # sink_token_length, + # softcap, + # deterministic, + # 0, # sm_margin + # ) + dq, dk, dv = torch.autograd.grad(out, (q, k, v), g) + # print(f"dO_O max diff: {(softmax_d - do_o).abs().max().item()}") + # assert (softmax_d - do_o).abs().max().item() <= 1e-5 + # assert dq_accum.abs().max().item() == 0.0 + + # dS = torch.einsum('bthd,bshd->bhts', g.float(), v.float()) + # P = torch.softmax(qk, -1) + # dP = P * (dS - do_o.transpose(1, 2).unsqueeze(1)) + # dQ = torch.einsum('bhts,bshd->bthd', dP, k.float()) + # dV = torch.einsum('bhts,bthd->bshd', P, g.float()) + # dK = torch.einsum('bhts,bthd->bshd', dP, q.float()) + + # dq, dk, dv = torch.autograd.grad(out, (q, k, v), g) + dq_ref, dk_ref, dv_ref = torch.autograd.grad(out_ref, (q_ref, k_ref, v_ref), g) + dq_pt, dk_pt, dv_pt = torch.autograd.grad(out_pt, (q_ref, k_ref, v_ref), g) + print(f"dQ max diff: {(dq - dq_ref).abs().max().item()}") + print(f"dK max diff: {(dk - dk_ref).abs().max().item()}") + print(f"dV max diff: {(dv - dv_ref).abs().max().item()}") + print(f"dQ mean diff: {(dq - dq_ref).abs().mean().item()}") + print(f"dK mean diff: {(dk - dk_ref).abs().mean().item()}") + print(f"dV mean diff: {(dv - dv_ref).abs().mean().item()}") + print(f"dQ Pytorch max diff: {(dq_pt - dq_ref).abs().max().item()}") + print(f"dK Pytorch max diff: {(dk_pt - dk_ref).abs().max().item()}") + print(f"dV Pytorch max diff: {(dv_pt - dv_ref).abs().max().item()}") + print(f"dQ Pytorch mean diff: {(dq_pt - dq_ref).abs().mean().item()}") + print(f"dK Pytorch mean diff: {(dk_pt - dk_ref).abs().mean().item()}") + print(f"dV Pytorch mean diff: {(dv_pt - dv_ref).abs().mean().item()}") + # breakpoint() + + + if not DISABLE_BACKWARD and dtype != torch.float8_e4m3fn and not V_colmajor: + dq_atol = 2 * (dq_ref + 0.3 - 0.3 - dq_ref).abs().max().item() + (0 if softcap == 0 else 3e-4) + assert (dq - dq_ref).abs().max().item() <= rtol * (dq_pt - dq_ref).abs().max().item() + dq_atol + dk_atol = 2 * (dk_ref + 0.3 - 0.3 - dk_ref).abs().max().item() + (0 if softcap == 0 else 3e-4) + assert (dk - dk_ref).abs().max().item() <= rtol * (dk_pt - dk_ref).abs().max().item() + dk_atol + dv_atol = 2 * (dv_ref + 0.3 - 0.3 - dv_ref).abs().max().item() + (0 if softcap == 0 else 3e-4) + assert (dv - dv_ref).abs().max().item() <= rtol * (dv_pt - dv_ref).abs().max().item() + dv_atol + + +# @pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float8_e4m3fn]) +@pytest.mark.parametrize("dtype", [torch.bfloat16] + ([torch.float16] if not DISABLE_FP16 else []) + ([torch.float8_e4m3fn] if not DISABLE_FP8 else [])) +# @pytest.mark.parametrize("dtype", [torch.bfloat16]) +# @pytest.mark.parametrize("dtype", [torch.float8_e4m3fn]) +@pytest.mark.parametrize("mha_type", ["mha", "mqa", "gqa"]) +# @pytest.mark.parametrize("mha_type", ["mha"]) +# @pytest.mark.parametrize("deterministic", [False, True]) +@pytest.mark.parametrize("deterministic", [False]) +@pytest.mark.parametrize("softcap", [0.0] + ([15.0] if not DISABLE_SOFTCAP else [])) +# @pytest.mark.parametrize("softcap", [0.0]) +@pytest.mark.parametrize("local", [False] + ([True] if not DISABLE_LOCAL else [])) +# @pytest.mark.parametrize("local", [False]) +@pytest.mark.parametrize("causal", [False, True]) +# @pytest.mark.parametrize("causal", [False]) +@pytest.mark.parametrize("add_unused_qkv", [False, True]) +# @pytest.mark.parametrize("add_unused_qkv", [True]) +# @pytest.mark.parametrize("d", [32, 64, 96, 128, 160, 192, 224, 256]) +# @pytest.mark.parametrize('d', [32, 40, 64, 80, 96, 128, 160, 192, 256]) +# @pytest.mark.parametrize('d', [32, 64, 96, 128, 160, 192]) +# @pytest.mark.parametrize('d', [56, 80]) +# @pytest.mark.parametrize('d', [32, 40, 64, 80, 96, 128]) +# @pytest.mark.parametrize("d", [64, 96, 128]) +@pytest.mark.parametrize("d", COMPILED_HDIMS) +# @pytest.mark.parametrize("d", [128]) +@pytest.mark.parametrize( + "seqlen_q,seqlen_k", + [ + (1, 1), + (1, 3), + (2, 1), + (511, 1), + (3, 513), + (64, 128), + (128, 128), + (256, 256), + (113, 203), + (128, 217), + (113, 211), + (108, 256), + (256, 512), + (307, 256), + (640, 128), + (512, 256), + (1024, 1024), + (1023, 1024), + (1024, 1023), + (2048, 2048), + ], +) +def test_flash_attn_varlen_output( + seqlen_q, seqlen_k, d, add_unused_qkv, causal, local, softcap, deterministic, mha_type, dtype +): + device = "cuda" + # set seed + torch.random.manual_seed(seqlen_q + seqlen_k + d + int(causal) * 2 + int(local)) + # batch_size = 40 + # nheads = 16 + batch_size = 9 if seqlen_q <= 2048 else 2 + nheads = 6 + # batch_size = 2 + # nheads = 1 + nheads_kv = nheads if mha_type == "mha" else (2 if mha_type == "gqa" else 1) + dtype_ref = torch.bfloat16 if dtype == torch.float8_e4m3fn else dtype + q_ref = torch.randn(batch_size, seqlen_q, nheads, d, device=device, dtype=dtype_ref) + if softcap > 0.0: + # Ensure the values of qk are at least within softcap range. + q_ref = (q_ref * softcap / 4).detach().requires_grad_() + q_ref = q_ref.to(dtype).to(dtype_ref).requires_grad_() + k_ref = torch.randn(batch_size, seqlen_k, nheads_kv, d, device=device, dtype=dtype_ref).to(dtype).to(dtype_ref).requires_grad_() + v_ref = torch.randn(batch_size, seqlen_k, nheads_kv, d, device=device, dtype=dtype_ref).to(dtype).to(dtype_ref).requires_grad_() + # Put window_size after QKV randn so that window_size changes from test to test + window_size = (-1, -1) if not local else torch.randint(0, seqlen_k, (2,)) + if dtype == torch.float8_e4m3fn: + q_descale, k_descale, v_descale = [torch.rand(batch_size, nheads_kv, device=device, dtype=torch.float32) * 2 for _ in range(3)] + else: + q_descale, k_descale, v_descale = None, None, None + q, k, v = [x.detach().requires_grad_() for x in (q_ref, k_ref, v_ref)] + query_padding_mask = generate_random_padding_mask( + seqlen_q, batch_size, device, mode="random", zero_lengths=False + ) + key_padding_mask = generate_random_padding_mask( + seqlen_k, batch_size, device, mode="random", zero_lengths=True + ) + + def _gen_unused_masks(padding_mask, add_unused, max_seq_len, bs, device): + if add_unused: + another_mask = generate_random_padding_mask(max_seq_len, bs, device) + attn_mask = torch.logical_and(padding_mask, another_mask) + unused_mask = torch.logical_xor( + torch.logical_or(padding_mask, another_mask), attn_mask + ) + else: + attn_mask = padding_mask + unused_mask = None + return attn_mask, unused_mask + + query_padding_mask, query_unused_mask = _gen_unused_masks( + query_padding_mask, add_unused_qkv, seqlen_q, batch_size, q.device + ) + key_padding_mask, key_unused_mask = _gen_unused_masks( + key_padding_mask, add_unused_qkv, seqlen_k, batch_size, k.device + ) + + ( + q_unpad, + k_unpad, + v_unpad, + cu_seqlens_q, + cu_seqlens_k, + seqused_q, + seqused_k, + max_seqlen_q, + max_seqlen_k, + q, + k, + v, + output_pad_fn, + dq_pad_fn, + dk_pad_fn, + ) = generate_qkv(q, k, v, query_padding_mask, key_padding_mask, kvpacked=False, + query_unused_mask=query_unused_mask, key_unused_mask=key_unused_mask) + q_unpad, k_unpad, v_unpad = [x.detach().to(dtype).requires_grad_() for x in (q_unpad, k_unpad, v_unpad)] + out_ref, attn_ref = attention_ref( + q_ref, + k_ref, + v_ref, + query_padding_mask, + key_padding_mask, + causal=causal, + q_descale=q_descale, k_descale=k_descale, v_descale=v_descale, + window_size=window_size, + softcap=softcap + ) + out_pt, attn_pt = attention_ref( + q_ref, + k_ref, + v_ref, + query_padding_mask, + key_padding_mask, + causal=causal, + q_descale=q_descale, k_descale=k_descale, v_descale=v_descale, + window_size=window_size, + softcap=softcap, + upcast=False, + reorder_ops=True, + intermediate_dtype=dtype if dtype == torch.float8_e4m3fn else None, + ) + + + print(f"Pytorch max diff: {(out_pt - out_ref).abs().max().item()}") + print(f"Pytorch mean diff: {(out_pt - out_ref).abs().mean().item()}") + + if query_unused_mask is not None: + q_zero_masking = rearrange(query_unused_mask, "b s -> b s 1 1") + + # Numerical error if we just do any arithmetic on out_ref + fwd_atol = 2 * (out_ref + 0.3 - 0.3 - out_ref).abs().max().item() + rtol = 2 if softcap == 0.0 else 3 + + pack_gqa_vals = [False, True] if not DISABLE_PACKGQA else [False] + num_splits_vals = [1, 3] if not DISABLE_SPLIT else [1] + for pack_gqa, num_splits in itertools.product(pack_gqa_vals, num_splits_vals): + out_unpad, lse = flash_attn_varlen_func( + q_unpad, + k_unpad, + v_unpad, + cu_seqlens_q, + cu_seqlens_k, + seqused_q, seqused_k, + max_seqlen_q, + max_seqlen_k, + causal=causal, + q_descale=q_descale, + k_descale=k_descale, v_descale=v_descale, + window_size=window_size, + softcap=softcap, + ) + out = output_pad_fn(out_unpad) + if query_unused_mask is not None: + out.masked_fill_(q_zero_masking, 0.0) + print(f"Output max diff: {(out - out_ref).abs().max().item()}") + print(f"Output mean diff: {(out - out_ref).abs().mean().item()}") + # if not causal: + # print(f"LSE max diff: {(lse - lse_ref).abs().max().item()}") + # breakpoint() + + # Check that FlashAttention's numerical error is at most 3x the numerical error + # of a Pytorch implementation. + assert (out - out_ref).abs().max().item() <= rtol * (out_pt - out_ref).abs().max().item() + fwd_atol + + + if not DISABLE_BACKWARD and dtype != torch.float8_e4m3fn: + g_unpad = torch.randn_like(out_unpad) + do_o = ((g_unpad.float() * out_unpad.float()).sum(-1)).transpose(-1, -2) + # import flash_attn_3_cuda + # dq_unpad, dk_unpad, dv_unpad, softmax_d, dq_accum, lse_log2 = flash_attn_3_cuda.bwd_varlen( + # g_unpad, + # q_unpad, + # k_unpad, + # v_unpad, + # out_unpad, + # lse, + # None, + # None, + # None, + # cu_seqlens_q, + # cu_seqlens_k, + # None, None, + # max_seqlen_q, + # max_seqlen_k, + # d ** (-0.5), + # causal, + # window_size[0], window_size[1], + # softcap, + # deterministic, + # 0, # sm_margin + # ) + dq_unpad, dk_unpad, dv_unpad = torch.autograd.grad(out_unpad, (q_unpad, k_unpad, v_unpad), g_unpad) + dq = dq_pad_fn(dq_unpad) + dk = dk_pad_fn(dk_unpad) + dv = dk_pad_fn(dv_unpad) + if key_unused_mask is not None: + k_zero_masking = rearrange(key_unused_mask, "b s -> b s 1 1") + dk.masked_fill_(k_zero_masking, 0.0) + dv.masked_fill_(k_zero_masking, 0.0) + if query_unused_mask is not None: + dq.masked_fill_(q_zero_masking, 0.0) + # print(f"dO_O max diff: {(softmax_d - do_o).abs().max().item()}") + # assert (softmax_d - do_o).abs().max().item() <= 1e-5 + # assert dq_accum.abs().max().item() == 0.0 + g = output_pad_fn(g_unpad) + + # qk = torch.einsum('bthd,bshd->bhts', q / (d ** 0.5), k).float() + # qk = torch.masked_fill(qk, rearrange(~key_padding_mask, "b s -> b 1 1 s"), float("-inf")) + # dS = torch.einsum('bthd,bshd->bhts', g.float(), v.float()) + # P = torch.softmax(qk, -1) + # dP = P * (dS - (g.float() * out.float()).sum(-1).transpose(1, 2).unsqueeze(-1)) + # dQ = torch.einsum('bhts,bshd->bthd', dP, k.float()) + # dV = torch.einsum('bhts,bthd->bshd', P, g.float()) + # dK = torch.einsum('bhts,bthd->bshd', dP, q.float()) + + + # dq, dk, dv = torch.autograd.grad(out, (q, k, v), g) + dq_ref, dk_ref, dv_ref = torch.autograd.grad(out_ref, (q_ref, k_ref, v_ref), g) + dq_pt, dk_pt, dv_pt = torch.autograd.grad(out_pt, (q_ref, k_ref, v_ref), g) + print(f"dQ max diff: {(dq - dq_ref).abs().max().item()}") + print(f"dK max diff: {(dk - dk_ref).abs().max().item()}") + print(f"dV max diff: {(dv - dv_ref).abs().max().item()}") + print(f"dQ mean diff: {(dq - dq_ref).abs().mean().item()}") + print(f"dK mean diff: {(dk - dk_ref).abs().mean().item()}") + print(f"dV mean diff: {(dv - dv_ref).abs().mean().item()}") + print(f"dQ Pytorch max diff: {(dq_pt - dq_ref).abs().max().item()}") + print(f"dK Pytorch max diff: {(dk_pt - dk_ref).abs().max().item()}") + print(f"dV Pytorch max diff: {(dv_pt - dv_ref).abs().max().item()}") + print(f"dQ Pytorch mean diff: {(dq_pt - dq_ref).abs().mean().item()}") + print(f"dK Pytorch mean diff: {(dk_pt - dk_ref).abs().mean().item()}") + print(f"dV Pytorch mean diff: {(dv_pt - dv_ref).abs().mean().item()}") + # breakpoint() + + if not DISABLE_BACKWARD and dtype != torch.float8_e4m3fn: + dq_atol = 2 * (dq_ref + 0.3 - 0.3 - dq_ref).abs().max().item() + (0 if softcap == 0 else 3e-4) + assert (dq - dq_ref).abs().max().item() <= rtol * (dq_pt - dq_ref).abs().max().item() + dq_atol + dk_atol = 2 * (dk_ref + 0.3 - 0.3 - dk_ref).abs().max().item() + (0 if softcap == 0 else 3e-4) + assert (dk - dk_ref).abs().max().item() <= rtol * (dk_pt - dk_ref).abs().max().item() + dk_atol + dv_atol = 2 * (dv_ref + 0.3 - 0.3 - dv_ref).abs().max().item() + (0 if softcap == 0 else 3e-4) + assert (dv - dv_ref).abs().max().item() <= rtol * (dv_pt - dv_ref).abs().max().item() + dv_atol + + +# @pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float8_e4m3fn]) +@pytest.mark.parametrize("dtype", [torch.bfloat16] + ([torch.float8_e4m3fn] if not DISABLE_FP8 else [])) +# @pytest.mark.parametrize("dtype", [torch.bfloat16]) +# @pytest.mark.parametrize("dtype", [torch.float8_e4m3fn]) +@pytest.mark.parametrize("num_splits", [1] + ([0] if not DISABLE_SPLIT else [])) +# @pytest.mark.parametrize("num_splits", [1]) +@pytest.mark.parametrize("mha_type", ["mha", "mqa", "gqa"]) +# @pytest.mark.parametrize("mha_type", ["mha"]) +@pytest.mark.parametrize("new_kv", [False] + ([True] if not DISABLE_APPENDKV else [])) +# @pytest.mark.parametrize("new_kv", [True]) +# @pytest.mark.parametrize("local", [False, True]) +@pytest.mark.parametrize("causal,local", [(False, False), (True, False)] + ([(False, True)] if not DISABLE_LOCAL else [])) +# @pytest.mark.parametrize("causal,local", [(False, False), (True, False)]) +# @pytest.mark.parametrize("causal,local", [(False, False)]) +@pytest.mark.parametrize("seqlen_new_eq_seqlen_q", [True, False] if not DISABLE_APPENDKV else [True]) +# @pytest.mark.parametrize("seqlen_new_eq_seqlen_q", [True]) +@pytest.mark.parametrize("rotary_interleaved", [False, True] if not DISABLE_APPENDKV else [False]) +# @pytest.mark.parametrize("rotary_interleaved", [True]) +@pytest.mark.parametrize("rotary_fraction", [0.0, 0.5, 1.0] if not DISABLE_APPENDKV else [0.0]) +# @pytest.mark.parametrize("rotary_fraction", [0.0]) +@pytest.mark.parametrize("page_size", [None] + ([1, 4, 128] if not DISABLE_PAGEDKV else [])) +# @pytest.mark.parametrize("page_size", [None]) +@pytest.mark.parametrize("has_leftpad", [False, True]) +# @pytest.mark.parametrize("has_leftpad", [False]) +@pytest.mark.parametrize("has_batch_idx", [False, True]) +# @pytest.mark.parametrize("has_batch_idx", [False]) +@pytest.mark.parametrize("varlen_q", [False, True]) +# @pytest.mark.parametrize("varlen_q", [True]) +# @pytest.mark.parametrize("d", [32, 59, 64, 80, 128, 256]) +# @pytest.mark.parametrize("d", [32, 64, 96, 128, 160, 192, 224, 256]) +# @pytest.mark.parametrize('d', [32, 40, 64, 80, 96, 128, 160, 192]) +# @pytest.mark.parametrize('d', [56, 80]) +@pytest.mark.parametrize("d", [128]) +@pytest.mark.parametrize( + "seqlen_q,seqlen_k", + [ + (1, 128), + (1, 339), + (3, 1024), + (64, 800), + (64, 256), + (3, 799), + (64, 2048), + (16, 20000), + (1, 128 * 1024), + (16, 128 * 1024), + (128, 128), + (256, 512), # To test appending KV with more than 1 block + (2048, 3577), # Enough tile to test persistent scheduler + ], +) +# @pytest.mark.parametrize('seqlen_q,seqlen_k', [(256, 128)]) +def test_flash_attn_kvcache( + seqlen_q, + seqlen_k, + d, + varlen_q, + has_batch_idx, + has_leftpad, + page_size, + rotary_fraction, + rotary_interleaved, + seqlen_new_eq_seqlen_q, + causal, + local, + new_kv, + mha_type, + num_splits, + dtype, +): + if page_size is not None and seqlen_k % page_size != 0: + pytest.skip() + if seqlen_q > seqlen_k and new_kv: + pytest.skip() + if not new_kv and rotary_fraction > 0.0: + pytest.skip() + device = "cuda" + # set seed + torch.random.manual_seed(0) + batch_size = 5 + # batch_size = 1 + batch_size_cache = batch_size if not has_batch_idx else batch_size * 2 + nheads = 6 + # nheads = 1 + # rotary_dim must be a multiple of 16, and must be <= d + rotary_dim = math.floor(int(rotary_fraction * d) / 16) * 16 + nheads_k = nheads if mha_type == "mha" else (1 if mha_type == "mqa" else 3) + assert nheads % nheads_k == 0 + dtype_ref = torch.bfloat16 if dtype == torch.float8_e4m3fn else dtype + q = torch.randn(batch_size, seqlen_q, nheads, d, device=device, dtype=dtype_ref).to(dtype).to(dtype_ref) + if varlen_q: + query_padding_mask = generate_random_padding_mask(seqlen_q, batch_size, device, mode="random") + q_unpad, indices_q, cu_seqlens_q, max_seqlen_q, *rest = unpad_input(q, query_padding_mask) + output_pad_fn = lambda output_unpad: pad_input( + output_unpad, indices_q, batch_size, seqlen_q + ) + else: + query_padding_mask = None + q_unpad = q + cu_seqlens_q, max_seqlen_q = None, None + # Put window_size after QKV randn so that window_size changes from test to test + window_size = (-1, -1) if not local else torch.randint(0, seqlen_k, (2,)) + + seqlen_new = seqlen_q if seqlen_new_eq_seqlen_q else torch.randint(1, seqlen_q + 1, (1,)).item() + cu_seqlens_k_new = None + key_new_padding_mask = None + if new_kv: + k = torch.randn(batch_size, seqlen_new, nheads_k, d, device=device, dtype=dtype_ref).to(dtype).to(dtype_ref) + v = torch.randn(batch_size, seqlen_new, nheads_k, d, device=device, dtype=dtype_ref).to(dtype).to(dtype_ref) + if varlen_q: # k & v are also varlen + key_new_padding_mask = generate_random_padding_mask(seqlen_new, batch_size, device, mode="random") + k_unpad, indices_k, cu_seqlens_k_new, *rest = unpad_input(k, key_new_padding_mask) + v_unpad, *rest = unpad_input(v, key_new_padding_mask) + else: + k_unpad, v_unpad = k, v + else: + k, v, k_unpad, v_unpad = None, None, None, None + if page_size is None: + k_cache = torch.randn(batch_size_cache, seqlen_k, nheads_k, d, device=device, dtype=dtype_ref).to(dtype).to(dtype_ref) + v_cache = torch.randn(batch_size_cache, seqlen_k, nheads_k, d, device=device, dtype=dtype_ref).to(dtype).to(dtype_ref) + page_table = None + else: + ( + k_cache, + v_cache, + page_table, + k_cache_paged, + v_cache_paged, + num_blocks, + ) = _generate_block_kvcache( + seqlen_k, page_size, batch_size_cache, nheads_k, d, device, dtype_ref + ) + cache_seqlens = torch.randint( + 0 if new_kv else 1, + # If we don't use seqlen_q in the case of causal and rotary, cos/sin won't be long enough + ( + (seqlen_k - (seqlen_q if (causal or local) and rotary_dim > 1 else seqlen_new) + 1) + if new_kv + else (seqlen_k + 1) + ), + (batch_size,), + dtype=torch.int32, + device=device, + ) + if has_leftpad: + cache_leftpad = torch.cat([torch.randint(0, cache_seqlens[i].item(), (1,), dtype=torch.int32, device=device) + if cache_seqlens[i].item() > 0 else torch.zeros(1, dtype=torch.int32, device=device) + for i in range(batch_size)]) + else: + cache_leftpad = None + if has_batch_idx: + cache_batch_idx = torch.randperm(batch_size_cache, dtype=torch.int32, device=device)[ + :batch_size + ] + else: + cache_batch_idx = None + arange = rearrange(torch.arange(seqlen_k, device=device), "s -> 1 s") + cache_seqlens_expanded = rearrange(cache_seqlens, "b -> b 1") + if not new_kv: + key_padding_mask = arange < cache_seqlens_expanded + else: + k_new_seqlens = key_new_padding_mask.sum(-1, keepdims=True) if varlen_q else seqlen_new + key_padding_mask = arange < cache_seqlens_expanded + k_new_seqlens + if has_leftpad: + key_padding_mask = torch.logical_and( + key_padding_mask, arange >= cache_leftpad.unsqueeze(-1).expand(-1, seqlen_k) + ) + # cache_seqlens = torch.tensor([64], dtype=torch.int32, device=device) + if rotary_dim > 0: + angle = ( + torch.rand( + seqlen_k if page_size is None else num_blocks * page_size, + rotary_dim // 2, + device=device, + ) + * 2 + * math.pi + ) + cos = torch.cos(angle).to(dtype=dtype_ref).to(dtype).to(dtype_ref) + sin = torch.sin(angle).to(dtype=dtype_ref).to(dtype).to(dtype_ref) + if causal or local: + q_ro = apply_rotary_emb( + q, cos, sin, seqlen_offsets=cache_seqlens, interleaved=rotary_interleaved + ) + else: + q_ro = rearrange( + apply_rotary_emb( + rearrange(q, "b s h d -> b 1 (s h) d"), + cos, + sin, + seqlen_offsets=cache_seqlens, + interleaved=rotary_interleaved, + ), + "b 1 (s h) d -> b s h d", + s=seqlen_q, + ) + # q_ro = q + k_ro = apply_rotary_emb( + k, cos, sin, seqlen_offsets=cache_seqlens, interleaved=rotary_interleaved + ) + else: + cos, sin = None, None + q_ro, k_ro = q, k + # k_cache[:, 64:] = -1 + k_cache_ref = (k_cache if not has_batch_idx else k_cache[cache_batch_idx]).clone() + v_cache_ref = (v_cache if not has_batch_idx else v_cache[cache_batch_idx]).clone() + if new_kv: + update_mask = torch.logical_and( + cache_seqlens_expanded <= arange, arange < cache_seqlens_expanded + k_new_seqlens + ) + k_to_update = rearrange(k_ro, "b s ... -> (b s) ...") + v_to_update = rearrange(v, "b s ... -> (b s) ...") + if varlen_q: + k_to_update = k_to_update[indices_k] + v_to_update = v_to_update[indices_k] + k_cache_ref[update_mask] = k_to_update + v_cache_ref[update_mask] = v_to_update + k_cache_rep = repeat(k_cache_ref, "b s h d -> b s (h g) d", g=nheads // nheads_k) + v_cache_rep = repeat(v_cache_ref, "b s h d -> b s (h g) d", g=nheads // nheads_k) + out_ref, _ = attention_ref( + q_ro, + k_cache_rep, + v_cache_rep, + query_padding_mask, + key_padding_mask, + causal=causal, + window_size=window_size, + key_leftpad=cache_leftpad, + ) + out_pt, _ = attention_ref( + q_ro, + k_cache_rep, + v_cache_rep, + query_padding_mask, + key_padding_mask, + causal=causal, + window_size=window_size, + upcast=False, + reorder_ops=True, + key_leftpad=cache_leftpad, + intermediate_dtype=dtype if dtype == torch.float8_e4m3fn else None + ) + q = q.to(dtype) + q_unpad = q_unpad.to(dtype) if varlen_q else None + k_cache = k_cache.to(dtype) + v_cache = v_cache.to(dtype) + k_cache_paged = k_cache_paged.to(dtype) if page_size is not None else None + v_cache_paged = v_cache_paged.to(dtype) if page_size is not None else None + k = k.to(dtype) if k is not None else None + v = v.to(dtype) if v is not None else None + k_unpad = k_unpad.to(dtype) if k_unpad is not None else None + v_unpad = v_unpad.to(dtype) if v_unpad is not None else None + cos = cos.to(dtype) if cos is not None else None + sin = sin.to(dtype) if sin is not None else None + out, lse, *rest = flash_attn_with_kvcache( + q if not varlen_q else q_unpad, + k_cache if page_size is None else k_cache_paged, + v_cache if page_size is None else v_cache_paged, + k if not new_kv or not varlen_q else k_unpad, + v if not new_kv or not varlen_q else v_unpad, + rotary_cos=cos, + rotary_sin=sin, + cache_seqlens=cache_seqlens, + cache_batch_idx=cache_batch_idx, + cache_leftpad=cache_leftpad, + page_table=page_table, + cu_seqlens_q=cu_seqlens_q, + cu_seqlens_k_new=cu_seqlens_k_new, + max_seqlen_q=max_seqlen_q, + causal=causal, + window_size=window_size, + rotary_interleaved=rotary_interleaved, + num_splits=num_splits, + return_softmax_lse=True + ) + if varlen_q: + out = output_pad_fn(out) + # out = flash_attn_with_kvcache( + # q, k_cache, v_cache, cache_seqlens=cache_seqlens, causal=causal, window_size=window_size + # ) + # out = flash_attn_with_kvcache(q, k_cache, v_cache, causal=causal, window_size=window_size) + # qk = torch.einsum("bqhd,bkhd->bhqk", q, k_cache_ref) + # m = qk.amax(-1, keepdim=True) + # s_tmp = torch.exp((qk - m) / math.sqrt(d)) + # o1 = torch.einsum('bhst,bthd->bshd', s_tmp, v_cache_ref) + # lse_ref = torch.logsumexp(qk / math.sqrt(d), -1) + # probs = torch.softmax(qk, dim=-1) + print(f"Output max diff: {(out - out_ref).abs().max().item()}") + print(f"Output mean diff: {(out - out_ref).abs().mean().item()}") + print(f"Pytorch max diff: {(out_pt - out_ref).abs().max().item()}") + print(f"Pytorch mean diff: {(out_pt - out_ref).abs().mean().item()}") + # breakpoint() + + # Check that FlashAttention's numerical error is at most twice the numerical error + # of a Pytorch implementation. + if new_kv: + if page_size is None: + k_cache_select = ( + k_cache.to(dtype_ref) if not has_batch_idx else k_cache.to(dtype_ref)[cache_batch_idx] + ) + v_cache_select = ( + v_cache.to(dtype_ref) if not has_batch_idx else v_cache.to(dtype_ref)[cache_batch_idx] + ) + else: + k_cache_select = rearrange( + k_cache_paged.to(dtype_ref)[(page_table if not has_batch_idx else page_table[cache_batch_idx]).flatten()], + "(b nblocks) block_size ... -> b (nblocks block_size) ...", + b=batch_size, + )[:, :seqlen_k].to(dtype_ref) + v_cache_select = rearrange( + v_cache_paged.to(dtype_ref)[(page_table if not has_batch_idx else page_table[cache_batch_idx]).flatten()], + "(b nblocks) block_size ... -> b (nblocks block_size) ...", + b=batch_size, + )[:, :seqlen_k].to(dtype_ref) + k_cache_ref = k_cache_ref.to(dtype).to(dtype_ref) + v_cache_ref = v_cache_ref.to(dtype).to(dtype_ref) + if dtype is not torch.float8_e4m3fn: + assert torch.equal(v_cache_select, v_cache_ref) + else: + assert torch.allclose(v_cache_select, v_cache_ref, rtol=1e-3, atol=1e-3) + # breakpoint() + # if rotary_dim == 0 and dtype is not torch.float8_e4m3fn: + if rotary_dim == 0: + assert torch.equal(k_cache_select, k_cache_ref) + else: + # if not torch.allclose(k_cache_select, k_cache_ref, rtol=1e-3, atol=1e-3): + # breakpoint() + if dtype is not torch.float8_e4m3fn: + assert torch.allclose(k_cache_select, k_cache_ref, rtol=1e-3, atol=1e-3) + else: + assert torch.allclose(k_cache_select, k_cache_ref, rtol=1e-1, atol=1e-1) + mult = 4 if dtype == torch.float8_e4m3fn else 2 + assert (out - out_ref).abs().max().item() <= mult * (out_pt - out_ref).abs().max().item() + 1e-5 + mult_mean = 3 if dtype == torch.float8_e4m3fn else 1.5 + assert (out - out_ref).abs().mean().item() <= mult_mean * (out_pt - out_ref).abs().mean().item() + + +def _generate_block_kvcache(seqlen_k, page_size, batch_size, nheads_k, d, device, dtype): + num_blocks = math.ceil(seqlen_k / page_size) * batch_size * 3 + k_cache_paged = torch.randn( + num_blocks, page_size, nheads_k, d, device=device, dtype=dtype + ) + v_cache_paged = torch.randn( + num_blocks, page_size, nheads_k, d, device=device, dtype=dtype + ) + page_table = rearrange( + torch.randperm(num_blocks, dtype=torch.int32, device=device), + "(b nblocks) -> b nblocks", + b=batch_size, + ) + k_cache = rearrange( + k_cache_paged[page_table.flatten()], + "(b nblocks) block_size ... -> b (nblocks block_size) ...", + b=batch_size, + )[:, :seqlen_k] + v_cache = rearrange( + v_cache_paged[page_table.flatten()], + "(b nblocks) block_size ... -> b (nblocks block_size) ...", + b=batch_size, + )[:, :seqlen_k] + return k_cache, v_cache, page_table, k_cache_paged, v_cache_paged, num_blocks + + +@pytest.mark.parametrize("dtype", [torch.bfloat16]) +@pytest.mark.parametrize("causal", [False, True]) +# @pytest.mark.parametrize('causal', [False]) +@pytest.mark.parametrize('d', [128]) +@pytest.mark.parametrize( + "seqlen_q,seqlen_k", + [ + (64, 8192), + ], +) +def test_flash_attn_cluster(seqlen_q, seqlen_k, d, causal, dtype): + device = "cuda" + torch.random.manual_seed(0) + batch_size = 2 + nheads = 16 + nheads_kv = 4 + # There was a bug where this would cause "unspecified launch failure" due to Cluster + q = torch.randn(batch_size, seqlen_q, nheads, d, device=device, dtype=dtype) + k = torch.randn(batch_size, seqlen_k, nheads_kv, d, device=device, dtype=dtype) + v = torch.randn(batch_size, seqlen_k, nheads_kv, d, device=device, dtype=dtype) + for _ in range(100): + flash_attn_func(q, k, v, causal=causal) + + +# @pytest.mark.parametrize("dtype", ([torch.float16] if is_sm75 else [torch.float16, torch.bfloat16])) +@pytest.mark.parametrize("dtype", [torch.bfloat16]) +@pytest.mark.parametrize("causal", [False, True]) +# @pytest.mark.parametrize('causal', [False]) +@pytest.mark.parametrize("d", [32, 40, 59, 64, 80, 96, 111, 128, 160, 192, 224, 256]) +# @pytest.mark.parametrize("d", [32, 40, 59, 64, 80, 96, 111, 128]) +# @pytest.mark.parametrize('d', [32, 56, 64, 80, 96, 128]) +# @pytest.mark.parametrize("d", [32, 64, 96, 128, 160, 192]) +# @pytest.mark.parametrize('d', [80]) +@pytest.mark.parametrize( + "seqlen_q,seqlen_k", + [ + (1, 239), + (239, 1), + (3, 799), + (799, 3), + (1024, 128), + (97, 97), + (128, 128), + (200, 200), + (256, 256), + (257, 257), + (384, 384), + (512, 512), + (768, 768), + (1024, 1024), + (2048, 2048), + ], +) +def test_flash_attn_race_condition(seqlen_q, seqlen_k, d, causal, dtype): + device = "cuda" + # set seed + torch.random.manual_seed(0) + # Simulate under memory load + dummy = torch.empty(70 * 1024 ** 3, dtype=torch.uint8, device=device) + batch_size = 60 # Sometimes we need large batch size for the race conditions to trigger + nheads = 4 + q = torch.randn(batch_size, seqlen_q, nheads, d, device=device, dtype=dtype, requires_grad=True) + k = torch.randn(batch_size, seqlen_k, nheads, d, device=device, dtype=dtype, requires_grad=True) + v = torch.randn(batch_size, seqlen_k, nheads, d, device=device, dtype=dtype, requires_grad=True) + torch.random.manual_seed(42) + out0, lse0 = flash_attn_func(q, k, v, causal=causal) + g = torch.randn_like(out0) + dq0, dk0, dv0 = torch.autograd.grad(out0, (q, k, v), g) + # Numerical error if we just do any arithmetic on dq + dq_atol = 2 * ((dq0 + 0.3 - 0.3) - dq0).abs().max().item() + + for i in range(1000): + torch.random.manual_seed(42) + out, lse = flash_attn_func(q, k, v, causal=causal) + assert torch.equal(out, out0) + assert torch.equal(lse, lse0) + + dq, dk, dv = torch.autograd.grad(out, (q, k, v), g) + dq_equal = torch.allclose(dq, dq0, atol=dq_atol) + if not dq_equal: + print(f"Iter {i}, {dq_atol = }, dQ max diff: {(dq - dq0).abs().max().item()}") + # breakpoint() + assert torch.equal(dv, dv0) + assert torch.equal(dk, dk0) + assert dq_equal + + +def attention_combine_ref(out_partial, lse_partial): + """ + out_partial: (num_splits, batch_size, seqlen, nheads, d) + lse_partial: (num_splits, batch_size, nheads, seqlen) + """ + lse = torch.logsumexp(lse_partial, dim=0) + scale = torch.exp(lse_partial - lse) + scale = torch.where(torch.isinf(scale) | torch.isnan(scale), torch.zeros_like(scale), scale) + out = (scale.unsqueeze(-1) * out_partial).sum(0) + return out, lse + + +@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16]) +# @pytest.mark.parametrize("dtype", [torch.float32]) +# @pytest.mark.parametrize("d", [32, 40, 59, 64, 80, 96, 111, 128, 160, 192, 224, 256]) +@pytest.mark.parametrize("d", [64, 96, 128, 192, 256]) +# @pytest.mark.parametrize("d", [128]) +@pytest.mark.parametrize("seqlen", [1, 2, 3, 32, 64, 256, 113, 108, 640, 1024, 2048]) +# @pytest.mark.parametrize("seqlen", [12, 32, 64, 256, 112, 108, 640, 1024, 2048, 8192]) +# @pytest.mark.parametrize("seqlen", [15]) +@pytest.mark.parametrize("num_splits", [1, 2, 3, 5, 17, 32, 55, 97, 155]) +# @pytest.mark.parametrize("num_splits", [1, 2, 3, 5, 11]) +# @pytest.mark.parametrize("num_splits", [128]) +def test_flash_attn_combine(num_splits, seqlen, d, dtype): + if DISABLE_SPLIT: + pytest.skip() + device = "cuda" + # set seed + torch.random.manual_seed(1) + batch_size = 5 + nheads = 16 + # batch_size = 1 + # nheads = 1 + out_partial = torch.randn(num_splits * 2, batch_size, nheads, seqlen, d, device=device, dtype=torch.float32).transpose(2, 3)[:num_splits] # To test non-contiguous tensor + lse_partial = torch.randn(num_splits, batch_size, nheads * 2, seqlen, device=device, dtype=torch.float32).transpose(-1, -2)[:, :, :, :nheads] # To test non-contiguous tensor + # To test short-circuiting based on num_splits + lse_partial[num_splits // 2:, :batch_size // 3] = -float("inf") + out, lse = flash_attn_combine(out_partial, lse_partial, out_dtype=dtype) + out_ref, lse_ref = attention_combine_ref(out_partial, lse_partial) + out_pt = out_ref.to(dtype) + + print(f"LSE max diff: {(lse - lse_ref).abs().max().item()}") + print(f"LSE mean diff: {(lse - lse_ref).abs().mean().item()}") + print(f"Output max diff: {(out - out_ref).abs().max().item()}") + print(f"Output mean diff: {(out - out_ref).abs().mean().item()}") + print(f"Pytorch max diff: {(out_pt - out_ref).abs().max().item()}") + print(f"Pytorch mean diff: {(out_pt - out_ref).abs().mean().item()}") + # breakpoint() + + assert torch.allclose(lse, lse_ref, atol=1e-5, rtol=1e-5) + multiple = 2 + assert ((out - out_ref).abs().max().item() <= multiple * (out_pt - out_ref).abs().max().item()) or torch.allclose(out, out_pt, atol=1e-5, rtol=1e-5) + + # from flash_attn.utils.benchmark import pytorch_profiler + # # pytorch_profiler(torch.sum, lse_partial) + # pytorch_profiler(flash_attn_combine, out_partial, lse_partial) + # pytorch_profiler(torch.sum, out_partial) diff --git a/lib/python3.12/site-packages/hopper/test_util.py b/lib/python3.12/site-packages/hopper/test_util.py new file mode 100644 index 0000000000000000000000000000000000000000..54eb195eb3603c3f3a70b6789a4a1553b86d5be8 --- /dev/null +++ b/lib/python3.12/site-packages/hopper/test_util.py @@ -0,0 +1,288 @@ +import math + +import torch +from einops import rearrange, repeat + +from padding import pad_input, unpad_input + + +def generate_random_padding_mask(max_seqlen, batch_size, device, mode="random", zero_lengths=False): + assert mode in ["full", "random", "third"] + if mode == "full": + lengths = torch.full((batch_size, 1), max_seqlen, device=device, dtype=torch.int32) + elif mode == "random": + lengths = torch.randint( + max(0 if zero_lengths else 1, max_seqlen - 20), max_seqlen + 1, (batch_size, 1), device=device + ) + elif mode == "third": + lengths = torch.randint(max_seqlen // 3, max_seqlen + 1, (batch_size, 1), device=device) + + if zero_lengths: + # Generate zero-lengths every 5 batches and the last batch. + for i in range(batch_size): + if i % 5 == 0: + lengths[i] = 0 + lengths[-1] = 0 + padding_mask = ( + repeat(torch.arange(max_seqlen, device=device), "s -> b s", b=batch_size) < lengths + ) + return padding_mask + + +def generate_qkv( + q, k, v, query_padding_mask=None, key_padding_mask=None, kvpacked=False, qkvpacked=False, + query_unused_mask=None, key_unused_mask=None, +): + """ + Arguments: + q: (batch_size, seqlen_q, nheads, d) + k: (batch_size, seqlen_k, nheads_k, d) + v: (batch_size, seqlen_k, nheads_k, d) + query_padding_mask: (batch_size, seqlen), bool + key_padding_mask: (batch_size, seqlen), bool + """ + assert not (kvpacked and qkvpacked) + batch_size, seqlen_q, nheads, d = q.shape + _, seqlen_k, nheads_k, _ = k.shape + assert k.shape == (batch_size, seqlen_k, nheads_k, d) + assert v.shape == (batch_size, seqlen_k, nheads_k, d) + if query_unused_mask is not None or key_unused_mask is not None: + assert not kvpacked + assert not qkvpacked + + if query_padding_mask is not None: + q_unpad, indices_q, cu_seqlens_q, max_seqlen_q, seqused_q = unpad_input( + q, query_padding_mask, query_unused_mask + ) + output_pad_fn = lambda output_unpad: pad_input( + output_unpad, indices_q, batch_size, seqlen_q + ) + else: + q_unpad = rearrange(q, "b s h d -> (b s) h d") + cu_seqlens_q = torch.arange( + 0, (batch_size + 1) * seqlen_q, step=seqlen_q, dtype=torch.int32, device=q_unpad.device + ) + seqused_q = None + max_seqlen_q = seqlen_q + output_pad_fn = lambda output_unpad: rearrange( + output_unpad, "(b s) h d -> b s h d", b=batch_size + ) + + if key_padding_mask is not None: + k_unpad, indices_k, cu_seqlens_k, max_seqlen_k, seqused_k = unpad_input( + k, key_padding_mask, key_unused_mask + ) + v_unpad, *rest = unpad_input(v, key_padding_mask, key_unused_mask) + else: + k_unpad = rearrange(k, "b s h d -> (b s) h d") + v_unpad = rearrange(v, "b s h d -> (b s) h d") + cu_seqlens_k = torch.arange( + 0, (batch_size + 1) * seqlen_k, step=seqlen_k, dtype=torch.int32, device=k_unpad.device + ) + seqused_k = None + max_seqlen_k = seqlen_k + + if qkvpacked: + assert (query_padding_mask == key_padding_mask).all() + assert nheads == nheads_k + qkv_unpad = torch.stack([q_unpad, k_unpad, v_unpad], dim=1) + qkv = torch.stack([q, k, v], dim=2) + if query_padding_mask is not None: + dqkv_pad_fn = lambda dqkv_unpad: pad_input(dqkv_unpad, indices_q, batch_size, seqlen_q) + else: + dqkv_pad_fn = lambda dqkv_unpad: rearrange( + dqkv_unpad, "(b s) t h d -> b s t h d", b=batch_size + ) + return ( + qkv_unpad.detach().requires_grad_(), + cu_seqlens_q, + max_seqlen_q, + qkv.detach().requires_grad_(), + output_pad_fn, + dqkv_pad_fn, + ) + elif kvpacked: + kv_unpad = torch.stack([k_unpad, v_unpad], dim=1) + kv = torch.stack([k, v], dim=2) + dq_pad_fn = output_pad_fn + if key_padding_mask is not None: + dkv_pad_fn = lambda dkv_unpad: pad_input(dkv_unpad, indices_k, batch_size, seqlen_k) + else: + dkv_pad_fn = lambda dkv_unpad: rearrange( + dkv_unpad, "(b s) t h d -> b s t h d", b=batch_size + ) + return ( + q_unpad.detach().requires_grad_(), + kv_unpad.detach().requires_grad_(), + cu_seqlens_q, + cu_seqlens_k, + max_seqlen_q, + max_seqlen_k, + q.detach().requires_grad_(), + kv.detach().requires_grad_(), + output_pad_fn, + dq_pad_fn, + dkv_pad_fn, + ) + else: + dq_pad_fn = output_pad_fn + if key_padding_mask is not None: + dk_pad_fn = lambda dk_unpad: pad_input(dk_unpad, indices_k, batch_size, seqlen_k) + else: + dk_pad_fn = lambda dk_unpad: rearrange(dk_unpad, "(b s) h d -> b s h d", b=batch_size) + return ( + q_unpad.detach().requires_grad_(), + k_unpad.detach().requires_grad_(), + v_unpad.detach().requires_grad_(), + cu_seqlens_q, + cu_seqlens_k, + seqused_q, + seqused_k, + max_seqlen_q, + max_seqlen_k, + q.detach().requires_grad_(), + k.detach().requires_grad_(), + v.detach().requires_grad_(), + output_pad_fn, + dq_pad_fn, + dk_pad_fn, + ) + + +def construct_local_mask( + seqlen_q, + seqlen_k, + window_size=(-1, -1), # -1 means infinite window size + sink_token_length=0, + query_padding_mask=None, + key_padding_mask=None, + key_leftpad=None, + device=None, +): + row_idx = rearrange(torch.arange(seqlen_q, device=device, dtype=torch.long), "s -> s 1") + col_idx = torch.arange(seqlen_k, device=device, dtype=torch.long) + if key_leftpad is not None: + key_leftpad = rearrange(key_leftpad, "b -> b 1 1 1") + col_idx = repeat(col_idx, "s -> b 1 1 s", b=key_leftpad.shape[0]) + col_idx = torch.where(col_idx >= key_leftpad, col_idx - key_leftpad, 2**32) + sk = ( + seqlen_k + if key_padding_mask is None + else rearrange(key_padding_mask.sum(-1), "b -> b 1 1 1") + ) + sq = ( + seqlen_q + if query_padding_mask is None + else rearrange(query_padding_mask.sum(-1), "b -> b 1 1 1") + ) + if window_size[0] < 0: + return col_idx > row_idx + sk - sq + window_size[1] + else: + sk = torch.full_like(col_idx, seqlen_k) if key_padding_mask is None else sk + return torch.logical_or( + col_idx > torch.minimum(row_idx + sk - sq + window_size[1], sk), + torch.logical_and(col_idx < row_idx + sk - sq - window_size[0], col_idx >= sink_token_length), + ) + + +def attention_ref( + q, + k, + v, + query_padding_mask=None, + key_padding_mask=None, + key_leftpad=None, + attn_bias=None, + dropout_p=0.0, + dropout_mask=None, + causal=False, + q_descale=None, k_descale=None, v_descale=None, + window_size=(-1, -1), # -1 means infinite window size + sink_token_length=0, + softcap=0.0, + upcast=True, + reorder_ops=False, + intermediate_dtype=None, +): + """ + Arguments: + q: (batch_size, seqlen_q, nheads, head_dim) + k: (batch_size, seqlen_k, nheads, head_dim) + v: (batch_size, seqlen_k, nheads, head_dim) + query_padding_mask: (batch_size, seqlen_q) + key_padding_mask: (batch_size, seqlen_k) + attn_bias: broadcastable to (batch_size, nheads, seqlen_q, seqlen_k) + dropout_p: float + dropout_mask: (batch_size, nheads, seqlen_q, seqlen_k) + causal: whether to apply causal masking + upcast: whether to cast all inputs to fp32, do all computation in fp32, then cast + output back to fp16/bf16. + reorder_ops: whether to change the order of operations (scaling k instead of scaling k, etc.) + without changing the math. This is to estimate the numerical error from operation + reordering. + Output: + output: (batch_size, seqlen_q, nheads, head_dim) + attention: (batch_size, nheads, seqlen_q, seqlen_k), softmax after dropout + """ + if causal: + window_size = (window_size[0], 0) + dtype_og = q.dtype + if upcast: + q, k, v = q.float(), k.float(), v.float() + if q_descale is not None: + q_descale = repeat(q_descale, "b h -> b (h g)", g = q.shape[2] // k.shape[2]) + q = (q.float() * rearrange(q_descale, "b h -> b 1 h 1")).to(dtype=q.dtype) + if k_descale is not None: + k = (k.float() * rearrange(k_descale, "b h -> b 1 h 1")).to(dtype=k.dtype) + if v_descale is not None: + v = (v.float() * rearrange(v_descale, "b h -> b 1 h 1")).to(dtype=v.dtype) + seqlen_q, seqlen_k = q.shape[1], k.shape[1] + k = repeat(k, "b s h d -> b s (h g) d", g=q.shape[2] // k.shape[2]) + v = repeat(v, "b s h d -> b s (h g) d", g=q.shape[2] // v.shape[2]) + d = q.shape[-1] + if not reorder_ops: + scores = torch.einsum("bthd,bshd->bhts", q / math.sqrt(d), k) + else: + scores = torch.einsum("bthd,bshd->bhts", q, k / math.sqrt(d)) + if softcap > 0: + scores = torch.tanh(scores / softcap) * softcap + if key_padding_mask is not None: + scores.masked_fill_(rearrange(~key_padding_mask, "b s -> b 1 1 s"), float("-inf")) + if window_size[0] >= 0 or window_size[1] >= 0: + local_mask = construct_local_mask( + seqlen_q, + seqlen_k, + window_size, + sink_token_length, + query_padding_mask, + key_padding_mask, + key_leftpad=key_leftpad, + device=q.device, + ) + scores.masked_fill_(local_mask, float("-inf")) + if attn_bias is not None: + scores = scores + attn_bias + attention = torch.softmax(scores, dim=-1).to(v.dtype) + # We want to mask here so that the attention matrix doesn't have any NaNs + # Otherwise we'll get NaN in dV + if query_padding_mask is not None: + attention = attention.masked_fill(rearrange(~query_padding_mask, "b s -> b 1 s 1"), 0.0) + # Without this we might get NaN in dv + if key_padding_mask is not None: + attention = attention.masked_fill(rearrange(~key_padding_mask, "b s -> b 1 1 s"), 0.0) + # Some rows might be completely masked out so we fill them with zero instead of NaN + if window_size[0] >= 0 or window_size[1] >= 0: + attention = attention.masked_fill(torch.all(local_mask, dim=-1, keepdim=True), 0.0) + dropout_scaling = 1.0 / (1 - dropout_p) + # attention_drop = attention.masked_fill(~dropout_mask, 0.0) * dropout_scaling + # output = torch.einsum('bhts,bshd->bthd', attention_drop , v) + if dropout_mask is not None: + attention_drop = attention.masked_fill(~dropout_mask, 0.0) + else: + attention_drop = attention + if intermediate_dtype is not None: + attention_drop = attention_drop.to(intermediate_dtype).to(attention_drop.dtype) + output = torch.einsum("bhts,bshd->bthd", attention_drop, v * dropout_scaling) + if query_padding_mask is not None: + output.masked_fill_(rearrange(~query_padding_mask, "b s -> b s 1 1"), 0.0) + return output.to(dtype=dtype_og), attention.to(dtype=dtype_og) diff --git a/lib/python3.12/site-packages/httpcore-1.0.9.dist-info/INSTALLER b/lib/python3.12/site-packages/httpcore-1.0.9.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68 --- /dev/null +++ b/lib/python3.12/site-packages/httpcore-1.0.9.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/lib/python3.12/site-packages/httpcore-1.0.9.dist-info/METADATA b/lib/python3.12/site-packages/httpcore-1.0.9.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..8056834e6714b089ef49847820064a1ae4b041fd --- /dev/null +++ b/lib/python3.12/site-packages/httpcore-1.0.9.dist-info/METADATA @@ -0,0 +1,625 @@ +Metadata-Version: 2.4 +Name: httpcore +Version: 1.0.9 +Summary: A minimal low-level HTTP client. +Project-URL: Documentation, https://www.encode.io/httpcore +Project-URL: Homepage, https://www.encode.io/httpcore/ +Project-URL: Source, https://github.com/encode/httpcore +Author-email: Tom Christie +License-Expression: BSD-3-Clause +License-File: LICENSE.md +Classifier: Development Status :: 3 - Alpha +Classifier: Environment :: Web Environment +Classifier: Framework :: AsyncIO +Classifier: Framework :: Trio +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: BSD License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3 :: Only +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Topic :: Internet :: WWW/HTTP +Requires-Python: >=3.8 +Requires-Dist: certifi +Requires-Dist: h11>=0.16 +Provides-Extra: asyncio +Requires-Dist: anyio<5.0,>=4.0; extra == 'asyncio' +Provides-Extra: http2 +Requires-Dist: h2<5,>=3; extra == 'http2' +Provides-Extra: socks +Requires-Dist: socksio==1.*; extra == 'socks' +Provides-Extra: trio +Requires-Dist: trio<1.0,>=0.22.0; extra == 'trio' +Description-Content-Type: text/markdown + +# HTTP Core + +[![Test Suite](https://github.com/encode/httpcore/workflows/Test%20Suite/badge.svg)](https://github.com/encode/httpcore/actions) +[![Package version](https://badge.fury.io/py/httpcore.svg)](https://pypi.org/project/httpcore/) + +> *Do one thing, and do it well.* + +The HTTP Core package provides a minimal low-level HTTP client, which does +one thing only. Sending HTTP requests. + +It does not provide any high level model abstractions over the API, +does not handle redirects, multipart uploads, building authentication headers, +transparent HTTP caching, URL parsing, session cookie handling, +content or charset decoding, handling JSON, environment based configuration +defaults, or any of that Jazz. + +Some things HTTP Core does do: + +* Sending HTTP requests. +* Thread-safe / task-safe connection pooling. +* HTTP(S) proxy & SOCKS proxy support. +* Supports HTTP/1.1 and HTTP/2. +* Provides both sync and async interfaces. +* Async backend support for `asyncio` and `trio`. + +## Requirements + +Python 3.8+ + +## Installation + +For HTTP/1.1 only support, install with: + +```shell +$ pip install httpcore +``` + +There are also a number of optional extras available... + +```shell +$ pip install httpcore['asyncio,trio,http2,socks'] +``` + +## Sending requests + +Send an HTTP request: + +```python +import httpcore + +response = httpcore.request("GET", "https://www.example.com/") + +print(response) +# +print(response.status) +# 200 +print(response.headers) +# [(b'Accept-Ranges', b'bytes'), (b'Age', b'557328'), (b'Cache-Control', b'max-age=604800'), ...] +print(response.content) +# b'\n\n\nExample Domain\n\n\n ...' +``` + +The top-level `httpcore.request()` function is provided for convenience. In practice whenever you're working with `httpcore` you'll want to use the connection pooling functionality that it provides. + +```python +import httpcore + +http = httpcore.ConnectionPool() +response = http.request("GET", "https://www.example.com/") +``` + +Once you're ready to get going, [head over to the documentation](https://www.encode.io/httpcore/). + +## Motivation + +You *probably* don't want to be using HTTP Core directly. It might make sense if +you're writing something like a proxy service in Python, and you just want +something at the lowest possible level, but more typically you'll want to use +a higher level client library, such as `httpx`. + +The motivation for `httpcore` is: + +* To provide a reusable low-level client library, that other packages can then build on top of. +* To provide a *really clear interface split* between the networking code and client logic, + so that each is easier to understand and reason about in isolation. + +## Dependencies + +The `httpcore` package has the following dependencies... + +* `h11` +* `certifi` + +And the following optional extras... + +* `anyio` - Required by `pip install httpcore['asyncio']`. +* `trio` - Required by `pip install httpcore['trio']`. +* `h2` - Required by `pip install httpcore['http2']`. +* `socksio` - Required by `pip install httpcore['socks']`. + +## Versioning + +We use [SEMVER for our versioning policy](https://semver.org/). + +For changes between package versions please see our [project changelog](CHANGELOG.md). + +We recommend pinning your requirements either the most current major version, or a more specific version range: + +```python +pip install 'httpcore==1.*' +``` +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). + +## Version 1.0.9 (April 24th, 2025) + +- Resolve https://github.com/advisories/GHSA-vqfr-h8mv-ghfj with h11 dependency update. (#1008) + +## Version 1.0.8 (April 11th, 2025) + +- Fix `AttributeError` when importing on Python 3.14. (#1005) + +## Version 1.0.7 (November 15th, 2024) + +- Support `proxy=…` configuration on `ConnectionPool()`. (#974) + +## Version 1.0.6 (October 1st, 2024) + +- Relax `trio` dependency pinning. (#956) +- Handle `trio` raising `NotImplementedError` on unsupported platforms. (#955) +- Handle mapping `ssl.SSLError` to `httpcore.ConnectError`. (#918) + +## 1.0.5 (March 27th, 2024) + +- Handle `EndOfStream` exception for anyio backend. (#899) +- Allow trio `0.25.*` series in package dependancies. (#903) + +## 1.0.4 (February 21st, 2024) + +- Add `target` request extension. (#888) +- Fix support for connection `Upgrade` and `CONNECT` when some data in the stream has been read. (#882) + +## 1.0.3 (February 13th, 2024) + +- Fix support for async cancellations. (#880) +- Fix trace extension when used with socks proxy. (#849) +- Fix SSL context for connections using the "wss" scheme (#869) + +## 1.0.2 (November 10th, 2023) + +- Fix `float("inf")` timeouts in `Event.wait` function. (#846) + +## 1.0.1 (November 3rd, 2023) + +- Fix pool timeout to account for the total time spent retrying. (#823) +- Raise a neater RuntimeError when the correct async deps are not installed. (#826) +- Add support for synchronous TLS-in-TLS streams. (#840) + +## 1.0.0 (October 6th, 2023) + +From version 1.0 our async support is now optional, as the package has minimal dependencies by default. + +For async support use either `pip install 'httpcore[asyncio]'` or `pip install 'httpcore[trio]'`. + +The project versioning policy is now explicitly governed by SEMVER. See https://semver.org/. + +- Async support becomes fully optional. (#809) +- Add support for Python 3.12. (#807) + +## 0.18.0 (September 8th, 2023) + +- Add support for HTTPS proxies. (#745, #786) +- Drop Python 3.7 support. (#727) +- Handle `sni_hostname` extension with SOCKS proxy. (#774) +- Handle HTTP/1.1 half-closed connections gracefully. (#641) +- Change the type of `Extensions` from `Mapping[Str, Any]` to `MutableMapping[Str, Any]`. (#762) + +## 0.17.3 (July 5th, 2023) + +- Support async cancellations, ensuring that the connection pool is left in a clean state when cancellations occur. (#726) +- The networking backend interface has [been added to the public API](https://www.encode.io/httpcore/network-backends). Some classes which were previously private implementation detail are now part of the top-level public API. (#699) +- Graceful handling of HTTP/2 GoAway frames, with requests being transparently retried on a new connection. (#730) +- Add exceptions when a synchronous `trace callback` is passed to an asynchronous request or an asynchronous `trace callback` is passed to a synchronous request. (#717) +- Drop Python 3.7 support. (#727) + +## 0.17.2 (May 23th, 2023) + +- Add `socket_options` argument to `ConnectionPool` and `HTTProxy` classes. (#668) +- Improve logging with per-module logger names. (#690) +- Add `sni_hostname` request extension. (#696) +- Resolve race condition during import of `anyio` package. (#692) +- Enable TCP_NODELAY for all synchronous sockets. (#651) + +## 0.17.1 (May 17th, 2023) + +- If 'retries' is set, then allow retries if an SSL handshake error occurs. (#669) +- Improve correctness of tracebacks on network exceptions, by raising properly chained exceptions. (#678) +- Prevent connection-hanging behaviour when HTTP/2 connections are closed by a server-sent 'GoAway' frame. (#679) +- Fix edge-case exception when removing requests from the connection pool. (#680) +- Fix pool timeout edge-case. (#688) + +## 0.17.0 (March 16th, 2023) + +- Add DEBUG level logging. (#648) +- Respect HTTP/2 max concurrent streams when settings updates are sent by server. (#652) +- Increase the allowable HTTP header size to 100kB. (#647) +- Add `retries` option to SOCKS proxy classes. (#643) + +## 0.16.3 (December 20th, 2022) + +- Allow `ws` and `wss` schemes. Allows us to properly support websocket upgrade connections. (#625) +- Forwarding HTTP proxies use a connection-per-remote-host. Required by some proxy implementations. (#637) +- Don't raise `RuntimeError` when closing a connection pool with active connections. Removes some error cases when cancellations are used. (#631) +- Lazy import `anyio`, so that it's no longer a hard dependancy, and isn't imported if unused. (#639) + +## 0.16.2 (November 25th, 2022) + +- Revert 'Fix async cancellation behaviour', which introduced race conditions. (#627) +- Raise `RuntimeError` if attempting to us UNIX domain sockets on Windows. (#619) + +## 0.16.1 (November 17th, 2022) + +- Fix HTTP/1.1 interim informational responses, such as "100 Continue". (#605) + +## 0.16.0 (October 11th, 2022) + +- Support HTTP/1.1 informational responses. (#581) +- Fix async cancellation behaviour. (#580) +- Support `h11` 0.14. (#579) + +## 0.15.0 (May 17th, 2022) + +- Drop Python 3.6 support (#535) +- Ensure HTTP proxy CONNECT requests include `timeout` configuration. (#506) +- Switch to explicit `typing.Optional` for type hints. (#513) +- For `trio` map OSError exceptions to `ConnectError`. (#543) + +## 0.14.7 (February 4th, 2022) + +- Requests which raise a PoolTimeout need to be removed from the pool queue. (#502) +- Fix AttributeError that happened when Socks5Connection were terminated. (#501) + +## 0.14.6 (February 1st, 2022) + +- Fix SOCKS support for `http://` URLs. (#492) +- Resolve race condition around exceptions during streaming a response. (#491) + +## 0.14.5 (January 18th, 2022) + +- SOCKS proxy support. (#478) +- Add proxy_auth argument to HTTPProxy. (#481) +- Improve error message on 'RemoteProtocolError' exception when server disconnects without sending a response. (#479) + +## 0.14.4 (January 5th, 2022) + +- Support HTTP/2 on HTTPS tunnelling proxies. (#468) +- Fix proxy headers missing on HTTP forwarding. (#456) +- Only instantiate SSL context if required. (#457) +- More robust HTTP/2 handling. (#253, #439, #440, #441) + +## 0.14.3 (November 17th, 2021) + +- Fix race condition when removing closed connections from the pool. (#437) + +## 0.14.2 (November 16th, 2021) + +- Failed connections no longer remain in the pool. (Pull #433) + +## 0.14.1 (November 12th, 2021) + +- `max_connections` becomes optional. (Pull #429) +- `certifi` is now included in the install dependancies. (Pull #428) +- `h2` is now strictly optional. (Pull #428) + +## 0.14.0 (November 11th, 2021) + +The 0.14 release is a complete reworking of `httpcore`, comprehensively addressing some underlying issues in the connection pooling, as well as substantially redesigning the API to be more user friendly. + +Some of the lower-level API design also makes the components more easily testable in isolation, and the package now has 100% test coverage. + +See [discussion #419](https://github.com/encode/httpcore/discussions/419) for a little more background. + +There's some other neat bits in there too, such as the "trace" extension, which gives a hook into inspecting the internal events that occur during the request/response cycle. This extension is needed for the HTTPX cli, in order to... + +* Log the point at which the connection is established, and the IP/port on which it is made. +* Determine if the outgoing request should log as HTTP/1.1 or HTTP/2, rather than having to assume it's HTTP/2 if the --http2 flag was passed. (Which may not actually be true.) +* Log SSL version info / certificate info. + +Note that `curio` support is not currently available in 0.14.0. If you're using `httpcore` with `curio` please get in touch, so we can assess if we ought to prioritize it as a feature or not. + +## 0.13.7 (September 13th, 2021) + +- Fix broken error messaging when URL scheme is missing, or a non HTTP(S) scheme is used. (Pull #403) + +## 0.13.6 (June 15th, 2021) + +### Fixed + +- Close sockets when read or write timeouts occur. (Pull #365) + +## 0.13.5 (June 14th, 2021) + +### Fixed + +- Resolved niggles with AnyIO EOF behaviours. (Pull #358, #362) + +## 0.13.4 (June 9th, 2021) + +### Added + +- Improved error messaging when URL scheme is missing, or a non HTTP(S) scheme is used. (Pull #354) + +### Fixed + +- Switched to `anyio` as the default backend implementation when running with `asyncio`. Resolves some awkward [TLS timeout issues](https://github.com/encode/httpx/discussions/1511). + +## 0.13.3 (May 6th, 2021) + +### Added + +- Support HTTP/2 prior knowledge, using `httpcore.SyncConnectionPool(http1=False)`. (Pull #333) + +### Fixed + +- Handle cases where environment does not provide `select.poll` support. (Pull #331) + +## 0.13.2 (April 29th, 2021) + +### Added + +- Improve error message for specific case of `RemoteProtocolError` where server disconnects without sending a response. (Pull #313) + +## 0.13.1 (April 28th, 2021) + +### Fixed + +- More resiliant testing for closed connections. (Pull #311) +- Don't raise exceptions on ungraceful connection closes. (Pull #310) + +## 0.13.0 (April 21st, 2021) + +The 0.13 release updates the core API in order to match the HTTPX Transport API, +introduced in HTTPX 0.18 onwards. + +An example of making requests with the new interface is: + +```python +with httpcore.SyncConnectionPool() as http: + status_code, headers, stream, extensions = http.handle_request( + method=b'GET', + url=(b'https', b'example.org', 443, b'/'), + headers=[(b'host', b'example.org'), (b'user-agent', b'httpcore')] + stream=httpcore.ByteStream(b''), + extensions={} + ) + body = stream.read() + print(status_code, body) +``` + +### Changed + +- The `.request()` method is now `handle_request()`. (Pull #296) +- The `.arequest()` method is now `.handle_async_request()`. (Pull #296) +- The `headers` argument is no longer optional. (Pull #296) +- The `stream` argument is no longer optional. (Pull #296) +- The `ext` argument is now named `extensions`, and is no longer optional. (Pull #296) +- The `"reason"` extension keyword is now named `"reason_phrase"`. (Pull #296) +- The `"reason_phrase"` and `"http_version"` extensions now use byte strings for their values. (Pull #296) +- The `httpcore.PlainByteStream()` class becomes `httpcore.ByteStream()`. (Pull #296) + +### Added + +- Streams now support a `.read()` interface. (Pull #296) + +### Fixed + +- Task cancellation no longer leaks connections from the connection pool. (Pull #305) + +## 0.12.3 (December 7th, 2020) + +### Fixed + +- Abort SSL connections on close rather than waiting for remote EOF when using `asyncio`. (Pull #167) +- Fix exception raised in case of connect timeouts when using the `anyio` backend. (Pull #236) +- Fix `Host` header precedence for `:authority` in HTTP/2. (Pull #241, #243) +- Handle extra edge case when detecting for socket readability when using `asyncio`. (Pull #242, #244) +- Fix `asyncio` SSL warning when using proxy tunneling. (Pull #249) + +## 0.12.2 (November 20th, 2020) + +### Fixed + +- Properly wrap connect errors on the asyncio backend. (Pull #235) +- Fix `ImportError` occurring on Python 3.9 when using the HTTP/1.1 sync client in a multithreaded context. (Pull #237) + +## 0.12.1 (November 7th, 2020) + +### Added + +- Add connect retries. (Pull #221) + +### Fixed + +- Tweak detection of dropped connections, resolving an issue with open files limits on Linux. (Pull #185) +- Avoid leaking connections when establishing an HTTP tunnel to a proxy has failed. (Pull #223) +- Properly wrap OS errors when using `trio`. (Pull #225) + +## 0.12.0 (October 6th, 2020) + +### Changed + +- HTTP header casing is now preserved, rather than always sent in lowercase. (#216 and python-hyper/h11#104) + +### Added + +- Add Python 3.9 to officially supported versions. + +### Fixed + +- Gracefully handle a stdlib asyncio bug when a connection is closed while it is in a paused-for-reading state. (#201) + +## 0.11.1 (September 28nd, 2020) + +### Fixed + +- Add await to async semaphore release() coroutine (#197) +- Drop incorrect curio classifier (#192) + +## 0.11.0 (September 22nd, 2020) + +The Transport API with 0.11.0 has a couple of significant changes. + +Firstly we've moved changed the request interface in order to allow extensions, which will later enable us to support features +such as trailing headers, HTTP/2 server push, and CONNECT/Upgrade connections. + +The interface changes from: + +```python +def request(method, url, headers, stream, timeout): + return (http_version, status_code, reason, headers, stream) +``` + +To instead including an optional dictionary of extensions on the request and response: + +```python +def request(method, url, headers, stream, ext): + return (status_code, headers, stream, ext) +``` + +Having an open-ended extensions point will allow us to add later support for various optional features, that wouldn't otherwise be supported without these API changes. + +In particular: + +* Trailing headers support. +* HTTP/2 Server Push +* sendfile. +* Exposing raw connection on CONNECT, Upgrade, HTTP/2 bi-di streaming. +* Exposing debug information out of the API, including template name, template context. + +Currently extensions are limited to: + +* request: `timeout` - Optional. Timeout dictionary. +* response: `http_version` - Optional. Include the HTTP version used on the response. +* response: `reason` - Optional. Include the reason phrase used on the response. Only valid with HTTP/1.*. + +See https://github.com/encode/httpx/issues/1274#issuecomment-694884553 for the history behind this. + +Secondly, the async version of `request` is now namespaced as `arequest`. + +This allows concrete transports to support both sync and async implementations on the same class. + +### Added + +- Add curio support. (Pull #168) +- Add anyio support, with `backend="anyio"`. (Pull #169) + +### Changed + +- Update the Transport API to use 'ext' for optional extensions. (Pull #190) +- Update the Transport API to use `.request` and `.arequest` so implementations can support both sync and async. (Pull #189) + +## 0.10.2 (August 20th, 2020) + +### Added + +- Added Unix Domain Socket support. (Pull #139) + +### Fixed + +- Always include the port on proxy CONNECT requests. (Pull #154) +- Fix `max_keepalive_connections` configuration. (Pull #153) +- Fixes behaviour in HTTP/1.1 where server disconnects can be used to signal the end of the response body. (Pull #164) + +## 0.10.1 (August 7th, 2020) + +- Include `max_keepalive_connections` on `AsyncHTTPProxy`/`SyncHTTPProxy` classes. + +## 0.10.0 (August 7th, 2020) + +The most notable change in the 0.10.0 release is that HTTP/2 support is now fully optional. + +Use either `pip install httpcore` for HTTP/1.1 support only, or `pip install httpcore[http2]` for HTTP/1.1 and HTTP/2 support. + +### Added + +- HTTP/2 support becomes optional. (Pull #121, #130) +- Add `local_address=...` support. (Pull #100, #134) +- Add `PlainByteStream`, `IteratorByteStream`, `AsyncIteratorByteStream`. The `AsyncByteSteam` and `SyncByteStream` classes are now pure interface classes. (#133) +- Add `LocalProtocolError`, `RemoteProtocolError` exceptions. (Pull #129) +- Add `UnsupportedProtocol` exception. (Pull #128) +- Add `.get_connection_info()` method. (Pull #102, #137) +- Add better TRACE logs. (Pull #101) + +### Changed + +- `max_keepalive` is deprecated in favour of `max_keepalive_connections`. (Pull #140) + +### Fixed + +- Improve handling of server disconnects. (Pull #112) + +## 0.9.1 (May 27th, 2020) + +### Fixed + +- Proper host resolution for sync case, including IPv6 support. (Pull #97) +- Close outstanding connections when connection pool is closed. (Pull #98) + +## 0.9.0 (May 21th, 2020) + +### Changed + +- URL port becomes an `Optional[int]` instead of `int`. (Pull #92) + +### Fixed + +- Honor HTTP/2 max concurrent streams settings. (Pull #89, #90) +- Remove incorrect debug log. (Pull #83) + +## 0.8.4 (May 11th, 2020) + +### Added + +- Logging via HTTPCORE_LOG_LEVEL and HTTPX_LOG_LEVEL environment variables +and TRACE level logging. (Pull #79) + +### Fixed + +- Reuse of connections on HTTP/2 in close concurrency situations. (Pull #81) + +## 0.8.3 (May 6rd, 2020) + +### Fixed + +- Include `Host` and `Accept` headers on proxy "CONNECT" requests. +- De-duplicate any headers also contained in proxy_headers. +- HTTP/2 flag not being passed down to proxy connections. + +## 0.8.2 (May 3rd, 2020) + +### Fixed + +- Fix connections using proxy forwarding requests not being added to the +connection pool properly. (Pull #70) + +## 0.8.1 (April 30th, 2020) + +### Changed + +- Allow inherintance of both `httpcore.AsyncByteStream`, `httpcore.SyncByteStream` without type conflicts. + +## 0.8.0 (April 30th, 2020) + +### Fixed + +- Fixed tunnel proxy support. + +### Added + +- New `TimeoutException` base class. + +## 0.7.0 (March 5th, 2020) + +- First integration with HTTPX. diff --git a/lib/python3.12/site-packages/httpcore-1.0.9.dist-info/RECORD b/lib/python3.12/site-packages/httpcore-1.0.9.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..44d4769e791312e8572bc5c24ed2efdc86819835 --- /dev/null +++ b/lib/python3.12/site-packages/httpcore-1.0.9.dist-info/RECORD @@ -0,0 +1,68 @@ +httpcore-1.0.9.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +httpcore-1.0.9.dist-info/METADATA,sha256=_i1P2mGZEol4d54M8n88BFxTGGP83Zh-rMdPOhjUHCE,21529 +httpcore-1.0.9.dist-info/RECORD,, +httpcore-1.0.9.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87 +httpcore-1.0.9.dist-info/licenses/LICENSE.md,sha256=_ctZFUx0y6uhahEkL3dAvqnyPW_rVUeRfYxflKgDkqU,1518 +httpcore/__init__.py,sha256=9kT_kqChCCJUTHww24ZmR_ezcdbpRYWksD-gYNzkZP8,3445 +httpcore/__pycache__/__init__.cpython-312.pyc,, +httpcore/__pycache__/_api.cpython-312.pyc,, +httpcore/__pycache__/_exceptions.cpython-312.pyc,, +httpcore/__pycache__/_models.cpython-312.pyc,, +httpcore/__pycache__/_ssl.cpython-312.pyc,, +httpcore/__pycache__/_synchronization.cpython-312.pyc,, +httpcore/__pycache__/_trace.cpython-312.pyc,, +httpcore/__pycache__/_utils.cpython-312.pyc,, +httpcore/_api.py,sha256=unZmeDschBWCGCPCwkS3Wot9euK6bg_kKxLtGTxw214,3146 +httpcore/_async/__init__.py,sha256=EWdl2v4thnAHzJpqjU4h2a8DUiGAvNiWrkii9pfhTf0,1221 +httpcore/_async/__pycache__/__init__.cpython-312.pyc,, +httpcore/_async/__pycache__/connection.cpython-312.pyc,, +httpcore/_async/__pycache__/connection_pool.cpython-312.pyc,, +httpcore/_async/__pycache__/http11.cpython-312.pyc,, +httpcore/_async/__pycache__/http2.cpython-312.pyc,, +httpcore/_async/__pycache__/http_proxy.cpython-312.pyc,, +httpcore/_async/__pycache__/interfaces.cpython-312.pyc,, +httpcore/_async/__pycache__/socks_proxy.cpython-312.pyc,, +httpcore/_async/connection.py,sha256=6OcPXqMEfc0BU38_-iHUNDd1vKSTc2UVT09XqNb_BOk,8449 +httpcore/_async/connection_pool.py,sha256=DOIQ2s2ZCf9qfwxhzMprTPLqCL8OxGXiKF6qRHxvVyY,17307 +httpcore/_async/http11.py,sha256=-qM9bV7PjSQF5vxs37-eUXOIFwbIjPcZbNliuX9TtBw,13880 +httpcore/_async/http2.py,sha256=azX1fcmtXaIwjputFlZ4vd92J8xwjGOa9ax9QIv4394,23936 +httpcore/_async/http_proxy.py,sha256=2zVkrlv-Ds-rWGaqaXlrhEJiAQFPo23BT3Gq_sWoBXU,14701 +httpcore/_async/interfaces.py,sha256=jTiaWL83pgpGC9ziv90ZfwaKNMmHwmOalzaKiuTxATo,4455 +httpcore/_async/socks_proxy.py,sha256=lLKgLlggPfhFlqi0ODeBkOWvt9CghBBUyqsnsU1tx6Q,13841 +httpcore/_backends/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +httpcore/_backends/__pycache__/__init__.cpython-312.pyc,, +httpcore/_backends/__pycache__/anyio.cpython-312.pyc,, +httpcore/_backends/__pycache__/auto.cpython-312.pyc,, +httpcore/_backends/__pycache__/base.cpython-312.pyc,, +httpcore/_backends/__pycache__/mock.cpython-312.pyc,, +httpcore/_backends/__pycache__/sync.cpython-312.pyc,, +httpcore/_backends/__pycache__/trio.cpython-312.pyc,, +httpcore/_backends/anyio.py,sha256=x8PgEhXRC8bVqsdzk_YJx8Y6d9Tub06CuUSwnbmtqoY,5252 +httpcore/_backends/auto.py,sha256=zO136PKZmsaTDK-HRk84eA-MUg8_2wJf4NvmK432Aio,1662 +httpcore/_backends/base.py,sha256=aShgRdZnMmRhFWHetjumlM73f8Kz1YOAyCUP_4kHslA,3042 +httpcore/_backends/mock.py,sha256=er9T436uSe7NLrfiLa4x6Nuqg5ivQ693CxWYCWsgbH4,4077 +httpcore/_backends/sync.py,sha256=bhE4d9iK9Umxdsdsgm2EfKnXaBms2WggGYU-7jmUujU,7977 +httpcore/_backends/trio.py,sha256=LHu4_Mr5MswQmmT3yE4oLgf9b_JJfeVS4BjDxeJc7Ro,5996 +httpcore/_exceptions.py,sha256=looCKga3_YVYu3s-d3L9RMPRJyhsY7fiuuGxvkOD0c0,1184 +httpcore/_models.py,sha256=IO2CcXcdpovRcLTdGFGB6RyBZdEm2h_TOmoCc4rEKho,17623 +httpcore/_ssl.py,sha256=srqmSNU4iOUvWF-SrJvb8G_YEbHFELOXQOwdDIBTS9c,187 +httpcore/_sync/__init__.py,sha256=JBDIgXt5la1LCJ1sLQeKhjKFpLnpNr8Svs6z2ni3fgg,1141 +httpcore/_sync/__pycache__/__init__.cpython-312.pyc,, +httpcore/_sync/__pycache__/connection.cpython-312.pyc,, +httpcore/_sync/__pycache__/connection_pool.cpython-312.pyc,, +httpcore/_sync/__pycache__/http11.cpython-312.pyc,, +httpcore/_sync/__pycache__/http2.cpython-312.pyc,, +httpcore/_sync/__pycache__/http_proxy.cpython-312.pyc,, +httpcore/_sync/__pycache__/interfaces.cpython-312.pyc,, +httpcore/_sync/__pycache__/socks_proxy.cpython-312.pyc,, +httpcore/_sync/connection.py,sha256=9exGOb3PB-Mp2T1-sckSeL2t-tJ_9-NXomV8ihmWCgU,8238 +httpcore/_sync/connection_pool.py,sha256=a-T8LTsUxc7r0Ww1atfHSDoWPjQ0fA8Ul7S3-F0Mj70,16955 +httpcore/_sync/http11.py,sha256=IFobD1Md5JFlJGKWnh1_Q3epikUryI8qo09v8MiJIEA,13476 +httpcore/_sync/http2.py,sha256=AxU4yhcq68Bn5vqdJYtiXKYUj7nvhYbxz3v4rT4xnvA,23400 +httpcore/_sync/http_proxy.py,sha256=_al_6crKuEZu2wyvu493RZImJdBJnj5oGKNjLOJL2Zo,14463 +httpcore/_sync/interfaces.py,sha256=snXON42vUDHO5JBJvo8D4VWk2Wat44z2OXXHDrjbl94,4344 +httpcore/_sync/socks_proxy.py,sha256=zegZW9Snqj2_992DFJa8_CppOVBkVL4AgwduRkStakQ,13614 +httpcore/_synchronization.py,sha256=zSi13mAColBnknjZBknUC6hKNDQT4C6ijnezZ-r0T2s,9434 +httpcore/_trace.py,sha256=ck6ZoIzYTkdNAIfq5MGeKqBXDtqjOX-qfYwmZFbrGco,3952 +httpcore/_utils.py,sha256=_RLgXYOAYC350ikALV59GZ68IJrdocRZxPs9PjmzdFY,1537 +httpcore/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 diff --git a/lib/python3.12/site-packages/httpcore-1.0.9.dist-info/WHEEL b/lib/python3.12/site-packages/httpcore-1.0.9.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..12228d414b6cfed7c39d3781c85c63256a1d7fb5 --- /dev/null +++ b/lib/python3.12/site-packages/httpcore-1.0.9.dist-info/WHEEL @@ -0,0 +1,4 @@ +Wheel-Version: 1.0 +Generator: hatchling 1.27.0 +Root-Is-Purelib: true +Tag: py3-none-any diff --git a/lib/python3.12/site-packages/httpcore-1.0.9.dist-info/licenses/LICENSE.md b/lib/python3.12/site-packages/httpcore-1.0.9.dist-info/licenses/LICENSE.md new file mode 100644 index 0000000000000000000000000000000000000000..311b2b56c53f678ab95fc0def708c675d521a807 --- /dev/null +++ b/lib/python3.12/site-packages/httpcore-1.0.9.dist-info/licenses/LICENSE.md @@ -0,0 +1,27 @@ +Copyright © 2020, [Encode OSS Ltd](https://www.encode.io/). +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/lib/python3.12/site-packages/jiter/__init__.py b/lib/python3.12/site-packages/jiter/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3d17192f352540e415a2982ca50e57a753b1a5b6 --- /dev/null +++ b/lib/python3.12/site-packages/jiter/__init__.py @@ -0,0 +1,5 @@ +from .jiter import * + +__doc__ = jiter.__doc__ +if hasattr(jiter, "__all__"): + __all__ = jiter.__all__ \ No newline at end of file diff --git a/lib/python3.12/site-packages/jiter/__init__.pyi b/lib/python3.12/site-packages/jiter/__init__.pyi new file mode 100644 index 0000000000000000000000000000000000000000..ab488b825e3127814e93536cfc36476bb06f663e --- /dev/null +++ b/lib/python3.12/site-packages/jiter/__init__.pyi @@ -0,0 +1,70 @@ +import decimal +from typing import Any, Literal + +def from_json( + json_data: bytes, + /, + *, + allow_inf_nan: bool = True, + cache_mode: Literal[True, False, "all", "keys", "none"] = "all", + partial_mode: Literal[True, False, "off", "on", "trailing-strings"] = False, + catch_duplicate_keys: bool = False, + float_mode: Literal["float", "decimal", "lossless-float"] = "float", +) -> Any: + """ + Parse input bytes into a JSON object. + + Arguments: + json_data: The JSON data to parse + allow_inf_nan: Whether to allow infinity (`Infinity` an `-Infinity`) and `NaN` values to float fields. + Defaults to True. + cache_mode: cache Python strings to improve performance at the cost of some memory usage + - True / 'all' - cache all strings + - 'keys' - cache only object keys + - False / 'none' - cache nothing + partial_mode: How to handle incomplete strings: + - False / 'off' - raise an exception if the input is incomplete + - True / 'on' - allow incomplete JSON but discard the last string if it is incomplete + - 'trailing-strings' - allow incomplete JSON, and include the last incomplete string in the output + catch_duplicate_keys: if True, raise an exception if objects contain the same key multiple times + float_mode: How to return floats: as a `float`, `Decimal` or `LosslessFloat` + + Returns: + Python object built from the JSON input. + """ + +def cache_clear() -> None: + """ + Reset the string cache. + """ + +def cache_usage() -> int: + """ + get the size of the string cache. + + Returns: + Size of the string cache in bytes. + """ + + +class LosslessFloat: + """ + Represents a float from JSON, by holding the underlying bytes representing a float from JSON. + """ + def __init__(self, json_float: bytes): + """Construct a LosslessFloat object from a JSON bytes slice""" + + def as_decimal(self) -> decimal.Decimal: + """Construct a Python Decimal from the JSON bytes slice""" + + def __float__(self) -> float: + """Construct a Python float from the JSON bytes slice""" + + def __bytes__(self) -> bytes: + """Return the JSON bytes slice as bytes""" + + def __str__(self): + """Return the JSON bytes slice as a string""" + + def __repr__(self): + ... diff --git a/lib/python3.12/site-packages/jiter/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/jiter/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..907cf4c9d09ddbe259e03a9e5a319313836f941a Binary files /dev/null and b/lib/python3.12/site-packages/jiter/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/jiter/py.typed b/lib/python3.12/site-packages/jiter/py.typed new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.12/site-packages/kiwisolver-1.4.9.dist-info/INSTALLER b/lib/python3.12/site-packages/kiwisolver-1.4.9.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68 --- /dev/null +++ b/lib/python3.12/site-packages/kiwisolver-1.4.9.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/lib/python3.12/site-packages/kiwisolver-1.4.9.dist-info/METADATA b/lib/python3.12/site-packages/kiwisolver-1.4.9.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..1f0723f8e6a0cf7cf4ce17a6988662cba4d90064 --- /dev/null +++ b/lib/python3.12/site-packages/kiwisolver-1.4.9.dist-info/METADATA @@ -0,0 +1,120 @@ +Metadata-Version: 2.4 +Name: kiwisolver +Version: 1.4.9 +Summary: A fast implementation of the Cassowary constraint solver +Author-email: The Nucleic Development Team +Maintainer-email: "Matthieu C. Dartiailh" +License: ========================= + The Kiwi licensing terms + ========================= + Kiwi is licensed under the terms of the Modified BSD License (also known as + New or Revised BSD), as follows: + + Copyright (c) 2013-2025, Nucleic Development Team + + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + + Neither the name of the Nucleic Development Team nor the names of its + contributors may be used to endorse or promote products derived from this + software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + About Kiwi + ---------- + Chris Colbert began the Kiwi project in December 2013 in an effort to + create a blisteringly fast UI constraint solver. Chris is still the + project lead. + + The Nucleic Development Team is the set of all contributors to the Nucleic + project and its subprojects. + + The core team that coordinates development on GitHub can be found here: + http://github.com/nucleic. The current team consists of: + + * Chris Colbert + + Our Copyright Policy + -------------------- + Nucleic uses a shared copyright model. Each contributor maintains copyright + over their contributions to Nucleic. But, it is important to note that these + contributions are typically only changes to the repositories. Thus, the Nucleic + source code, in its entirety is not the copyright of any single person or + institution. Instead, it is the collective copyright of the entire Nucleic + Development Team. If individual contributors want to maintain a record of what + changes/contributions they have specific copyright on, they should indicate + their copyright in the commit message of the change, when they commit the + change to one of the Nucleic repositories. + + With this in mind, the following banner should be used in any source code file + to indicate the copyright and license terms: + + #------------------------------------------------------------------------------ + # Copyright (c) 2013-2025, Nucleic Development Team. + # + # Distributed under the terms of the Modified BSD License. + # + # The full license is in the file LICENSE, distributed with this software. + #------------------------------------------------------------------------------ + +Project-URL: homepage, https://github.com/nucleic/kiwi +Project-URL: documentation, https://kiwisolver.readthedocs.io/en/latest/ +Project-URL: repository, https://github.com/nucleic/kiwi +Project-URL: changelog, https://github.com/nucleic/kiwi/blob/main/releasenotes.rst +Classifier: License :: OSI Approved :: BSD License +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 +Classifier: Programming Language :: Python :: 3.14 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Requires-Python: >=3.10 +Description-Content-Type: text/x-rst +License-File: LICENSE +Dynamic: license-file + +Welcome to Kiwi +=============== + +.. image:: https://github.com/nucleic/kiwi/workflows/Continuous%20Integration/badge.svg + :target: https://github.com/nucleic/kiwi/actions +.. image:: https://github.com/nucleic/kiwi/workflows/Documentation%20building/badge.svg + :target: https://github.com/nucleic/kiwi/actions +.. image:: https://codecov.io/gh/nucleic/kiwi/branch/main/graph/badge.svg + :target: https://codecov.io/gh/nucleic/kiwi +.. image:: https://readthedocs.org/projects/kiwisolver/badge/?version=latest + :target: https://kiwisolver.readthedocs.io/en/latest/?badge=latest + :alt: Documentation Status + +Kiwi is an efficient C++ implementation of the Cassowary constraint solving +algorithm. Kiwi is an implementation of the algorithm based on the +`seminal Cassowary paper `_. +It is *not* a refactoring of the original C++ solver. Kiwi has been designed +from the ground up to be lightweight and fast. Kiwi ranges from 10x to 500x +faster than the original Cassowary solver with typical use cases gaining a 40x +improvement. Memory savings are consistently > 5x. + +In addition to the C++ solver, Kiwi ships with hand-rolled Python bindings for +Python 3.7+. diff --git a/lib/python3.12/site-packages/kiwisolver-1.4.9.dist-info/RECORD b/lib/python3.12/site-packages/kiwisolver-1.4.9.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..11419f9e51bc71657289f8c26d48c6032149aed1 --- /dev/null +++ b/lib/python3.12/site-packages/kiwisolver-1.4.9.dist-info/RECORD @@ -0,0 +1,13 @@ +kiwisolver-1.4.9.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +kiwisolver-1.4.9.dist-info/METADATA,sha256=ytNKN7XLZs-huWdvM3IQN64thQubDHE3dHih1r8MfUA,6260 +kiwisolver-1.4.9.dist-info/RECORD,, +kiwisolver-1.4.9.dist-info/WHEEL,sha256=aSgG0F4rGPZtV0iTEIfy6dtHq6g67Lze3uLfk0vWn88,151 +kiwisolver-1.4.9.dist-info/licenses/LICENSE,sha256=zyB5nTLeDu-i6pBOOsUSL0eu1dNSkwdR62FyR2LEnZA,3289 +kiwisolver-1.4.9.dist-info/top_level.txt,sha256=xqwWj7oSHlpIjcw2QMJb8puTFPdjDBO78AZp9gjTh9c,11 +kiwisolver/__init__.py,sha256=4Sa-MNI1lRh3K1n9LEgKBeZXVVI-uJrD-xsk16jYR2c,1013 +kiwisolver/__pycache__/__init__.cpython-312.pyc,, +kiwisolver/__pycache__/exceptions.cpython-312.pyc,, +kiwisolver/_cext.cpython-312-x86_64-linux-gnu.so,sha256=NT3Dc1krBmxzCVpMcramDcNcDufchhN8jda7K_oWfzg,5728472 +kiwisolver/_cext.pyi,sha256=-w7Otijw7d-9Rh_1-EiYtRn2L-STdvjoArh02szr8tA,8657 +kiwisolver/exceptions.py,sha256=haGECAifFjVqwT5esQ1sEiqsajW6Jydip16ieHPeL04,1242 +kiwisolver/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 diff --git a/lib/python3.12/site-packages/kiwisolver-1.4.9.dist-info/WHEEL b/lib/python3.12/site-packages/kiwisolver-1.4.9.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..e21e9f2f89caabbb0f5e84c1775fbe781dd45a63 --- /dev/null +++ b/lib/python3.12/site-packages/kiwisolver-1.4.9.dist-info/WHEEL @@ -0,0 +1,6 @@ +Wheel-Version: 1.0 +Generator: setuptools (80.9.0) +Root-Is-Purelib: false +Tag: cp312-cp312-manylinux_2_17_x86_64 +Tag: cp312-cp312-manylinux2014_x86_64 + diff --git a/lib/python3.12/site-packages/kiwisolver-1.4.9.dist-info/licenses/LICENSE b/lib/python3.12/site-packages/kiwisolver-1.4.9.dist-info/licenses/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..cd42fdb38b4561314d90d82200ff198ff384a7c1 --- /dev/null +++ b/lib/python3.12/site-packages/kiwisolver-1.4.9.dist-info/licenses/LICENSE @@ -0,0 +1,71 @@ +========================= + The Kiwi licensing terms +========================= +Kiwi is licensed under the terms of the Modified BSD License (also known as +New or Revised BSD), as follows: + +Copyright (c) 2013-2025, Nucleic Development Team + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +Redistributions in binary form must reproduce the above copyright notice, this +list of conditions and the following disclaimer in the documentation and/or +other materials provided with the distribution. + +Neither the name of the Nucleic Development Team nor the names of its +contributors may be used to endorse or promote products derived from this +software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +About Kiwi +---------- +Chris Colbert began the Kiwi project in December 2013 in an effort to +create a blisteringly fast UI constraint solver. Chris is still the +project lead. + +The Nucleic Development Team is the set of all contributors to the Nucleic +project and its subprojects. + +The core team that coordinates development on GitHub can be found here: +http://github.com/nucleic. The current team consists of: + +* Chris Colbert + +Our Copyright Policy +-------------------- +Nucleic uses a shared copyright model. Each contributor maintains copyright +over their contributions to Nucleic. But, it is important to note that these +contributions are typically only changes to the repositories. Thus, the Nucleic +source code, in its entirety is not the copyright of any single person or +institution. Instead, it is the collective copyright of the entire Nucleic +Development Team. If individual contributors want to maintain a record of what +changes/contributions they have specific copyright on, they should indicate +their copyright in the commit message of the change, when they commit the +change to one of the Nucleic repositories. + +With this in mind, the following banner should be used in any source code file +to indicate the copyright and license terms: + +#------------------------------------------------------------------------------ +# Copyright (c) 2013-2025, Nucleic Development Team. +# +# Distributed under the terms of the Modified BSD License. +# +# The full license is in the file LICENSE, distributed with this software. +#------------------------------------------------------------------------------ diff --git a/lib/python3.12/site-packages/kiwisolver-1.4.9.dist-info/top_level.txt b/lib/python3.12/site-packages/kiwisolver-1.4.9.dist-info/top_level.txt new file mode 100644 index 0000000000000000000000000000000000000000..9b85884d1a91ed346d4f939d15d27ad1e33a5894 --- /dev/null +++ b/lib/python3.12/site-packages/kiwisolver-1.4.9.dist-info/top_level.txt @@ -0,0 +1 @@ +kiwisolver diff --git a/lib/python3.12/site-packages/ptyprocess-0.7.0.dist-info/INSTALLER b/lib/python3.12/site-packages/ptyprocess-0.7.0.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68 --- /dev/null +++ b/lib/python3.12/site-packages/ptyprocess-0.7.0.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/lib/python3.12/site-packages/ptyprocess-0.7.0.dist-info/LICENSE b/lib/python3.12/site-packages/ptyprocess-0.7.0.dist-info/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..9c772742de96e441674e7ee221058d369c5d7976 --- /dev/null +++ b/lib/python3.12/site-packages/ptyprocess-0.7.0.dist-info/LICENSE @@ -0,0 +1,16 @@ +Ptyprocess is under the ISC license, as code derived from Pexpect. + http://opensource.org/licenses/ISC + +Copyright (c) 2013-2014, Pexpect development team +Copyright (c) 2012, Noah Spurrier + +PERMISSION TO USE, COPY, MODIFY, AND/OR DISTRIBUTE THIS SOFTWARE FOR ANY PURPOSE +WITH OR WITHOUT FEE IS HEREBY GRANTED, PROVIDED THAT THE ABOVE COPYRIGHT NOTICE +AND THIS PERMISSION NOTICE APPEAR IN ALL COPIES. THE SOFTWARE IS PROVIDED +"AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE +INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT +SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL +DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, +WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING +OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + diff --git a/lib/python3.12/site-packages/ptyprocess-0.7.0.dist-info/METADATA b/lib/python3.12/site-packages/ptyprocess-0.7.0.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..ab1d4e0efedc5bb136ba8f41f83d5f7903c24a89 --- /dev/null +++ b/lib/python3.12/site-packages/ptyprocess-0.7.0.dist-info/METADATA @@ -0,0 +1,37 @@ +Metadata-Version: 2.1 +Name: ptyprocess +Version: 0.7.0 +Summary: Run a subprocess in a pseudo terminal +Home-page: https://github.com/pexpect/ptyprocess +License: UNKNOWN +Author: Thomas Kluyver +Author-email: thomas@kluyver.me.uk +Description-Content-Type: text/x-rst +Classifier: Development Status :: 5 - Production/Stable +Classifier: Environment :: Console +Classifier: Intended Audience :: Developers +Classifier: Intended Audience :: System Administrators +Classifier: License :: OSI Approved :: ISC License (ISCL) +Classifier: Operating System :: POSIX +Classifier: Operating System :: MacOS :: MacOS X +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Topic :: Terminals + +Launch a subprocess in a pseudo terminal (pty), and interact with both the +process and its pty. + +Sometimes, piping stdin and stdout is not enough. There might be a password +prompt that doesn't read from stdin, output that changes when it's going to a +pipe rather than a terminal, or curses-style interfaces that rely on a terminal. +If you need to automate these things, running the process in a pseudo terminal +(pty) is the answer. + +Interface:: + + p = PtyProcessUnicode.spawn(['python']) + p.read(20) + p.write('6+6\n') + p.read(20) + diff --git a/lib/python3.12/site-packages/ptyprocess-0.7.0.dist-info/RECORD b/lib/python3.12/site-packages/ptyprocess-0.7.0.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..c594bd58481b8ae24acee019dca848492777bf44 --- /dev/null +++ b/lib/python3.12/site-packages/ptyprocess-0.7.0.dist-info/RECORD @@ -0,0 +1,13 @@ +ptyprocess-0.7.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +ptyprocess-0.7.0.dist-info/LICENSE,sha256=yCLThbGnMymEYkF5m-zxhpC11Edkwb7WkwC1NqQFAwo,905 +ptyprocess-0.7.0.dist-info/METADATA,sha256=w8K5a12aVdpZWMNNCMGKCEn1ZgkCbMRtXJW4t3_PPgw,1312 +ptyprocess-0.7.0.dist-info/RECORD,, +ptyprocess-0.7.0.dist-info/WHEEL,sha256=NLqmsx-ZFZ6gDavYgh2oH0ZSN-KRmpcdEXIZDnYy9Pg,99 +ptyprocess/__init__.py,sha256=sn-W_1nNRTuIOi2aCEHVL06wCVJcR-LOZdgpXzwFuTU,138 +ptyprocess/__pycache__/__init__.cpython-312.pyc,, +ptyprocess/__pycache__/_fork_pty.cpython-312.pyc,, +ptyprocess/__pycache__/ptyprocess.cpython-312.pyc,, +ptyprocess/__pycache__/util.cpython-312.pyc,, +ptyprocess/_fork_pty.py,sha256=VVvMy8c4ZpjDMiIMSg8T1BQ1g3SBexDpey_cxi0n5aw,2362 +ptyprocess/ptyprocess.py,sha256=sk2sU2I22Yyl1gU3FjFmpWL3B43o0KqG3d3CI8r0Nq8,31686 +ptyprocess/util.py,sha256=rQAdDRZfoOiOn6vykWth0wI6FFKAp7aJtBSdt-KBWdU,2785 diff --git a/lib/python3.12/site-packages/ptyprocess-0.7.0.dist-info/WHEEL b/lib/python3.12/site-packages/ptyprocess-0.7.0.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..3825653907a25e6c53a51b0e07f01029da5f7683 --- /dev/null +++ b/lib/python3.12/site-packages/ptyprocess-0.7.0.dist-info/WHEEL @@ -0,0 +1,5 @@ +Wheel-Version: 1.0 +Generator: flit 3.0.0 +Root-Is-Purelib: true +Tag: py2-none-any +Tag: py3-none-any diff --git a/lib/python3.12/site-packages/pydantic_core/__init__.py b/lib/python3.12/site-packages/pydantic_core/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..98b64b86e89a1b2b3233add2f1de0eed37ee10a2 --- /dev/null +++ b/lib/python3.12/site-packages/pydantic_core/__init__.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +import sys as _sys +from typing import Any as _Any + +from ._pydantic_core import ( + ArgsKwargs, + MultiHostUrl, + PydanticCustomError, + PydanticKnownError, + PydanticOmit, + PydanticSerializationError, + PydanticSerializationUnexpectedValue, + PydanticUndefined, + PydanticUndefinedType, + PydanticUseDefault, + SchemaError, + SchemaSerializer, + SchemaValidator, + Some, + TzInfo, + Url, + ValidationError, + __version__, + from_json, + to_json, + to_jsonable_python, + validate_core_schema, +) +from .core_schema import CoreConfig, CoreSchema, CoreSchemaType, ErrorType + +if _sys.version_info < (3, 11): + from typing_extensions import NotRequired as _NotRequired +else: + from typing import NotRequired as _NotRequired + +if _sys.version_info < (3, 12): + from typing_extensions import TypedDict as _TypedDict +else: + from typing import TypedDict as _TypedDict + +__all__ = [ + '__version__', + 'CoreConfig', + 'CoreSchema', + 'CoreSchemaType', + 'SchemaValidator', + 'SchemaSerializer', + 'Some', + 'Url', + 'MultiHostUrl', + 'ArgsKwargs', + 'PydanticUndefined', + 'PydanticUndefinedType', + 'SchemaError', + 'ErrorDetails', + 'InitErrorDetails', + 'ValidationError', + 'PydanticCustomError', + 'PydanticKnownError', + 'PydanticOmit', + 'PydanticUseDefault', + 'PydanticSerializationError', + 'PydanticSerializationUnexpectedValue', + 'TzInfo', + 'to_json', + 'from_json', + 'to_jsonable_python', + 'validate_core_schema', +] + + +class ErrorDetails(_TypedDict): + type: str + """ + The type of error that occurred, this is an identifier designed for + programmatic use that will change rarely or never. + + `type` is unique for each error message, and can hence be used as an identifier to build custom error messages. + """ + loc: tuple[int | str, ...] + """Tuple of strings and ints identifying where in the schema the error occurred.""" + msg: str + """A human readable error message.""" + input: _Any + """The input data at this `loc` that caused the error.""" + ctx: _NotRequired[dict[str, _Any]] + """ + Values which are required to render the error message, and could hence be useful in rendering custom error messages. + Also useful for passing custom error data forward. + """ + url: _NotRequired[str] + """ + The documentation URL giving information about the error. No URL is available if + a [`PydanticCustomError`][pydantic_core.PydanticCustomError] is used. + """ + + +class InitErrorDetails(_TypedDict): + type: str | PydanticCustomError + """The type of error that occurred, this should be a "slug" identifier that changes rarely or never.""" + loc: _NotRequired[tuple[int | str, ...]] + """Tuple of strings and ints identifying where in the schema the error occurred.""" + input: _Any + """The input data at this `loc` that caused the error.""" + ctx: _NotRequired[dict[str, _Any]] + """ + Values which are required to render the error message, and could hence be useful in rendering custom error messages. + Also useful for passing custom error data forward. + """ + + +class ErrorTypeInfo(_TypedDict): + """ + Gives information about errors. + """ + + type: ErrorType + """The type of error that occurred, this should a "slug" identifier that changes rarely or never.""" + message_template_python: str + """String template to render a human readable error message from using context, when the input is Python.""" + example_message_python: str + """Example of a human readable error message, when the input is Python.""" + message_template_json: _NotRequired[str] + """String template to render a human readable error message from using context, when the input is JSON data.""" + example_message_json: _NotRequired[str] + """Example of a human readable error message, when the input is JSON data.""" + example_context: dict[str, _Any] | None + """Example of context values.""" + + +class MultiHostHost(_TypedDict): + """ + A host part of a multi-host URL. + """ + + username: str | None + """The username part of this host, or `None`.""" + password: str | None + """The password part of this host, or `None`.""" + host: str | None + """The host part of this host, or `None`.""" + port: int | None + """The port part of this host, or `None`.""" diff --git a/lib/python3.12/site-packages/pydantic_core/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/pydantic_core/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fb7a06b11070b386512c0976150ce047943b5372 Binary files /dev/null and b/lib/python3.12/site-packages/pydantic_core/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/pydantic_core/_pydantic_core.pyi b/lib/python3.12/site-packages/pydantic_core/_pydantic_core.pyi new file mode 100644 index 0000000000000000000000000000000000000000..17098cca580d7a19c1705987bc06172bc3b39b8d --- /dev/null +++ b/lib/python3.12/site-packages/pydantic_core/_pydantic_core.pyi @@ -0,0 +1,1039 @@ +import datetime +from collections.abc import Mapping +from typing import Any, Callable, Generic, Literal, TypeVar, final + +from _typeshed import SupportsAllComparisons +from typing_extensions import LiteralString, Self, TypeAlias + +from pydantic_core import ErrorDetails, ErrorTypeInfo, InitErrorDetails, MultiHostHost +from pydantic_core.core_schema import CoreConfig, CoreSchema, ErrorType + +__all__ = [ + '__version__', + 'build_profile', + 'build_info', + '_recursion_limit', + 'ArgsKwargs', + 'SchemaValidator', + 'SchemaSerializer', + 'Url', + 'MultiHostUrl', + 'SchemaError', + 'ValidationError', + 'PydanticCustomError', + 'PydanticKnownError', + 'PydanticOmit', + 'PydanticUseDefault', + 'PydanticSerializationError', + 'PydanticSerializationUnexpectedValue', + 'PydanticUndefined', + 'PydanticUndefinedType', + 'Some', + 'to_json', + 'from_json', + 'to_jsonable_python', + 'list_all_errors', + 'TzInfo', + 'validate_core_schema', +] +__version__: str +build_profile: str +build_info: str +_recursion_limit: int + +_T = TypeVar('_T', default=Any, covariant=True) + +_StringInput: TypeAlias = 'dict[str, _StringInput]' + +@final +class Some(Generic[_T]): + """ + Similar to Rust's [`Option::Some`](https://doc.rust-lang.org/std/option/enum.Option.html) type, this + identifies a value as being present, and provides a way to access it. + + Generally used in a union with `None` to different between "some value which could be None" and no value. + """ + + __match_args__ = ('value',) + + @property + def value(self) -> _T: + """ + Returns the value wrapped by `Some`. + """ + @classmethod + def __class_getitem__(cls, item: Any, /) -> type[Self]: ... + +@final +class SchemaValidator: + """ + `SchemaValidator` is the Python wrapper for `pydantic-core`'s Rust validation logic, internally it owns one + `CombinedValidator` which may in turn own more `CombinedValidator`s which make up the full schema validator. + """ + + # note: pyo3 currently supports __new__, but not __init__, though we include __init__ stubs + # and docstrings here (and in the following classes) for documentation purposes + + def __init__(self, schema: CoreSchema, config: CoreConfig | None = None) -> None: + """Initializes the `SchemaValidator`. + + Arguments: + schema: The `CoreSchema` to use for validation. + config: Optionally a [`CoreConfig`][pydantic_core.core_schema.CoreConfig] to configure validation. + """ + + def __new__(cls, schema: CoreSchema, config: CoreConfig | None = None) -> Self: ... + @property + def title(self) -> str: + """ + The title of the schema, as used in the heading of [`ValidationError.__str__()`][pydantic_core.ValidationError]. + """ + def validate_python( + self, + input: Any, + *, + strict: bool | None = None, + from_attributes: bool | None = None, + context: Any | None = None, + self_instance: Any | None = None, + allow_partial: bool | Literal['off', 'on', 'trailing-strings'] = False, + by_alias: bool | None = None, + by_name: bool | None = None, + ) -> Any: + """ + Validate a Python object against the schema and return the validated object. + + Arguments: + input: The Python object to validate. + strict: Whether to validate the object in strict mode. + If `None`, the value of [`CoreConfig.strict`][pydantic_core.core_schema.CoreConfig] is used. + from_attributes: Whether to validate objects as inputs to models by extracting attributes. + If `None`, the value of [`CoreConfig.from_attributes`][pydantic_core.core_schema.CoreConfig] is used. + context: The context to use for validation, this is passed to functional validators as + [`info.context`][pydantic_core.core_schema.ValidationInfo.context]. + self_instance: An instance of a model set attributes on from validation, this is used when running + validation from the `__init__` method of a model. + allow_partial: Whether to allow partial validation; if `True` errors in the last element of sequences + and mappings are ignored. + `'trailing-strings'` means any final unfinished JSON string is included in the result. + by_alias: Whether to use the field's alias when validating against the provided input data. + by_name: Whether to use the field's name when validating against the provided input data. + + Raises: + ValidationError: If validation fails. + Exception: Other error types maybe raised if internal errors occur. + + Returns: + The validated object. + """ + def isinstance_python( + self, + input: Any, + *, + strict: bool | None = None, + from_attributes: bool | None = None, + context: Any | None = None, + self_instance: Any | None = None, + by_alias: bool | None = None, + by_name: bool | None = None, + ) -> bool: + """ + Similar to [`validate_python()`][pydantic_core.SchemaValidator.validate_python] but returns a boolean. + + Arguments match `validate_python()`. This method will not raise `ValidationError`s but will raise internal + errors. + + Returns: + `True` if validation succeeds, `False` if validation fails. + """ + def validate_json( + self, + input: str | bytes | bytearray, + *, + strict: bool | None = None, + context: Any | None = None, + self_instance: Any | None = None, + allow_partial: bool | Literal['off', 'on', 'trailing-strings'] = False, + by_alias: bool | None = None, + by_name: bool | None = None, + ) -> Any: + """ + Validate JSON data directly against the schema and return the validated Python object. + + This method should be significantly faster than `validate_python(json.loads(json_data))` as it avoids the + need to create intermediate Python objects + + It also handles constructing the correct Python type even in strict mode, where + `validate_python(json.loads(json_data))` would fail validation. + + Arguments: + input: The JSON data to validate. + strict: Whether to validate the object in strict mode. + If `None`, the value of [`CoreConfig.strict`][pydantic_core.core_schema.CoreConfig] is used. + context: The context to use for validation, this is passed to functional validators as + [`info.context`][pydantic_core.core_schema.ValidationInfo.context]. + self_instance: An instance of a model set attributes on from validation. + allow_partial: Whether to allow partial validation; if `True` incomplete JSON will be parsed successfully + and errors in the last element of sequences and mappings are ignored. + `'trailing-strings'` means any final unfinished JSON string is included in the result. + by_alias: Whether to use the field's alias when validating against the provided input data. + by_name: Whether to use the field's name when validating against the provided input data. + + Raises: + ValidationError: If validation fails or if the JSON data is invalid. + Exception: Other error types maybe raised if internal errors occur. + + Returns: + The validated Python object. + """ + def validate_strings( + self, + input: _StringInput, + *, + strict: bool | None = None, + context: Any | None = None, + allow_partial: bool | Literal['off', 'on', 'trailing-strings'] = False, + by_alias: bool | None = None, + by_name: bool | None = None, + ) -> Any: + """ + Validate a string against the schema and return the validated Python object. + + This is similar to `validate_json` but applies to scenarios where the input will be a string but not + JSON data, e.g. URL fragments, query parameters, etc. + + Arguments: + input: The input as a string, or bytes/bytearray if `strict=False`. + strict: Whether to validate the object in strict mode. + If `None`, the value of [`CoreConfig.strict`][pydantic_core.core_schema.CoreConfig] is used. + context: The context to use for validation, this is passed to functional validators as + [`info.context`][pydantic_core.core_schema.ValidationInfo.context]. + allow_partial: Whether to allow partial validation; if `True` errors in the last element of sequences + and mappings are ignored. + `'trailing-strings'` means any final unfinished JSON string is included in the result. + by_alias: Whether to use the field's alias when validating against the provided input data. + by_name: Whether to use the field's name when validating against the provided input data. + + Raises: + ValidationError: If validation fails or if the JSON data is invalid. + Exception: Other error types maybe raised if internal errors occur. + + Returns: + The validated Python object. + """ + def validate_assignment( + self, + obj: Any, + field_name: str, + field_value: Any, + *, + strict: bool | None = None, + from_attributes: bool | None = None, + context: Any | None = None, + by_alias: bool | None = None, + by_name: bool | None = None, + ) -> dict[str, Any] | tuple[dict[str, Any], dict[str, Any] | None, set[str]]: + """ + Validate an assignment to a field on a model. + + Arguments: + obj: The model instance being assigned to. + field_name: The name of the field to validate assignment for. + field_value: The value to assign to the field. + strict: Whether to validate the object in strict mode. + If `None`, the value of [`CoreConfig.strict`][pydantic_core.core_schema.CoreConfig] is used. + from_attributes: Whether to validate objects as inputs to models by extracting attributes. + If `None`, the value of [`CoreConfig.from_attributes`][pydantic_core.core_schema.CoreConfig] is used. + context: The context to use for validation, this is passed to functional validators as + [`info.context`][pydantic_core.core_schema.ValidationInfo.context]. + by_alias: Whether to use the field's alias when validating against the provided input data. + by_name: Whether to use the field's name when validating against the provided input data. + + Raises: + ValidationError: If validation fails. + Exception: Other error types maybe raised if internal errors occur. + + Returns: + Either the model dict or a tuple of `(model_data, model_extra, fields_set)` + """ + def get_default_value(self, *, strict: bool | None = None, context: Any = None) -> Some | None: + """ + Get the default value for the schema, including running default value validation. + + Arguments: + strict: Whether to validate the default value in strict mode. + If `None`, the value of [`CoreConfig.strict`][pydantic_core.core_schema.CoreConfig] is used. + context: The context to use for validation, this is passed to functional validators as + [`info.context`][pydantic_core.core_schema.ValidationInfo.context]. + + Raises: + ValidationError: If validation fails. + Exception: Other error types maybe raised if internal errors occur. + + Returns: + `None` if the schema has no default value, otherwise a [`Some`][pydantic_core.Some] containing the default. + """ + +# In reality, `bool` should be replaced by `Literal[True]` but mypy fails to correctly apply bidirectional type inference +# (e.g. when using `{'a': {'b': True}}`). +_IncEx: TypeAlias = set[int] | set[str] | Mapping[int, _IncEx | bool] | Mapping[str, _IncEx | bool] + +@final +class SchemaSerializer: + """ + `SchemaSerializer` is the Python wrapper for `pydantic-core`'s Rust serialization logic, internally it owns one + `CombinedSerializer` which may in turn own more `CombinedSerializer`s which make up the full schema serializer. + """ + + def __init__(self, schema: CoreSchema, config: CoreConfig | None = None) -> None: + """Initializes the `SchemaSerializer`. + + Arguments: + schema: The `CoreSchema` to use for serialization. + config: Optionally a [`CoreConfig`][pydantic_core.core_schema.CoreConfig] to to configure serialization. + """ + + def __new__(cls, schema: CoreSchema, config: CoreConfig | None = None) -> Self: ... + def to_python( + self, + value: Any, + *, + mode: str | None = None, + include: _IncEx | None = None, + exclude: _IncEx | None = None, + by_alias: bool | None = None, + exclude_unset: bool = False, + exclude_defaults: bool = False, + exclude_none: bool = False, + round_trip: bool = False, + warnings: bool | Literal['none', 'warn', 'error'] = True, + fallback: Callable[[Any], Any] | None = None, + serialize_as_any: bool = False, + context: Any | None = None, + ) -> Any: + """ + Serialize/marshal a Python object to a Python object including transforming and filtering data. + + Arguments: + value: The Python object to serialize. + mode: The serialization mode to use, either `'python'` or `'json'`, defaults to `'python'`. In JSON mode, + all values are converted to JSON compatible types, e.g. `None`, `int`, `float`, `str`, `list`, `dict`. + include: A set of fields to include, if `None` all fields are included. + exclude: A set of fields to exclude, if `None` no fields are excluded. + by_alias: Whether to use the alias names of fields. + exclude_unset: Whether to exclude fields that are not set, + e.g. are not included in `__pydantic_fields_set__`. + exclude_defaults: Whether to exclude fields that are equal to their default value. + exclude_none: Whether to exclude fields that have a value of `None`. + round_trip: Whether to enable serialization and validation round-trip support. + warnings: How to handle invalid fields. False/"none" ignores them, True/"warn" logs errors, + "error" raises a [`PydanticSerializationError`][pydantic_core.PydanticSerializationError]. + fallback: A function to call when an unknown value is encountered, + if `None` a [`PydanticSerializationError`][pydantic_core.PydanticSerializationError] error is raised. + serialize_as_any: Whether to serialize fields with duck-typing serialization behavior. + context: The context to use for serialization, this is passed to functional serializers as + [`info.context`][pydantic_core.core_schema.SerializationInfo.context]. + + Raises: + PydanticSerializationError: If serialization fails and no `fallback` function is provided. + + Returns: + The serialized Python object. + """ + def to_json( + self, + value: Any, + *, + indent: int | None = None, + include: _IncEx | None = None, + exclude: _IncEx | None = None, + by_alias: bool | None = None, + exclude_unset: bool = False, + exclude_defaults: bool = False, + exclude_none: bool = False, + round_trip: bool = False, + warnings: bool | Literal['none', 'warn', 'error'] = True, + fallback: Callable[[Any], Any] | None = None, + serialize_as_any: bool = False, + context: Any | None = None, + ) -> bytes: + """ + Serialize a Python object to JSON including transforming and filtering data. + + Arguments: + value: The Python object to serialize. + indent: If `None`, the JSON will be compact, otherwise it will be pretty-printed with the indent provided. + include: A set of fields to include, if `None` all fields are included. + exclude: A set of fields to exclude, if `None` no fields are excluded. + by_alias: Whether to use the alias names of fields. + exclude_unset: Whether to exclude fields that are not set, + e.g. are not included in `__pydantic_fields_set__`. + exclude_defaults: Whether to exclude fields that are equal to their default value. + exclude_none: Whether to exclude fields that have a value of `None`. + round_trip: Whether to enable serialization and validation round-trip support. + warnings: How to handle invalid fields. False/"none" ignores them, True/"warn" logs errors, + "error" raises a [`PydanticSerializationError`][pydantic_core.PydanticSerializationError]. + fallback: A function to call when an unknown value is encountered, + if `None` a [`PydanticSerializationError`][pydantic_core.PydanticSerializationError] error is raised. + serialize_as_any: Whether to serialize fields with duck-typing serialization behavior. + context: The context to use for serialization, this is passed to functional serializers as + [`info.context`][pydantic_core.core_schema.SerializationInfo.context]. + + Raises: + PydanticSerializationError: If serialization fails and no `fallback` function is provided. + + Returns: + JSON bytes. + """ + +def to_json( + value: Any, + *, + indent: int | None = None, + include: _IncEx | None = None, + exclude: _IncEx | None = None, + # Note: In Pydantic 2.11, the default value of `by_alias` on `SchemaSerializer` was changed from `True` to `None`, + # to be consistent with the Pydantic "dump" methods. However, the default of `True` was kept here for + # backwards compatibility. In Pydantic V3, `by_alias` is expected to default to `True` everywhere: + by_alias: bool = True, + exclude_none: bool = False, + round_trip: bool = False, + timedelta_mode: Literal['iso8601', 'float'] = 'iso8601', + bytes_mode: Literal['utf8', 'base64', 'hex'] = 'utf8', + inf_nan_mode: Literal['null', 'constants', 'strings'] = 'constants', + serialize_unknown: bool = False, + fallback: Callable[[Any], Any] | None = None, + serialize_as_any: bool = False, + context: Any | None = None, +) -> bytes: + """ + Serialize a Python object to JSON including transforming and filtering data. + + This is effectively a standalone version of [`SchemaSerializer.to_json`][pydantic_core.SchemaSerializer.to_json]. + + Arguments: + value: The Python object to serialize. + indent: If `None`, the JSON will be compact, otherwise it will be pretty-printed with the indent provided. + include: A set of fields to include, if `None` all fields are included. + exclude: A set of fields to exclude, if `None` no fields are excluded. + by_alias: Whether to use the alias names of fields. + exclude_none: Whether to exclude fields that have a value of `None`. + round_trip: Whether to enable serialization and validation round-trip support. + timedelta_mode: How to serialize `timedelta` objects, either `'iso8601'` or `'float'`. + bytes_mode: How to serialize `bytes` objects, either `'utf8'`, `'base64'`, or `'hex'`. + inf_nan_mode: How to serialize `Infinity`, `-Infinity` and `NaN` values, either `'null'`, `'constants'`, or `'strings'`. + serialize_unknown: Attempt to serialize unknown types, `str(value)` will be used, if that fails + `""` will be used. + fallback: A function to call when an unknown value is encountered, + if `None` a [`PydanticSerializationError`][pydantic_core.PydanticSerializationError] error is raised. + serialize_as_any: Whether to serialize fields with duck-typing serialization behavior. + context: The context to use for serialization, this is passed to functional serializers as + [`info.context`][pydantic_core.core_schema.SerializationInfo.context]. + + Raises: + PydanticSerializationError: If serialization fails and no `fallback` function is provided. + + Returns: + JSON bytes. + """ + +def from_json( + data: str | bytes | bytearray, + *, + allow_inf_nan: bool = True, + cache_strings: bool | Literal['all', 'keys', 'none'] = True, + allow_partial: bool | Literal['off', 'on', 'trailing-strings'] = False, +) -> Any: + """ + Deserialize JSON data to a Python object. + + This is effectively a faster version of `json.loads()`, with some extra functionality. + + Arguments: + data: The JSON data to deserialize. + allow_inf_nan: Whether to allow `Infinity`, `-Infinity` and `NaN` values as `json.loads()` does by default. + cache_strings: Whether to cache strings to avoid constructing new Python objects, + this should have a significant impact on performance while increasing memory usage slightly, + `all/True` means cache all strings, `keys` means cache only dict keys, `none/False` means no caching. + allow_partial: Whether to allow partial deserialization, if `True` JSON data is returned if the end of the + input is reached before the full object is deserialized, e.g. `["aa", "bb", "c` would return `['aa', 'bb']`. + `'trailing-strings'` means any final unfinished JSON string is included in the result. + + Raises: + ValueError: If deserialization fails. + + Returns: + The deserialized Python object. + """ + +def to_jsonable_python( + value: Any, + *, + include: _IncEx | None = None, + exclude: _IncEx | None = None, + # Note: In Pydantic 2.11, the default value of `by_alias` on `SchemaSerializer` was changed from `True` to `None`, + # to be consistent with the Pydantic "dump" methods. However, the default of `True` was kept here for + # backwards compatibility. In Pydantic V3, `by_alias` is expected to default to `True` everywhere: + by_alias: bool = True, + exclude_none: bool = False, + round_trip: bool = False, + timedelta_mode: Literal['iso8601', 'float'] = 'iso8601', + bytes_mode: Literal['utf8', 'base64', 'hex'] = 'utf8', + inf_nan_mode: Literal['null', 'constants', 'strings'] = 'constants', + serialize_unknown: bool = False, + fallback: Callable[[Any], Any] | None = None, + serialize_as_any: bool = False, + context: Any | None = None, +) -> Any: + """ + Serialize/marshal a Python object to a JSON-serializable Python object including transforming and filtering data. + + This is effectively a standalone version of + [`SchemaSerializer.to_python(mode='json')`][pydantic_core.SchemaSerializer.to_python]. + + Args: + value: The Python object to serialize. + include: A set of fields to include, if `None` all fields are included. + exclude: A set of fields to exclude, if `None` no fields are excluded. + by_alias: Whether to use the alias names of fields. + exclude_none: Whether to exclude fields that have a value of `None`. + round_trip: Whether to enable serialization and validation round-trip support. + timedelta_mode: How to serialize `timedelta` objects, either `'iso8601'` or `'float'`. + bytes_mode: How to serialize `bytes` objects, either `'utf8'`, `'base64'`, or `'hex'`. + inf_nan_mode: How to serialize `Infinity`, `-Infinity` and `NaN` values, either `'null'`, `'constants'`, or `'strings'`. + serialize_unknown: Attempt to serialize unknown types, `str(value)` will be used, if that fails + `""` will be used. + fallback: A function to call when an unknown value is encountered, + if `None` a [`PydanticSerializationError`][pydantic_core.PydanticSerializationError] error is raised. + serialize_as_any: Whether to serialize fields with duck-typing serialization behavior. + context: The context to use for serialization, this is passed to functional serializers as + [`info.context`][pydantic_core.core_schema.SerializationInfo.context]. + + Raises: + PydanticSerializationError: If serialization fails and no `fallback` function is provided. + + Returns: + The serialized Python object. + """ + +class Url(SupportsAllComparisons): + """ + A URL type, internal logic uses the [url rust crate](https://docs.rs/url/latest/url/) originally developed + by Mozilla. + """ + + def __init__(self, url: str) -> None: ... + def __new__(cls, url: str) -> Self: ... + @property + def scheme(self) -> str: ... + @property + def username(self) -> str | None: ... + @property + def password(self) -> str | None: ... + @property + def host(self) -> str | None: ... + def unicode_host(self) -> str | None: ... + @property + def port(self) -> int | None: ... + @property + def path(self) -> str | None: ... + @property + def query(self) -> str | None: ... + def query_params(self) -> list[tuple[str, str]]: ... + @property + def fragment(self) -> str | None: ... + def unicode_string(self) -> str: ... + def __repr__(self) -> str: ... + def __str__(self) -> str: ... + def __deepcopy__(self, memo: dict) -> str: ... + @classmethod + def build( + cls, + *, + scheme: str, + username: str | None = None, + password: str | None = None, + host: str, + port: int | None = None, + path: str | None = None, + query: str | None = None, + fragment: str | None = None, + ) -> Self: ... + +class MultiHostUrl(SupportsAllComparisons): + """ + A URL type with support for multiple hosts, as used by some databases for DSNs, e.g. `https://foo.com,bar.com/path`. + + Internal URL logic uses the [url rust crate](https://docs.rs/url/latest/url/) originally developed + by Mozilla. + """ + + def __init__(self, url: str) -> None: ... + def __new__(cls, url: str) -> Self: ... + @property + def scheme(self) -> str: ... + @property + def path(self) -> str | None: ... + @property + def query(self) -> str | None: ... + def query_params(self) -> list[tuple[str, str]]: ... + @property + def fragment(self) -> str | None: ... + def hosts(self) -> list[MultiHostHost]: ... + def unicode_string(self) -> str: ... + def __repr__(self) -> str: ... + def __str__(self) -> str: ... + def __deepcopy__(self, memo: dict) -> Self: ... + @classmethod + def build( + cls, + *, + scheme: str, + hosts: list[MultiHostHost] | None = None, + username: str | None = None, + password: str | None = None, + host: str | None = None, + port: int | None = None, + path: str | None = None, + query: str | None = None, + fragment: str | None = None, + ) -> Self: ... + +@final +class SchemaError(Exception): + """ + Information about errors that occur while building a [`SchemaValidator`][pydantic_core.SchemaValidator] + or [`SchemaSerializer`][pydantic_core.SchemaSerializer]. + """ + + def error_count(self) -> int: + """ + Returns: + The number of errors in the schema. + """ + def errors(self) -> list[ErrorDetails]: + """ + Returns: + A list of [`ErrorDetails`][pydantic_core.ErrorDetails] for each error in the schema. + """ + +class ValidationError(ValueError): + """ + `ValidationError` is the exception raised by `pydantic-core` when validation fails, it contains a list of errors + which detail why validation failed. + """ + @classmethod + def from_exception_data( + cls, + title: str, + line_errors: list[InitErrorDetails], + input_type: Literal['python', 'json'] = 'python', + hide_input: bool = False, + ) -> Self: + """ + Python constructor for a Validation Error. + + The API for constructing validation errors will probably change in the future, + hence the static method rather than `__init__`. + + Arguments: + title: The title of the error, as used in the heading of `str(validation_error)` + line_errors: A list of [`InitErrorDetails`][pydantic_core.InitErrorDetails] which contain information + about errors that occurred during validation. + input_type: Whether the error is for a Python object or JSON. + hide_input: Whether to hide the input value in the error message. + """ + @property + def title(self) -> str: + """ + The title of the error, as used in the heading of `str(validation_error)`. + """ + def error_count(self) -> int: + """ + Returns: + The number of errors in the validation error. + """ + def errors( + self, *, include_url: bool = True, include_context: bool = True, include_input: bool = True + ) -> list[ErrorDetails]: + """ + Details about each error in the validation error. + + Args: + include_url: Whether to include a URL to documentation on the error each error. + include_context: Whether to include the context of each error. + include_input: Whether to include the input value of each error. + + Returns: + A list of [`ErrorDetails`][pydantic_core.ErrorDetails] for each error in the validation error. + """ + def json( + self, + *, + indent: int | None = None, + include_url: bool = True, + include_context: bool = True, + include_input: bool = True, + ) -> str: + """ + Same as [`errors()`][pydantic_core.ValidationError.errors] but returns a JSON string. + + Args: + indent: The number of spaces to indent the JSON by, or `None` for no indentation - compact JSON. + include_url: Whether to include a URL to documentation on the error each error. + include_context: Whether to include the context of each error. + include_input: Whether to include the input value of each error. + + Returns: + a JSON string. + """ + + def __repr__(self) -> str: + """ + A string representation of the validation error. + + Whether or not documentation URLs are included in the repr is controlled by the + environment variable `PYDANTIC_ERRORS_INCLUDE_URL` being set to `1` or + `true`; by default, URLs are shown. + + Due to implementation details, this environment variable can only be set once, + before the first validation error is created. + """ + +class PydanticCustomError(ValueError): + """A custom exception providing flexible error handling for Pydantic validators. + + You can raise this error in custom validators when you'd like flexibility in regards to the error type, message, and context. + + Example: + ```py + from pydantic_core import PydanticCustomError + + def custom_validator(v) -> None: + if v <= 10: + raise PydanticCustomError('custom_value_error', 'Value must be greater than {value}', {'value': 10, 'extra_context': 'extra_data'}) + return v + ``` + """ + + def __init__( + self, error_type: LiteralString, message_template: LiteralString, context: dict[str, Any] | None = None + ) -> None: + """Initializes the `PydanticCustomError`. + + Arguments: + error_type: The error type. + message_template: The message template. + context: The data to inject into the message template. + """ + + def __new__( + cls, error_type: LiteralString, message_template: LiteralString, context: dict[str, Any] | None = None + ) -> Self: ... + @property + def context(self) -> dict[str, Any] | None: + """Values which are required to render the error message, and could hence be useful in passing error data forward.""" + + @property + def type(self) -> str: + """The error type associated with the error. For consistency with Pydantic, this is typically a snake_case string.""" + + @property + def message_template(self) -> str: + """The message template associated with the error. This is a string that can be formatted with context variables in `{curly_braces}`.""" + + def message(self) -> str: + """The formatted message associated with the error. This presents as the message template with context variables appropriately injected.""" + +@final +class PydanticKnownError(ValueError): + """A helper class for raising exceptions that mimic Pydantic's built-in exceptions, with more flexibility in regards to context. + + Unlike [`PydanticCustomError`][pydantic_core.PydanticCustomError], the `error_type` argument must be a known `ErrorType`. + + Example: + ```py + from pydantic_core import PydanticKnownError + + def custom_validator(v) -> None: + if v <= 10: + raise PydanticKnownError(error_type='greater_than', context={'gt': 10}) + return v + ``` + """ + + def __init__(self, error_type: ErrorType, context: dict[str, Any] | None = None) -> None: + """Initializes the `PydanticKnownError`. + + Arguments: + error_type: The error type. + context: The data to inject into the message template. + """ + + def __new__(cls, error_type: ErrorType, context: dict[str, Any] | None = None) -> Self: ... + @property + def context(self) -> dict[str, Any] | None: + """Values which are required to render the error message, and could hence be useful in passing error data forward.""" + + @property + def type(self) -> ErrorType: + """The type of the error.""" + + @property + def message_template(self) -> str: + """The message template associated with the provided error type. This is a string that can be formatted with context variables in `{curly_braces}`.""" + + def message(self) -> str: + """The formatted message associated with the error. This presents as the message template with context variables appropriately injected.""" + +@final +class PydanticOmit(Exception): + """An exception to signal that a field should be omitted from a generated result. + + This could span from omitting a field from a JSON Schema to omitting a field from a serialized result. + Upcoming: more robust support for using PydanticOmit in custom serializers is still in development. + Right now, this is primarily used in the JSON Schema generation process. + + Example: + ```py + from typing import Callable + + from pydantic_core import PydanticOmit + + from pydantic import BaseModel + from pydantic.json_schema import GenerateJsonSchema, JsonSchemaValue + + + class MyGenerateJsonSchema(GenerateJsonSchema): + def handle_invalid_for_json_schema(self, schema, error_info) -> JsonSchemaValue: + raise PydanticOmit + + + class Predicate(BaseModel): + name: str = 'no-op' + func: Callable = lambda x: x + + + instance_example = Predicate() + + validation_schema = instance_example.model_json_schema(schema_generator=MyGenerateJsonSchema, mode='validation') + print(validation_schema) + ''' + {'properties': {'name': {'default': 'no-op', 'title': 'Name', 'type': 'string'}}, 'title': 'Predicate', 'type': 'object'} + ''' + ``` + + For a more in depth example / explanation, see the [customizing JSON schema](../concepts/json_schema.md#customizing-the-json-schema-generation-process) docs. + """ + + def __new__(cls) -> Self: ... + +@final +class PydanticUseDefault(Exception): + """An exception to signal that standard validation either failed or should be skipped, and the default value should be used instead. + + This warning can be raised in custom valiation functions to redirect the flow of validation. + + Example: + ```py + from pydantic_core import PydanticUseDefault + from datetime import datetime + from pydantic import BaseModel, field_validator + + + class Event(BaseModel): + name: str = 'meeting' + time: datetime + + @field_validator('name', mode='plain') + def name_must_be_present(cls, v) -> str: + if not v or not isinstance(v, str): + raise PydanticUseDefault() + return v + + + event1 = Event(name='party', time=datetime(2024, 1, 1, 12, 0, 0)) + print(repr(event1)) + # > Event(name='party', time=datetime.datetime(2024, 1, 1, 12, 0)) + event2 = Event(time=datetime(2024, 1, 1, 12, 0, 0)) + print(repr(event2)) + # > Event(name='meeting', time=datetime.datetime(2024, 1, 1, 12, 0)) + ``` + + For an additional example, see the [validating partial json data](../concepts/json.md#partial-json-parsing) section of the Pydantic documentation. + """ + + def __new__(cls) -> Self: ... + +@final +class PydanticSerializationError(ValueError): + """An error raised when an issue occurs during serialization. + + In custom serializers, this error can be used to indicate that serialization has failed. + """ + + def __init__(self, message: str) -> None: + """Initializes the `PydanticSerializationError`. + + Arguments: + message: The message associated with the error. + """ + + def __new__(cls, message: str) -> Self: ... + +@final +class PydanticSerializationUnexpectedValue(ValueError): + """An error raised when an unexpected value is encountered during serialization. + + This error is often caught and coerced into a warning, as `pydantic-core` generally makes a best attempt + at serializing values, in contrast with validation where errors are eagerly raised. + + Example: + ```py + from pydantic import BaseModel, field_serializer + from pydantic_core import PydanticSerializationUnexpectedValue + + class BasicPoint(BaseModel): + x: int + y: int + + @field_serializer('*') + def serialize(self, v): + if not isinstance(v, int): + raise PydanticSerializationUnexpectedValue(f'Expected type `int`, got {type(v)} with value {v}') + return v + + point = BasicPoint(x=1, y=2) + # some sort of mutation + point.x = 'a' + + print(point.model_dump()) + ''' + UserWarning: Pydantic serializer warnings: + PydanticSerializationUnexpectedValue(Expected type `int`, got with value a) + return self.__pydantic_serializer__.to_python( + {'x': 'a', 'y': 2} + ''' + ``` + + This is often used internally in `pydantic-core` when unexpected types are encountered during serialization, + but it can also be used by users in custom serializers, as seen above. + """ + + def __init__(self, message: str) -> None: + """Initializes the `PydanticSerializationUnexpectedValue`. + + Arguments: + message: The message associated with the unexpected value. + """ + + def __new__(cls, message: str | None = None) -> Self: ... + +@final +class ArgsKwargs: + """A construct used to store arguments and keyword arguments for a function call. + + This data structure is generally used to store information for core schemas associated with functions (like in an arguments schema). + This data structure is also currently used for some validation against dataclasses. + + Example: + ```py + from pydantic.dataclasses import dataclass + from pydantic import model_validator + + + @dataclass + class Model: + a: int + b: int + + @model_validator(mode="before") + @classmethod + def no_op_validator(cls, values): + print(values) + return values + + Model(1, b=2) + #> ArgsKwargs((1,), {"b": 2}) + + Model(1, 2) + #> ArgsKwargs((1, 2), {}) + + Model(a=1, b=2) + #> ArgsKwargs((), {"a": 1, "b": 2}) + ``` + """ + + def __init__(self, args: tuple[Any, ...], kwargs: dict[str, Any] | None = None) -> None: + """Initializes the `ArgsKwargs`. + + Arguments: + args: The arguments (inherently ordered) for a function call. + kwargs: The keyword arguments for a function call + """ + + def __new__(cls, args: tuple[Any, ...], kwargs: dict[str, Any] | None = None) -> Self: ... + @property + def args(self) -> tuple[Any, ...]: + """The arguments (inherently ordered) for a function call.""" + + @property + def kwargs(self) -> dict[str, Any] | None: + """The keyword arguments for a function call.""" + +@final +class PydanticUndefinedType: + """A type used as a sentinel for undefined values.""" + + def __copy__(self) -> Self: ... + def __deepcopy__(self, memo: Any) -> Self: ... + +PydanticUndefined: PydanticUndefinedType + +def list_all_errors() -> list[ErrorTypeInfo]: + """ + Get information about all built-in errors. + + Returns: + A list of `ErrorTypeInfo` typed dicts. + """ +@final +class TzInfo(datetime.tzinfo): + """An `pydantic-core` implementation of the abstract [`datetime.tzinfo`][] class.""" + + # def __new__(cls, seconds: float) -> Self: ... + + # Docstrings for attributes sourced from the abstract base class, [`datetime.tzinfo`](https://docs.python.org/3/library/datetime.html#datetime.tzinfo). + + def tzname(self, dt: datetime.datetime | None) -> str | None: + """Return the time zone name corresponding to the [`datetime`][datetime.datetime] object _dt_, as a string. + + For more info, see [`tzinfo.tzname`][datetime.tzinfo.tzname]. + """ + + def utcoffset(self, dt: datetime.datetime | None) -> datetime.timedelta | None: + """Return offset of local time from UTC, as a [`timedelta`][datetime.timedelta] object that is positive east of UTC. If local time is west of UTC, this should be negative. + + More info can be found at [`tzinfo.utcoffset`][datetime.tzinfo.utcoffset]. + """ + + def dst(self, dt: datetime.datetime | None) -> datetime.timedelta | None: + """Return the daylight saving time (DST) adjustment, as a [`timedelta`][datetime.timedelta] object or `None` if DST information isn’t known. + + More info can be found at[`tzinfo.dst`][datetime.tzinfo.dst].""" + + def fromutc(self, dt: datetime.datetime) -> datetime.datetime: + """Adjust the date and time data associated datetime object _dt_, returning an equivalent datetime in self’s local time. + + More info can be found at [`tzinfo.fromutc`][datetime.tzinfo.fromutc].""" + + def __deepcopy__(self, _memo: dict[Any, Any]) -> TzInfo: ... + +def validate_core_schema(schema: CoreSchema, *, strict: bool | None = None) -> CoreSchema: + """Validate a core schema. + + This currently uses lax mode for validation (i.e. will coerce strings to dates and such) + but may use strict mode in the future. + We may also remove this function altogether, do not rely on it being present if you are + using pydantic-core directly. + """ diff --git a/lib/python3.12/site-packages/pydantic_core/core_schema.py b/lib/python3.12/site-packages/pydantic_core/core_schema.py new file mode 100644 index 0000000000000000000000000000000000000000..0ab3dd9474dedd946540a4cc01ceed577afddf5e --- /dev/null +++ b/lib/python3.12/site-packages/pydantic_core/core_schema.py @@ -0,0 +1,4325 @@ +""" +This module contains definitions to build schemas which `pydantic_core` can +validate and serialize. +""" + +from __future__ import annotations as _annotations + +import sys +import warnings +from collections.abc import Hashable, Mapping +from datetime import date, datetime, time, timedelta +from decimal import Decimal +from re import Pattern +from typing import TYPE_CHECKING, Any, Callable, Literal, Union + +from typing_extensions import deprecated + +if sys.version_info < (3, 12): + from typing_extensions import TypedDict +else: + from typing import TypedDict + +if sys.version_info < (3, 11): + from typing_extensions import Protocol, Required, TypeAlias +else: + from typing import Protocol, Required, TypeAlias + +if TYPE_CHECKING: + from pydantic_core import PydanticUndefined +else: + # The initial build of pydantic_core requires PydanticUndefined to generate + # the core schema; so we need to conditionally skip it. mypy doesn't like + # this at all, hence the TYPE_CHECKING branch above. + try: + from pydantic_core import PydanticUndefined + except ImportError: + PydanticUndefined = object() + + +ExtraBehavior = Literal['allow', 'forbid', 'ignore'] + + +class CoreConfig(TypedDict, total=False): + """ + Base class for schema configuration options. + + Attributes: + title: The name of the configuration. + strict: Whether the configuration should strictly adhere to specified rules. + extra_fields_behavior: The behavior for handling extra fields. + typed_dict_total: Whether the TypedDict should be considered total. Default is `True`. + from_attributes: Whether to use attributes for models, dataclasses, and tagged union keys. + loc_by_alias: Whether to use the used alias (or first alias for "field required" errors) instead of + `field_names` to construct error `loc`s. Default is `True`. + revalidate_instances: Whether instances of models and dataclasses should re-validate. Default is 'never'. + validate_default: Whether to validate default values during validation. Default is `False`. + str_max_length: The maximum length for string fields. + str_min_length: The minimum length for string fields. + str_strip_whitespace: Whether to strip whitespace from string fields. + str_to_lower: Whether to convert string fields to lowercase. + str_to_upper: Whether to convert string fields to uppercase. + allow_inf_nan: Whether to allow infinity and NaN values for float fields. Default is `True`. + ser_json_timedelta: The serialization option for `timedelta` values. Default is 'iso8601'. + ser_json_bytes: The serialization option for `bytes` values. Default is 'utf8'. + ser_json_inf_nan: The serialization option for infinity and NaN values + in float fields. Default is 'null'. + val_json_bytes: The validation option for `bytes` values, complementing ser_json_bytes. Default is 'utf8'. + hide_input_in_errors: Whether to hide input data from `ValidationError` representation. + validation_error_cause: Whether to add user-python excs to the __cause__ of a ValidationError. + Requires exceptiongroup backport pre Python 3.11. + coerce_numbers_to_str: Whether to enable coercion of any `Number` type to `str` (not applicable in `strict` mode). + regex_engine: The regex engine to use for regex pattern validation. Default is 'rust-regex'. See `StringSchema`. + cache_strings: Whether to cache strings. Default is `True`, `True` or `'all'` is required to cache strings + during general validation since validators don't know if they're in a key or a value. + validate_by_alias: Whether to use the field's alias when validating against the provided input data. Default is `True`. + validate_by_name: Whether to use the field's name when validating against the provided input data. Default is `False`. Replacement for `populate_by_name`. + serialize_by_alias: Whether to serialize by alias. Default is `False`, expected to change to `True` in V3. + """ + + title: str + strict: bool + # settings related to typed dicts, model fields, dataclass fields + extra_fields_behavior: ExtraBehavior + typed_dict_total: bool # default: True + # used for models, dataclasses, and tagged union keys + from_attributes: bool + # whether to use the used alias (or first alias for "field required" errors) instead of field_names + # to construct error `loc`s, default True + loc_by_alias: bool + # whether instances of models and dataclasses (including subclass instances) should re-validate, default 'never' + revalidate_instances: Literal['always', 'never', 'subclass-instances'] + # whether to validate default values during validation, default False + validate_default: bool + # used on typed-dicts and arguments + # fields related to string fields only + str_max_length: int + str_min_length: int + str_strip_whitespace: bool + str_to_lower: bool + str_to_upper: bool + # fields related to float fields only + allow_inf_nan: bool # default: True + # the config options are used to customise serialization to JSON + ser_json_timedelta: Literal['iso8601', 'float'] # default: 'iso8601' + ser_json_bytes: Literal['utf8', 'base64', 'hex'] # default: 'utf8' + ser_json_inf_nan: Literal['null', 'constants', 'strings'] # default: 'null' + val_json_bytes: Literal['utf8', 'base64', 'hex'] # default: 'utf8' + # used to hide input data from ValidationError repr + hide_input_in_errors: bool + validation_error_cause: bool # default: False + coerce_numbers_to_str: bool # default: False + regex_engine: Literal['rust-regex', 'python-re'] # default: 'rust-regex' + cache_strings: Union[bool, Literal['all', 'keys', 'none']] # default: 'True' + validate_by_alias: bool # default: True + validate_by_name: bool # default: False + serialize_by_alias: bool # default: False + + +IncExCall: TypeAlias = 'set[int | str] | dict[int | str, IncExCall] | None' + + +class SerializationInfo(Protocol): + @property + def include(self) -> IncExCall: ... + + @property + def exclude(self) -> IncExCall: ... + + @property + def context(self) -> Any | None: + """Current serialization context.""" + + @property + def mode(self) -> str: ... + + @property + def by_alias(self) -> bool: ... + + @property + def exclude_unset(self) -> bool: ... + + @property + def exclude_defaults(self) -> bool: ... + + @property + def exclude_none(self) -> bool: ... + + @property + def serialize_as_any(self) -> bool: ... + + @property + def round_trip(self) -> bool: ... + + def mode_is_json(self) -> bool: ... + + def __str__(self) -> str: ... + + def __repr__(self) -> str: ... + + +class FieldSerializationInfo(SerializationInfo, Protocol): + @property + def field_name(self) -> str: ... + + +class ValidationInfo(Protocol): + """ + Argument passed to validation functions. + """ + + @property + def context(self) -> Any | None: + """Current validation context.""" + ... + + @property + def config(self) -> CoreConfig | None: + """The CoreConfig that applies to this validation.""" + ... + + @property + def mode(self) -> Literal['python', 'json']: + """The type of input data we are currently validating""" + ... + + @property + def data(self) -> dict[str, Any]: + """The data being validated for this model.""" + ... + + @property + def field_name(self) -> str | None: + """ + The name of the current field being validated if this validator is + attached to a model field. + """ + ... + + +ExpectedSerializationTypes = Literal[ + 'none', + 'int', + 'bool', + 'float', + 'str', + 'bytes', + 'bytearray', + 'list', + 'tuple', + 'set', + 'frozenset', + 'generator', + 'dict', + 'datetime', + 'date', + 'time', + 'timedelta', + 'url', + 'multi-host-url', + 'json', + 'uuid', + 'any', +] + + +class SimpleSerSchema(TypedDict, total=False): + type: Required[ExpectedSerializationTypes] + + +def simple_ser_schema(type: ExpectedSerializationTypes) -> SimpleSerSchema: + """ + Returns a schema for serialization with a custom type. + + Args: + type: The type to use for serialization + """ + return SimpleSerSchema(type=type) + + +# (input_value: Any, /) -> Any +GeneralPlainNoInfoSerializerFunction = Callable[[Any], Any] +# (input_value: Any, info: FieldSerializationInfo, /) -> Any +GeneralPlainInfoSerializerFunction = Callable[[Any, SerializationInfo], Any] +# (model: Any, input_value: Any, /) -> Any +FieldPlainNoInfoSerializerFunction = Callable[[Any, Any], Any] +# (model: Any, input_value: Any, info: FieldSerializationInfo, /) -> Any +FieldPlainInfoSerializerFunction = Callable[[Any, Any, FieldSerializationInfo], Any] +SerializerFunction = Union[ + GeneralPlainNoInfoSerializerFunction, + GeneralPlainInfoSerializerFunction, + FieldPlainNoInfoSerializerFunction, + FieldPlainInfoSerializerFunction, +] + +WhenUsed = Literal['always', 'unless-none', 'json', 'json-unless-none'] +""" +Values have the following meanings: + +* `'always'` means always use +* `'unless-none'` means use unless the value is `None` +* `'json'` means use when serializing to JSON +* `'json-unless-none'` means use when serializing to JSON and the value is not `None` +""" + + +class PlainSerializerFunctionSerSchema(TypedDict, total=False): + type: Required[Literal['function-plain']] + function: Required[SerializerFunction] + is_field_serializer: bool # default False + info_arg: bool # default False + return_schema: CoreSchema # if omitted, AnySchema is used + when_used: WhenUsed # default: 'always' + + +def plain_serializer_function_ser_schema( + function: SerializerFunction, + *, + is_field_serializer: bool | None = None, + info_arg: bool | None = None, + return_schema: CoreSchema | None = None, + when_used: WhenUsed = 'always', +) -> PlainSerializerFunctionSerSchema: + """ + Returns a schema for serialization with a function, can be either a "general" or "field" function. + + Args: + function: The function to use for serialization + is_field_serializer: Whether the serializer is for a field, e.g. takes `model` as the first argument, + and `info` includes `field_name` + info_arg: Whether the function takes an `info` argument + return_schema: Schema to use for serializing return value + when_used: When the function should be called + """ + if when_used == 'always': + # just to avoid extra elements in schema, and to use the actual default defined in rust + when_used = None # type: ignore + return _dict_not_none( + type='function-plain', + function=function, + is_field_serializer=is_field_serializer, + info_arg=info_arg, + return_schema=return_schema, + when_used=when_used, + ) + + +class SerializerFunctionWrapHandler(Protocol): # pragma: no cover + def __call__(self, input_value: Any, index_key: int | str | None = None, /) -> Any: ... + + +# (input_value: Any, serializer: SerializerFunctionWrapHandler, /) -> Any +GeneralWrapNoInfoSerializerFunction = Callable[[Any, SerializerFunctionWrapHandler], Any] +# (input_value: Any, serializer: SerializerFunctionWrapHandler, info: SerializationInfo, /) -> Any +GeneralWrapInfoSerializerFunction = Callable[[Any, SerializerFunctionWrapHandler, SerializationInfo], Any] +# (model: Any, input_value: Any, serializer: SerializerFunctionWrapHandler, /) -> Any +FieldWrapNoInfoSerializerFunction = Callable[[Any, Any, SerializerFunctionWrapHandler], Any] +# (model: Any, input_value: Any, serializer: SerializerFunctionWrapHandler, info: FieldSerializationInfo, /) -> Any +FieldWrapInfoSerializerFunction = Callable[[Any, Any, SerializerFunctionWrapHandler, FieldSerializationInfo], Any] +WrapSerializerFunction = Union[ + GeneralWrapNoInfoSerializerFunction, + GeneralWrapInfoSerializerFunction, + FieldWrapNoInfoSerializerFunction, + FieldWrapInfoSerializerFunction, +] + + +class WrapSerializerFunctionSerSchema(TypedDict, total=False): + type: Required[Literal['function-wrap']] + function: Required[WrapSerializerFunction] + is_field_serializer: bool # default False + info_arg: bool # default False + schema: CoreSchema # if omitted, the schema on which this serializer is defined is used + return_schema: CoreSchema # if omitted, AnySchema is used + when_used: WhenUsed # default: 'always' + + +def wrap_serializer_function_ser_schema( + function: WrapSerializerFunction, + *, + is_field_serializer: bool | None = None, + info_arg: bool | None = None, + schema: CoreSchema | None = None, + return_schema: CoreSchema | None = None, + when_used: WhenUsed = 'always', +) -> WrapSerializerFunctionSerSchema: + """ + Returns a schema for serialization with a wrap function, can be either a "general" or "field" function. + + Args: + function: The function to use for serialization + is_field_serializer: Whether the serializer is for a field, e.g. takes `model` as the first argument, + and `info` includes `field_name` + info_arg: Whether the function takes an `info` argument + schema: The schema to use for the inner serialization + return_schema: Schema to use for serializing return value + when_used: When the function should be called + """ + if when_used == 'always': + # just to avoid extra elements in schema, and to use the actual default defined in rust + when_used = None # type: ignore + return _dict_not_none( + type='function-wrap', + function=function, + is_field_serializer=is_field_serializer, + info_arg=info_arg, + schema=schema, + return_schema=return_schema, + when_used=when_used, + ) + + +class FormatSerSchema(TypedDict, total=False): + type: Required[Literal['format']] + formatting_string: Required[str] + when_used: WhenUsed # default: 'json-unless-none' + + +def format_ser_schema(formatting_string: str, *, when_used: WhenUsed = 'json-unless-none') -> FormatSerSchema: + """ + Returns a schema for serialization using python's `format` method. + + Args: + formatting_string: String defining the format to use + when_used: Same meaning as for [general_function_plain_ser_schema], but with a different default + """ + if when_used == 'json-unless-none': + # just to avoid extra elements in schema, and to use the actual default defined in rust + when_used = None # type: ignore + return _dict_not_none(type='format', formatting_string=formatting_string, when_used=when_used) + + +class ToStringSerSchema(TypedDict, total=False): + type: Required[Literal['to-string']] + when_used: WhenUsed # default: 'json-unless-none' + + +def to_string_ser_schema(*, when_used: WhenUsed = 'json-unless-none') -> ToStringSerSchema: + """ + Returns a schema for serialization using python's `str()` / `__str__` method. + + Args: + when_used: Same meaning as for [general_function_plain_ser_schema], but with a different default + """ + s = dict(type='to-string') + if when_used != 'json-unless-none': + # just to avoid extra elements in schema, and to use the actual default defined in rust + s['when_used'] = when_used + return s # type: ignore + + +class ModelSerSchema(TypedDict, total=False): + type: Required[Literal['model']] + cls: Required[type[Any]] + schema: Required[CoreSchema] + + +def model_ser_schema(cls: type[Any], schema: CoreSchema) -> ModelSerSchema: + """ + Returns a schema for serialization using a model. + + Args: + cls: The expected class type, used to generate warnings if the wrong type is passed + schema: Internal schema to use to serialize the model dict + """ + return ModelSerSchema(type='model', cls=cls, schema=schema) + + +SerSchema = Union[ + SimpleSerSchema, + PlainSerializerFunctionSerSchema, + WrapSerializerFunctionSerSchema, + FormatSerSchema, + ToStringSerSchema, + ModelSerSchema, +] + + +class InvalidSchema(TypedDict, total=False): + type: Required[Literal['invalid']] + ref: str + metadata: dict[str, Any] + # note, we never plan to use this, but include it for type checking purposes to match + # all other CoreSchema union members + serialization: SerSchema + + +def invalid_schema(ref: str | None = None, metadata: dict[str, Any] | None = None) -> InvalidSchema: + """ + Returns an invalid schema, used to indicate that a schema is invalid. + + Returns a schema that matches any value, e.g.: + + Args: + ref: optional unique identifier of the schema, used to reference the schema in other places + metadata: Any other information you want to include with the schema, not used by pydantic-core + """ + + return _dict_not_none(type='invalid', ref=ref, metadata=metadata) + + +class ComputedField(TypedDict, total=False): + type: Required[Literal['computed-field']] + property_name: Required[str] + return_schema: Required[CoreSchema] + alias: str + metadata: dict[str, Any] + + +def computed_field( + property_name: str, return_schema: CoreSchema, *, alias: str | None = None, metadata: dict[str, Any] | None = None +) -> ComputedField: + """ + ComputedFields are properties of a model or dataclass that are included in serialization. + + Args: + property_name: The name of the property on the model or dataclass + return_schema: The schema used for the type returned by the computed field + alias: The name to use in the serialized output + metadata: Any other information you want to include with the schema, not used by pydantic-core + """ + return _dict_not_none( + type='computed-field', property_name=property_name, return_schema=return_schema, alias=alias, metadata=metadata + ) + + +class AnySchema(TypedDict, total=False): + type: Required[Literal['any']] + ref: str + metadata: dict[str, Any] + serialization: SerSchema + + +def any_schema( + *, ref: str | None = None, metadata: dict[str, Any] | None = None, serialization: SerSchema | None = None +) -> AnySchema: + """ + Returns a schema that matches any value, e.g.: + + ```py + from pydantic_core import SchemaValidator, core_schema + + schema = core_schema.any_schema() + v = SchemaValidator(schema) + assert v.validate_python(1) == 1 + ``` + + Args: + ref: optional unique identifier of the schema, used to reference the schema in other places + metadata: Any other information you want to include with the schema, not used by pydantic-core + serialization: Custom serialization schema + """ + return _dict_not_none(type='any', ref=ref, metadata=metadata, serialization=serialization) + + +class NoneSchema(TypedDict, total=False): + type: Required[Literal['none']] + ref: str + metadata: dict[str, Any] + serialization: SerSchema + + +def none_schema( + *, ref: str | None = None, metadata: dict[str, Any] | None = None, serialization: SerSchema | None = None +) -> NoneSchema: + """ + Returns a schema that matches a None value, e.g.: + + ```py + from pydantic_core import SchemaValidator, core_schema + + schema = core_schema.none_schema() + v = SchemaValidator(schema) + assert v.validate_python(None) is None + ``` + + Args: + ref: optional unique identifier of the schema, used to reference the schema in other places + metadata: Any other information you want to include with the schema, not used by pydantic-core + serialization: Custom serialization schema + """ + return _dict_not_none(type='none', ref=ref, metadata=metadata, serialization=serialization) + + +class BoolSchema(TypedDict, total=False): + type: Required[Literal['bool']] + strict: bool + ref: str + metadata: dict[str, Any] + serialization: SerSchema + + +def bool_schema( + strict: bool | None = None, + ref: str | None = None, + metadata: dict[str, Any] | None = None, + serialization: SerSchema | None = None, +) -> BoolSchema: + """ + Returns a schema that matches a bool value, e.g.: + + ```py + from pydantic_core import SchemaValidator, core_schema + + schema = core_schema.bool_schema() + v = SchemaValidator(schema) + assert v.validate_python('True') is True + ``` + + Args: + strict: Whether the value should be a bool or a value that can be converted to a bool + ref: optional unique identifier of the schema, used to reference the schema in other places + metadata: Any other information you want to include with the schema, not used by pydantic-core + serialization: Custom serialization schema + """ + return _dict_not_none(type='bool', strict=strict, ref=ref, metadata=metadata, serialization=serialization) + + +class IntSchema(TypedDict, total=False): + type: Required[Literal['int']] + multiple_of: int + le: int + ge: int + lt: int + gt: int + strict: bool + ref: str + metadata: dict[str, Any] + serialization: SerSchema + + +def int_schema( + *, + multiple_of: int | None = None, + le: int | None = None, + ge: int | None = None, + lt: int | None = None, + gt: int | None = None, + strict: bool | None = None, + ref: str | None = None, + metadata: dict[str, Any] | None = None, + serialization: SerSchema | None = None, +) -> IntSchema: + """ + Returns a schema that matches a int value, e.g.: + + ```py + from pydantic_core import SchemaValidator, core_schema + + schema = core_schema.int_schema(multiple_of=2, le=6, ge=2) + v = SchemaValidator(schema) + assert v.validate_python('4') == 4 + ``` + + Args: + multiple_of: The value must be a multiple of this number + le: The value must be less than or equal to this number + ge: The value must be greater than or equal to this number + lt: The value must be strictly less than this number + gt: The value must be strictly greater than this number + strict: Whether the value should be a int or a value that can be converted to a int + ref: optional unique identifier of the schema, used to reference the schema in other places + metadata: Any other information you want to include with the schema, not used by pydantic-core + serialization: Custom serialization schema + """ + return _dict_not_none( + type='int', + multiple_of=multiple_of, + le=le, + ge=ge, + lt=lt, + gt=gt, + strict=strict, + ref=ref, + metadata=metadata, + serialization=serialization, + ) + + +class FloatSchema(TypedDict, total=False): + type: Required[Literal['float']] + allow_inf_nan: bool # whether 'NaN', '+inf', '-inf' should be forbidden. default: True + multiple_of: float + le: float + ge: float + lt: float + gt: float + strict: bool + ref: str + metadata: dict[str, Any] + serialization: SerSchema + + +def float_schema( + *, + allow_inf_nan: bool | None = None, + multiple_of: float | None = None, + le: float | None = None, + ge: float | None = None, + lt: float | None = None, + gt: float | None = None, + strict: bool | None = None, + ref: str | None = None, + metadata: dict[str, Any] | None = None, + serialization: SerSchema | None = None, +) -> FloatSchema: + """ + Returns a schema that matches a float value, e.g.: + + ```py + from pydantic_core import SchemaValidator, core_schema + + schema = core_schema.float_schema(le=0.8, ge=0.2) + v = SchemaValidator(schema) + assert v.validate_python('0.5') == 0.5 + ``` + + Args: + allow_inf_nan: Whether to allow inf and nan values + multiple_of: The value must be a multiple of this number + le: The value must be less than or equal to this number + ge: The value must be greater than or equal to this number + lt: The value must be strictly less than this number + gt: The value must be strictly greater than this number + strict: Whether the value should be a float or a value that can be converted to a float + ref: optional unique identifier of the schema, used to reference the schema in other places + metadata: Any other information you want to include with the schema, not used by pydantic-core + serialization: Custom serialization schema + """ + return _dict_not_none( + type='float', + allow_inf_nan=allow_inf_nan, + multiple_of=multiple_of, + le=le, + ge=ge, + lt=lt, + gt=gt, + strict=strict, + ref=ref, + metadata=metadata, + serialization=serialization, + ) + + +class DecimalSchema(TypedDict, total=False): + type: Required[Literal['decimal']] + allow_inf_nan: bool # whether 'NaN', '+inf', '-inf' should be forbidden. default: False + multiple_of: Decimal + le: Decimal + ge: Decimal + lt: Decimal + gt: Decimal + max_digits: int + decimal_places: int + strict: bool + ref: str + metadata: dict[str, Any] + serialization: SerSchema + + +def decimal_schema( + *, + allow_inf_nan: bool | None = None, + multiple_of: Decimal | None = None, + le: Decimal | None = None, + ge: Decimal | None = None, + lt: Decimal | None = None, + gt: Decimal | None = None, + max_digits: int | None = None, + decimal_places: int | None = None, + strict: bool | None = None, + ref: str | None = None, + metadata: dict[str, Any] | None = None, + serialization: SerSchema | None = None, +) -> DecimalSchema: + """ + Returns a schema that matches a decimal value, e.g.: + + ```py + from decimal import Decimal + from pydantic_core import SchemaValidator, core_schema + + schema = core_schema.decimal_schema(le=0.8, ge=0.2) + v = SchemaValidator(schema) + assert v.validate_python('0.5') == Decimal('0.5') + ``` + + Args: + allow_inf_nan: Whether to allow inf and nan values + multiple_of: The value must be a multiple of this number + le: The value must be less than or equal to this number + ge: The value must be greater than or equal to this number + lt: The value must be strictly less than this number + gt: The value must be strictly greater than this number + max_digits: The maximum number of decimal digits allowed + decimal_places: The maximum number of decimal places allowed + strict: Whether the value should be a float or a value that can be converted to a float + ref: optional unique identifier of the schema, used to reference the schema in other places + metadata: Any other information you want to include with the schema, not used by pydantic-core + serialization: Custom serialization schema + """ + return _dict_not_none( + type='decimal', + gt=gt, + ge=ge, + lt=lt, + le=le, + max_digits=max_digits, + decimal_places=decimal_places, + multiple_of=multiple_of, + allow_inf_nan=allow_inf_nan, + strict=strict, + ref=ref, + metadata=metadata, + serialization=serialization, + ) + + +class ComplexSchema(TypedDict, total=False): + type: Required[Literal['complex']] + strict: bool + ref: str + metadata: dict[str, Any] + serialization: SerSchema + + +def complex_schema( + *, + strict: bool | None = None, + ref: str | None = None, + metadata: dict[str, Any] | None = None, + serialization: SerSchema | None = None, +) -> ComplexSchema: + """ + Returns a schema that matches a complex value, e.g.: + + ```py + from pydantic_core import SchemaValidator, core_schema + + schema = core_schema.complex_schema() + v = SchemaValidator(schema) + assert v.validate_python('1+2j') == complex(1, 2) + assert v.validate_python(complex(1, 2)) == complex(1, 2) + ``` + + Args: + strict: Whether the value should be a complex object instance or a value that can be converted to a complex object + ref: optional unique identifier of the schema, used to reference the schema in other places + metadata: Any other information you want to include with the schema, not used by pydantic-core + serialization: Custom serialization schema + """ + return _dict_not_none( + type='complex', + strict=strict, + ref=ref, + metadata=metadata, + serialization=serialization, + ) + + +class StringSchema(TypedDict, total=False): + type: Required[Literal['str']] + pattern: Union[str, Pattern[str]] + max_length: int + min_length: int + strip_whitespace: bool + to_lower: bool + to_upper: bool + regex_engine: Literal['rust-regex', 'python-re'] # default: 'rust-regex' + strict: bool + coerce_numbers_to_str: bool + ref: str + metadata: dict[str, Any] + serialization: SerSchema + + +def str_schema( + *, + pattern: str | Pattern[str] | None = None, + max_length: int | None = None, + min_length: int | None = None, + strip_whitespace: bool | None = None, + to_lower: bool | None = None, + to_upper: bool | None = None, + regex_engine: Literal['rust-regex', 'python-re'] | None = None, + strict: bool | None = None, + coerce_numbers_to_str: bool | None = None, + ref: str | None = None, + metadata: dict[str, Any] | None = None, + serialization: SerSchema | None = None, +) -> StringSchema: + """ + Returns a schema that matches a string value, e.g.: + + ```py + from pydantic_core import SchemaValidator, core_schema + + schema = core_schema.str_schema(max_length=10, min_length=2) + v = SchemaValidator(schema) + assert v.validate_python('hello') == 'hello' + ``` + + Args: + pattern: A regex pattern that the value must match + max_length: The value must be at most this length + min_length: The value must be at least this length + strip_whitespace: Whether to strip whitespace from the value + to_lower: Whether to convert the value to lowercase + to_upper: Whether to convert the value to uppercase + regex_engine: The regex engine to use for pattern validation. Default is 'rust-regex'. + - `rust-regex` uses the [`regex`](https://docs.rs/regex) Rust + crate, which is non-backtracking and therefore more DDoS + resistant, but does not support all regex features. + - `python-re` use the [`re`](https://docs.python.org/3/library/re.html) module, + which supports all regex features, but may be slower. + strict: Whether the value should be a string or a value that can be converted to a string + coerce_numbers_to_str: Whether to enable coercion of any `Number` type to `str` (not applicable in `strict` mode). + ref: optional unique identifier of the schema, used to reference the schema in other places + metadata: Any other information you want to include with the schema, not used by pydantic-core + serialization: Custom serialization schema + """ + return _dict_not_none( + type='str', + pattern=pattern, + max_length=max_length, + min_length=min_length, + strip_whitespace=strip_whitespace, + to_lower=to_lower, + to_upper=to_upper, + regex_engine=regex_engine, + strict=strict, + coerce_numbers_to_str=coerce_numbers_to_str, + ref=ref, + metadata=metadata, + serialization=serialization, + ) + + +class BytesSchema(TypedDict, total=False): + type: Required[Literal['bytes']] + max_length: int + min_length: int + strict: bool + ref: str + metadata: dict[str, Any] + serialization: SerSchema + + +def bytes_schema( + *, + max_length: int | None = None, + min_length: int | None = None, + strict: bool | None = None, + ref: str | None = None, + metadata: dict[str, Any] | None = None, + serialization: SerSchema | None = None, +) -> BytesSchema: + """ + Returns a schema that matches a bytes value, e.g.: + + ```py + from pydantic_core import SchemaValidator, core_schema + + schema = core_schema.bytes_schema(max_length=10, min_length=2) + v = SchemaValidator(schema) + assert v.validate_python(b'hello') == b'hello' + ``` + + Args: + max_length: The value must be at most this length + min_length: The value must be at least this length + strict: Whether the value should be a bytes or a value that can be converted to a bytes + ref: optional unique identifier of the schema, used to reference the schema in other places + metadata: Any other information you want to include with the schema, not used by pydantic-core + serialization: Custom serialization schema + """ + return _dict_not_none( + type='bytes', + max_length=max_length, + min_length=min_length, + strict=strict, + ref=ref, + metadata=metadata, + serialization=serialization, + ) + + +class DateSchema(TypedDict, total=False): + type: Required[Literal['date']] + strict: bool + le: date + ge: date + lt: date + gt: date + now_op: Literal['past', 'future'] + # defaults to current local utc offset from `time.localtime().tm_gmtoff` + # value is restricted to -86_400 < offset < 86_400 by bounds in generate_self_schema.py + now_utc_offset: int + ref: str + metadata: dict[str, Any] + serialization: SerSchema + + +def date_schema( + *, + strict: bool | None = None, + le: date | None = None, + ge: date | None = None, + lt: date | None = None, + gt: date | None = None, + now_op: Literal['past', 'future'] | None = None, + now_utc_offset: int | None = None, + ref: str | None = None, + metadata: dict[str, Any] | None = None, + serialization: SerSchema | None = None, +) -> DateSchema: + """ + Returns a schema that matches a date value, e.g.: + + ```py + from datetime import date + from pydantic_core import SchemaValidator, core_schema + + schema = core_schema.date_schema(le=date(2020, 1, 1), ge=date(2019, 1, 1)) + v = SchemaValidator(schema) + assert v.validate_python(date(2019, 6, 1)) == date(2019, 6, 1) + ``` + + Args: + strict: Whether the value should be a date or a value that can be converted to a date + le: The value must be less than or equal to this date + ge: The value must be greater than or equal to this date + lt: The value must be strictly less than this date + gt: The value must be strictly greater than this date + now_op: The value must be in the past or future relative to the current date + now_utc_offset: The value must be in the past or future relative to the current date with this utc offset + ref: optional unique identifier of the schema, used to reference the schema in other places + metadata: Any other information you want to include with the schema, not used by pydantic-core + serialization: Custom serialization schema + """ + return _dict_not_none( + type='date', + strict=strict, + le=le, + ge=ge, + lt=lt, + gt=gt, + now_op=now_op, + now_utc_offset=now_utc_offset, + ref=ref, + metadata=metadata, + serialization=serialization, + ) + + +class TimeSchema(TypedDict, total=False): + type: Required[Literal['time']] + strict: bool + le: time + ge: time + lt: time + gt: time + tz_constraint: Union[Literal['aware', 'naive'], int] + microseconds_precision: Literal['truncate', 'error'] + ref: str + metadata: dict[str, Any] + serialization: SerSchema + + +def time_schema( + *, + strict: bool | None = None, + le: time | None = None, + ge: time | None = None, + lt: time | None = None, + gt: time | None = None, + tz_constraint: Literal['aware', 'naive'] | int | None = None, + microseconds_precision: Literal['truncate', 'error'] = 'truncate', + ref: str | None = None, + metadata: dict[str, Any] | None = None, + serialization: SerSchema | None = None, +) -> TimeSchema: + """ + Returns a schema that matches a time value, e.g.: + + ```py + from datetime import time + from pydantic_core import SchemaValidator, core_schema + + schema = core_schema.time_schema(le=time(12, 0, 0), ge=time(6, 0, 0)) + v = SchemaValidator(schema) + assert v.validate_python(time(9, 0, 0)) == time(9, 0, 0) + ``` + + Args: + strict: Whether the value should be a time or a value that can be converted to a time + le: The value must be less than or equal to this time + ge: The value must be greater than or equal to this time + lt: The value must be strictly less than this time + gt: The value must be strictly greater than this time + tz_constraint: The value must be timezone aware or naive, or an int to indicate required tz offset + microseconds_precision: The behavior when seconds have more than 6 digits or microseconds is too large + ref: optional unique identifier of the schema, used to reference the schema in other places + metadata: Any other information you want to include with the schema, not used by pydantic-core + serialization: Custom serialization schema + """ + return _dict_not_none( + type='time', + strict=strict, + le=le, + ge=ge, + lt=lt, + gt=gt, + tz_constraint=tz_constraint, + microseconds_precision=microseconds_precision, + ref=ref, + metadata=metadata, + serialization=serialization, + ) + + +class DatetimeSchema(TypedDict, total=False): + type: Required[Literal['datetime']] + strict: bool + le: datetime + ge: datetime + lt: datetime + gt: datetime + now_op: Literal['past', 'future'] + tz_constraint: Union[Literal['aware', 'naive'], int] + # defaults to current local utc offset from `time.localtime().tm_gmtoff` + # value is restricted to -86_400 < offset < 86_400 by bounds in generate_self_schema.py + now_utc_offset: int + microseconds_precision: Literal['truncate', 'error'] # default: 'truncate' + ref: str + metadata: dict[str, Any] + serialization: SerSchema + + +def datetime_schema( + *, + strict: bool | None = None, + le: datetime | None = None, + ge: datetime | None = None, + lt: datetime | None = None, + gt: datetime | None = None, + now_op: Literal['past', 'future'] | None = None, + tz_constraint: Literal['aware', 'naive'] | int | None = None, + now_utc_offset: int | None = None, + microseconds_precision: Literal['truncate', 'error'] = 'truncate', + ref: str | None = None, + metadata: dict[str, Any] | None = None, + serialization: SerSchema | None = None, +) -> DatetimeSchema: + """ + Returns a schema that matches a datetime value, e.g.: + + ```py + from datetime import datetime + from pydantic_core import SchemaValidator, core_schema + + schema = core_schema.datetime_schema() + v = SchemaValidator(schema) + now = datetime.now() + assert v.validate_python(str(now)) == now + ``` + + Args: + strict: Whether the value should be a datetime or a value that can be converted to a datetime + le: The value must be less than or equal to this datetime + ge: The value must be greater than or equal to this datetime + lt: The value must be strictly less than this datetime + gt: The value must be strictly greater than this datetime + now_op: The value must be in the past or future relative to the current datetime + tz_constraint: The value must be timezone aware or naive, or an int to indicate required tz offset + TODO: use of a tzinfo where offset changes based on the datetime is not yet supported + now_utc_offset: The value must be in the past or future relative to the current datetime with this utc offset + microseconds_precision: The behavior when seconds have more than 6 digits or microseconds is too large + ref: optional unique identifier of the schema, used to reference the schema in other places + metadata: Any other information you want to include with the schema, not used by pydantic-core + serialization: Custom serialization schema + """ + return _dict_not_none( + type='datetime', + strict=strict, + le=le, + ge=ge, + lt=lt, + gt=gt, + now_op=now_op, + tz_constraint=tz_constraint, + now_utc_offset=now_utc_offset, + microseconds_precision=microseconds_precision, + ref=ref, + metadata=metadata, + serialization=serialization, + ) + + +class TimedeltaSchema(TypedDict, total=False): + type: Required[Literal['timedelta']] + strict: bool + le: timedelta + ge: timedelta + lt: timedelta + gt: timedelta + microseconds_precision: Literal['truncate', 'error'] + ref: str + metadata: dict[str, Any] + serialization: SerSchema + + +def timedelta_schema( + *, + strict: bool | None = None, + le: timedelta | None = None, + ge: timedelta | None = None, + lt: timedelta | None = None, + gt: timedelta | None = None, + microseconds_precision: Literal['truncate', 'error'] = 'truncate', + ref: str | None = None, + metadata: dict[str, Any] | None = None, + serialization: SerSchema | None = None, +) -> TimedeltaSchema: + """ + Returns a schema that matches a timedelta value, e.g.: + + ```py + from datetime import timedelta + from pydantic_core import SchemaValidator, core_schema + + schema = core_schema.timedelta_schema(le=timedelta(days=1), ge=timedelta(days=0)) + v = SchemaValidator(schema) + assert v.validate_python(timedelta(hours=12)) == timedelta(hours=12) + ``` + + Args: + strict: Whether the value should be a timedelta or a value that can be converted to a timedelta + le: The value must be less than or equal to this timedelta + ge: The value must be greater than or equal to this timedelta + lt: The value must be strictly less than this timedelta + gt: The value must be strictly greater than this timedelta + microseconds_precision: The behavior when seconds have more than 6 digits or microseconds is too large + ref: optional unique identifier of the schema, used to reference the schema in other places + metadata: Any other information you want to include with the schema, not used by pydantic-core + serialization: Custom serialization schema + """ + return _dict_not_none( + type='timedelta', + strict=strict, + le=le, + ge=ge, + lt=lt, + gt=gt, + microseconds_precision=microseconds_precision, + ref=ref, + metadata=metadata, + serialization=serialization, + ) + + +class LiteralSchema(TypedDict, total=False): + type: Required[Literal['literal']] + expected: Required[list[Any]] + ref: str + metadata: dict[str, Any] + serialization: SerSchema + + +def literal_schema( + expected: list[Any], + *, + ref: str | None = None, + metadata: dict[str, Any] | None = None, + serialization: SerSchema | None = None, +) -> LiteralSchema: + """ + Returns a schema that matches a literal value, e.g.: + + ```py + from pydantic_core import SchemaValidator, core_schema + + schema = core_schema.literal_schema(['hello', 'world']) + v = SchemaValidator(schema) + assert v.validate_python('hello') == 'hello' + ``` + + Args: + expected: The value must be one of these values + ref: optional unique identifier of the schema, used to reference the schema in other places + metadata: Any other information you want to include with the schema, not used by pydantic-core + serialization: Custom serialization schema + """ + return _dict_not_none(type='literal', expected=expected, ref=ref, metadata=metadata, serialization=serialization) + + +class EnumSchema(TypedDict, total=False): + type: Required[Literal['enum']] + cls: Required[Any] + members: Required[list[Any]] + sub_type: Literal['str', 'int', 'float'] + missing: Callable[[Any], Any] + strict: bool + ref: str + metadata: dict[str, Any] + serialization: SerSchema + + +def enum_schema( + cls: Any, + members: list[Any], + *, + sub_type: Literal['str', 'int', 'float'] | None = None, + missing: Callable[[Any], Any] | None = None, + strict: bool | None = None, + ref: str | None = None, + metadata: dict[str, Any] | None = None, + serialization: SerSchema | None = None, +) -> EnumSchema: + """ + Returns a schema that matches an enum value, e.g.: + + ```py + from enum import Enum + from pydantic_core import SchemaValidator, core_schema + + class Color(Enum): + RED = 1 + GREEN = 2 + BLUE = 3 + + schema = core_schema.enum_schema(Color, list(Color.__members__.values())) + v = SchemaValidator(schema) + assert v.validate_python(2) is Color.GREEN + ``` + + Args: + cls: The enum class + members: The members of the enum, generally `list(MyEnum.__members__.values())` + sub_type: The type of the enum, either 'str' or 'int' or None for plain enums + missing: A function to use when the value is not found in the enum, from `_missing_` + strict: Whether to use strict mode, defaults to False + ref: optional unique identifier of the schema, used to reference the schema in other places + metadata: Any other information you want to include with the schema, not used by pydantic-core + serialization: Custom serialization schema + """ + return _dict_not_none( + type='enum', + cls=cls, + members=members, + sub_type=sub_type, + missing=missing, + strict=strict, + ref=ref, + metadata=metadata, + serialization=serialization, + ) + + +# must match input/parse_json.rs::JsonType::try_from +JsonType = Literal['null', 'bool', 'int', 'float', 'str', 'list', 'dict'] + + +class IsInstanceSchema(TypedDict, total=False): + type: Required[Literal['is-instance']] + cls: Required[Any] + cls_repr: str + ref: str + metadata: dict[str, Any] + serialization: SerSchema + + +def is_instance_schema( + cls: Any, + *, + cls_repr: str | None = None, + ref: str | None = None, + metadata: dict[str, Any] | None = None, + serialization: SerSchema | None = None, +) -> IsInstanceSchema: + """ + Returns a schema that checks if a value is an instance of a class, equivalent to python's `isinstance` method, e.g.: + + ```py + from pydantic_core import SchemaValidator, core_schema + + class A: + pass + + schema = core_schema.is_instance_schema(cls=A) + v = SchemaValidator(schema) + v.validate_python(A()) + ``` + + Args: + cls: The value must be an instance of this class + cls_repr: If provided this string is used in the validator name instead of `repr(cls)` + ref: optional unique identifier of the schema, used to reference the schema in other places + metadata: Any other information you want to include with the schema, not used by pydantic-core + serialization: Custom serialization schema + """ + return _dict_not_none( + type='is-instance', cls=cls, cls_repr=cls_repr, ref=ref, metadata=metadata, serialization=serialization + ) + + +class IsSubclassSchema(TypedDict, total=False): + type: Required[Literal['is-subclass']] + cls: Required[type[Any]] + cls_repr: str + ref: str + metadata: dict[str, Any] + serialization: SerSchema + + +def is_subclass_schema( + cls: type[Any], + *, + cls_repr: str | None = None, + ref: str | None = None, + metadata: dict[str, Any] | None = None, + serialization: SerSchema | None = None, +) -> IsInstanceSchema: + """ + Returns a schema that checks if a value is a subtype of a class, equivalent to python's `issubclass` method, e.g.: + + ```py + from pydantic_core import SchemaValidator, core_schema + + class A: + pass + + class B(A): + pass + + schema = core_schema.is_subclass_schema(cls=A) + v = SchemaValidator(schema) + v.validate_python(B) + ``` + + Args: + cls: The value must be a subclass of this class + cls_repr: If provided this string is used in the validator name instead of `repr(cls)` + ref: optional unique identifier of the schema, used to reference the schema in other places + metadata: Any other information you want to include with the schema, not used by pydantic-core + serialization: Custom serialization schema + """ + return _dict_not_none( + type='is-subclass', cls=cls, cls_repr=cls_repr, ref=ref, metadata=metadata, serialization=serialization + ) + + +class CallableSchema(TypedDict, total=False): + type: Required[Literal['callable']] + ref: str + metadata: dict[str, Any] + serialization: SerSchema + + +def callable_schema( + *, ref: str | None = None, metadata: dict[str, Any] | None = None, serialization: SerSchema | None = None +) -> CallableSchema: + """ + Returns a schema that checks if a value is callable, equivalent to python's `callable` method, e.g.: + + ```py + from pydantic_core import SchemaValidator, core_schema + + schema = core_schema.callable_schema() + v = SchemaValidator(schema) + v.validate_python(min) + ``` + + Args: + ref: optional unique identifier of the schema, used to reference the schema in other places + metadata: Any other information you want to include with the schema, not used by pydantic-core + serialization: Custom serialization schema + """ + return _dict_not_none(type='callable', ref=ref, metadata=metadata, serialization=serialization) + + +class UuidSchema(TypedDict, total=False): + type: Required[Literal['uuid']] + version: Literal[1, 3, 4, 5, 7] + strict: bool + ref: str + metadata: dict[str, Any] + serialization: SerSchema + + +def uuid_schema( + *, + version: Literal[1, 3, 4, 5, 6, 7, 8] | None = None, + strict: bool | None = None, + ref: str | None = None, + metadata: dict[str, Any] | None = None, + serialization: SerSchema | None = None, +) -> UuidSchema: + return _dict_not_none( + type='uuid', version=version, strict=strict, ref=ref, metadata=metadata, serialization=serialization + ) + + +class IncExSeqSerSchema(TypedDict, total=False): + type: Required[Literal['include-exclude-sequence']] + include: set[int] + exclude: set[int] + + +def filter_seq_schema(*, include: set[int] | None = None, exclude: set[int] | None = None) -> IncExSeqSerSchema: + return _dict_not_none(type='include-exclude-sequence', include=include, exclude=exclude) + + +IncExSeqOrElseSerSchema = Union[IncExSeqSerSchema, SerSchema] + + +class ListSchema(TypedDict, total=False): + type: Required[Literal['list']] + items_schema: CoreSchema + min_length: int + max_length: int + fail_fast: bool + strict: bool + ref: str + metadata: dict[str, Any] + serialization: IncExSeqOrElseSerSchema + + +def list_schema( + items_schema: CoreSchema | None = None, + *, + min_length: int | None = None, + max_length: int | None = None, + fail_fast: bool | None = None, + strict: bool | None = None, + ref: str | None = None, + metadata: dict[str, Any] | None = None, + serialization: IncExSeqOrElseSerSchema | None = None, +) -> ListSchema: + """ + Returns a schema that matches a list value, e.g.: + + ```py + from pydantic_core import SchemaValidator, core_schema + + schema = core_schema.list_schema(core_schema.int_schema(), min_length=0, max_length=10) + v = SchemaValidator(schema) + assert v.validate_python(['4']) == [4] + ``` + + Args: + items_schema: The value must be a list of items that match this schema + min_length: The value must be a list with at least this many items + max_length: The value must be a list with at most this many items + fail_fast: Stop validation on the first error + strict: The value must be a list with exactly this many items + ref: optional unique identifier of the schema, used to reference the schema in other places + metadata: Any other information you want to include with the schema, not used by pydantic-core + serialization: Custom serialization schema + """ + return _dict_not_none( + type='list', + items_schema=items_schema, + min_length=min_length, + max_length=max_length, + fail_fast=fail_fast, + strict=strict, + ref=ref, + metadata=metadata, + serialization=serialization, + ) + + +# @deprecated('tuple_positional_schema is deprecated. Use pydantic_core.core_schema.tuple_schema instead.') +def tuple_positional_schema( + items_schema: list[CoreSchema], + *, + extras_schema: CoreSchema | None = None, + strict: bool | None = None, + ref: str | None = None, + metadata: dict[str, Any] | None = None, + serialization: IncExSeqOrElseSerSchema | None = None, +) -> TupleSchema: + """ + Returns a schema that matches a tuple of schemas, e.g.: + + ```py + from pydantic_core import SchemaValidator, core_schema + + schema = core_schema.tuple_positional_schema( + [core_schema.int_schema(), core_schema.str_schema()] + ) + v = SchemaValidator(schema) + assert v.validate_python((1, 'hello')) == (1, 'hello') + ``` + + Args: + items_schema: The value must be a tuple with items that match these schemas + extras_schema: The value must be a tuple with items that match this schema + This was inspired by JSON schema's `prefixItems` and `items` fields. + In python's `typing.Tuple`, you can't specify a type for "extra" items -- they must all be the same type + if the length is variable. So this field won't be set from a `typing.Tuple` annotation on a pydantic model. + strict: The value must be a tuple with exactly this many items + ref: optional unique identifier of the schema, used to reference the schema in other places + metadata: Any other information you want to include with the schema, not used by pydantic-core + serialization: Custom serialization schema + """ + if extras_schema is not None: + variadic_item_index = len(items_schema) + items_schema = items_schema + [extras_schema] + else: + variadic_item_index = None + return tuple_schema( + items_schema=items_schema, + variadic_item_index=variadic_item_index, + strict=strict, + ref=ref, + metadata=metadata, + serialization=serialization, + ) + + +# @deprecated('tuple_variable_schema is deprecated. Use pydantic_core.core_schema.tuple_schema instead.') +def tuple_variable_schema( + items_schema: CoreSchema | None = None, + *, + min_length: int | None = None, + max_length: int | None = None, + strict: bool | None = None, + ref: str | None = None, + metadata: dict[str, Any] | None = None, + serialization: IncExSeqOrElseSerSchema | None = None, +) -> TupleSchema: + """ + Returns a schema that matches a tuple of a given schema, e.g.: + + ```py + from pydantic_core import SchemaValidator, core_schema + + schema = core_schema.tuple_variable_schema( + items_schema=core_schema.int_schema(), min_length=0, max_length=10 + ) + v = SchemaValidator(schema) + assert v.validate_python(('1', 2, 3)) == (1, 2, 3) + ``` + + Args: + items_schema: The value must be a tuple with items that match this schema + min_length: The value must be a tuple with at least this many items + max_length: The value must be a tuple with at most this many items + strict: The value must be a tuple with exactly this many items + ref: Optional unique identifier of the schema, used to reference the schema in other places + metadata: Any other information you want to include with the schema, not used by pydantic-core + serialization: Custom serialization schema + """ + return tuple_schema( + items_schema=[items_schema or any_schema()], + variadic_item_index=0, + min_length=min_length, + max_length=max_length, + strict=strict, + ref=ref, + metadata=metadata, + serialization=serialization, + ) + + +class TupleSchema(TypedDict, total=False): + type: Required[Literal['tuple']] + items_schema: Required[list[CoreSchema]] + variadic_item_index: int + min_length: int + max_length: int + fail_fast: bool + strict: bool + ref: str + metadata: dict[str, Any] + serialization: IncExSeqOrElseSerSchema + + +def tuple_schema( + items_schema: list[CoreSchema], + *, + variadic_item_index: int | None = None, + min_length: int | None = None, + max_length: int | None = None, + fail_fast: bool | None = None, + strict: bool | None = None, + ref: str | None = None, + metadata: dict[str, Any] | None = None, + serialization: IncExSeqOrElseSerSchema | None = None, +) -> TupleSchema: + """ + Returns a schema that matches a tuple of schemas, with an optional variadic item, e.g.: + + ```py + from pydantic_core import SchemaValidator, core_schema + + schema = core_schema.tuple_schema( + [core_schema.int_schema(), core_schema.str_schema(), core_schema.float_schema()], + variadic_item_index=1, + ) + v = SchemaValidator(schema) + assert v.validate_python((1, 'hello', 'world', 1.5)) == (1, 'hello', 'world', 1.5) + ``` + + Args: + items_schema: The value must be a tuple with items that match these schemas + variadic_item_index: The index of the schema in `items_schema` to be treated as variadic (following PEP 646) + min_length: The value must be a tuple with at least this many items + max_length: The value must be a tuple with at most this many items + fail_fast: Stop validation on the first error + strict: The value must be a tuple with exactly this many items + ref: Optional unique identifier of the schema, used to reference the schema in other places + metadata: Any other information you want to include with the schema, not used by pydantic-core + serialization: Custom serialization schema + """ + return _dict_not_none( + type='tuple', + items_schema=items_schema, + variadic_item_index=variadic_item_index, + min_length=min_length, + max_length=max_length, + fail_fast=fail_fast, + strict=strict, + ref=ref, + metadata=metadata, + serialization=serialization, + ) + + +class SetSchema(TypedDict, total=False): + type: Required[Literal['set']] + items_schema: CoreSchema + min_length: int + max_length: int + fail_fast: bool + strict: bool + ref: str + metadata: dict[str, Any] + serialization: SerSchema + + +def set_schema( + items_schema: CoreSchema | None = None, + *, + min_length: int | None = None, + max_length: int | None = None, + fail_fast: bool | None = None, + strict: bool | None = None, + ref: str | None = None, + metadata: dict[str, Any] | None = None, + serialization: SerSchema | None = None, +) -> SetSchema: + """ + Returns a schema that matches a set of a given schema, e.g.: + + ```py + from pydantic_core import SchemaValidator, core_schema + + schema = core_schema.set_schema( + items_schema=core_schema.int_schema(), min_length=0, max_length=10 + ) + v = SchemaValidator(schema) + assert v.validate_python({1, '2', 3}) == {1, 2, 3} + ``` + + Args: + items_schema: The value must be a set with items that match this schema + min_length: The value must be a set with at least this many items + max_length: The value must be a set with at most this many items + fail_fast: Stop validation on the first error + strict: The value must be a set with exactly this many items + ref: optional unique identifier of the schema, used to reference the schema in other places + metadata: Any other information you want to include with the schema, not used by pydantic-core + serialization: Custom serialization schema + """ + return _dict_not_none( + type='set', + items_schema=items_schema, + min_length=min_length, + max_length=max_length, + fail_fast=fail_fast, + strict=strict, + ref=ref, + metadata=metadata, + serialization=serialization, + ) + + +class FrozenSetSchema(TypedDict, total=False): + type: Required[Literal['frozenset']] + items_schema: CoreSchema + min_length: int + max_length: int + fail_fast: bool + strict: bool + ref: str + metadata: dict[str, Any] + serialization: SerSchema + + +def frozenset_schema( + items_schema: CoreSchema | None = None, + *, + min_length: int | None = None, + max_length: int | None = None, + fail_fast: bool | None = None, + strict: bool | None = None, + ref: str | None = None, + metadata: dict[str, Any] | None = None, + serialization: SerSchema | None = None, +) -> FrozenSetSchema: + """ + Returns a schema that matches a frozenset of a given schema, e.g.: + + ```py + from pydantic_core import SchemaValidator, core_schema + + schema = core_schema.frozenset_schema( + items_schema=core_schema.int_schema(), min_length=0, max_length=10 + ) + v = SchemaValidator(schema) + assert v.validate_python(frozenset(range(3))) == frozenset({0, 1, 2}) + ``` + + Args: + items_schema: The value must be a frozenset with items that match this schema + min_length: The value must be a frozenset with at least this many items + max_length: The value must be a frozenset with at most this many items + fail_fast: Stop validation on the first error + strict: The value must be a frozenset with exactly this many items + ref: optional unique identifier of the schema, used to reference the schema in other places + metadata: Any other information you want to include with the schema, not used by pydantic-core + serialization: Custom serialization schema + """ + return _dict_not_none( + type='frozenset', + items_schema=items_schema, + min_length=min_length, + max_length=max_length, + fail_fast=fail_fast, + strict=strict, + ref=ref, + metadata=metadata, + serialization=serialization, + ) + + +class GeneratorSchema(TypedDict, total=False): + type: Required[Literal['generator']] + items_schema: CoreSchema + min_length: int + max_length: int + ref: str + metadata: dict[str, Any] + serialization: IncExSeqOrElseSerSchema + + +def generator_schema( + items_schema: CoreSchema | None = None, + *, + min_length: int | None = None, + max_length: int | None = None, + ref: str | None = None, + metadata: dict[str, Any] | None = None, + serialization: IncExSeqOrElseSerSchema | None = None, +) -> GeneratorSchema: + """ + Returns a schema that matches a generator value, e.g.: + + ```py + from typing import Iterator + from pydantic_core import SchemaValidator, core_schema + + def gen() -> Iterator[int]: + yield 1 + + schema = core_schema.generator_schema(items_schema=core_schema.int_schema()) + v = SchemaValidator(schema) + v.validate_python(gen()) + ``` + + Unlike other types, validated generators do not raise ValidationErrors eagerly, + but instead will raise a ValidationError when a violating value is actually read from the generator. + This is to ensure that "validated" generators retain the benefit of lazy evaluation. + + Args: + items_schema: The value must be a generator with items that match this schema + min_length: The value must be a generator that yields at least this many items + max_length: The value must be a generator that yields at most this many items + ref: optional unique identifier of the schema, used to reference the schema in other places + metadata: Any other information you want to include with the schema, not used by pydantic-core + serialization: Custom serialization schema + """ + return _dict_not_none( + type='generator', + items_schema=items_schema, + min_length=min_length, + max_length=max_length, + ref=ref, + metadata=metadata, + serialization=serialization, + ) + + +IncExDict = set[Union[int, str]] + + +class IncExDictSerSchema(TypedDict, total=False): + type: Required[Literal['include-exclude-dict']] + include: IncExDict + exclude: IncExDict + + +def filter_dict_schema(*, include: IncExDict | None = None, exclude: IncExDict | None = None) -> IncExDictSerSchema: + return _dict_not_none(type='include-exclude-dict', include=include, exclude=exclude) + + +IncExDictOrElseSerSchema = Union[IncExDictSerSchema, SerSchema] + + +class DictSchema(TypedDict, total=False): + type: Required[Literal['dict']] + keys_schema: CoreSchema # default: AnySchema + values_schema: CoreSchema # default: AnySchema + min_length: int + max_length: int + strict: bool + ref: str + metadata: dict[str, Any] + serialization: IncExDictOrElseSerSchema + + +def dict_schema( + keys_schema: CoreSchema | None = None, + values_schema: CoreSchema | None = None, + *, + min_length: int | None = None, + max_length: int | None = None, + strict: bool | None = None, + ref: str | None = None, + metadata: dict[str, Any] | None = None, + serialization: SerSchema | None = None, +) -> DictSchema: + """ + Returns a schema that matches a dict value, e.g.: + + ```py + from pydantic_core import SchemaValidator, core_schema + + schema = core_schema.dict_schema( + keys_schema=core_schema.str_schema(), values_schema=core_schema.int_schema() + ) + v = SchemaValidator(schema) + assert v.validate_python({'a': '1', 'b': 2}) == {'a': 1, 'b': 2} + ``` + + Args: + keys_schema: The value must be a dict with keys that match this schema + values_schema: The value must be a dict with values that match this schema + min_length: The value must be a dict with at least this many items + max_length: The value must be a dict with at most this many items + strict: Whether the keys and values should be validated with strict mode + ref: optional unique identifier of the schema, used to reference the schema in other places + metadata: Any other information you want to include with the schema, not used by pydantic-core + serialization: Custom serialization schema + """ + return _dict_not_none( + type='dict', + keys_schema=keys_schema, + values_schema=values_schema, + min_length=min_length, + max_length=max_length, + strict=strict, + ref=ref, + metadata=metadata, + serialization=serialization, + ) + + +# (input_value: Any, /) -> Any +NoInfoValidatorFunction = Callable[[Any], Any] + + +class NoInfoValidatorFunctionSchema(TypedDict): + type: Literal['no-info'] + function: NoInfoValidatorFunction + + +# (input_value: Any, info: ValidationInfo, /) -> Any +WithInfoValidatorFunction = Callable[[Any, ValidationInfo], Any] + + +class WithInfoValidatorFunctionSchema(TypedDict, total=False): + type: Required[Literal['with-info']] + function: Required[WithInfoValidatorFunction] + field_name: str + + +ValidationFunction = Union[NoInfoValidatorFunctionSchema, WithInfoValidatorFunctionSchema] + + +class _ValidatorFunctionSchema(TypedDict, total=False): + function: Required[ValidationFunction] + schema: Required[CoreSchema] + ref: str + metadata: dict[str, Any] + serialization: SerSchema + + +class BeforeValidatorFunctionSchema(_ValidatorFunctionSchema, total=False): + type: Required[Literal['function-before']] + json_schema_input_schema: CoreSchema + + +def no_info_before_validator_function( + function: NoInfoValidatorFunction, + schema: CoreSchema, + *, + ref: str | None = None, + json_schema_input_schema: CoreSchema | None = None, + metadata: dict[str, Any] | None = None, + serialization: SerSchema | None = None, +) -> BeforeValidatorFunctionSchema: + """ + Returns a schema that calls a validator function before validating, no `info` argument is provided, e.g.: + + ```py + from pydantic_core import SchemaValidator, core_schema + + def fn(v: bytes) -> str: + return v.decode() + 'world' + + func_schema = core_schema.no_info_before_validator_function( + function=fn, schema=core_schema.str_schema() + ) + schema = core_schema.typed_dict_schema({'a': core_schema.typed_dict_field(func_schema)}) + + v = SchemaValidator(schema) + assert v.validate_python({'a': b'hello '}) == {'a': 'hello world'} + ``` + + Args: + function: The validator function to call + schema: The schema to validate the output of the validator function + ref: optional unique identifier of the schema, used to reference the schema in other places + json_schema_input_schema: The core schema to be used to generate the corresponding JSON Schema input type + metadata: Any other information you want to include with the schema, not used by pydantic-core + serialization: Custom serialization schema + """ + return _dict_not_none( + type='function-before', + function={'type': 'no-info', 'function': function}, + schema=schema, + ref=ref, + json_schema_input_schema=json_schema_input_schema, + metadata=metadata, + serialization=serialization, + ) + + +def with_info_before_validator_function( + function: WithInfoValidatorFunction, + schema: CoreSchema, + *, + field_name: str | None = None, + ref: str | None = None, + json_schema_input_schema: CoreSchema | None = None, + metadata: dict[str, Any] | None = None, + serialization: SerSchema | None = None, +) -> BeforeValidatorFunctionSchema: + """ + Returns a schema that calls a validator function before validation, the function is called with + an `info` argument, e.g.: + + ```py + from pydantic_core import SchemaValidator, core_schema + + def fn(v: bytes, info: core_schema.ValidationInfo) -> str: + assert info.data is not None + assert info.field_name is not None + return v.decode() + 'world' + + func_schema = core_schema.with_info_before_validator_function( + function=fn, schema=core_schema.str_schema(), field_name='a' + ) + schema = core_schema.typed_dict_schema({'a': core_schema.typed_dict_field(func_schema)}) + + v = SchemaValidator(schema) + assert v.validate_python({'a': b'hello '}) == {'a': 'hello world'} + ``` + + Args: + function: The validator function to call + field_name: The name of the field + schema: The schema to validate the output of the validator function + ref: optional unique identifier of the schema, used to reference the schema in other places + json_schema_input_schema: The core schema to be used to generate the corresponding JSON Schema input type + metadata: Any other information you want to include with the schema, not used by pydantic-core + serialization: Custom serialization schema + """ + return _dict_not_none( + type='function-before', + function=_dict_not_none(type='with-info', function=function, field_name=field_name), + schema=schema, + ref=ref, + json_schema_input_schema=json_schema_input_schema, + metadata=metadata, + serialization=serialization, + ) + + +class AfterValidatorFunctionSchema(_ValidatorFunctionSchema, total=False): + type: Required[Literal['function-after']] + + +def no_info_after_validator_function( + function: NoInfoValidatorFunction, + schema: CoreSchema, + *, + ref: str | None = None, + json_schema_input_schema: CoreSchema | None = None, + metadata: dict[str, Any] | None = None, + serialization: SerSchema | None = None, +) -> AfterValidatorFunctionSchema: + """ + Returns a schema that calls a validator function after validating, no `info` argument is provided, e.g.: + + ```py + from pydantic_core import SchemaValidator, core_schema + + def fn(v: str) -> str: + return v + 'world' + + func_schema = core_schema.no_info_after_validator_function(fn, core_schema.str_schema()) + schema = core_schema.typed_dict_schema({'a': core_schema.typed_dict_field(func_schema)}) + + v = SchemaValidator(schema) + assert v.validate_python({'a': b'hello '}) == {'a': 'hello world'} + ``` + + Args: + function: The validator function to call after the schema is validated + schema: The schema to validate before the validator function + ref: optional unique identifier of the schema, used to reference the schema in other places + json_schema_input_schema: The core schema to be used to generate the corresponding JSON Schema input type + metadata: Any other information you want to include with the schema, not used by pydantic-core + serialization: Custom serialization schema + """ + return _dict_not_none( + type='function-after', + function={'type': 'no-info', 'function': function}, + schema=schema, + ref=ref, + json_schema_input_schema=json_schema_input_schema, + metadata=metadata, + serialization=serialization, + ) + + +def with_info_after_validator_function( + function: WithInfoValidatorFunction, + schema: CoreSchema, + *, + field_name: str | None = None, + ref: str | None = None, + metadata: dict[str, Any] | None = None, + serialization: SerSchema | None = None, +) -> AfterValidatorFunctionSchema: + """ + Returns a schema that calls a validator function after validation, the function is called with + an `info` argument, e.g.: + + ```py + from pydantic_core import SchemaValidator, core_schema + + def fn(v: str, info: core_schema.ValidationInfo) -> str: + assert info.data is not None + assert info.field_name is not None + return v + 'world' + + func_schema = core_schema.with_info_after_validator_function( + function=fn, schema=core_schema.str_schema(), field_name='a' + ) + schema = core_schema.typed_dict_schema({'a': core_schema.typed_dict_field(func_schema)}) + + v = SchemaValidator(schema) + assert v.validate_python({'a': b'hello '}) == {'a': 'hello world'} + ``` + + Args: + function: The validator function to call after the schema is validated + schema: The schema to validate before the validator function + field_name: The name of the field this validators is applied to, if any + ref: optional unique identifier of the schema, used to reference the schema in other places + metadata: Any other information you want to include with the schema, not used by pydantic-core + serialization: Custom serialization schema + """ + return _dict_not_none( + type='function-after', + function=_dict_not_none(type='with-info', function=function, field_name=field_name), + schema=schema, + ref=ref, + metadata=metadata, + serialization=serialization, + ) + + +class ValidatorFunctionWrapHandler(Protocol): + def __call__(self, input_value: Any, outer_location: str | int | None = None, /) -> Any: # pragma: no cover + ... + + +# (input_value: Any, validator: ValidatorFunctionWrapHandler, /) -> Any +NoInfoWrapValidatorFunction = Callable[[Any, ValidatorFunctionWrapHandler], Any] + + +class NoInfoWrapValidatorFunctionSchema(TypedDict): + type: Literal['no-info'] + function: NoInfoWrapValidatorFunction + + +# (input_value: Any, validator: ValidatorFunctionWrapHandler, info: ValidationInfo, /) -> Any +WithInfoWrapValidatorFunction = Callable[[Any, ValidatorFunctionWrapHandler, ValidationInfo], Any] + + +class WithInfoWrapValidatorFunctionSchema(TypedDict, total=False): + type: Required[Literal['with-info']] + function: Required[WithInfoWrapValidatorFunction] + field_name: str + + +WrapValidatorFunction = Union[NoInfoWrapValidatorFunctionSchema, WithInfoWrapValidatorFunctionSchema] + + +class WrapValidatorFunctionSchema(TypedDict, total=False): + type: Required[Literal['function-wrap']] + function: Required[WrapValidatorFunction] + schema: Required[CoreSchema] + ref: str + json_schema_input_schema: CoreSchema + metadata: dict[str, Any] + serialization: SerSchema + + +def no_info_wrap_validator_function( + function: NoInfoWrapValidatorFunction, + schema: CoreSchema, + *, + ref: str | None = None, + json_schema_input_schema: CoreSchema | None = None, + metadata: dict[str, Any] | None = None, + serialization: SerSchema | None = None, +) -> WrapValidatorFunctionSchema: + """ + Returns a schema which calls a function with a `validator` callable argument which can + optionally be used to call inner validation with the function logic, this is much like the + "onion" implementation of middleware in many popular web frameworks, no `info` argument is passed, e.g.: + + ```py + from pydantic_core import SchemaValidator, core_schema + + def fn( + v: str, + validator: core_schema.ValidatorFunctionWrapHandler, + ) -> str: + return validator(input_value=v) + 'world' + + schema = core_schema.no_info_wrap_validator_function( + function=fn, schema=core_schema.str_schema() + ) + v = SchemaValidator(schema) + assert v.validate_python('hello ') == 'hello world' + ``` + + Args: + function: The validator function to call + schema: The schema to validate the output of the validator function + ref: optional unique identifier of the schema, used to reference the schema in other places + json_schema_input_schema: The core schema to be used to generate the corresponding JSON Schema input type + metadata: Any other information you want to include with the schema, not used by pydantic-core + serialization: Custom serialization schema + """ + return _dict_not_none( + type='function-wrap', + function={'type': 'no-info', 'function': function}, + schema=schema, + json_schema_input_schema=json_schema_input_schema, + ref=ref, + metadata=metadata, + serialization=serialization, + ) + + +def with_info_wrap_validator_function( + function: WithInfoWrapValidatorFunction, + schema: CoreSchema, + *, + field_name: str | None = None, + json_schema_input_schema: CoreSchema | None = None, + ref: str | None = None, + metadata: dict[str, Any] | None = None, + serialization: SerSchema | None = None, +) -> WrapValidatorFunctionSchema: + """ + Returns a schema which calls a function with a `validator` callable argument which can + optionally be used to call inner validation with the function logic, this is much like the + "onion" implementation of middleware in many popular web frameworks, an `info` argument is also passed, e.g.: + + ```py + from pydantic_core import SchemaValidator, core_schema + + def fn( + v: str, + validator: core_schema.ValidatorFunctionWrapHandler, + info: core_schema.ValidationInfo, + ) -> str: + return validator(input_value=v) + 'world' + + schema = core_schema.with_info_wrap_validator_function( + function=fn, schema=core_schema.str_schema() + ) + v = SchemaValidator(schema) + assert v.validate_python('hello ') == 'hello world' + ``` + + Args: + function: The validator function to call + schema: The schema to validate the output of the validator function + field_name: The name of the field this validators is applied to, if any + json_schema_input_schema: The core schema to be used to generate the corresponding JSON Schema input type + ref: optional unique identifier of the schema, used to reference the schema in other places + metadata: Any other information you want to include with the schema, not used by pydantic-core + serialization: Custom serialization schema + """ + return _dict_not_none( + type='function-wrap', + function=_dict_not_none(type='with-info', function=function, field_name=field_name), + schema=schema, + json_schema_input_schema=json_schema_input_schema, + ref=ref, + metadata=metadata, + serialization=serialization, + ) + + +class PlainValidatorFunctionSchema(TypedDict, total=False): + type: Required[Literal['function-plain']] + function: Required[ValidationFunction] + ref: str + json_schema_input_schema: CoreSchema + metadata: dict[str, Any] + serialization: SerSchema + + +def no_info_plain_validator_function( + function: NoInfoValidatorFunction, + *, + ref: str | None = None, + json_schema_input_schema: CoreSchema | None = None, + metadata: dict[str, Any] | None = None, + serialization: SerSchema | None = None, +) -> PlainValidatorFunctionSchema: + """ + Returns a schema that uses the provided function for validation, no `info` argument is passed, e.g.: + + ```py + from pydantic_core import SchemaValidator, core_schema + + def fn(v: str) -> str: + assert 'hello' in v + return v + 'world' + + schema = core_schema.no_info_plain_validator_function(function=fn) + v = SchemaValidator(schema) + assert v.validate_python('hello ') == 'hello world' + ``` + + Args: + function: The validator function to call + ref: optional unique identifier of the schema, used to reference the schema in other places + json_schema_input_schema: The core schema to be used to generate the corresponding JSON Schema input type + metadata: Any other information you want to include with the schema, not used by pydantic-core + serialization: Custom serialization schema + """ + return _dict_not_none( + type='function-plain', + function={'type': 'no-info', 'function': function}, + ref=ref, + json_schema_input_schema=json_schema_input_schema, + metadata=metadata, + serialization=serialization, + ) + + +def with_info_plain_validator_function( + function: WithInfoValidatorFunction, + *, + field_name: str | None = None, + ref: str | None = None, + json_schema_input_schema: CoreSchema | None = None, + metadata: dict[str, Any] | None = None, + serialization: SerSchema | None = None, +) -> PlainValidatorFunctionSchema: + """ + Returns a schema that uses the provided function for validation, an `info` argument is passed, e.g.: + + ```py + from pydantic_core import SchemaValidator, core_schema + + def fn(v: str, info: core_schema.ValidationInfo) -> str: + assert 'hello' in v + return v + 'world' + + schema = core_schema.with_info_plain_validator_function(function=fn) + v = SchemaValidator(schema) + assert v.validate_python('hello ') == 'hello world' + ``` + + Args: + function: The validator function to call + field_name: The name of the field this validators is applied to, if any + ref: optional unique identifier of the schema, used to reference the schema in other places + json_schema_input_schema: The core schema to be used to generate the corresponding JSON Schema input type + metadata: Any other information you want to include with the schema, not used by pydantic-core + serialization: Custom serialization schema + """ + return _dict_not_none( + type='function-plain', + function=_dict_not_none(type='with-info', function=function, field_name=field_name), + ref=ref, + json_schema_input_schema=json_schema_input_schema, + metadata=metadata, + serialization=serialization, + ) + + +class WithDefaultSchema(TypedDict, total=False): + type: Required[Literal['default']] + schema: Required[CoreSchema] + default: Any + default_factory: Union[Callable[[], Any], Callable[[dict[str, Any]], Any]] + default_factory_takes_data: bool + on_error: Literal['raise', 'omit', 'default'] # default: 'raise' + validate_default: bool # default: False + strict: bool + ref: str + metadata: dict[str, Any] + serialization: SerSchema + + +def with_default_schema( + schema: CoreSchema, + *, + default: Any = PydanticUndefined, + default_factory: Union[Callable[[], Any], Callable[[dict[str, Any]], Any], None] = None, + default_factory_takes_data: bool | None = None, + on_error: Literal['raise', 'omit', 'default'] | None = None, + validate_default: bool | None = None, + strict: bool | None = None, + ref: str | None = None, + metadata: dict[str, Any] | None = None, + serialization: SerSchema | None = None, +) -> WithDefaultSchema: + """ + Returns a schema that adds a default value to the given schema, e.g.: + + ```py + from pydantic_core import SchemaValidator, core_schema + + schema = core_schema.with_default_schema(core_schema.str_schema(), default='hello') + wrapper_schema = core_schema.typed_dict_schema( + {'a': core_schema.typed_dict_field(schema)} + ) + v = SchemaValidator(wrapper_schema) + assert v.validate_python({}) == v.validate_python({'a': 'hello'}) + ``` + + Args: + schema: The schema to add a default value to + default: The default value to use + default_factory: A callable that returns the default value to use + default_factory_takes_data: Whether the default factory takes a validated data argument + on_error: What to do if the schema validation fails. One of 'raise', 'omit', 'default' + validate_default: Whether the default value should be validated + strict: Whether the underlying schema should be validated with strict mode + ref: optional unique identifier of the schema, used to reference the schema in other places + metadata: Any other information you want to include with the schema, not used by pydantic-core + serialization: Custom serialization schema + """ + s = _dict_not_none( + type='default', + schema=schema, + default_factory=default_factory, + default_factory_takes_data=default_factory_takes_data, + on_error=on_error, + validate_default=validate_default, + strict=strict, + ref=ref, + metadata=metadata, + serialization=serialization, + ) + if default is not PydanticUndefined: + s['default'] = default + return s + + +class NullableSchema(TypedDict, total=False): + type: Required[Literal['nullable']] + schema: Required[CoreSchema] + strict: bool + ref: str + metadata: dict[str, Any] + serialization: SerSchema + + +def nullable_schema( + schema: CoreSchema, + *, + strict: bool | None = None, + ref: str | None = None, + metadata: dict[str, Any] | None = None, + serialization: SerSchema | None = None, +) -> NullableSchema: + """ + Returns a schema that matches a nullable value, e.g.: + + ```py + from pydantic_core import SchemaValidator, core_schema + + schema = core_schema.nullable_schema(core_schema.str_schema()) + v = SchemaValidator(schema) + assert v.validate_python(None) is None + ``` + + Args: + schema: The schema to wrap + strict: Whether the underlying schema should be validated with strict mode + ref: optional unique identifier of the schema, used to reference the schema in other places + metadata: Any other information you want to include with the schema, not used by pydantic-core + serialization: Custom serialization schema + """ + return _dict_not_none( + type='nullable', schema=schema, strict=strict, ref=ref, metadata=metadata, serialization=serialization + ) + + +class UnionSchema(TypedDict, total=False): + type: Required[Literal['union']] + choices: Required[list[Union[CoreSchema, tuple[CoreSchema, str]]]] + # default true, whether to automatically collapse unions with one element to the inner validator + auto_collapse: bool + custom_error_type: str + custom_error_message: str + custom_error_context: dict[str, Union[str, int, float]] + mode: Literal['smart', 'left_to_right'] # default: 'smart' + strict: bool + ref: str + metadata: dict[str, Any] + serialization: SerSchema + + +def union_schema( + choices: list[CoreSchema | tuple[CoreSchema, str]], + *, + auto_collapse: bool | None = None, + custom_error_type: str | None = None, + custom_error_message: str | None = None, + custom_error_context: dict[str, str | int] | None = None, + mode: Literal['smart', 'left_to_right'] | None = None, + ref: str | None = None, + metadata: dict[str, Any] | None = None, + serialization: SerSchema | None = None, +) -> UnionSchema: + """ + Returns a schema that matches a union value, e.g.: + + ```py + from pydantic_core import SchemaValidator, core_schema + + schema = core_schema.union_schema([core_schema.str_schema(), core_schema.int_schema()]) + v = SchemaValidator(schema) + assert v.validate_python('hello') == 'hello' + assert v.validate_python(1) == 1 + ``` + + Args: + choices: The schemas to match. If a tuple, the second item is used as the label for the case. + auto_collapse: whether to automatically collapse unions with one element to the inner validator, default true + custom_error_type: The custom error type to use if the validation fails + custom_error_message: The custom error message to use if the validation fails + custom_error_context: The custom error context to use if the validation fails + mode: How to select which choice to return + * `smart` (default) will try to return the choice which is the closest match to the input value + * `left_to_right` will return the first choice in `choices` which succeeds validation + ref: optional unique identifier of the schema, used to reference the schema in other places + metadata: Any other information you want to include with the schema, not used by pydantic-core + serialization: Custom serialization schema + """ + return _dict_not_none( + type='union', + choices=choices, + auto_collapse=auto_collapse, + custom_error_type=custom_error_type, + custom_error_message=custom_error_message, + custom_error_context=custom_error_context, + mode=mode, + ref=ref, + metadata=metadata, + serialization=serialization, + ) + + +class TaggedUnionSchema(TypedDict, total=False): + type: Required[Literal['tagged-union']] + choices: Required[dict[Hashable, CoreSchema]] + discriminator: Required[Union[str, list[Union[str, int]], list[list[Union[str, int]]], Callable[[Any], Hashable]]] + custom_error_type: str + custom_error_message: str + custom_error_context: dict[str, Union[str, int, float]] + strict: bool + from_attributes: bool # default: True + ref: str + metadata: dict[str, Any] + serialization: SerSchema + + +def tagged_union_schema( + choices: dict[Any, CoreSchema], + discriminator: str | list[str | int] | list[list[str | int]] | Callable[[Any], Any], + *, + custom_error_type: str | None = None, + custom_error_message: str | None = None, + custom_error_context: dict[str, int | str | float] | None = None, + strict: bool | None = None, + from_attributes: bool | None = None, + ref: str | None = None, + metadata: dict[str, Any] | None = None, + serialization: SerSchema | None = None, +) -> TaggedUnionSchema: + """ + Returns a schema that matches a tagged union value, e.g.: + + ```py + from pydantic_core import SchemaValidator, core_schema + + apple_schema = core_schema.typed_dict_schema( + { + 'foo': core_schema.typed_dict_field(core_schema.str_schema()), + 'bar': core_schema.typed_dict_field(core_schema.int_schema()), + } + ) + banana_schema = core_schema.typed_dict_schema( + { + 'foo': core_schema.typed_dict_field(core_schema.str_schema()), + 'spam': core_schema.typed_dict_field( + core_schema.list_schema(items_schema=core_schema.int_schema()) + ), + } + ) + schema = core_schema.tagged_union_schema( + choices={ + 'apple': apple_schema, + 'banana': banana_schema, + }, + discriminator='foo', + ) + v = SchemaValidator(schema) + assert v.validate_python({'foo': 'apple', 'bar': '123'}) == {'foo': 'apple', 'bar': 123} + assert v.validate_python({'foo': 'banana', 'spam': [1, 2, 3]}) == { + 'foo': 'banana', + 'spam': [1, 2, 3], + } + ``` + + Args: + choices: The schemas to match + When retrieving a schema from `choices` using the discriminator value, if the value is a str, + it should be fed back into the `choices` map until a schema is obtained + (This approach is to prevent multiple ownership of a single schema in Rust) + discriminator: The discriminator to use to determine the schema to use + * If `discriminator` is a str, it is the name of the attribute to use as the discriminator + * If `discriminator` is a list of int/str, it should be used as a "path" to access the discriminator + * If `discriminator` is a list of lists, each inner list is a path, and the first path that exists is used + * If `discriminator` is a callable, it should return the discriminator when called on the value to validate; + the callable can return `None` to indicate that there is no matching discriminator present on the input + custom_error_type: The custom error type to use if the validation fails + custom_error_message: The custom error message to use if the validation fails + custom_error_context: The custom error context to use if the validation fails + strict: Whether the underlying schemas should be validated with strict mode + from_attributes: Whether to use the attributes of the object to retrieve the discriminator value + ref: optional unique identifier of the schema, used to reference the schema in other places + metadata: Any other information you want to include with the schema, not used by pydantic-core + serialization: Custom serialization schema + """ + return _dict_not_none( + type='tagged-union', + choices=choices, + discriminator=discriminator, + custom_error_type=custom_error_type, + custom_error_message=custom_error_message, + custom_error_context=custom_error_context, + strict=strict, + from_attributes=from_attributes, + ref=ref, + metadata=metadata, + serialization=serialization, + ) + + +class ChainSchema(TypedDict, total=False): + type: Required[Literal['chain']] + steps: Required[list[CoreSchema]] + ref: str + metadata: dict[str, Any] + serialization: SerSchema + + +def chain_schema( + steps: list[CoreSchema], + *, + ref: str | None = None, + metadata: dict[str, Any] | None = None, + serialization: SerSchema | None = None, +) -> ChainSchema: + """ + Returns a schema that chains the provided validation schemas, e.g.: + + ```py + from pydantic_core import SchemaValidator, core_schema + + def fn(v: str, info: core_schema.ValidationInfo) -> str: + assert 'hello' in v + return v + ' world' + + fn_schema = core_schema.with_info_plain_validator_function(function=fn) + schema = core_schema.chain_schema( + [fn_schema, fn_schema, fn_schema, core_schema.str_schema()] + ) + v = SchemaValidator(schema) + assert v.validate_python('hello') == 'hello world world world' + ``` + + Args: + steps: The schemas to chain + ref: optional unique identifier of the schema, used to reference the schema in other places + metadata: Any other information you want to include with the schema, not used by pydantic-core + serialization: Custom serialization schema + """ + return _dict_not_none(type='chain', steps=steps, ref=ref, metadata=metadata, serialization=serialization) + + +class LaxOrStrictSchema(TypedDict, total=False): + type: Required[Literal['lax-or-strict']] + lax_schema: Required[CoreSchema] + strict_schema: Required[CoreSchema] + strict: bool + ref: str + metadata: dict[str, Any] + serialization: SerSchema + + +def lax_or_strict_schema( + lax_schema: CoreSchema, + strict_schema: CoreSchema, + *, + strict: bool | None = None, + ref: str | None = None, + metadata: dict[str, Any] | None = None, + serialization: SerSchema | None = None, +) -> LaxOrStrictSchema: + """ + Returns a schema that uses the lax or strict schema, e.g.: + + ```py + from pydantic_core import SchemaValidator, core_schema + + def fn(v: str, info: core_schema.ValidationInfo) -> str: + assert 'hello' in v + return v + ' world' + + lax_schema = core_schema.int_schema(strict=False) + strict_schema = core_schema.int_schema(strict=True) + + schema = core_schema.lax_or_strict_schema( + lax_schema=lax_schema, strict_schema=strict_schema, strict=True + ) + v = SchemaValidator(schema) + assert v.validate_python(123) == 123 + + schema = core_schema.lax_or_strict_schema( + lax_schema=lax_schema, strict_schema=strict_schema, strict=False + ) + v = SchemaValidator(schema) + assert v.validate_python('123') == 123 + ``` + + Args: + lax_schema: The lax schema to use + strict_schema: The strict schema to use + strict: Whether the strict schema should be used + ref: optional unique identifier of the schema, used to reference the schema in other places + metadata: Any other information you want to include with the schema, not used by pydantic-core + serialization: Custom serialization schema + """ + return _dict_not_none( + type='lax-or-strict', + lax_schema=lax_schema, + strict_schema=strict_schema, + strict=strict, + ref=ref, + metadata=metadata, + serialization=serialization, + ) + + +class JsonOrPythonSchema(TypedDict, total=False): + type: Required[Literal['json-or-python']] + json_schema: Required[CoreSchema] + python_schema: Required[CoreSchema] + ref: str + metadata: dict[str, Any] + serialization: SerSchema + + +def json_or_python_schema( + json_schema: CoreSchema, + python_schema: CoreSchema, + *, + ref: str | None = None, + metadata: dict[str, Any] | None = None, + serialization: SerSchema | None = None, +) -> JsonOrPythonSchema: + """ + Returns a schema that uses the Json or Python schema depending on the input: + + ```py + from pydantic_core import SchemaValidator, ValidationError, core_schema + + v = SchemaValidator( + core_schema.json_or_python_schema( + json_schema=core_schema.int_schema(), + python_schema=core_schema.int_schema(strict=True), + ) + ) + + assert v.validate_json('"123"') == 123 + + try: + v.validate_python('123') + except ValidationError: + pass + else: + raise AssertionError('Validation should have failed') + ``` + + Args: + json_schema: The schema to use for Json inputs + python_schema: The schema to use for Python inputs + ref: optional unique identifier of the schema, used to reference the schema in other places + metadata: Any other information you want to include with the schema, not used by pydantic-core + serialization: Custom serialization schema + """ + return _dict_not_none( + type='json-or-python', + json_schema=json_schema, + python_schema=python_schema, + ref=ref, + metadata=metadata, + serialization=serialization, + ) + + +class TypedDictField(TypedDict, total=False): + type: Required[Literal['typed-dict-field']] + schema: Required[CoreSchema] + required: bool + validation_alias: Union[str, list[Union[str, int]], list[list[Union[str, int]]]] + serialization_alias: str + serialization_exclude: bool # default: False + metadata: dict[str, Any] + + +def typed_dict_field( + schema: CoreSchema, + *, + required: bool | None = None, + validation_alias: str | list[str | int] | list[list[str | int]] | None = None, + serialization_alias: str | None = None, + serialization_exclude: bool | None = None, + metadata: dict[str, Any] | None = None, +) -> TypedDictField: + """ + Returns a schema that matches a typed dict field, e.g.: + + ```py + from pydantic_core import core_schema + + field = core_schema.typed_dict_field(schema=core_schema.int_schema(), required=True) + ``` + + Args: + schema: The schema to use for the field + required: Whether the field is required, otherwise uses the value from `total` on the typed dict + validation_alias: The alias(es) to use to find the field in the validation data + serialization_alias: The alias to use as a key when serializing + serialization_exclude: Whether to exclude the field when serializing + metadata: Any other information you want to include with the schema, not used by pydantic-core + """ + return _dict_not_none( + type='typed-dict-field', + schema=schema, + required=required, + validation_alias=validation_alias, + serialization_alias=serialization_alias, + serialization_exclude=serialization_exclude, + metadata=metadata, + ) + + +class TypedDictSchema(TypedDict, total=False): + type: Required[Literal['typed-dict']] + fields: Required[dict[str, TypedDictField]] + cls: type[Any] + cls_name: str + computed_fields: list[ComputedField] + strict: bool + extras_schema: CoreSchema + # all these values can be set via config, equivalent fields have `typed_dict_` prefix + extra_behavior: ExtraBehavior + total: bool # default: True + ref: str + metadata: dict[str, Any] + serialization: SerSchema + config: CoreConfig + + +def typed_dict_schema( + fields: dict[str, TypedDictField], + *, + cls: type[Any] | None = None, + cls_name: str | None = None, + computed_fields: list[ComputedField] | None = None, + strict: bool | None = None, + extras_schema: CoreSchema | None = None, + extra_behavior: ExtraBehavior | None = None, + total: bool | None = None, + ref: str | None = None, + metadata: dict[str, Any] | None = None, + serialization: SerSchema | None = None, + config: CoreConfig | None = None, +) -> TypedDictSchema: + """ + Returns a schema that matches a typed dict, e.g.: + + ```py + from typing_extensions import TypedDict + + from pydantic_core import SchemaValidator, core_schema + + class MyTypedDict(TypedDict): + a: str + + wrapper_schema = core_schema.typed_dict_schema( + {'a': core_schema.typed_dict_field(core_schema.str_schema())}, cls=MyTypedDict + ) + v = SchemaValidator(wrapper_schema) + assert v.validate_python({'a': 'hello'}) == {'a': 'hello'} + ``` + + Args: + fields: The fields to use for the typed dict + cls: The class to use for the typed dict + cls_name: The name to use in error locations. Falls back to `cls.__name__`, or the validator name if no class + is provided. + computed_fields: Computed fields to use when serializing the model, only applies when directly inside a model + strict: Whether the typed dict is strict + extras_schema: The extra validator to use for the typed dict + ref: optional unique identifier of the schema, used to reference the schema in other places + metadata: Any other information you want to include with the schema, not used by pydantic-core + extra_behavior: The extra behavior to use for the typed dict + total: Whether the typed dict is total, otherwise uses `typed_dict_total` from config + serialization: Custom serialization schema + """ + return _dict_not_none( + type='typed-dict', + fields=fields, + cls=cls, + cls_name=cls_name, + computed_fields=computed_fields, + strict=strict, + extras_schema=extras_schema, + extra_behavior=extra_behavior, + total=total, + ref=ref, + metadata=metadata, + serialization=serialization, + config=config, + ) + + +class ModelField(TypedDict, total=False): + type: Required[Literal['model-field']] + schema: Required[CoreSchema] + validation_alias: Union[str, list[Union[str, int]], list[list[Union[str, int]]]] + serialization_alias: str + serialization_exclude: bool # default: False + frozen: bool + metadata: dict[str, Any] + + +def model_field( + schema: CoreSchema, + *, + validation_alias: str | list[str | int] | list[list[str | int]] | None = None, + serialization_alias: str | None = None, + serialization_exclude: bool | None = None, + frozen: bool | None = None, + metadata: dict[str, Any] | None = None, +) -> ModelField: + """ + Returns a schema for a model field, e.g.: + + ```py + from pydantic_core import core_schema + + field = core_schema.model_field(schema=core_schema.int_schema()) + ``` + + Args: + schema: The schema to use for the field + validation_alias: The alias(es) to use to find the field in the validation data + serialization_alias: The alias to use as a key when serializing + serialization_exclude: Whether to exclude the field when serializing + frozen: Whether the field is frozen + metadata: Any other information you want to include with the schema, not used by pydantic-core + """ + return _dict_not_none( + type='model-field', + schema=schema, + validation_alias=validation_alias, + serialization_alias=serialization_alias, + serialization_exclude=serialization_exclude, + frozen=frozen, + metadata=metadata, + ) + + +class ModelFieldsSchema(TypedDict, total=False): + type: Required[Literal['model-fields']] + fields: Required[dict[str, ModelField]] + model_name: str + computed_fields: list[ComputedField] + strict: bool + extras_schema: CoreSchema + extras_keys_schema: CoreSchema + extra_behavior: ExtraBehavior + from_attributes: bool + ref: str + metadata: dict[str, Any] + serialization: SerSchema + + +def model_fields_schema( + fields: dict[str, ModelField], + *, + model_name: str | None = None, + computed_fields: list[ComputedField] | None = None, + strict: bool | None = None, + extras_schema: CoreSchema | None = None, + extras_keys_schema: CoreSchema | None = None, + extra_behavior: ExtraBehavior | None = None, + from_attributes: bool | None = None, + ref: str | None = None, + metadata: dict[str, Any] | None = None, + serialization: SerSchema | None = None, +) -> ModelFieldsSchema: + """ + Returns a schema that matches the fields of a Pydantic model, e.g.: + + ```py + from pydantic_core import SchemaValidator, core_schema + + wrapper_schema = core_schema.model_fields_schema( + {'a': core_schema.model_field(core_schema.str_schema())} + ) + v = SchemaValidator(wrapper_schema) + print(v.validate_python({'a': 'hello'})) + #> ({'a': 'hello'}, None, {'a'}) + ``` + + Args: + fields: The fields of the model + model_name: The name of the model, used for error messages, defaults to "Model" + computed_fields: Computed fields to use when serializing the model, only applies when directly inside a model + strict: Whether the model is strict + extras_schema: The schema to use when validating extra input data + extras_keys_schema: The schema to use when validating the keys of extra input data + ref: optional unique identifier of the schema, used to reference the schema in other places + metadata: Any other information you want to include with the schema, not used by pydantic-core + extra_behavior: The extra behavior to use for the model fields + from_attributes: Whether the model fields should be populated from attributes + serialization: Custom serialization schema + """ + return _dict_not_none( + type='model-fields', + fields=fields, + model_name=model_name, + computed_fields=computed_fields, + strict=strict, + extras_schema=extras_schema, + extras_keys_schema=extras_keys_schema, + extra_behavior=extra_behavior, + from_attributes=from_attributes, + ref=ref, + metadata=metadata, + serialization=serialization, + ) + + +class ModelSchema(TypedDict, total=False): + type: Required[Literal['model']] + cls: Required[type[Any]] + generic_origin: type[Any] + schema: Required[CoreSchema] + custom_init: bool + root_model: bool + post_init: str + revalidate_instances: Literal['always', 'never', 'subclass-instances'] # default: 'never' + strict: bool + frozen: bool + extra_behavior: ExtraBehavior + config: CoreConfig + ref: str + metadata: dict[str, Any] + serialization: SerSchema + + +def model_schema( + cls: type[Any], + schema: CoreSchema, + *, + generic_origin: type[Any] | None = None, + custom_init: bool | None = None, + root_model: bool | None = None, + post_init: str | None = None, + revalidate_instances: Literal['always', 'never', 'subclass-instances'] | None = None, + strict: bool | None = None, + frozen: bool | None = None, + extra_behavior: ExtraBehavior | None = None, + config: CoreConfig | None = None, + ref: str | None = None, + metadata: dict[str, Any] | None = None, + serialization: SerSchema | None = None, +) -> ModelSchema: + """ + A model schema generally contains a typed-dict schema. + It will run the typed dict validator, then create a new class + and set the dict and fields set returned from the typed dict validator + to `__dict__` and `__pydantic_fields_set__` respectively. + + Example: + + ```py + from pydantic_core import CoreConfig, SchemaValidator, core_schema + + class MyModel: + __slots__ = ( + '__dict__', + '__pydantic_fields_set__', + '__pydantic_extra__', + '__pydantic_private__', + ) + + schema = core_schema.model_schema( + cls=MyModel, + config=CoreConfig(str_max_length=5), + schema=core_schema.model_fields_schema( + fields={'a': core_schema.model_field(core_schema.str_schema())}, + ), + ) + v = SchemaValidator(schema) + assert v.isinstance_python({'a': 'hello'}) is True + assert v.isinstance_python({'a': 'too long'}) is False + ``` + + Args: + cls: The class to use for the model + schema: The schema to use for the model + generic_origin: The origin type used for this model, if it's a parametrized generic. Ex, + if this model schema represents `SomeModel[int]`, generic_origin is `SomeModel` + custom_init: Whether the model has a custom init method + root_model: Whether the model is a `RootModel` + post_init: The call after init to use for the model + revalidate_instances: whether instances of models and dataclasses (including subclass instances) + should re-validate defaults to config.revalidate_instances, else 'never' + strict: Whether the model is strict + frozen: Whether the model is frozen + extra_behavior: The extra behavior to use for the model, used in serialization + config: The config to use for the model + ref: optional unique identifier of the schema, used to reference the schema in other places + metadata: Any other information you want to include with the schema, not used by pydantic-core + serialization: Custom serialization schema + """ + return _dict_not_none( + type='model', + cls=cls, + generic_origin=generic_origin, + schema=schema, + custom_init=custom_init, + root_model=root_model, + post_init=post_init, + revalidate_instances=revalidate_instances, + strict=strict, + frozen=frozen, + extra_behavior=extra_behavior, + config=config, + ref=ref, + metadata=metadata, + serialization=serialization, + ) + + +class DataclassField(TypedDict, total=False): + type: Required[Literal['dataclass-field']] + name: Required[str] + schema: Required[CoreSchema] + kw_only: bool # default: True + init: bool # default: True + init_only: bool # default: False + frozen: bool # default: False + validation_alias: Union[str, list[Union[str, int]], list[list[Union[str, int]]]] + serialization_alias: str + serialization_exclude: bool # default: False + metadata: dict[str, Any] + + +def dataclass_field( + name: str, + schema: CoreSchema, + *, + kw_only: bool | None = None, + init: bool | None = None, + init_only: bool | None = None, + validation_alias: str | list[str | int] | list[list[str | int]] | None = None, + serialization_alias: str | None = None, + serialization_exclude: bool | None = None, + metadata: dict[str, Any] | None = None, + frozen: bool | None = None, +) -> DataclassField: + """ + Returns a schema for a dataclass field, e.g.: + + ```py + from pydantic_core import SchemaValidator, core_schema + + field = core_schema.dataclass_field( + name='a', schema=core_schema.str_schema(), kw_only=False + ) + schema = core_schema.dataclass_args_schema('Foobar', [field]) + v = SchemaValidator(schema) + assert v.validate_python({'a': 'hello'}) == ({'a': 'hello'}, None) + ``` + + Args: + name: The name to use for the argument parameter + schema: The schema to use for the argument parameter + kw_only: Whether the field can be set with a positional argument as well as a keyword argument + init: Whether the field should be validated during initialization + init_only: Whether the field should be omitted from `__dict__` and passed to `__post_init__` + validation_alias: The alias(es) to use to find the field in the validation data + serialization_alias: The alias to use as a key when serializing + serialization_exclude: Whether to exclude the field when serializing + metadata: Any other information you want to include with the schema, not used by pydantic-core + frozen: Whether the field is frozen + """ + return _dict_not_none( + type='dataclass-field', + name=name, + schema=schema, + kw_only=kw_only, + init=init, + init_only=init_only, + validation_alias=validation_alias, + serialization_alias=serialization_alias, + serialization_exclude=serialization_exclude, + metadata=metadata, + frozen=frozen, + ) + + +class DataclassArgsSchema(TypedDict, total=False): + type: Required[Literal['dataclass-args']] + dataclass_name: Required[str] + fields: Required[list[DataclassField]] + computed_fields: list[ComputedField] + collect_init_only: bool # default: False + ref: str + metadata: dict[str, Any] + serialization: SerSchema + extra_behavior: ExtraBehavior + + +def dataclass_args_schema( + dataclass_name: str, + fields: list[DataclassField], + *, + computed_fields: list[ComputedField] | None = None, + collect_init_only: bool | None = None, + ref: str | None = None, + metadata: dict[str, Any] | None = None, + serialization: SerSchema | None = None, + extra_behavior: ExtraBehavior | None = None, +) -> DataclassArgsSchema: + """ + Returns a schema for validating dataclass arguments, e.g.: + + ```py + from pydantic_core import SchemaValidator, core_schema + + field_a = core_schema.dataclass_field( + name='a', schema=core_schema.str_schema(), kw_only=False + ) + field_b = core_schema.dataclass_field( + name='b', schema=core_schema.bool_schema(), kw_only=False + ) + schema = core_schema.dataclass_args_schema('Foobar', [field_a, field_b]) + v = SchemaValidator(schema) + assert v.validate_python({'a': 'hello', 'b': True}) == ({'a': 'hello', 'b': True}, None) + ``` + + Args: + dataclass_name: The name of the dataclass being validated + fields: The fields to use for the dataclass + computed_fields: Computed fields to use when serializing the dataclass + collect_init_only: Whether to collect init only fields into a dict to pass to `__post_init__` + ref: optional unique identifier of the schema, used to reference the schema in other places + metadata: Any other information you want to include with the schema, not used by pydantic-core + serialization: Custom serialization schema + extra_behavior: How to handle extra fields + """ + return _dict_not_none( + type='dataclass-args', + dataclass_name=dataclass_name, + fields=fields, + computed_fields=computed_fields, + collect_init_only=collect_init_only, + ref=ref, + metadata=metadata, + serialization=serialization, + extra_behavior=extra_behavior, + ) + + +class DataclassSchema(TypedDict, total=False): + type: Required[Literal['dataclass']] + cls: Required[type[Any]] + generic_origin: type[Any] + schema: Required[CoreSchema] + fields: Required[list[str]] + cls_name: str + post_init: bool # default: False + revalidate_instances: Literal['always', 'never', 'subclass-instances'] # default: 'never' + strict: bool # default: False + frozen: bool # default False + ref: str + metadata: dict[str, Any] + serialization: SerSchema + slots: bool + config: CoreConfig + + +def dataclass_schema( + cls: type[Any], + schema: CoreSchema, + fields: list[str], + *, + generic_origin: type[Any] | None = None, + cls_name: str | None = None, + post_init: bool | None = None, + revalidate_instances: Literal['always', 'never', 'subclass-instances'] | None = None, + strict: bool | None = None, + ref: str | None = None, + metadata: dict[str, Any] | None = None, + serialization: SerSchema | None = None, + frozen: bool | None = None, + slots: bool | None = None, + config: CoreConfig | None = None, +) -> DataclassSchema: + """ + Returns a schema for a dataclass. As with `ModelSchema`, this schema can only be used as a field within + another schema, not as the root type. + + Args: + cls: The dataclass type, used to perform subclass checks + schema: The schema to use for the dataclass fields + fields: Fields of the dataclass, this is used in serialization and in validation during re-validation + and while validating assignment + generic_origin: The origin type used for this dataclass, if it's a parametrized generic. Ex, + if this model schema represents `SomeDataclass[int]`, generic_origin is `SomeDataclass` + cls_name: The name to use in error locs, etc; this is useful for generics (default: `cls.__name__`) + post_init: Whether to call `__post_init__` after validation + revalidate_instances: whether instances of models and dataclasses (including subclass instances) + should re-validate defaults to config.revalidate_instances, else 'never' + strict: Whether to require an exact instance of `cls` + ref: optional unique identifier of the schema, used to reference the schema in other places + metadata: Any other information you want to include with the schema, not used by pydantic-core + serialization: Custom serialization schema + frozen: Whether the dataclass is frozen + slots: Whether `slots=True` on the dataclass, means each field is assigned independently, rather than + simply setting `__dict__`, default false + """ + return _dict_not_none( + type='dataclass', + cls=cls, + generic_origin=generic_origin, + fields=fields, + cls_name=cls_name, + schema=schema, + post_init=post_init, + revalidate_instances=revalidate_instances, + strict=strict, + ref=ref, + metadata=metadata, + serialization=serialization, + frozen=frozen, + slots=slots, + config=config, + ) + + +class ArgumentsParameter(TypedDict, total=False): + name: Required[str] + schema: Required[CoreSchema] + mode: Literal['positional_only', 'positional_or_keyword', 'keyword_only'] # default positional_or_keyword + alias: Union[str, list[Union[str, int]], list[list[Union[str, int]]]] + + +def arguments_parameter( + name: str, + schema: CoreSchema, + *, + mode: Literal['positional_only', 'positional_or_keyword', 'keyword_only'] | None = None, + alias: str | list[str | int] | list[list[str | int]] | None = None, +) -> ArgumentsParameter: + """ + Returns a schema that matches an argument parameter, e.g.: + + ```py + from pydantic_core import SchemaValidator, core_schema + + param = core_schema.arguments_parameter( + name='a', schema=core_schema.str_schema(), mode='positional_only' + ) + schema = core_schema.arguments_schema([param]) + v = SchemaValidator(schema) + assert v.validate_python(('hello',)) == (('hello',), {}) + ``` + + Args: + name: The name to use for the argument parameter + schema: The schema to use for the argument parameter + mode: The mode to use for the argument parameter + alias: The alias to use for the argument parameter + """ + return _dict_not_none(name=name, schema=schema, mode=mode, alias=alias) + + +VarKwargsMode: TypeAlias = Literal['uniform', 'unpacked-typed-dict'] + + +class ArgumentsSchema(TypedDict, total=False): + type: Required[Literal['arguments']] + arguments_schema: Required[list[ArgumentsParameter]] + validate_by_name: bool + validate_by_alias: bool + var_args_schema: CoreSchema + var_kwargs_mode: VarKwargsMode + var_kwargs_schema: CoreSchema + ref: str + metadata: dict[str, Any] + serialization: SerSchema + + +def arguments_schema( + arguments: list[ArgumentsParameter], + *, + validate_by_name: bool | None = None, + validate_by_alias: bool | None = None, + var_args_schema: CoreSchema | None = None, + var_kwargs_mode: VarKwargsMode | None = None, + var_kwargs_schema: CoreSchema | None = None, + ref: str | None = None, + metadata: dict[str, Any] | None = None, + serialization: SerSchema | None = None, +) -> ArgumentsSchema: + """ + Returns a schema that matches an arguments schema, e.g.: + + ```py + from pydantic_core import SchemaValidator, core_schema + + param_a = core_schema.arguments_parameter( + name='a', schema=core_schema.str_schema(), mode='positional_only' + ) + param_b = core_schema.arguments_parameter( + name='b', schema=core_schema.bool_schema(), mode='positional_only' + ) + schema = core_schema.arguments_schema([param_a, param_b]) + v = SchemaValidator(schema) + assert v.validate_python(('hello', True)) == (('hello', True), {}) + ``` + + Args: + arguments: The arguments to use for the arguments schema + validate_by_name: Whether to populate by the parameter names, defaults to `False`. + validate_by_alias: Whether to populate by the parameter aliases, defaults to `True`. + var_args_schema: The variable args schema to use for the arguments schema + var_kwargs_mode: The validation mode to use for variadic keyword arguments. If `'uniform'`, every value of the + keyword arguments will be validated against the `var_kwargs_schema` schema. If `'unpacked-typed-dict'`, + the `var_kwargs_schema` argument must be a [`typed_dict_schema`][pydantic_core.core_schema.typed_dict_schema] + var_kwargs_schema: The variable kwargs schema to use for the arguments schema + ref: optional unique identifier of the schema, used to reference the schema in other places + metadata: Any other information you want to include with the schema, not used by pydantic-core + serialization: Custom serialization schema + """ + return _dict_not_none( + type='arguments', + arguments_schema=arguments, + validate_by_name=validate_by_name, + validate_by_alias=validate_by_alias, + var_args_schema=var_args_schema, + var_kwargs_mode=var_kwargs_mode, + var_kwargs_schema=var_kwargs_schema, + ref=ref, + metadata=metadata, + serialization=serialization, + ) + + +class ArgumentsV3Parameter(TypedDict, total=False): + name: Required[str] + schema: Required[CoreSchema] + mode: Literal[ + 'positional_only', + 'positional_or_keyword', + 'keyword_only', + 'var_args', + 'var_kwargs_uniform', + 'var_kwargs_unpacked_typed_dict', + ] # default positional_or_keyword + alias: Union[str, list[Union[str, int]], list[list[Union[str, int]]]] + + +def arguments_v3_parameter( + name: str, + schema: CoreSchema, + *, + mode: Literal[ + 'positional_only', + 'positional_or_keyword', + 'keyword_only', + 'var_args', + 'var_kwargs_uniform', + 'var_kwargs_unpacked_typed_dict', + ] + | None = None, + alias: str | list[str | int] | list[list[str | int]] | None = None, +) -> ArgumentsV3Parameter: + """ + Returns a schema that matches an argument parameter, e.g.: + + ```py + from pydantic_core import SchemaValidator, core_schema + + param = core_schema.arguments_v3_parameter( + name='a', schema=core_schema.str_schema(), mode='positional_only' + ) + schema = core_schema.arguments_v3_schema([param]) + v = SchemaValidator(schema) + assert v.validate_python({'a': 'hello'}) == (('hello',), {}) + ``` + + Args: + name: The name to use for the argument parameter + schema: The schema to use for the argument parameter + mode: The mode to use for the argument parameter + alias: The alias to use for the argument parameter + """ + return _dict_not_none(name=name, schema=schema, mode=mode, alias=alias) + + +class ArgumentsV3Schema(TypedDict, total=False): + type: Required[Literal['arguments-v3']] + arguments_schema: Required[list[ArgumentsV3Parameter]] + validate_by_name: bool + validate_by_alias: bool + extra_behavior: Literal['forbid', 'ignore'] # 'allow' doesn't make sense here. + ref: str + metadata: dict[str, Any] + serialization: SerSchema + + +def arguments_v3_schema( + arguments: list[ArgumentsV3Parameter], + *, + validate_by_name: bool | None = None, + validate_by_alias: bool | None = None, + extra_behavior: Literal['forbid', 'ignore'] | None = None, + ref: str | None = None, + metadata: dict[str, Any] | None = None, + serialization: SerSchema | None = None, +) -> ArgumentsV3Schema: + """ + Returns a schema that matches an arguments schema, e.g.: + + ```py + from pydantic_core import SchemaValidator, core_schema + + param_a = core_schema.arguments_v3_parameter( + name='a', schema=core_schema.str_schema(), mode='positional_only' + ) + param_b = core_schema.arguments_v3_parameter( + name='kwargs', schema=core_schema.bool_schema(), mode='var_kwargs_uniform' + ) + schema = core_schema.arguments_v3_schema([param_a, param_b]) + v = SchemaValidator(schema) + assert v.validate_python({'a': 'hi', 'kwargs': {'b': True}}) == (('hi',), {'b': True}) + ``` + + This schema is currently not used by other Pydantic components. In V3, it will most likely + become the default arguments schema for the `'call'` schema. + + Args: + arguments: The arguments to use for the arguments schema. + validate_by_name: Whether to populate by the parameter names, defaults to `False`. + validate_by_alias: Whether to populate by the parameter aliases, defaults to `True`. + extra_behavior: The extra behavior to use. + ref: optional unique identifier of the schema, used to reference the schema in other places. + metadata: Any other information you want to include with the schema, not used by pydantic-core. + serialization: Custom serialization schema. + """ + return _dict_not_none( + type='arguments-v3', + arguments_schema=arguments, + validate_by_name=validate_by_name, + validate_by_alias=validate_by_alias, + extra_behavior=extra_behavior, + ref=ref, + metadata=metadata, + serialization=serialization, + ) + + +class CallSchema(TypedDict, total=False): + type: Required[Literal['call']] + arguments_schema: Required[CoreSchema] + function: Required[Callable[..., Any]] + function_name: str # default function.__name__ + return_schema: CoreSchema + ref: str + metadata: dict[str, Any] + serialization: SerSchema + + +def call_schema( + arguments: CoreSchema, + function: Callable[..., Any], + *, + function_name: str | None = None, + return_schema: CoreSchema | None = None, + ref: str | None = None, + metadata: dict[str, Any] | None = None, + serialization: SerSchema | None = None, +) -> CallSchema: + """ + Returns a schema that matches an arguments schema, then calls a function, e.g.: + + ```py + from pydantic_core import SchemaValidator, core_schema + + param_a = core_schema.arguments_parameter( + name='a', schema=core_schema.str_schema(), mode='positional_only' + ) + param_b = core_schema.arguments_parameter( + name='b', schema=core_schema.bool_schema(), mode='positional_only' + ) + args_schema = core_schema.arguments_schema([param_a, param_b]) + + schema = core_schema.call_schema( + arguments=args_schema, + function=lambda a, b: a + str(not b), + return_schema=core_schema.str_schema(), + ) + v = SchemaValidator(schema) + assert v.validate_python((('hello', True))) == 'helloFalse' + ``` + + Args: + arguments: The arguments to use for the arguments schema + function: The function to use for the call schema + function_name: The function name to use for the call schema, if not provided `function.__name__` is used + return_schema: The return schema to use for the call schema + ref: optional unique identifier of the schema, used to reference the schema in other places + metadata: Any other information you want to include with the schema, not used by pydantic-core + serialization: Custom serialization schema + """ + return _dict_not_none( + type='call', + arguments_schema=arguments, + function=function, + function_name=function_name, + return_schema=return_schema, + ref=ref, + metadata=metadata, + serialization=serialization, + ) + + +class CustomErrorSchema(TypedDict, total=False): + type: Required[Literal['custom-error']] + schema: Required[CoreSchema] + custom_error_type: Required[str] + custom_error_message: str + custom_error_context: dict[str, Union[str, int, float]] + ref: str + metadata: dict[str, Any] + serialization: SerSchema + + +def custom_error_schema( + schema: CoreSchema, + custom_error_type: str, + *, + custom_error_message: str | None = None, + custom_error_context: dict[str, Any] | None = None, + ref: str | None = None, + metadata: dict[str, Any] | None = None, + serialization: SerSchema | None = None, +) -> CustomErrorSchema: + """ + Returns a schema that matches a custom error value, e.g.: + + ```py + from pydantic_core import SchemaValidator, core_schema + + schema = core_schema.custom_error_schema( + schema=core_schema.int_schema(), + custom_error_type='MyError', + custom_error_message='Error msg', + ) + v = SchemaValidator(schema) + v.validate_python(1) + ``` + + Args: + schema: The schema to use for the custom error schema + custom_error_type: The custom error type to use for the custom error schema + custom_error_message: The custom error message to use for the custom error schema + custom_error_context: The custom error context to use for the custom error schema + ref: optional unique identifier of the schema, used to reference the schema in other places + metadata: Any other information you want to include with the schema, not used by pydantic-core + serialization: Custom serialization schema + """ + return _dict_not_none( + type='custom-error', + schema=schema, + custom_error_type=custom_error_type, + custom_error_message=custom_error_message, + custom_error_context=custom_error_context, + ref=ref, + metadata=metadata, + serialization=serialization, + ) + + +class JsonSchema(TypedDict, total=False): + type: Required[Literal['json']] + schema: CoreSchema + ref: str + metadata: dict[str, Any] + serialization: SerSchema + + +def json_schema( + schema: CoreSchema | None = None, + *, + ref: str | None = None, + metadata: dict[str, Any] | None = None, + serialization: SerSchema | None = None, +) -> JsonSchema: + """ + Returns a schema that matches a JSON value, e.g.: + + ```py + from pydantic_core import SchemaValidator, core_schema + + dict_schema = core_schema.model_fields_schema( + { + 'field_a': core_schema.model_field(core_schema.str_schema()), + 'field_b': core_schema.model_field(core_schema.bool_schema()), + }, + ) + + class MyModel: + __slots__ = ( + '__dict__', + '__pydantic_fields_set__', + '__pydantic_extra__', + '__pydantic_private__', + ) + field_a: str + field_b: bool + + json_schema = core_schema.json_schema(schema=dict_schema) + schema = core_schema.model_schema(cls=MyModel, schema=json_schema) + v = SchemaValidator(schema) + m = v.validate_python('{"field_a": "hello", "field_b": true}') + assert isinstance(m, MyModel) + ``` + + Args: + schema: The schema to use for the JSON schema + ref: optional unique identifier of the schema, used to reference the schema in other places + metadata: Any other information you want to include with the schema, not used by pydantic-core + serialization: Custom serialization schema + """ + return _dict_not_none(type='json', schema=schema, ref=ref, metadata=metadata, serialization=serialization) + + +class UrlSchema(TypedDict, total=False): + type: Required[Literal['url']] + max_length: int + allowed_schemes: list[str] + host_required: bool # default False + default_host: str + default_port: int + default_path: str + strict: bool + ref: str + metadata: dict[str, Any] + serialization: SerSchema + + +def url_schema( + *, + max_length: int | None = None, + allowed_schemes: list[str] | None = None, + host_required: bool | None = None, + default_host: str | None = None, + default_port: int | None = None, + default_path: str | None = None, + strict: bool | None = None, + ref: str | None = None, + metadata: dict[str, Any] | None = None, + serialization: SerSchema | None = None, +) -> UrlSchema: + """ + Returns a schema that matches a URL value, e.g.: + + ```py + from pydantic_core import SchemaValidator, core_schema + + schema = core_schema.url_schema() + v = SchemaValidator(schema) + print(v.validate_python('https://example.com')) + #> https://example.com/ + ``` + + Args: + max_length: The maximum length of the URL + allowed_schemes: The allowed URL schemes + host_required: Whether the URL must have a host + default_host: The default host to use if the URL does not have a host + default_port: The default port to use if the URL does not have a port + default_path: The default path to use if the URL does not have a path + strict: Whether to use strict URL parsing + ref: optional unique identifier of the schema, used to reference the schema in other places + metadata: Any other information you want to include with the schema, not used by pydantic-core + serialization: Custom serialization schema + """ + return _dict_not_none( + type='url', + max_length=max_length, + allowed_schemes=allowed_schemes, + host_required=host_required, + default_host=default_host, + default_port=default_port, + default_path=default_path, + strict=strict, + ref=ref, + metadata=metadata, + serialization=serialization, + ) + + +class MultiHostUrlSchema(TypedDict, total=False): + type: Required[Literal['multi-host-url']] + max_length: int + allowed_schemes: list[str] + host_required: bool # default False + default_host: str + default_port: int + default_path: str + strict: bool + ref: str + metadata: dict[str, Any] + serialization: SerSchema + + +def multi_host_url_schema( + *, + max_length: int | None = None, + allowed_schemes: list[str] | None = None, + host_required: bool | None = None, + default_host: str | None = None, + default_port: int | None = None, + default_path: str | None = None, + strict: bool | None = None, + ref: str | None = None, + metadata: dict[str, Any] | None = None, + serialization: SerSchema | None = None, +) -> MultiHostUrlSchema: + """ + Returns a schema that matches a URL value with possibly multiple hosts, e.g.: + + ```py + from pydantic_core import SchemaValidator, core_schema + + schema = core_schema.multi_host_url_schema() + v = SchemaValidator(schema) + print(v.validate_python('redis://localhost,0.0.0.0,127.0.0.1')) + #> redis://localhost,0.0.0.0,127.0.0.1 + ``` + + Args: + max_length: The maximum length of the URL + allowed_schemes: The allowed URL schemes + host_required: Whether the URL must have a host + default_host: The default host to use if the URL does not have a host + default_port: The default port to use if the URL does not have a port + default_path: The default path to use if the URL does not have a path + strict: Whether to use strict URL parsing + ref: optional unique identifier of the schema, used to reference the schema in other places + metadata: Any other information you want to include with the schema, not used by pydantic-core + serialization: Custom serialization schema + """ + return _dict_not_none( + type='multi-host-url', + max_length=max_length, + allowed_schemes=allowed_schemes, + host_required=host_required, + default_host=default_host, + default_port=default_port, + default_path=default_path, + strict=strict, + ref=ref, + metadata=metadata, + serialization=serialization, + ) + + +class DefinitionsSchema(TypedDict, total=False): + type: Required[Literal['definitions']] + schema: Required[CoreSchema] + definitions: Required[list[CoreSchema]] + metadata: dict[str, Any] + serialization: SerSchema + + +def definitions_schema(schema: CoreSchema, definitions: list[CoreSchema]) -> DefinitionsSchema: + """ + Build a schema that contains both an inner schema and a list of definitions which can be used + within the inner schema. + + ```py + from pydantic_core import SchemaValidator, core_schema + + schema = core_schema.definitions_schema( + core_schema.list_schema(core_schema.definition_reference_schema('foobar')), + [core_schema.int_schema(ref='foobar')], + ) + v = SchemaValidator(schema) + assert v.validate_python([1, 2, '3']) == [1, 2, 3] + ``` + + Args: + schema: The inner schema + definitions: List of definitions which can be referenced within inner schema + """ + return DefinitionsSchema(type='definitions', schema=schema, definitions=definitions) + + +class DefinitionReferenceSchema(TypedDict, total=False): + type: Required[Literal['definition-ref']] + schema_ref: Required[str] + ref: str + metadata: dict[str, Any] + serialization: SerSchema + + +def definition_reference_schema( + schema_ref: str, + ref: str | None = None, + metadata: dict[str, Any] | None = None, + serialization: SerSchema | None = None, +) -> DefinitionReferenceSchema: + """ + Returns a schema that points to a schema stored in "definitions", this is useful for nested recursive + models and also when you want to define validators separately from the main schema, e.g.: + + ```py + from pydantic_core import SchemaValidator, core_schema + + schema_definition = core_schema.definition_reference_schema('list-schema') + schema = core_schema.definitions_schema( + schema=schema_definition, + definitions=[ + core_schema.list_schema(items_schema=schema_definition, ref='list-schema'), + ], + ) + v = SchemaValidator(schema) + assert v.validate_python([()]) == [[]] + ``` + + Args: + schema_ref: The schema ref to use for the definition reference schema + metadata: Any other information you want to include with the schema, not used by pydantic-core + serialization: Custom serialization schema + """ + return _dict_not_none( + type='definition-ref', schema_ref=schema_ref, ref=ref, metadata=metadata, serialization=serialization + ) + + +MYPY = False +# See https://github.com/python/mypy/issues/14034 for details, in summary mypy is extremely slow to process this +# union which kills performance not just for pydantic, but even for code using pydantic +if not MYPY: + CoreSchema = Union[ + InvalidSchema, + AnySchema, + NoneSchema, + BoolSchema, + IntSchema, + FloatSchema, + DecimalSchema, + StringSchema, + BytesSchema, + DateSchema, + TimeSchema, + DatetimeSchema, + TimedeltaSchema, + LiteralSchema, + EnumSchema, + IsInstanceSchema, + IsSubclassSchema, + CallableSchema, + ListSchema, + TupleSchema, + SetSchema, + FrozenSetSchema, + GeneratorSchema, + DictSchema, + AfterValidatorFunctionSchema, + BeforeValidatorFunctionSchema, + WrapValidatorFunctionSchema, + PlainValidatorFunctionSchema, + WithDefaultSchema, + NullableSchema, + UnionSchema, + TaggedUnionSchema, + ChainSchema, + LaxOrStrictSchema, + JsonOrPythonSchema, + TypedDictSchema, + ModelFieldsSchema, + ModelSchema, + DataclassArgsSchema, + DataclassSchema, + ArgumentsSchema, + ArgumentsV3Schema, + CallSchema, + CustomErrorSchema, + JsonSchema, + UrlSchema, + MultiHostUrlSchema, + DefinitionsSchema, + DefinitionReferenceSchema, + UuidSchema, + ComplexSchema, + ] +elif False: + CoreSchema: TypeAlias = Mapping[str, Any] + + +# to update this, call `pytest -k test_core_schema_type_literal` and copy the output +CoreSchemaType = Literal[ + 'invalid', + 'any', + 'none', + 'bool', + 'int', + 'float', + 'decimal', + 'str', + 'bytes', + 'date', + 'time', + 'datetime', + 'timedelta', + 'literal', + 'enum', + 'is-instance', + 'is-subclass', + 'callable', + 'list', + 'tuple', + 'set', + 'frozenset', + 'generator', + 'dict', + 'function-after', + 'function-before', + 'function-wrap', + 'function-plain', + 'default', + 'nullable', + 'union', + 'tagged-union', + 'chain', + 'lax-or-strict', + 'json-or-python', + 'typed-dict', + 'model-fields', + 'model', + 'dataclass-args', + 'dataclass', + 'arguments', + 'arguments-v3', + 'call', + 'custom-error', + 'json', + 'url', + 'multi-host-url', + 'definitions', + 'definition-ref', + 'uuid', + 'complex', +] + +CoreSchemaFieldType = Literal['model-field', 'dataclass-field', 'typed-dict-field', 'computed-field'] + + +# used in _pydantic_core.pyi::PydanticKnownError +# to update this, call `pytest -k test_all_errors` and copy the output +ErrorType = Literal[ + 'no_such_attribute', + 'json_invalid', + 'json_type', + 'needs_python_object', + 'recursion_loop', + 'missing', + 'frozen_field', + 'frozen_instance', + 'extra_forbidden', + 'invalid_key', + 'get_attribute_error', + 'model_type', + 'model_attributes_type', + 'dataclass_type', + 'dataclass_exact_type', + 'none_required', + 'greater_than', + 'greater_than_equal', + 'less_than', + 'less_than_equal', + 'multiple_of', + 'finite_number', + 'too_short', + 'too_long', + 'iterable_type', + 'iteration_error', + 'string_type', + 'string_sub_type', + 'string_unicode', + 'string_too_short', + 'string_too_long', + 'string_pattern_mismatch', + 'enum', + 'dict_type', + 'mapping_type', + 'list_type', + 'tuple_type', + 'set_type', + 'set_item_not_hashable', + 'bool_type', + 'bool_parsing', + 'int_type', + 'int_parsing', + 'int_parsing_size', + 'int_from_float', + 'float_type', + 'float_parsing', + 'bytes_type', + 'bytes_too_short', + 'bytes_too_long', + 'bytes_invalid_encoding', + 'value_error', + 'assertion_error', + 'literal_error', + 'date_type', + 'date_parsing', + 'date_from_datetime_parsing', + 'date_from_datetime_inexact', + 'date_past', + 'date_future', + 'time_type', + 'time_parsing', + 'datetime_type', + 'datetime_parsing', + 'datetime_object_invalid', + 'datetime_from_date_parsing', + 'datetime_past', + 'datetime_future', + 'timezone_naive', + 'timezone_aware', + 'timezone_offset', + 'time_delta_type', + 'time_delta_parsing', + 'frozen_set_type', + 'is_instance_of', + 'is_subclass_of', + 'callable_type', + 'union_tag_invalid', + 'union_tag_not_found', + 'arguments_type', + 'missing_argument', + 'unexpected_keyword_argument', + 'missing_keyword_only_argument', + 'unexpected_positional_argument', + 'missing_positional_only_argument', + 'multiple_argument_values', + 'url_type', + 'url_parsing', + 'url_syntax_violation', + 'url_too_long', + 'url_scheme', + 'uuid_type', + 'uuid_parsing', + 'uuid_version', + 'decimal_type', + 'decimal_parsing', + 'decimal_max_digits', + 'decimal_max_places', + 'decimal_whole_digits', + 'complex_type', + 'complex_str_parsing', +] + + +def _dict_not_none(**kwargs: Any) -> Any: + return {k: v for k, v in kwargs.items() if v is not None} + + +############################################################################### +# All this stuff is deprecated by #980 and will be removed eventually +# They're kept because some code external code will be using them + + +@deprecated('`field_before_validator_function` is deprecated, use `with_info_before_validator_function` instead.') +def field_before_validator_function(function: WithInfoValidatorFunction, field_name: str, schema: CoreSchema, **kwargs): + warnings.warn( + '`field_before_validator_function` is deprecated, use `with_info_before_validator_function` instead.', + DeprecationWarning, + ) + return with_info_before_validator_function(function, schema, field_name=field_name, **kwargs) + + +@deprecated('`general_before_validator_function` is deprecated, use `with_info_before_validator_function` instead.') +def general_before_validator_function(*args, **kwargs): + warnings.warn( + '`general_before_validator_function` is deprecated, use `with_info_before_validator_function` instead.', + DeprecationWarning, + ) + return with_info_before_validator_function(*args, **kwargs) + + +@deprecated('`field_after_validator_function` is deprecated, use `with_info_after_validator_function` instead.') +def field_after_validator_function(function: WithInfoValidatorFunction, field_name: str, schema: CoreSchema, **kwargs): + warnings.warn( + '`field_after_validator_function` is deprecated, use `with_info_after_validator_function` instead.', + DeprecationWarning, + ) + return with_info_after_validator_function(function, schema, field_name=field_name, **kwargs) + + +@deprecated('`general_after_validator_function` is deprecated, use `with_info_after_validator_function` instead.') +def general_after_validator_function(*args, **kwargs): + warnings.warn( + '`general_after_validator_function` is deprecated, use `with_info_after_validator_function` instead.', + DeprecationWarning, + ) + return with_info_after_validator_function(*args, **kwargs) + + +@deprecated('`field_wrap_validator_function` is deprecated, use `with_info_wrap_validator_function` instead.') +def field_wrap_validator_function( + function: WithInfoWrapValidatorFunction, field_name: str, schema: CoreSchema, **kwargs +): + warnings.warn( + '`field_wrap_validator_function` is deprecated, use `with_info_wrap_validator_function` instead.', + DeprecationWarning, + ) + return with_info_wrap_validator_function(function, schema, field_name=field_name, **kwargs) + + +@deprecated('`general_wrap_validator_function` is deprecated, use `with_info_wrap_validator_function` instead.') +def general_wrap_validator_function(*args, **kwargs): + warnings.warn( + '`general_wrap_validator_function` is deprecated, use `with_info_wrap_validator_function` instead.', + DeprecationWarning, + ) + return with_info_wrap_validator_function(*args, **kwargs) + + +@deprecated('`field_plain_validator_function` is deprecated, use `with_info_plain_validator_function` instead.') +def field_plain_validator_function(function: WithInfoValidatorFunction, field_name: str, **kwargs): + warnings.warn( + '`field_plain_validator_function` is deprecated, use `with_info_plain_validator_function` instead.', + DeprecationWarning, + ) + return with_info_plain_validator_function(function, field_name=field_name, **kwargs) + + +@deprecated('`general_plain_validator_function` is deprecated, use `with_info_plain_validator_function` instead.') +def general_plain_validator_function(*args, **kwargs): + warnings.warn( + '`general_plain_validator_function` is deprecated, use `with_info_plain_validator_function` instead.', + DeprecationWarning, + ) + return with_info_plain_validator_function(*args, **kwargs) + + +_deprecated_import_lookup = { + 'FieldValidationInfo': ValidationInfo, + 'FieldValidatorFunction': WithInfoValidatorFunction, + 'GeneralValidatorFunction': WithInfoValidatorFunction, + 'FieldWrapValidatorFunction': WithInfoWrapValidatorFunction, +} + +if TYPE_CHECKING: + FieldValidationInfo = ValidationInfo + + +def __getattr__(attr_name: str) -> object: + new_attr = _deprecated_import_lookup.get(attr_name) + if new_attr is None: + raise AttributeError(f"module 'pydantic_core' has no attribute '{attr_name}'") + else: + import warnings + + msg = f'`{attr_name}` is deprecated, use `{new_attr.__name__}` instead.' + warnings.warn(msg, DeprecationWarning, stacklevel=1) + return new_attr diff --git a/lib/python3.12/site-packages/pydantic_core/py.typed b/lib/python3.12/site-packages/pydantic_core/py.typed new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391