diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/_distutils_hack/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/_distutils_hack/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..94f71b99ecefd27e6cf3fd06fc3c0eef5fd73910 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/_distutils_hack/__init__.py @@ -0,0 +1,239 @@ +# don't import any costly modules +import os +import sys + +report_url = ( + "https://github.com/pypa/setuptools/issues/new?template=distutils-deprecation.yml" +) + + +def warn_distutils_present(): + if 'distutils' not in sys.modules: + return + import warnings + + warnings.warn( + "Distutils was imported before Setuptools, but importing Setuptools " + "also replaces the `distutils` module in `sys.modules`. This may lead " + "to undesirable behaviors or errors. To avoid these issues, avoid " + "using distutils directly, ensure that setuptools is installed in the " + "traditional way (e.g. not an editable install), and/or make sure " + "that setuptools is always imported before distutils." + ) + + +def clear_distutils(): + if 'distutils' not in sys.modules: + return + import warnings + + warnings.warn( + "Setuptools is replacing distutils. Support for replacing " + "an already imported distutils is deprecated. In the future, " + "this condition will fail. " + f"Register concerns at {report_url}" + ) + mods = [ + name + for name in sys.modules + if name == "distutils" or name.startswith("distutils.") + ] + for name in mods: + del sys.modules[name] + + +def enabled(): + """ + Allow selection of distutils by environment variable. + """ + which = os.environ.get('SETUPTOOLS_USE_DISTUTILS', 'local') + if which == 'stdlib': + import warnings + + warnings.warn( + "Reliance on distutils from stdlib is deprecated. Users " + "must rely on setuptools to provide the distutils module. " + "Avoid importing distutils or import setuptools first, " + "and avoid setting SETUPTOOLS_USE_DISTUTILS=stdlib. " + f"Register concerns at {report_url}" + ) + return which == 'local' + + +def ensure_local_distutils(): + import importlib + + clear_distutils() + + # With the DistutilsMetaFinder in place, + # perform an import to cause distutils to be + # loaded from setuptools._distutils. Ref #2906. + with shim(): + importlib.import_module('distutils') + + # check that submodules load as expected + core = importlib.import_module('distutils.core') + assert '_distutils' in core.__file__, core.__file__ + assert 'setuptools._distutils.log' not in sys.modules + + +def do_override(): + """ + Ensure that the local copy of distutils is preferred over stdlib. + + See https://github.com/pypa/setuptools/issues/417#issuecomment-392298401 + for more motivation. + """ + if enabled(): + warn_distutils_present() + ensure_local_distutils() + + +class _TrivialRe: + def __init__(self, *patterns) -> None: + self._patterns = patterns + + def match(self, string): + return all(pat in string for pat in self._patterns) + + +class DistutilsMetaFinder: + def find_spec(self, fullname, path, target=None): + # optimization: only consider top level modules and those + # found in the CPython test suite. + if path is not None and not fullname.startswith('test.'): + return None + + method_name = 'spec_for_{fullname}'.format(**locals()) + method = getattr(self, method_name, lambda: None) + return method() + + def spec_for_distutils(self): + if self.is_cpython(): + return None + + import importlib + import importlib.abc + import importlib.util + + try: + mod = importlib.import_module('setuptools._distutils') + except Exception: + # There are a couple of cases where setuptools._distutils + # may not be present: + # - An older Setuptools without a local distutils is + # taking precedence. Ref #2957. + # - Path manipulation during sitecustomize removes + # setuptools from the path but only after the hook + # has been loaded. Ref #2980. + # In either case, fall back to stdlib behavior. + return None + + class DistutilsLoader(importlib.abc.Loader): + def create_module(self, spec): + mod.__name__ = 'distutils' + return mod + + def exec_module(self, module): + pass + + return importlib.util.spec_from_loader( + 'distutils', DistutilsLoader(), origin=mod.__file__ + ) + + @staticmethod + def is_cpython(): + """ + Suppress supplying distutils for CPython (build and tests). + Ref #2965 and #3007. + """ + return os.path.isfile('pybuilddir.txt') + + def spec_for_pip(self): + """ + Ensure stdlib distutils when running under pip. + See pypa/pip#8761 for rationale. + """ + if sys.version_info >= (3, 12) or self.pip_imported_during_build(): + return + clear_distutils() + self.spec_for_distutils = lambda: None + + @classmethod + def pip_imported_during_build(cls): + """ + Detect if pip is being imported in a build script. Ref #2355. + """ + import traceback + + return any( + cls.frame_file_is_setup(frame) for frame, line in traceback.walk_stack(None) + ) + + @staticmethod + def frame_file_is_setup(frame): + """ + Return True if the indicated frame suggests a setup.py file. + """ + # some frames may not have __file__ (#2940) + return frame.f_globals.get('__file__', '').endswith('setup.py') + + def spec_for_sensitive_tests(self): + """ + Ensure stdlib distutils when running select tests under CPython. + + python/cpython#91169 + """ + clear_distutils() + self.spec_for_distutils = lambda: None + + sensitive_tests = ( + [ + 'test.test_distutils', + 'test.test_peg_generator', + 'test.test_importlib', + ] + if sys.version_info < (3, 10) + else [ + 'test.test_distutils', + ] + ) + + +for name in DistutilsMetaFinder.sensitive_tests: + setattr( + DistutilsMetaFinder, + f'spec_for_{name}', + DistutilsMetaFinder.spec_for_sensitive_tests, + ) + + +DISTUTILS_FINDER = DistutilsMetaFinder() + + +def add_shim(): + DISTUTILS_FINDER in sys.meta_path or insert_shim() + + +class shim: + def __enter__(self) -> None: + insert_shim() + + def __exit__(self, exc: object, value: object, tb: object) -> None: + _remove_shim() + + +def insert_shim(): + sys.meta_path.insert(0, DISTUTILS_FINDER) + + +def _remove_shim(): + try: + sys.meta_path.remove(DISTUTILS_FINDER) + except ValueError: + pass + + +if sys.version_info < (3, 12): + # DistutilsMetaFinder can only be disabled in Python < 3.12 (PEP 632) + remove_shim = _remove_shim diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/_distutils_hack/override.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/_distutils_hack/override.py new file mode 100644 index 0000000000000000000000000000000000000000..2cc433a4a55e3b41fa31089918fb62096092f89f --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/_distutils_hack/override.py @@ -0,0 +1 @@ +__import__('_distutils_hack').do_override() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/annotated_doc/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/annotated_doc/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a0152a7d12abc2db37fb26e764a61e0c894a43f3 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/annotated_doc/__init__.py @@ -0,0 +1,3 @@ +from .main import Doc as Doc + +__version__ = "0.0.4" diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/annotated_doc/main.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/annotated_doc/main.py new file mode 100644 index 0000000000000000000000000000000000000000..7063c59e4500a1d02bfc9b41887f9e95f8163507 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/annotated_doc/main.py @@ -0,0 +1,36 @@ +class Doc: + """Define the documentation of a type annotation using `Annotated`, to be + used in class attributes, function and method parameters, return values, + and variables. + + The value should be a positional-only string literal to allow static tools + like editors and documentation generators to use it. + + This complements docstrings. + + The string value passed is available in the attribute `documentation`. + + Example: + + ```Python + from typing import Annotated + from annotated_doc import Doc + + def hi(name: Annotated[str, Doc("Who to say hi to")]) -> None: + print(f"Hi, {name}!") + ``` + """ + + def __init__(self, documentation: str, /) -> None: + self.documentation = documentation + + def __repr__(self) -> str: + return f"Doc({self.documentation!r})" + + def __hash__(self) -> int: + return hash(self.documentation) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, Doc): + return NotImplemented + return self.documentation == other.documentation diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/annotated_doc/py.typed b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/annotated_doc/py.typed new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dataproperty/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dataproperty/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..beb08988c3b9ad602ba5ac7a361514252ee2dbef --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dataproperty/__init__.py @@ -0,0 +1,45 @@ +""" +.. codeauthor:: Tsuyoshi Hombashi +""" + +from .__version__ import __author__, __copyright__, __email__, __license__, __version__ +from ._align import Align +from ._align_getter import align_getter +from ._column import ColumnDataProperty +from ._common import MAX_STRICT_LEVEL_MAP, MIN_STRICT_LEVEL_MAP, NOT_QUOTING_FLAGS, DefaultValue +from ._container import MinMaxContainer +from ._dataproperty import DataProperty +from ._extractor import DataPropertyExtractor, DataPropertyMatrix, MatrixFormatting +from ._formatter import Format +from ._function import calc_ascii_char_width, get_integer_digit, get_number_of_digit +from ._line_break import LineBreakHandling +from ._preprocessor import Preprocessor +from .logger import set_logger + + +__all__ = ( + "Align", + "align_getter", + "ColumnDataProperty", + "DataProperty", + "DataPropertyExtractor", + "DataPropertyMatrix", + "Format", + "LineBreakHandling", + "MatrixFormatting", + "MinMaxContainer", + "Preprocessor", + "calc_ascii_char_width", + "get_integer_digit", + "get_number_of_digit", + "MAX_STRICT_LEVEL_MAP", + "MIN_STRICT_LEVEL_MAP", + "NOT_QUOTING_FLAGS", + "DefaultValue", + "set_logger", + "__author__", + "__copyright__", + "__email__", + "__license__", + "__version__", +) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dataproperty/__version__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dataproperty/__version__.py new file mode 100644 index 0000000000000000000000000000000000000000..6b02049e0e9ae8a2e497d783aa86167c4dc34f49 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dataproperty/__version__.py @@ -0,0 +1,9 @@ +from typing import Final + + +__author__: Final = "Tsuyoshi Hombashi" +__copyright__: Final = f"Copyright 2016-2024, {__author__}" +__license__: Final = "MIT License" +__version__ = "1.1.0" +__maintainer__: Final = __author__ +__email__: Final = "tsuyoshi.hombashi@gmail.com" diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dataproperty/_align.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dataproperty/_align.py new file mode 100644 index 0000000000000000000000000000000000000000..7d8a318349fc4cdacf4397fcabc62b5fe3e2552a --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dataproperty/_align.py @@ -0,0 +1,25 @@ +""" +.. codeauthor:: Tsuyoshi Hombashi +""" + +import enum + + +@enum.unique +class Align(enum.Enum): + AUTO = (1 << 0, "auto") + LEFT = (1 << 1, "left") + RIGHT = (1 << 2, "right") + CENTER = (1 << 3, "center") + + @property + def align_code(self) -> int: + return self.__align_code + + @property + def align_string(self) -> str: + return self.__align_string + + def __init__(self, code: int, string: str) -> None: + self.__align_code = code + self.__align_string = string diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dataproperty/_align_getter.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dataproperty/_align_getter.py new file mode 100644 index 0000000000000000000000000000000000000000..3550e7681cb74b699e0856bee5bad141ba288ca2 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dataproperty/_align_getter.py @@ -0,0 +1,33 @@ +""" +.. codeauthor:: Tsuyoshi Hombashi +""" + +from typing import Dict + +from typepy import Typecode + +from ._align import Align + + +class AlignGetter: + @property + def typecode_align_table(self): + raise NotImplementedError() + + @typecode_align_table.setter + def typecode_align_table(self, x: Dict[Typecode, Align]) -> None: + self.__typecode_align_table = x + + def get_align_from_typecode(self, typecode: Typecode) -> Align: + return self.__typecode_align_table.get(typecode, self.default_align) + + def __init__(self) -> None: + self.typecode_align_table = { + Typecode.STRING: Align.LEFT, + Typecode.INTEGER: Align.RIGHT, + Typecode.REAL_NUMBER: Align.RIGHT, + } + self.default_align = Align.LEFT + + +align_getter = AlignGetter() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dataproperty/_base.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dataproperty/_base.py new file mode 100644 index 0000000000000000000000000000000000000000..2b25b5cd805ecd3412a69f6558b189dae382e9d8 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dataproperty/_base.py @@ -0,0 +1,98 @@ +from typing import Final, Optional + +from typepy import ( + Bool, + DateTime, + Dictionary, + Infinity, + Integer, + IpAddress, + List, + Nan, + NoneType, + NullString, + RealNumber, + String, + Typecode, +) +from typepy.type import AbstractType + +from ._formatter import Formatter +from ._interface import DataPeropertyInterface + + +class DataPeropertyBase(DataPeropertyInterface): + __slots__ = ( + "_datetime_format_str", + "_decimal_places", + "_east_asian_ambiguous_width", + "_formatter", + "_typecode", + "__format_str", + ) + + __TYPE_CLASS_TABLE: Final[dict[Typecode, type[AbstractType]]] = { + Typecode.BOOL: Bool, + Typecode.DATETIME: DateTime, + Typecode.DICTIONARY: Dictionary, + Typecode.INTEGER: Integer, + Typecode.INFINITY: Infinity, + Typecode.IP_ADDRESS: IpAddress, + Typecode.LIST: List, + Typecode.NAN: Nan, + Typecode.NONE: NoneType, + Typecode.NULL_STRING: NullString, + Typecode.REAL_NUMBER: RealNumber, + Typecode.STRING: String, + } + + @property + def type_class(self) -> type[AbstractType]: + return self.__TYPE_CLASS_TABLE[self.typecode] + + @property + def typecode(self) -> Typecode: + """ + ``typepy.Typecode`` that corresponds to the type of the ``data``. + + :return: + One of the Enum value that are defined ``typepy.Typecode``. + :rtype: typepy.Typecode + """ + + assert self._typecode + + return self._typecode + + @property + def typename(self) -> str: + return self.typecode.name + + def __init__( + self, + format_flags: Optional[int], + is_formatting_float: bool, + datetime_format_str: str, + east_asian_ambiguous_width: int, + ) -> None: + self._decimal_places: Optional[int] = None + self._east_asian_ambiguous_width = east_asian_ambiguous_width + self._typecode: Optional[Typecode] = None + + self._datetime_format_str = datetime_format_str + self.__format_str = "" + + self._formatter = Formatter( + format_flags=format_flags, + datetime_format_str=self._datetime_format_str, + is_formatting_float=is_formatting_float, + ) + + @property + def format_str(self) -> str: + if self.__format_str: + return self.__format_str + + self.__format_str = self._formatter.make_format_str(self.typecode, self.decimal_places) + + return self.__format_str diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dataproperty/_column.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dataproperty/_column.py new file mode 100644 index 0000000000000000000000000000000000000000..dff22531cbfa858b3dcdb7911b62b5579e2cb2bf --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dataproperty/_column.py @@ -0,0 +1,352 @@ +from typing import Any, Optional + +from mbstrdecoder import MultiByteStrDecoder +from typepy import Integer, StrictLevel, Typecode, TypeConversionError + +from ._align import Align +from ._align_getter import align_getter +from ._base import DataPeropertyBase +from ._common import DefaultValue +from ._container import ListContainer, MinMaxContainer +from ._dataproperty import DataProperty +from ._function import calc_ascii_char_width +from .typing import FloatType + + +class ColumnDataProperty(DataPeropertyBase): + __slots__ = ( + "__header_ascii_char_width", + "__body_ascii_char_width", + "__column_index", + "__dp_list", + "__float_type", + "__format_map", + "__is_calculate", + "__max_precision", + "__minmax_integer_digits", + "__minmax_decimal_places", + "__minmax_additional_format_len", + "__typecode_bitmap", + ) + + @property + def align(self) -> Align: + return align_getter.get_align_from_typecode(self.typecode) + + @property + def bit_length(self) -> Optional[int]: + if self.typecode != Typecode.INTEGER: + return None + + bit_length = 0 + for value_dp in self.__dp_list: + try: + bit_length = max(bit_length, int.bit_length(value_dp.data)) + except TypeError: + pass + + return bit_length + + @property + def column_index(self) -> int: + return self.__column_index + + @property + def decimal_places(self) -> Optional[int]: + return self._decimal_places + + @property + def ascii_char_width(self) -> int: + return max(self.__header_ascii_char_width, self.__body_ascii_char_width) + + @property + def minmax_integer_digits(self) -> MinMaxContainer: + return self.__minmax_integer_digits + + @property + def minmax_decimal_places(self) -> ListContainer: + return self.__minmax_decimal_places + + @property + def minmax_additional_format_len(self) -> MinMaxContainer: + return self.__minmax_additional_format_len + + def __init__( + self, + column_index: int, + float_type: Optional[FloatType], + min_width: int = 0, + format_flags: Optional[int] = None, + is_formatting_float: bool = True, + datetime_format_str: str = DefaultValue.DATETIME_FORMAT, + east_asian_ambiguous_width: int = 1, + max_precision: int = DefaultValue.MAX_PRECISION, + ) -> None: + super().__init__( + format_flags=format_flags, + is_formatting_float=is_formatting_float, + datetime_format_str=datetime_format_str, + east_asian_ambiguous_width=east_asian_ambiguous_width, + ) + + self.__header_ascii_char_width = 0 + self.__body_ascii_char_width = min_width + self.__column_index = column_index + + self.__float_type = float_type + + self.__is_calculate = True + self.__dp_list: list[DataProperty] = [] + self.__minmax_integer_digits = MinMaxContainer() + self.__minmax_decimal_places = ListContainer() + self.__minmax_additional_format_len = MinMaxContainer() + self.__max_precision = max_precision + + self.__typecode_bitmap = Typecode.NONE.value + self.__calc_typecode_from_bitmap() + + self.__format_map: dict[Typecode, str] = self._formatter.make_format_map( + decimal_places=self._decimal_places + ) + + def __repr__(self) -> str: + element_list = [] + + if self.column_index is not None: + element_list.append(f"column={self.column_index}") + + element_list.extend( + [ + f"type={self.typename}", + f"align={self.align.align_string}", + f"ascii_width={self.ascii_char_width}", + ] + ) + + if Integer(self.bit_length).is_type(): + element_list.append(f"bit_len={self.bit_length}") + + if self.minmax_integer_digits.has_value(): + if self.minmax_integer_digits.is_same_value(): + value = f"int_digits={self.minmax_integer_digits.min_value}" + else: + value = f"int_digits=({self.minmax_integer_digits})" + + element_list.append(value) + + if self.minmax_decimal_places.has_value(): + if self.minmax_decimal_places.is_same_value(): + value = f"decimal_places={self.minmax_decimal_places.min_value}" + else: + value = f"decimal_places=({self.minmax_decimal_places})" + + element_list.append(value) + + if not self.minmax_additional_format_len.is_zero(): + if self.minmax_additional_format_len.is_same_value(): + value = f"extra_len={self.minmax_additional_format_len.min_value}" + else: + value = f"extra_len=({self.minmax_additional_format_len})" + + element_list.append(value) + + return ", ".join(element_list) + + def dp_to_str(self, value_dp: DataProperty) -> str: + if value_dp.typecode == Typecode.STRING: + return str(value_dp.data) + + try: + value = self.__preprocess_value_before_tostring(value_dp) + except TypeConversionError: + return self.__format_map.get(value_dp.typecode, "{:s}").format(value_dp.data) + + to_string_format_str = self.__get_tostring_format(value_dp) + + try: + return to_string_format_str.format(value) + except (ValueError, TypeError): + pass + + try: + return MultiByteStrDecoder(value).unicode_str + except ValueError: + pass + + return str(value) + + def extend_width(self, ascii_char_width: int) -> None: + self.extend_header_width(ascii_char_width) + self.extend_body_width(ascii_char_width) + + def extend_header_width(self, ascii_char_width: int) -> None: + self.__header_ascii_char_width += ascii_char_width + + def extend_body_width(self, ascii_char_width: int) -> None: + self.__body_ascii_char_width += ascii_char_width + + def update_header(self, header_db: DataProperty) -> None: + self.__header_ascii_char_width = header_db.ascii_char_width + + def update_body(self, value_dp: DataProperty) -> None: + if value_dp.is_include_ansi_escape: + assert value_dp.no_ansi_escape_dp + value_dp = value_dp.no_ansi_escape_dp + + self.__typecode_bitmap |= value_dp.typecode.value + self.__calc_typecode_from_bitmap() + + if value_dp.typecode in (Typecode.REAL_NUMBER, Typecode.INTEGER): + self.__minmax_integer_digits.update(value_dp.integer_digits) + self.__minmax_decimal_places.update(value_dp.decimal_places) + self.__update_decimal_places() + + self.__minmax_additional_format_len.update(value_dp.additional_format_len) + + self.__dp_list.append(value_dp) + self.__update_ascii_char_width() + + def merge(self, column_dp: "ColumnDataProperty") -> None: + self.__typecode_bitmap |= column_dp.typecode.value + self.__calc_typecode_from_bitmap() + + self.__minmax_integer_digits.merge(column_dp.minmax_integer_digits) + self.__minmax_decimal_places.merge(column_dp.minmax_decimal_places) + self.__update_decimal_places() + + self.__minmax_additional_format_len.merge(column_dp.minmax_additional_format_len) + + self.__body_ascii_char_width = max(self.__body_ascii_char_width, column_dp.ascii_char_width) + self.__update_ascii_char_width() + + def begin_update(self) -> None: + self.__is_calculate = False + + def end_update(self) -> None: + self.__is_calculate = True + + self.__calc_typecode_from_bitmap() + self.__update_decimal_places() + self.__update_ascii_char_width() + + def __is_not_single_typecode(self, typecode_bitmap: int) -> bool: + return bool( + self.__typecode_bitmap & typecode_bitmap and self.__typecode_bitmap & ~typecode_bitmap + ) + + def __is_float_typecode(self) -> bool: + FLOAT_TYPECODE_BMP = ( + Typecode.REAL_NUMBER.value | Typecode.INFINITY.value | Typecode.NAN.value + ) + NUMBER_TYPECODE_BMP = FLOAT_TYPECODE_BMP | Typecode.INTEGER.value + + if self.__is_not_single_typecode(NUMBER_TYPECODE_BMP | Typecode.NULL_STRING.value): + return False + + if ( + bin(self.__typecode_bitmap & (FLOAT_TYPECODE_BMP | Typecode.NULL_STRING.value)).count( + "1" + ) + >= 2 + ): + return True + + if bin(self.__typecode_bitmap & NUMBER_TYPECODE_BMP).count("1") >= 2: + return True + + return False + + def __calc_body_ascii_char_width(self) -> int: + width_list = [self.__body_ascii_char_width] + + for value_dp in self.__dp_list: + if value_dp.is_include_ansi_escape: + assert value_dp.no_ansi_escape_dp + value_dp = value_dp.no_ansi_escape_dp + + width_list.append( + calc_ascii_char_width(self.dp_to_str(value_dp), self._east_asian_ambiguous_width) + ) + + return max(width_list) + + def __calc_decimal_places(self) -> Optional[int]: + if self.minmax_decimal_places.max_value is None: + return None + + return min(self.__max_precision, int(self.minmax_decimal_places.max_value)) + + def __get_tostring_format(self, value_dp: DataProperty) -> str: + if self.typecode == Typecode.STRING: + return self.__format_map.get(value_dp.typecode, "{:s}") + + return self.__format_map.get(self.typecode, "{:s}") + + def __get_typecode_from_bitmap(self) -> Typecode: + if self.__is_float_typecode(): + return Typecode.REAL_NUMBER + + if any( + [ + self.__is_not_single_typecode(Typecode.BOOL.value), + self.__is_not_single_typecode(Typecode.DATETIME.value), + ] + ): + return Typecode.STRING + + typecode_list = [ + Typecode.STRING, + Typecode.REAL_NUMBER, + Typecode.INTEGER, + Typecode.DATETIME, + Typecode.DICTIONARY, + Typecode.IP_ADDRESS, + Typecode.LIST, + Typecode.BOOL, + Typecode.INFINITY, + Typecode.NAN, + Typecode.NULL_STRING, + ] + + for typecode in typecode_list: + if self.__typecode_bitmap & typecode.value: + return typecode + + if self.__typecode_bitmap == Typecode.NONE.value: + return Typecode.NONE + + return Typecode.STRING + + def __update_ascii_char_width(self) -> None: + if not self.__is_calculate: + return + + self.__body_ascii_char_width = self.__calc_body_ascii_char_width() + + def __update_decimal_places(self) -> None: + if not self.__is_calculate: + return + + self._decimal_places = self.__calc_decimal_places() + self.__format_map = self._formatter.make_format_map(decimal_places=self._decimal_places) + + def __calc_typecode_from_bitmap(self) -> None: + if not self.__is_calculate: + return + + self._typecode = self.__get_typecode_from_bitmap() + + def __preprocess_value_before_tostring(self, value_dp: DataProperty) -> Any: + if self.typecode == value_dp.typecode or self.typecode in [ + Typecode.STRING, + Typecode.BOOL, + Typecode.DATETIME, + ]: + return value_dp.data + + return self.type_class( + value_dp.data, + strict_level=StrictLevel.MIN, + float_type=self.__float_type, + strip_ansi_escape=False, + ).convert() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dataproperty/_common.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dataproperty/_common.py new file mode 100644 index 0000000000000000000000000000000000000000..7f0b689b66d220d744b0ffab887c75b707cdcc94 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dataproperty/_common.py @@ -0,0 +1,74 @@ +""" +.. codeauthor:: Tsuyoshi Hombashi +""" + +import copy +import itertools +from datetime import datetime +from decimal import Decimal +from typing import Final + +from typepy import StrictLevel, Typecode + +from .typing import StrictLevelMap, TypeValueMap + + +NOT_QUOTING_FLAGS: Final = { + Typecode.BOOL: False, + Typecode.DATETIME: False, + Typecode.DICTIONARY: False, + Typecode.INFINITY: False, + Typecode.INTEGER: False, + Typecode.IP_ADDRESS: False, + Typecode.LIST: False, + Typecode.NAN: False, + Typecode.NULL_STRING: False, + Typecode.NONE: False, + Typecode.REAL_NUMBER: False, + Typecode.STRING: False, +} + +MAX_STRICT_LEVEL_MAP: Final[StrictLevelMap] = dict( + itertools.product(list(Typecode), [StrictLevel.MAX]) +) +MIN_STRICT_LEVEL_MAP: Final[StrictLevelMap] = dict( + itertools.product(list(Typecode), [StrictLevel.MIN]) +) + + +class DefaultValue: + DATETIME_FORMAT: Final = "%Y-%m-%dT%H:%M:%S%z" + FLOAT_TYPE: Final = Decimal + INF_VALUE: Final = FLOAT_TYPE("inf") + NAN_VALUE: Final = FLOAT_TYPE("nan") + + QUOTING_FLAGS: Final = copy.deepcopy(NOT_QUOTING_FLAGS) + + STRICT_LEVEL_MAP: Final[StrictLevelMap] = { + "default": StrictLevel.MAX, + Typecode.BOOL: StrictLevel.MAX, + Typecode.DATETIME: StrictLevel.MAX, + Typecode.DICTIONARY: StrictLevel.MAX, + Typecode.REAL_NUMBER: 1, + Typecode.INFINITY: StrictLevel.MIN, + Typecode.INTEGER: 1, + Typecode.IP_ADDRESS: StrictLevel.MAX, + Typecode.LIST: StrictLevel.MAX, + Typecode.NAN: StrictLevel.MIN, + Typecode.NONE: StrictLevel.MAX, + Typecode.NULL_STRING: StrictLevel.MIN, + Typecode.STRING: StrictLevel.MIN, + } + + TYPE_VALUE_MAP: Final[TypeValueMap] = { + Typecode.NONE: None, + Typecode.INFINITY: INF_VALUE, + Typecode.NAN: NAN_VALUE, + } + + MAX_WORKERS: Final = 1 + MAX_PRECISION: Final = 100 + + +def default_datetime_formatter(value: datetime) -> str: + return value.strftime(DefaultValue.DATETIME_FORMAT) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dataproperty/_container.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dataproperty/_container.py new file mode 100644 index 0000000000000000000000000000000000000000..543576395cabcc830071aa8d3e9f5c68663aebe5 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dataproperty/_container.py @@ -0,0 +1,196 @@ +""" +.. codeauthor:: Tsuyoshi Hombashi +""" + +import abc +from collections.abc import Sequence +from decimal import Decimal +from typing import Any, Final, Optional, Union + +from typepy import RealNumber + + +T = Union[int, float, Decimal] +NAN: Final = Decimal("nan") + + +class AbstractContainer(metaclass=abc.ABCMeta): + @property + @abc.abstractmethod + def min_value(self) -> Optional[Decimal]: # pragma: no cover + pass + + @property + @abc.abstractmethod + def max_value(self) -> Optional[Decimal]: # pragma: no cover + pass + + @abc.abstractmethod + def mean(self) -> Decimal: # pragma: no cover + pass + + @abc.abstractmethod + def update(self, value: Optional[T]) -> None: # pragma: no cover + pass + + @abc.abstractmethod + def merge(self, value: "AbstractContainer") -> None: # pragma: no cover + pass + + def __repr__(self) -> str: + if not self.has_value(): + return "None" + + return ", ".join([f"min={self.min_value}", f"max={self.max_value}"]) + + def has_value(self) -> bool: + return self.min_value is not None and self.max_value is not None + + def is_same_value(self) -> bool: + return self.has_value() and self.min_value == self.max_value + + def is_zero(self) -> bool: + return self.has_value() and self.min_value == 0 and self.max_value == 0 + + +class ListContainer(AbstractContainer): + __slots__ = ("__value_list",) + + @property + def min_value(self) -> Optional[Decimal]: + try: + return min(self.__value_list) + except ValueError: + return None + + @property + def max_value(self) -> Optional[Decimal]: + try: + return max(self.__value_list) + except ValueError: + return None + + @property + def value_list(self) -> list[Decimal]: + return self.__value_list + + def __init__(self, value_list: Optional[list[Decimal]] = None) -> None: + if value_list is None: + self.__value_list: list[Decimal] = [] + return + + for value in value_list: + self.update(value) + + def mean(self) -> Decimal: + try: + return Decimal(sum(self.__value_list) / len(self.__value_list)) + except ZeroDivisionError: + return NAN + + def update(self, value: Union[int, float, Decimal, None]) -> None: + if value is None: + return + + store_value = RealNumber(value).try_convert() + if store_value is None: + return + + self.__value_list.append(store_value) + + def merge(self, value: "AbstractContainer") -> None: + if not isinstance(value, ListContainer): + return + + for v in value.value_list: + self.update(v) + + +class MinMaxContainer(AbstractContainer): + __slots__ = ("__min_value", "__max_value") + + def __init__(self, value_list: Optional[Sequence[Decimal]] = None) -> None: + self.__min_value: Optional[Decimal] = None + self.__max_value: Optional[Decimal] = None + + if value_list is None: + return + + for value in value_list: + self.update(value) + + @property + def min_value(self) -> Optional[Decimal]: + return self.__min_value + + @property + def max_value(self) -> Optional[Decimal]: + return self.__max_value + + def __eq__(self, other: Any) -> bool: + if not isinstance(other, MinMaxContainer): + return False + + return all([self.min_value == other.min_value, self.max_value == other.max_value]) + + def __ne__(self, other: Any) -> bool: + if not isinstance(other, MinMaxContainer): + return True + + return any([self.min_value != other.min_value, self.max_value != other.max_value]) + + def __contains__(self, x: T) -> bool: + if self.min_value is None: + return False + + if self.max_value is None: + return False + + return self.min_value <= x <= self.max_value + + def diff(self) -> Decimal: + if self.min_value is None: + return NAN + + if self.max_value is None: + return NAN + + try: + return self.max_value - self.min_value + except TypeError: + return NAN + + def mean(self) -> Decimal: + if self.min_value is None: + return NAN + + if self.max_value is None: + return NAN + + try: + return (self.max_value + self.min_value) * Decimal("0.5") + except TypeError: + return NAN + + def update(self, value: Optional[T]) -> None: + if value is None: + return + + decimal_value = Decimal(value) + + if self.__min_value is None: + self.__min_value = decimal_value + else: + self.__min_value = min(self.__min_value, decimal_value) + + if self.__max_value is None: + self.__max_value = decimal_value + else: + self.__max_value = max(self.__max_value, decimal_value) + + def merge(self, value: "AbstractContainer") -> None: + if not isinstance(value, MinMaxContainer): + return + + self.update(value.min_value) + self.update(value.max_value) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dataproperty/_converter.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dataproperty/_converter.py new file mode 100644 index 0000000000000000000000000000000000000000..40757c5abcbf8c6f1b90952694eee5aae4576101 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dataproperty/_converter.py @@ -0,0 +1,90 @@ +""" +.. codeauthor:: Tsuyoshi Hombashi +""" + +import re +from typing import Any, Final, Optional + +from typepy import Typecode, TypeConversionError + +from ._common import MAX_STRICT_LEVEL_MAP, DefaultValue +from ._dataproperty import DataProperty +from ._preprocessor import Preprocessor +from .typing import DateTimeFormatter, FloatType, StrictLevelMap, TypeValueMap + + +class DataPropertyConverter: + __RE_QUOTE_LINE: Final = re.compile(r"^\s*[\"'].*[\"']\s*$") # noqa: w605 + __RE_QUOTE_CHAR: Final = re.compile("[\"']") + + def __init__( + self, + preprocessor: Preprocessor, + datetime_format_str: str, + datetime_formatter: Optional[DateTimeFormatter] = None, + type_value_map: Optional[TypeValueMap] = None, + quoting_flags: Optional[dict[Typecode, bool]] = None, + float_type: Optional[FloatType] = None, + strict_level_map: Optional[StrictLevelMap] = None, + ) -> None: + self.__preprocessor = preprocessor + self.__type_value_map: TypeValueMap = ( + type_value_map if type_value_map else DefaultValue.TYPE_VALUE_MAP + ) + self.__quoting_flags: dict[Typecode, bool] = ( + quoting_flags if quoting_flags else DefaultValue.QUOTING_FLAGS + ) + + self.__datetime_formatter = datetime_formatter + self.__datetime_format_str = datetime_format_str + self.__float_type = float_type + self.__strict_level_map = strict_level_map + + def convert(self, dp_value: DataProperty) -> DataProperty: + try: + return self.__create_dataproperty(self.__convert_value(dp_value)) + except TypeConversionError: + pass + + if not self.__quoting_flags.get(dp_value.typecode): + if self.__preprocessor.is_escape_html_tag: + return self.__create_dataproperty(dp_value.to_str()) + + return dp_value + + return self.__create_dataproperty(self.__apply_quote(dp_value.typecode, dp_value.to_str())) + + def __create_dataproperty(self, value: Any) -> DataProperty: + return DataProperty( + value, + preprocessor=self.__preprocessor, + float_type=self.__float_type, + datetime_format_str=self.__datetime_format_str, + strict_level_map=MAX_STRICT_LEVEL_MAP, + ) + + def __apply_quote(self, typecode: Typecode, data: Any) -> Any: + if not self.__quoting_flags.get(typecode): + return data + + try: + if self.__RE_QUOTE_LINE.search(data): + return data + except TypeError: + return data + + return '"{}"'.format(self.__RE_QUOTE_CHAR.sub('\\"', data.replace("\\", "\\\\"))) + + def __convert_value(self, dp_value: DataProperty) -> Any: + if dp_value.typecode in self.__type_value_map: + return self.__apply_quote(dp_value.typecode, self.__type_value_map[dp_value.typecode]) + + if dp_value.typecode == Typecode.DATETIME and self.__datetime_formatter: + try: + return self.__apply_quote( + dp_value.typecode, self.__datetime_formatter(dp_value.data) + ) + except TypeError: + raise TypeConversionError + + raise TypeConversionError("no need to convert") diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dataproperty/_dataproperty.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dataproperty/_dataproperty.py new file mode 100644 index 0000000000000000000000000000000000000000..20c9887c300d60cc1fcbb662e1e19f375f284d02 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dataproperty/_dataproperty.py @@ -0,0 +1,381 @@ +""" +.. codeauthor:: Tsuyoshi Hombashi +""" + +from decimal import Decimal +from typing import Any, Final, Optional, cast + +import typepy +from mbstrdecoder import MultiByteStrDecoder +from typepy import ( + Bool, + DateTime, + Dictionary, + Infinity, + Integer, + IpAddress, + Nan, + NoneType, + NullString, + RealNumber, + StrictLevel, + String, + Typecode, + TypeConversionError, +) +from typepy.type import AbstractType + +from ._align import Align +from ._align_getter import align_getter +from ._base import DataPeropertyBase +from ._common import DefaultValue +from ._function import calc_ascii_char_width, get_number_of_digit +from ._preprocessor import Preprocessor +from .typing import FloatType, StrictLevelMap, TypeHint + + +class DataProperty(DataPeropertyBase): + __slots__ = ( + "__data", + "__no_ansi_escape_data", + "__align", + "__integer_digits", + "__additional_format_len", + "__length", + "__ascii_char_width", + ) + + __type_class_list: Final[list[type[AbstractType]]] = [ + NoneType, + Integer, + Infinity, + Nan, + IpAddress, + RealNumber, + Bool, + typepy.List, + Dictionary, + DateTime, + NullString, + String, + ] + + def __init__( + self, + data: Any, + preprocessor: Optional[Preprocessor] = None, + type_hint: TypeHint = None, + float_type: Optional[FloatType] = None, + format_flags: Optional[int] = None, + datetime_format_str: str = DefaultValue.DATETIME_FORMAT, + strict_level_map: Optional[StrictLevelMap] = None, + east_asian_ambiguous_width: int = 1, + ) -> None: + super().__init__( + format_flags=format_flags, + is_formatting_float=True, + datetime_format_str=datetime_format_str, + east_asian_ambiguous_width=east_asian_ambiguous_width, + ) + + self.__additional_format_len: Optional[int] = None + self.__align: Optional[Align] = None + self.__ascii_char_width: Optional[int] = None + self.__integer_digits: Optional[int] = None + self.__length: Optional[int] = None + + if preprocessor is None: + preprocessor = Preprocessor() + + data, no_ansi_escape_data = preprocessor.preprocess(data) + + self.__set_data(data, type_hint, float_type, strict_level_map) + + if no_ansi_escape_data is None or len(data) == len(no_ansi_escape_data): + self.__no_ansi_escape_data: Optional[DataProperty] = None + else: + self.__no_ansi_escape_data = DataProperty(no_ansi_escape_data, float_type=float_type) + + def __eq__(self, other: Any) -> bool: + if not isinstance(other, DataProperty): + return False + + if self.typecode != other.typecode: + return False + + if self.typecode == Typecode.NAN: + return True + + return self.data == other.data + + def __ne__(self, other: Any) -> bool: + if not isinstance(other, DataProperty): + return True + + if self.typecode != other.typecode: + return True + + if self.typecode == Typecode.NAN: + return False + + return self.data != other.data + + def __repr__(self) -> str: + element_list = [] + + if self.typecode == Typecode.DATETIME: + element_list.append(f"data={str(self.data):s}") + else: + try: + element_list.append("data=" + self.to_str()) + except UnicodeEncodeError: + element_list.append(f"data={MultiByteStrDecoder(self.data).unicode_str}") + + element_list.extend( + [ + f"type={self.typename:s}", + f"align={self.align.align_string}", + f"ascii_width={self.ascii_char_width:d}", + ] + ) + + if Integer(self.length).is_type(): + element_list.append(f"length={self.length}") + + if Integer(self.integer_digits).is_type(): + element_list.append(f"int_digits={self.integer_digits}") + + if Integer(self.decimal_places).is_type(): + element_list.append(f"decimal_places={self.decimal_places}") + + if Integer(self.additional_format_len).is_type(): + element_list.append(f"extra_len={self.additional_format_len}") + + return ", ".join(element_list) + + @property + def align(self) -> Align: + if not self.__align: + if self.is_include_ansi_escape: + assert self.no_ansi_escape_dp + self.__align = self.no_ansi_escape_dp.align + else: + self.__align = align_getter.get_align_from_typecode(self.typecode) + + assert self.__align + + return self.__align + + @property + def decimal_places(self) -> Optional[int]: + """ + :return: + Decimal places if the ``data`` type either ``float`` or + ``decimal.Decimal``. Returns ``0`` if the ``data`` type is ``int``. + Otherwise, returns ``float("nan")``. + :rtype: int + """ + + if self._decimal_places is None: + self.__set_digit() + + return self._decimal_places + + @property + def data(self) -> Any: + """ + :return: Original data value. + :rtype: Original data type. + """ + + return self.__data + + @property + def is_include_ansi_escape(self) -> bool: + if self.no_ansi_escape_dp is None: + return False + + return self.length != self.no_ansi_escape_dp.length + + @property + def no_ansi_escape_dp(self) -> Optional["DataProperty"]: + return self.__no_ansi_escape_data + + @property + def length(self) -> Optional[int]: + """ + :return: Length of the ``data``. + :rtype: int + """ + + if self.__length is None: + self.__length = self.__get_length() + + return self.__length + + @property + def ascii_char_width(self) -> int: + if self.__ascii_char_width is None: + self.__ascii_char_width = self.__calc_ascii_char_width() + + return self.__ascii_char_width + + @property + def integer_digits(self) -> Optional[int]: + """ + :return: + Integer digits if the ``data`` type either + ``int``/``float``/``decimal.Decimal``. + Otherwise, returns ``None``. + :rtype: int + """ + + if self.__integer_digits is None: + self.__set_digit() + + return self.__integer_digits + + @property + def additional_format_len(self) -> int: + if self.__additional_format_len is None: + self.__additional_format_len = self.__get_additional_format_len() + + return self.__additional_format_len + + def get_padding_len(self, ascii_char_width: int) -> int: + if self.typecode in (Typecode.LIST, Typecode.DICTIONARY): + unicode_str_len = DataProperty(MultiByteStrDecoder(str(self.data)).unicode_str).length + assert unicode_str_len + return max( + ascii_char_width - (self.ascii_char_width - unicode_str_len), + 0, + ) + + try: + return max(ascii_char_width - (self.ascii_char_width - cast(int, self.length)), 0) + except TypeError: + return ascii_char_width + + def to_str(self) -> str: + return self.format_str.format(self.data) + + def __get_additional_format_len(self) -> int: + if not RealNumber(self.data, strip_ansi_escape=False).is_type(): + return 0 + + format_len = 0 + + if Decimal(self.data) < 0: + # for minus character + format_len += 1 + + return format_len + + def __get_base_float_len(self) -> int: + assert self.integer_digits is not None + assert self.decimal_places is not None + + if any([self.integer_digits < 0, self.decimal_places < 0]): + raise ValueError("integer digits and decimal places must be greater or equals to zero") + + float_len = self.integer_digits + self.decimal_places + if self.decimal_places > 0: + # for dot + float_len += 1 + + return float_len + + def __get_length(self) -> Optional[int]: + if self.typecode in (Typecode.DICTIONARY, Typecode.LIST, Typecode.STRING): + return len(self.data) + + return None + + def __calc_ascii_char_width(self) -> int: + if self.typecode == Typecode.INTEGER: + return cast(int, self.integer_digits) + self.additional_format_len + + if self.typecode == Typecode.REAL_NUMBER: + return self.__get_base_float_len() + self.additional_format_len + + if self.typecode == Typecode.DATETIME: + try: + return len(self.to_str()) + except ValueError: + # reach to this line if the year <1900. + # the datetime strftime() methods require year >= 1900. + return len(str(self.data)) + + if self.is_include_ansi_escape: + assert self.no_ansi_escape_dp + return self.no_ansi_escape_dp.ascii_char_width + + try: + unicode_str = MultiByteStrDecoder(self.data).unicode_str + except ValueError: + unicode_str = self.to_str() + + return calc_ascii_char_width(unicode_str, self._east_asian_ambiguous_width) + + def __set_data( + self, + data: Any, + type_hint: TypeHint, + float_type: Optional[FloatType], + strict_level_map: Optional[StrictLevelMap], + ) -> None: + if float_type is None: + float_type = DefaultValue.FLOAT_TYPE + + if strict_level_map is None: + strict_level_map = DefaultValue.STRICT_LEVEL_MAP + + if type_hint: + type_obj = type_hint( + data, strict_level=StrictLevel.MIN, float_type=float_type, strip_ansi_escape=False + ) + self._typecode = type_obj.typecode + self.__data = type_obj.try_convert() + + if type_hint( + self.__data, + strict_level=StrictLevel.MAX, + float_type=float_type, + strip_ansi_escape=False, + ).is_type(): + return + + for type_class in self.__type_class_list: + strict_level = strict_level_map.get( + type_class(None, 0).typecode, strict_level_map.get("default", StrictLevel.MAX) + ) + + if self.__try_convert_type(data, type_class, strict_level, float_type): + return + + raise TypeConversionError( + f"failed to convert: data={data}, strict_level={strict_level_map}" + ) + + def __set_digit(self) -> None: + integer_digits, decimal_places = get_number_of_digit(self.__data) + self.__integer_digits = integer_digits + self._decimal_places = decimal_places + + def __try_convert_type( + self, + data: Any, + type_class: type[AbstractType], + strict_level: int, + float_type: Optional[FloatType], + ) -> bool: + type_obj = type_class(data, strict_level, float_type=float_type, strip_ansi_escape=False) + + try: + self.__data = type_obj.convert() + except TypeConversionError: + return False + + self._typecode = type_obj.typecode + + return True diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dataproperty/_extractor.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dataproperty/_extractor.py new file mode 100644 index 0000000000000000000000000000000000000000..0bc9ea444ab1ac9fa3ceb814d5aa378ecb50b0ab --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dataproperty/_extractor.py @@ -0,0 +1,817 @@ +""" +.. codeauthor:: Tsuyoshi Hombashi +""" + +import copy +import enum +import sys +import typing +from collections import Counter +from collections.abc import Sequence +from decimal import Decimal +from typing import Any, Optional, Union, cast + +import typepy +from typepy import ( + Bool, + DateTime, + Dictionary, + Infinity, + Integer, + IpAddress, + Nan, + NoneType, + NullString, + RealNumber, + StrictLevel, + String, + Typecode, + is_empty_sequence, +) +from typepy.type import AbstractType + +from ._column import ColumnDataProperty +from ._common import MIN_STRICT_LEVEL_MAP, DefaultValue +from ._converter import DataPropertyConverter +from ._dataproperty import DataProperty +from ._formatter import Format +from ._preprocessor import Preprocessor +from .logger import logger # type: ignore +from .typing import ( + DateTimeFormatter, + StrictLevelMap, + TransFunc, + TypeHint, + TypeValueMap, + normalize_type_hint, +) + + +DataPropertyMatrix = list[list[DataProperty]] + + +@enum.unique +class MatrixFormatting(enum.Enum): + # raise exception if the matrix is not properly formatted + EXCEPTION = 1 << 1 + + # trim to the minimum size column + TRIM = 1 << 2 + + # Append None values to columns so that it is the same as the maximum + # column size. + FILL_NONE = 1 << 3 + + HEADER_ALIGNED = 1 << 4 + + +class DataPropertyExtractor: + """ + .. py:attribute:: quoting_flags + + Configurations to add double quote to for each items in a matrix, + where |Typecode| of table-value is |True| in the ``quote_flag_table`` + mapping table. ``quote_flag_table`` should be a dictionary. + And is ``{ Typecode : bool }``. Defaults to: + + .. code-block:: json + :caption: The default values + + { + Typecode.BOOL: False, + Typecode.DATETIME: False, + Typecode.DICTIONARY: False, + Typecode.INFINITY: False, + Typecode.INTEGER: False, + Typecode.IP_ADDRESS: False, + Typecode.LIST: False, + Typecode.NAN: False, + Typecode.NULL_STRING: False, + Typecode.NONE: False, + Typecode.REAL_NUMBER: False, + Typecode.STRING: False, + } + """ + + def __init__(self, max_precision: Optional[int] = None) -> None: + self.max_workers = DefaultValue.MAX_WORKERS + + if max_precision is None: + self.__max_precision = DefaultValue.MAX_PRECISION + else: + self.__max_precision = max_precision + + self.__headers: Sequence[str] = [] + self.__default_type_hint: TypeHint = None + self.__col_type_hints: list[TypeHint] = [] + + self.__strip_str_header: Optional[str] = None + self.__is_formatting_float = True + self.__min_col_ascii_char_width = 0 + self.__default_format_flags = Format.NONE + self.__format_flags_list: Sequence[int] = [] + self.__float_type: Union[type[float], type[Decimal], None] = None + self.__datetime_format_str = DefaultValue.DATETIME_FORMAT + self.__strict_level_map = copy.deepcopy( + cast(dict[Union[Typecode, str], int], DefaultValue.STRICT_LEVEL_MAP) + ) + self.__east_asian_ambiguous_width = 1 + + self.__preprocessor = Preprocessor() + + self.__type_value_map: TypeValueMap = copy.deepcopy(DefaultValue.TYPE_VALUE_MAP) + + self.__trans_func_list: list[TransFunc] = [] + self.__quoting_flags = copy.deepcopy(DefaultValue.QUOTING_FLAGS) + self.__datetime_formatter: Optional[DateTimeFormatter] = None + self.__matrix_formatting = MatrixFormatting.TRIM + self.__dp_converter: DataPropertyConverter + + self.__clear_cache() + + def __clear_cache(self) -> None: + self.__update_dp_converter() + self.__dp_cache_zero = self.__to_dp_raw(0) + self.__dp_cache_one = self.__to_dp_raw(1) + self.__dp_cache_true = self.__to_dp_raw(True) + self.__dp_cache_false = self.__to_dp_raw(False) + self.__dp_cache_map = {None: self.__to_dp_raw(None), "": self.__to_dp_raw("")} + + @property + def headers(self) -> Sequence[str]: + return self.__headers + + @headers.setter + def headers(self, value: Sequence[str]) -> None: + if self.__headers == value: + return + + self.__headers = value + self.__clear_cache() + + @property + def default_type_hint(self) -> TypeHint: + return self.__default_type_hint + + @default_type_hint.setter + def default_type_hint(self, value: TypeHint) -> None: + if self.__default_type_hint == value: + return + + self.__default_type_hint = value + self.__clear_cache() + + @property + def column_type_hints(self) -> list[TypeHint]: + return self.__col_type_hints + + @column_type_hints.setter + def column_type_hints(self, value: Sequence[Union[str, TypeHint]]) -> None: + normalized_type_hints: list[TypeHint] = [] + + for type_hint in value: + type_hint = normalize_type_hint(type_hint) + if type_hint not in ( + Bool, + DateTime, + Dictionary, + Infinity, + Integer, + IpAddress, + typepy.List, + Nan, + NoneType, + RealNumber, + String, + NullString, + None, + ): + raise ValueError(f"invalid type hint: {type(type_hint)}") + + normalized_type_hints.append(type_hint) + + if self.__col_type_hints == normalized_type_hints: + return + + self.__col_type_hints = normalized_type_hints + self.__clear_cache() + + @property + def is_formatting_float(self) -> bool: + return self.__is_formatting_float + + @is_formatting_float.setter + def is_formatting_float(self, value: bool) -> None: + self.__is_formatting_float = value + + @property + def max_precision(self) -> int: + return self.__max_precision + + @max_precision.setter + def max_precision(self, value: int) -> None: + if self.__max_precision == value: + return + + self.__max_precision = value + self.__clear_cache() + + @property + def preprocessor(self) -> Preprocessor: + return self.__preprocessor + + @preprocessor.setter + def preprocessor(self, value: Preprocessor) -> None: + if self.preprocessor == value: + return + + self.__preprocessor = value + self.__update_dp_converter() + + @property + def strip_str_header(self) -> Optional[str]: + return self.__strip_str_header + + @strip_str_header.setter + def strip_str_header(self, value: str) -> None: + if self.__strip_str_header == value: + return + + self.__strip_str_header = value + self.__clear_cache() + + @property + def min_column_width(self) -> int: + return self.__min_col_ascii_char_width + + @min_column_width.setter + def min_column_width(self, value: int) -> None: + if self.__min_col_ascii_char_width == value: + return + + self.__min_col_ascii_char_width = value + self.__clear_cache() + + @property + def default_format_flags(self) -> int: + return self.__default_format_flags + + @default_format_flags.setter + def default_format_flags(self, value: int) -> None: + if self.__default_format_flags == value: + return + + self.__default_format_flags = value + self.__clear_cache() + + @property + def format_flags_list(self) -> Sequence[int]: + return self.__format_flags_list + + @format_flags_list.setter + def format_flags_list(self, value: Sequence[int]) -> None: + if self.__format_flags_list == value: + return + + self.__format_flags_list = value + self.__clear_cache() + + @property + def float_type(self) -> Union[type[float], type[Decimal], None]: + return self.__float_type + + @float_type.setter + def float_type(self, value: Union[type[float], type[Decimal]]) -> None: + if self.__float_type == value: + return + + self.__float_type = value + self.__clear_cache() + + @property + def datetime_format_str(self) -> str: + return self.__datetime_format_str + + @datetime_format_str.setter + def datetime_format_str(self, value: str) -> None: + if self.__datetime_format_str == value: + return + + self.__datetime_format_str = value + self.__clear_cache() + + @property + def strict_level_map(self) -> StrictLevelMap: + return self.__strict_level_map + + @strict_level_map.setter + def strict_level_map(self, value: StrictLevelMap) -> None: + if self.__strict_level_map == value: + return + + self.__strict_level_map = cast(dict[Union[Typecode, str], int], value) + self.__clear_cache() + + @property + def east_asian_ambiguous_width(self) -> int: + return self.__east_asian_ambiguous_width + + @east_asian_ambiguous_width.setter + def east_asian_ambiguous_width(self, value: int) -> None: + if self.__east_asian_ambiguous_width == value: + return + + self.__east_asian_ambiguous_width = value + self.__clear_cache() + + @property + def type_value_map(self) -> TypeValueMap: + return self.__type_value_map + + @type_value_map.setter + def type_value_map(self, value: TypeValueMap) -> None: + if self.__type_value_map == value: + return + + self.__type_value_map = value + self.__clear_cache() + + def set_type_value(self, key: Typecode, value: Union[float, str, Decimal, None]) -> None: + self.__type_value_map[key] = value + self.__clear_cache() + + def register_trans_func(self, trans_func: TransFunc) -> None: + self.__trans_func_list.insert(0, trans_func) + self.__clear_cache() + + @property + def quoting_flags(self) -> dict[Typecode, bool]: + return self.__quoting_flags + + @quoting_flags.setter + def quoting_flags(self, value: dict[Typecode, bool]) -> None: + if self.__quoting_flags == value: + return + + self.__quoting_flags = value + self.__clear_cache() + + @property + def datetime_formatter(self) -> Optional[DateTimeFormatter]: + return self.__datetime_formatter + + @datetime_formatter.setter + def datetime_formatter(self, value: Optional[DateTimeFormatter]) -> None: + if self.__datetime_formatter == value: + return + + self.__datetime_formatter = value + self.__clear_cache() + + @property + def matrix_formatting(self) -> MatrixFormatting: + return self.__matrix_formatting + + @matrix_formatting.setter + def matrix_formatting(self, value: MatrixFormatting) -> None: + if self.__matrix_formatting == value: + return + + self.__matrix_formatting = value + self.__clear_cache() + + @property + def max_workers(self) -> int: + assert self.__max_workers + + return self.__max_workers + + @max_workers.setter + def max_workers(self, value: Optional[int]) -> None: + try: + from _multiprocessing import SemLock, sem_unlink # noqa + except ImportError: + logger.debug("This platform lacks a functioning sem_open implementation") + value = 1 + + if "pytest" in sys.modules and value != 1: + logger.debug("set max_workers to 1 to avoid deadlock when executed from pytest") + value = 1 + + self.__max_workers = value + if not self.__max_workers: + self.__max_workers = DefaultValue.MAX_WORKERS + + def to_dp(self, value: Any) -> DataProperty: + self.__update_dp_converter() + + return self.__to_dp(value) + + def to_dp_list(self, values: Sequence[Any]) -> list[DataProperty]: + if is_empty_sequence(values): + return [] + + self.__update_dp_converter() + + return self._to_dp_list(values) + + def to_column_dp_list( + self, + value_dp_matrix: Any, + previous_column_dp_list: Optional[Sequence[ColumnDataProperty]] = None, + ) -> list[ColumnDataProperty]: + col_dp_list = self.__get_col_dp_list_base() + + logger.debug("converting to column dataproperty:") + + logs = [" params:"] + if self.headers: + logs.append(f" headers={len(self.headers)}") + logs.extend( + [ + " prev_col_count={}".format( + len(previous_column_dp_list) if previous_column_dp_list else None + ), + f" matrix_formatting={self.matrix_formatting}", + ] + ) + if self.column_type_hints: + logs.append( + " column_type_hints=({})".format( + ", ".join( + [ + type_hint.__name__ if type_hint else "none" + for type_hint in self.column_type_hints + ] + ) + ) + ) + else: + logs.append(" column_type_hints=()") + + for log in logs: + logger.debug(log) + + logger.debug(" results:") + for col_idx, value_dp_list in enumerate(zip(*value_dp_matrix)): + try: + col_dp_list[col_idx] + except IndexError: + col_dp_list.append( + ColumnDataProperty( + column_index=col_idx, + float_type=self.float_type, + min_width=self.min_column_width, + format_flags=self.__get_format_flags(col_idx), + is_formatting_float=self.is_formatting_float, + datetime_format_str=self.datetime_format_str, + east_asian_ambiguous_width=self.east_asian_ambiguous_width, + max_precision=self.__max_precision, + ) + ) + + col_dp = col_dp_list[col_idx] + col_dp.begin_update() + + try: + col_dp.merge(previous_column_dp_list[col_idx]) # type: ignore + except (TypeError, IndexError): + pass + + for value_dp in value_dp_list: + col_dp.update_body(value_dp) + + col_dp.end_update() + + logger.debug(f" {str(col_dp):s}") + + return col_dp_list + + def to_dp_matrix(self, value_matrix: Sequence[Sequence[Any]]) -> DataPropertyMatrix: + self.__update_dp_converter() + logger.debug(f"max_workers={self.max_workers}, preprocessor={self.__preprocessor}") + + value_matrix = self.__strip_data_matrix(value_matrix) + + if self.__is_dp_matrix(value_matrix): + logger.debug("already a dataproperty matrix") + return value_matrix # type: ignore + + if self.max_workers <= 1: + return self.__to_dp_matrix_st(value_matrix) + + return self.__to_dp_matrix_mt(value_matrix) + + def to_header_dp_list(self) -> list[DataProperty]: + self.__update_dp_converter() + + preprocessor = copy.deepcopy(self.__preprocessor) + preprocessor.strip_str = self.strip_str_header + + return self._to_dp_list( + self.headers, + type_hint=String, + preprocessor=preprocessor, + strict_level_map=MIN_STRICT_LEVEL_MAP, + ) + + def update_preprocessor(self, **kwargs: Any) -> bool: + is_updated = self.__preprocessor.update(**kwargs) + self.__update_dp_converter() + + return is_updated + + def update_strict_level_map(self, value: StrictLevelMap) -> bool: + org = copy.deepcopy(self.__strict_level_map) + self.__strict_level_map.update(value) + + if org == self.__strict_level_map: + return False + + self.__clear_cache() + + return True + + """ + def update_dict(self, lhs: Mapping, rhs: Mapping) -> bool: + is_updated = False + + for key, value in rhs.items(): + if key not in lhs: + lhs[] + continue + + if getattr(lhs, key) == value: + continue + + setattr(lhs, key, value) + is_updated = True + + return is_updated + """ + + @staticmethod + def __is_dp_matrix(value: Any) -> bool: + try: + return isinstance(value[0][0], DataProperty) + except (TypeError, IndexError): + return False + + def __get_col_type_hint(self, col_idx: int) -> TypeHint: + try: + return self.column_type_hints[col_idx] + except (TypeError, IndexError): + return self.default_type_hint + + def __get_format_flags(self, col_idx: int) -> int: + try: + return self.format_flags_list[col_idx] + except (TypeError, IndexError): + return self.__default_format_flags + + def __to_dp( + self, + data: Any, + type_hint: TypeHint = None, + preprocessor: Optional[Preprocessor] = None, + strict_level_map: Optional[StrictLevelMap] = None, + ) -> DataProperty: + for trans_func in self.__trans_func_list: + data = trans_func(data) + + if type_hint: + return self.__to_dp_raw( + data, + type_hint=type_hint, + preprocessor=preprocessor, + strict_level_map=strict_level_map, + ) + + try: + if data in self.__dp_cache_map: + return self.__dp_cache_map[data] + except TypeError: + # unhashable type + pass + + if data == 0: + if data is False: + return self.__dp_cache_false + return self.__dp_cache_zero + if data == 1: + if data is True: + return self.__dp_cache_true + return self.__dp_cache_one + + return self.__to_dp_raw( + data, type_hint=type_hint, preprocessor=preprocessor, strict_level_map=strict_level_map + ) + + def __to_dp_raw( + self, + data: Any, + type_hint: TypeHint = None, + preprocessor: Optional[Preprocessor] = None, + strict_level_map: Optional[StrictLevelMap] = None, + ) -> DataProperty: + if preprocessor: + preprocessor = Preprocessor( + dequote=preprocessor.dequote, + line_break_handling=preprocessor.line_break_handling, + line_break_repl=preprocessor.line_break_repl, + strip_str=preprocessor.strip_str, + is_escape_formula_injection=preprocessor.is_escape_formula_injection, + ) + else: + preprocessor = Preprocessor( + dequote=self.preprocessor.dequote, + line_break_handling=self.preprocessor.line_break_handling, + line_break_repl=self.preprocessor.line_break_repl, + strip_str=self.preprocessor.strip_str, + is_escape_formula_injection=self.__preprocessor.is_escape_formula_injection, + ) + + value_dp = DataProperty( + data, + preprocessor=preprocessor, + type_hint=(type_hint if type_hint is not None else self.default_type_hint), + float_type=self.float_type, + datetime_format_str=self.datetime_format_str, + strict_level_map=(strict_level_map if type_hint is not None else self.strict_level_map), + east_asian_ambiguous_width=self.east_asian_ambiguous_width, + ) + + return self.__dp_converter.convert(value_dp) + + def __to_dp_matrix_st(self, value_matrix: Sequence[Sequence[Any]]) -> DataPropertyMatrix: + return list( + zip( # type: ignore + *( + _to_dp_list_helper( + self, + col_idx, + values, + self.__get_col_type_hint(col_idx), + self.__preprocessor, + )[1] + for col_idx, values in enumerate(zip(*value_matrix)) + ) + ) + ) + + def __to_dp_matrix_mt(self, value_matrix: Sequence[Sequence[Any]]) -> DataPropertyMatrix: + from concurrent import futures + + col_data_map = {} + + with futures.ProcessPoolExecutor(self.max_workers) as executor: + future_list = [ + executor.submit( + _to_dp_list_helper, + self, + col_idx, + values, + self.__get_col_type_hint(col_idx), + self.__preprocessor, + ) + for col_idx, values in enumerate(zip(*value_matrix)) + ] + + for future in futures.as_completed(future_list): + col_idx, value_dp_list = future.result() + col_data_map[col_idx] = value_dp_list + + return list( + zip(*(col_data_map[col_idx] for col_idx in sorted(col_data_map))) # type: ignore + ) + + def _to_dp_list( + self, + data_list: Sequence[Any], + type_hint: TypeHint = None, + preprocessor: Optional[Preprocessor] = None, + strict_level_map: Optional[StrictLevelMap] = None, + ) -> list[DataProperty]: + if is_empty_sequence(data_list): + return [] + + type_counter: typing.Counter[type[AbstractType]] = Counter() + + dp_list = [] + for data in data_list: + expect_type_hint: TypeHint = type_hint + if type_hint is None: + try: + expect_type_hint, _count = type_counter.most_common(1)[0] + if not expect_type_hint( + data, float_type=self.float_type, strict_level=StrictLevel.MAX + ).is_type(): + expect_type_hint = None + except IndexError: + pass + + dataprop = self.__to_dp( + data=data, + type_hint=expect_type_hint, + preprocessor=preprocessor if preprocessor else self.__preprocessor, + strict_level_map=strict_level_map, + ) + type_counter[dataprop.type_class] += 1 + + dp_list.append(dataprop) + + return dp_list + + def __strip_data_matrix(self, data_matrix: Sequence[Sequence[Any]]) -> Sequence[Sequence[Any]]: + header_col_size = len(self.headers) if self.headers else 0 + try: + col_size_list = [len(data_list) for data_list in data_matrix] + except TypeError: + return [] + + if self.headers: + min_col_size = min([header_col_size] + col_size_list) + max_col_size = max([header_col_size] + col_size_list) + elif col_size_list: + min_col_size = min(col_size_list) + max_col_size = max(col_size_list) + else: + min_col_size = 0 + max_col_size = 0 + + if self.matrix_formatting == MatrixFormatting.EXCEPTION: + if min_col_size != max_col_size: + raise ValueError( + "nonuniform column size found: min={}, max={}".format( + min_col_size, max_col_size + ) + ) + + return data_matrix + + if self.matrix_formatting == MatrixFormatting.HEADER_ALIGNED: + if header_col_size > 0: + format_col_size = header_col_size + else: + format_col_size = max_col_size + elif self.matrix_formatting == MatrixFormatting.TRIM: + format_col_size = min_col_size + elif self.matrix_formatting == MatrixFormatting.FILL_NONE: + format_col_size = max_col_size + else: + raise ValueError(f"unknown matrix formatting: {self.matrix_formatting}") + + return [ + list(data_matrix[row_idx][:format_col_size]) + [None] * (format_col_size - col_size) + for row_idx, col_size in enumerate(col_size_list) + ] + + def __get_col_dp_list_base(self) -> list[ColumnDataProperty]: + header_dp_list = self.to_header_dp_list() + col_dp_list = [] + + for col_idx, header_dp in enumerate(header_dp_list): + col_dp = ColumnDataProperty( + column_index=col_idx, + float_type=self.float_type, + min_width=self.min_column_width, + format_flags=self.__get_format_flags(col_idx), + is_formatting_float=self.is_formatting_float, + datetime_format_str=self.datetime_format_str, + east_asian_ambiguous_width=self.east_asian_ambiguous_width, + max_precision=self.__max_precision, + ) + col_dp.update_header(header_dp) + col_dp_list.append(col_dp) + + return col_dp_list + + def __update_dp_converter(self) -> None: + preprocessor = Preprocessor( + line_break_handling=self.__preprocessor.line_break_handling, + line_break_repl=self.preprocessor.line_break_repl, + is_escape_html_tag=self.__preprocessor.is_escape_html_tag, + is_escape_formula_injection=self.__preprocessor.is_escape_formula_injection, + ) + self.__dp_converter = DataPropertyConverter( + preprocessor=preprocessor, + type_value_map=self.type_value_map, + quoting_flags=self.quoting_flags, + datetime_formatter=self.datetime_formatter, + datetime_format_str=self.datetime_format_str, + float_type=self.float_type, + strict_level_map=self.strict_level_map, + ) + + +def _to_dp_list_helper( + extractor: DataPropertyExtractor, + col_idx: int, + data_list: Sequence[Any], + type_hint: TypeHint, + preprocessor: Preprocessor, +) -> tuple[int, list[DataProperty]]: + return ( + col_idx, + extractor._to_dp_list(data_list, type_hint=type_hint, preprocessor=preprocessor), + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dataproperty/_formatter.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dataproperty/_formatter.py new file mode 100644 index 0000000000000000000000000000000000000000..088edded65da9d7a0f04adac7927ebb6cf72504d --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dataproperty/_formatter.py @@ -0,0 +1,98 @@ +import copy +from decimal import Decimal +from typing import Final, Optional, Union + +from typepy import Nan, Typecode + + +DecimalPlaces = Union[float, Decimal] + + +class Format: + NONE: Final = 0 + THOUSAND_SEPARATOR: Final = 1 + + +class Formatter: + __slots__ = ("__is_formatting_float", "__format_flags", "__datetime_format_str") + + _BLANK_CURLY_BRACES_FORMAT_MAP: Final[dict[Typecode, str]] = { + Typecode.NONE: "{}", + Typecode.IP_ADDRESS: "{}", + Typecode.BOOL: "{}", + Typecode.DICTIONARY: "{}", + Typecode.LIST: "{}", + } + + def __init__( + self, + datetime_format_str: str, + is_formatting_float: Optional[bool] = True, + format_flags: Optional[int] = None, + ) -> None: + if format_flags is not None: + self.__format_flags = format_flags + else: + self.__format_flags = Format.NONE + + self.__datetime_format_str = datetime_format_str + self.__is_formatting_float = is_formatting_float + + def make_format_map( + self, decimal_places: Optional[DecimalPlaces] = None + ) -> dict[Typecode, str]: + format_map = copy.copy(self._BLANK_CURLY_BRACES_FORMAT_MAP) + format_map.update( + { + Typecode.INTEGER: self.make_format_str(Typecode.INTEGER), + Typecode.REAL_NUMBER: self.make_format_str(Typecode.REAL_NUMBER, decimal_places), + Typecode.INFINITY: self.make_format_str(Typecode.INFINITY), + Typecode.NAN: self.make_format_str(Typecode.NAN), + Typecode.DATETIME: self.make_format_str(Typecode.DATETIME), + } + ) + + return format_map + + def make_format_str( + self, typecode: Typecode, decimal_places: Optional[DecimalPlaces] = None + ) -> str: + format_str = self._BLANK_CURLY_BRACES_FORMAT_MAP.get(typecode) + if format_str is not None: + return format_str + + if typecode == Typecode.INTEGER: + return self.__get_integer_format() + + if typecode in (Typecode.REAL_NUMBER, Typecode.INFINITY, Typecode.NAN): + return self.__get_realnumber_format(decimal_places) + + if typecode == Typecode.DATETIME: + return "{:" + self.__datetime_format_str + "}" + + return "{:s}" + + def __get_base_format_str(self) -> str: + if self.__format_flags & Format.THOUSAND_SEPARATOR: + return "," + + return "" + + def __get_integer_format(self) -> str: + return "{:" + self.__get_base_format_str() + "d}" + + def __get_realnumber_format(self, decimal_places: Optional[DecimalPlaces]) -> str: + if not self.__is_formatting_float: + return "{}" + + base_format = self.__get_base_format_str() + + if decimal_places is None or Nan(decimal_places).is_type(): + return "{:" + base_format + "f}" + + try: + return "{:" + f"{base_format:s}.{decimal_places:d}f" + "}" + except ValueError: + pass + + return "{:" + base_format + "f}" diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dataproperty/_function.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dataproperty/_function.py new file mode 100644 index 0000000000000000000000000000000000000000..40d3fa4d4138581c278c89194efef826784c7ea2 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dataproperty/_function.py @@ -0,0 +1,116 @@ +""" +.. codeauthor:: Tsuyoshi Hombashi +""" + +import decimal +import re +from decimal import Decimal +from typing import Any, Final, Optional, Union + +from typepy import Integer, RealNumber, TypeConversionError + + +_ansi_escape: Final = re.compile(r"(\x9b|\x1b\[)[0-?]*[ -\/]*[@-~]", re.IGNORECASE) + + +def get_integer_digit(value: Any) -> int: + float_type: Final = RealNumber(value) + + with decimal.localcontext() as ctx: + ctx.prec = 60 + ctx.rounding = decimal.ROUND_HALF_DOWN + + try: + abs_value = abs(float_type.convert()) + except TypeConversionError: + try: + abs_value = abs(Integer(value).convert()) + except TypeConversionError: + raise ValueError( + f"the value must be a number: value='{value}' type='{type(value)}'" + ) + + return len(str(abs_value)) + + if abs_value.is_zero(): + return 1 + + try: + return len(str(abs_value.quantize(Decimal("1."), rounding=decimal.ROUND_DOWN))) + except decimal.InvalidOperation: + return len(str(abs_value)) + + +class DigitCalculator: + REGEXP_COMMON_LOG: Final = re.compile(r"[\d\.]+[eE]\-\d+") + REGEXP_SPLIT: Final = re.compile(r"[eE]\-") + + def get_decimal_places(self, value: Union[str, float, int, Decimal]) -> int: + if Integer(value).is_type(): + return 0 + + float_digit_len = 0 + abs_value = abs(float(value)) + text_value = str(abs_value) + float_text = "0" + if text_value.find(".") != -1: + float_text = text_value.split(".")[1] + float_digit_len = len(float_text) + elif self.REGEXP_COMMON_LOG.search(text_value): + float_text = self.REGEXP_SPLIT.split(text_value)[1] + float_digit_len = int(float_text) + + return float_digit_len + + +_digit_calculator = DigitCalculator() + + +def get_number_of_digit( + value: Any, max_decimal_places: int = 99 +) -> tuple[Optional[int], Optional[int]]: + try: + integer_digits = get_integer_digit(value) + except (ValueError, TypeError, OverflowError): + return (None, None) + + try: + decimal_places: Optional[int] = min( + _digit_calculator.get_decimal_places(value), max_decimal_places + ) + except (ValueError, TypeError): + decimal_places = None + + return (integer_digits, decimal_places) + + +def _validate_eaaw(east_asian_ambiguous_width: int) -> None: + if east_asian_ambiguous_width in (1, 2): + return + + raise ValueError( + "invalid east_asian_ambiguous_width: expected=1 or 2, actual={}".format( + east_asian_ambiguous_width + ) + ) + + +def strip_ansi_escape(unicode_str: str) -> str: + return _ansi_escape.sub("", unicode_str) + + +def calc_ascii_char_width(unicode_str: str, east_asian_ambiguous_width: int = 1) -> int: + import unicodedata + + width = 0 + for char in unicode_str: + char_width = unicodedata.east_asian_width(char) + if char_width in "WF": + width += 2 + elif char_width == "A": + _validate_eaaw(east_asian_ambiguous_width) + width += east_asian_ambiguous_width + else: + width += 1 + + return width diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dataproperty/_interface.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dataproperty/_interface.py new file mode 100644 index 0000000000000000000000000000000000000000..bab43415753cf5029310fda35ad6cae661e9b153 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dataproperty/_interface.py @@ -0,0 +1,34 @@ +""" +.. codeauthor:: Tsuyoshi Hombashi +""" + +import abc +from typing import Optional + +from typepy import Typecode + +from ._align import Align + + +class DataPeropertyInterface(metaclass=abc.ABCMeta): + __slots__ = () + + @property + @abc.abstractmethod + def align(self) -> Align: # pragma: no cover + pass + + @property + @abc.abstractmethod + def decimal_places(self) -> Optional[int]: # pragma: no cover + pass + + @property + @abc.abstractmethod + def typecode(self) -> Typecode: # pragma: no cover + pass + + @property + @abc.abstractmethod + def typename(self) -> str: # pragma: no cover + pass diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dataproperty/_line_break.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dataproperty/_line_break.py new file mode 100644 index 0000000000000000000000000000000000000000..e98d3e830b8174a7f784067d28cc74ee0d92f8f9 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dataproperty/_line_break.py @@ -0,0 +1,8 @@ +from enum import Enum, unique + + +@unique +class LineBreakHandling(Enum): + NOP = 0 + REPLACE = 1 + ESCAPE = 2 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dataproperty/_preprocessor.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dataproperty/_preprocessor.py new file mode 100644 index 0000000000000000000000000000000000000000..784941375501f6dd8895c1eb3da19f75b83fa229 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dataproperty/_preprocessor.py @@ -0,0 +1,173 @@ +import html +import re +from typing import Any, Final, Optional, Union + +from mbstrdecoder import MultiByteStrDecoder + +from ._function import strip_ansi_escape +from ._line_break import LineBreakHandling + + +_RE_LINE_BREAK: Final = re.compile(r"\r\n|\n") +_RE_FORMULA_PREFIX: Final = re.compile(r"^[-\+=@]") + + +def normalize_lbh(value: Optional[LineBreakHandling]) -> LineBreakHandling: + if isinstance(value, LineBreakHandling): + return value + + if value is None: + return LineBreakHandling.NOP + + return LineBreakHandling[value.upper()] # type: ignore + + +class Preprocessor: + @property + def line_break_handling(self) -> Optional[LineBreakHandling]: + return self.__line_break_handling + + @line_break_handling.setter + def line_break_handling(self, value: Optional[LineBreakHandling]) -> None: + self.__line_break_handling = normalize_lbh(value) + + def __init__( + self, + strip_str: Optional[Union[str, bytes]] = None, + replace_tabs_with_spaces: bool = True, + tab_length: int = 2, + line_break_handling: Optional[LineBreakHandling] = None, + line_break_repl: str = " ", + dequote: bool = False, + is_escape_html_tag: bool = False, + is_escape_formula_injection: bool = False, + ) -> None: + self.strip_str = strip_str + self.replace_tabs_with_spaces = replace_tabs_with_spaces + self.tab_length = tab_length + self.line_break_handling = line_break_handling + self.line_break_repl = line_break_repl + self.dequote = dequote + self.is_escape_html_tag = is_escape_html_tag + self.is_escape_formula_injection = is_escape_formula_injection + + def __repr__(self) -> str: + return ", ".join( + [ + f"strip_str={self.strip_str!r}", + f"replace_tabs_with_spaces={self.replace_tabs_with_spaces}", + f"tab_length={self.tab_length}", + f"line_break_handling={self.line_break_handling}", + f"line_break_repl={self.line_break_repl}", + f"escape_html_tag={self.is_escape_html_tag}", + f"escape_formula_injection={self.is_escape_formula_injection}", + ] + ) + + def preprocess(self, data: Any) -> tuple: + data, no_ansi_escape_data = self.__preprocess_string( + self.__preprocess_data(data, self.strip_str), + ) + return (data, no_ansi_escape_data) + + def update(self, **kwargs: Any) -> bool: + is_updated = False + + for key, value in kwargs.items(): + if not hasattr(self, key): + continue + + if getattr(self, key) == value: + continue + + setattr(self, key, value) + is_updated = True + + return is_updated + + def __preprocess_string(self, raw_data: Any) -> tuple[Any, Optional[str]]: + data = raw_data + + if not isinstance(data, str): + return (data, None) + + if self.replace_tabs_with_spaces: + try: + data = data.replace("\t", " " * self.tab_length) + except (TypeError, AttributeError, ValueError): + pass + + if self.is_escape_html_tag: + try: + data = html.escape(data) + except AttributeError: + return (data, None) + + data = self.__process_line_break(data) + data = self.__escape_formula_injection(data) + data = self.__dequote(data) + + try: + return (data, strip_ansi_escape(data)) + except TypeError: + return (data, None) + + @staticmethod + def __preprocess_data(data: Any, strip_str: Optional[Union[str, bytes]]) -> Any: + if strip_str is None: + return data + + try: + return data.strip(strip_str) + except AttributeError: + return data + except UnicodeDecodeError: + return MultiByteStrDecoder(data).unicode_str.strip(str(strip_str)) + except TypeError: + # reach here when data and strip_str type are different + if isinstance(data, bytes): + return MultiByteStrDecoder(data).unicode_str.strip(str(strip_str)) + elif isinstance(strip_str, bytes): + return data.strip(MultiByteStrDecoder(strip_str).unicode_str) + + def __dequote(self, s: str) -> str: + if not self.dequote or not s: + return s + + try: + if (s[0] == s[-1]) and s.startswith(("'", '"')): + if s.count(s[0]) == 2: + return s[1:-1] + except TypeError: + pass + + return s + + def __process_line_break(self, data: str) -> str: + lbh = self.line_break_handling + + if lbh == LineBreakHandling.NOP: + return data + + try: + if lbh == LineBreakHandling.REPLACE: + return _RE_LINE_BREAK.sub(self.line_break_repl, data) + + if lbh == LineBreakHandling.ESCAPE: + return data.replace("\n", "\\n").replace("\r", "\\r") + except (TypeError, AttributeError): + return data + + raise ValueError(f"unexpected line_break_handling: {lbh}") + + def __escape_formula_injection(self, data: str) -> str: + if not self.is_escape_formula_injection: + return data + + try: + if _RE_FORMULA_PREFIX.search(data): + return "'" + data + except (TypeError, AttributeError): + return data + + return data diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dataproperty/py.typed b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dataproperty/py.typed new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dataproperty/typing.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dataproperty/typing.py new file mode 100644 index 0000000000000000000000000000000000000000..ff3ab01cdcd2ee06713dad8a2df5ac6aa81f0eb0 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dataproperty/typing.py @@ -0,0 +1,63 @@ +from collections.abc import Mapping +from datetime import datetime +from decimal import Decimal +from typing import Any, Callable, Final, Optional, Union + +from typepy import ( + Bool, + DateTime, + Dictionary, + Infinity, + Integer, + IpAddress, + List, + Nan, + NoneType, + NullString, + RealNumber, + String, + Typecode, +) +from typepy.type import AbstractType + + +TypeHint = Optional[type[AbstractType]] +TransFunc = Callable[[Any], Any] +DateTimeFormatter = Callable[[datetime], str] + +FloatType = Union[type[Decimal], type[float]] +StrictLevelMap = Mapping[Union[str, Typecode], int] +TypeValueMap = dict[Typecode, Union[float, str, Decimal, None]] + +_type_hint_map: Final = { + # high frequently used types + "int": Integer, + "float": RealNumber, + "realnumber": RealNumber, + "str": String, + # low frequently used types + "bool": Bool, + "datetime": DateTime, + "dict": Dictionary, + "inf": Infinity, + "ip": IpAddress, + "list": List, + "nan": Nan, + "none": NoneType, + "nullstr": NullString, +} + + +def normalize_type_hint(type_hint: Union[str, TypeHint]) -> TypeHint: + if not type_hint: + return None + + if not isinstance(type_hint, str): + return type_hint + + type_hint = type_hint.strip().casefold() + for key, value in _type_hint_map.items(): + if type_hint.startswith(key): + return value + + raise ValueError(f"unknown typehint: {type_hint}") diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill-0.3.8.dist-info/INSTALLER b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill-0.3.8.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill-0.3.8.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill-0.3.8.dist-info/LICENSE b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill-0.3.8.dist-info/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..1eb999e0babe28897c4544d034b36f5f0fe77ca6 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill-0.3.8.dist-info/LICENSE @@ -0,0 +1,35 @@ +Copyright (c) 2004-2016 California Institute of Technology. +Copyright (c) 2016-2024 The Uncertainty Quantification Foundation. +All rights reserved. + +This software is available subject to the conditions and terms laid +out below. By downloading and using this software you are agreeing +to the following conditions. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + - Neither the names of the copyright holders nor the names of any of + the contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; +OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill-0.3.8.dist-info/METADATA b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill-0.3.8.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..79509850401450817ad338cd856c35cf78c2b3e4 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill-0.3.8.dist-info/METADATA @@ -0,0 +1,280 @@ +Metadata-Version: 2.1 +Name: dill +Version: 0.3.8 +Summary: serialize all of Python +Home-page: https://github.com/uqfoundation/dill +Author: Mike McKerns +Author-email: mmckerns@uqfoundation.org +Maintainer: Mike McKerns +Maintainer-email: mmckerns@uqfoundation.org +License: BSD-3-Clause +Download-URL: https://pypi.org/project/dill/#files +Project-URL: Documentation, http://dill.rtfd.io +Project-URL: Source Code, https://github.com/uqfoundation/dill +Project-URL: Bug Tracker, https://github.com/uqfoundation/dill/issues +Platform: Linux +Platform: Windows +Platform: Mac +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: Intended Audience :: Science/Research +Classifier: License :: OSI Approved :: BSD License +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: Topic :: Scientific/Engineering +Classifier: Topic :: Software Development +Requires-Python: >=3.8 +Provides-Extra: graph +Requires-Dist: objgraph (>=1.7.2) ; extra == 'graph' +Provides-Extra: profile +Requires-Dist: gprof2dot (>=2022.7.29) ; extra == 'profile' +Provides-Extra: readline + +----------------------------- +dill: serialize all of Python +----------------------------- + +About Dill +========== + +``dill`` extends Python's ``pickle`` module for serializing and de-serializing +Python objects to the majority of the built-in Python types. Serialization +is the process of converting an object to a byte stream, and the inverse +of which is converting a byte stream back to a Python object hierarchy. + +``dill`` provides the user the same interface as the ``pickle`` module, and +also includes some additional features. In addition to pickling Python +objects, ``dill`` provides the ability to save the state of an interpreter +session in a single command. Hence, it would be feasible to save an +interpreter session, close the interpreter, ship the pickled file to +another computer, open a new interpreter, unpickle the session and +thus continue from the 'saved' state of the original interpreter +session. + +``dill`` can be used to store Python objects to a file, but the primary +usage is to send Python objects across the network as a byte stream. +``dill`` is quite flexible, and allows arbitrary user defined classes +and functions to be serialized. Thus ``dill`` is not intended to be +secure against erroneously or maliciously constructed data. It is +left to the user to decide whether the data they unpickle is from +a trustworthy source. + +``dill`` is part of ``pathos``, a Python framework for heterogeneous computing. +``dill`` is in active development, so any user feedback, bug reports, comments, +or suggestions are highly appreciated. A list of issues is located at +https://github.com/uqfoundation/dill/issues, with a legacy list maintained at +https://uqfoundation.github.io/project/pathos/query. + + +Major Features +============== + +``dill`` can pickle the following standard types: + + - none, type, bool, int, float, complex, bytes, str, + - tuple, list, dict, file, buffer, builtin, + - Python classes, namedtuples, dataclasses, metaclasses, + - instances of classes, + - set, frozenset, array, functions, exceptions + +``dill`` can also pickle more 'exotic' standard types: + + - functions with yields, nested functions, lambdas, + - cell, method, unboundmethod, module, code, methodwrapper, + - methoddescriptor, getsetdescriptor, memberdescriptor, wrapperdescriptor, + - dictproxy, slice, notimplemented, ellipsis, quit + +``dill`` cannot yet pickle these standard types: + + - frame, generator, traceback + +``dill`` also provides the capability to: + + - save and load Python interpreter sessions + - save and extract the source code from functions and classes + - interactively diagnose pickling errors + + +Current Release +=============== + +The latest released version of ``dill`` is available from: + + https://pypi.org/project/dill + +``dill`` is distributed under a 3-clause BSD license. + + +Development Version +=================== + +You can get the latest development version with all the shiny new features at: + + https://github.com/uqfoundation + +If you have a new contribution, please submit a pull request. + + +Installation +============ + +``dill`` can be installed with ``pip``:: + + $ pip install dill + +To optionally include the ``objgraph`` diagnostic tool in the install:: + + $ pip install dill[graph] + +To optionally include the ``gprof2dot`` diagnostic tool in the install:: + + $ pip install dill[profile] + +For windows users, to optionally install session history tools:: + + $ pip install dill[readline] + + +Requirements +============ + +``dill`` requires: + + - ``python`` (or ``pypy``), **>=3.8** + - ``setuptools``, **>=42** + +Optional requirements: + + - ``objgraph``, **>=1.7.2** + - ``gprof2dot``, **>=2022.7.29** + - ``pyreadline``, **>=1.7.1** (on windows) + + +Basic Usage +=========== + +``dill`` is a drop-in replacement for ``pickle``. Existing code can be +updated to allow complete pickling using:: + + >>> import dill as pickle + +or:: + + >>> from dill import dumps, loads + +``dumps`` converts the object to a unique byte string, and ``loads`` performs +the inverse operation:: + + >>> squared = lambda x: x**2 + >>> loads(dumps(squared))(3) + 9 + +There are a number of options to control serialization which are provided +as keyword arguments to several ``dill`` functions: + +* with *protocol*, the pickle protocol level can be set. This uses the + same value as the ``pickle`` module, *DEFAULT_PROTOCOL*. +* with *byref=True*, ``dill`` to behave a lot more like pickle with + certain objects (like modules) pickled by reference as opposed to + attempting to pickle the object itself. +* with *recurse=True*, objects referred to in the global dictionary are + recursively traced and pickled, instead of the default behavior of + attempting to store the entire global dictionary. +* with *fmode*, the contents of the file can be pickled along with the file + handle, which is useful if the object is being sent over the wire to a + remote system which does not have the original file on disk. Options are + *HANDLE_FMODE* for just the handle, *CONTENTS_FMODE* for the file content + and *FILE_FMODE* for content and handle. +* with *ignore=False*, objects reconstructed with types defined in the + top-level script environment use the existing type in the environment + rather than a possibly different reconstructed type. + +The default serialization can also be set globally in *dill.settings*. +Thus, we can modify how ``dill`` handles references to the global dictionary +locally or globally:: + + >>> import dill.settings + >>> dumps(absolute) == dumps(absolute, recurse=True) + False + >>> dill.settings['recurse'] = True + >>> dumps(absolute) == dumps(absolute, recurse=True) + True + +``dill`` also includes source code inspection, as an alternate to pickling:: + + >>> import dill.source + >>> print(dill.source.getsource(squared)) + squared = lambda x:x**2 + +To aid in debugging pickling issues, use *dill.detect* which provides +tools like pickle tracing:: + + >>> import dill.detect + >>> with dill.detect.trace(): + >>> dumps(squared) + ┬ F1: at 0x7fe074f8c280> + ├┬ F2: + │└ # F2 [34 B] + ├┬ Co: at 0x7fe07501eb30, file "", line 1> + │├┬ F2: + ││└ # F2 [19 B] + │└ # Co [87 B] + ├┬ D1: + │└ # D1 [22 B] + ├┬ D2: + │└ # D2 [2 B] + ├┬ D2: + │├┬ D2: + ││└ # D2 [2 B] + │└ # D2 [23 B] + └ # F1 [180 B] + +With trace, we see how ``dill`` stored the lambda (``F1``) by first storing +``_create_function``, the underlying code object (``Co``) and ``_create_code`` +(which is used to handle code objects), then we handle the reference to +the global dict (``D2``) plus other dictionaries (``D1`` and ``D2``) that +save the lambda object's state. A ``#`` marks when the object is actually stored. + + +More Information +================ + +Probably the best way to get started is to look at the documentation at +http://dill.rtfd.io. Also see ``dill.tests`` for a set of scripts that +demonstrate how ``dill`` can serialize different Python objects. You can +run the test suite with ``python -m dill.tests``. The contents of any +pickle file can be examined with ``undill``. As ``dill`` conforms to +the ``pickle`` interface, the examples and documentation found at +http://docs.python.org/library/pickle.html also apply to ``dill`` +if one will ``import dill as pickle``. The source code is also generally +well documented, so further questions may be resolved by inspecting the +code itself. Please feel free to submit a ticket on github, or ask a +question on stackoverflow (**@Mike McKerns**). +If you would like to share how you use ``dill`` in your work, please send +an email (to **mmckerns at uqfoundation dot org**). + + +Citation +======== + +If you use ``dill`` to do research that leads to publication, we ask that you +acknowledge use of ``dill`` by citing the following in your publication:: + + M.M. McKerns, L. Strand, T. Sullivan, A. Fang, M.A.G. Aivazis, + "Building a framework for predictive science", Proceedings of + the 10th Python in Science Conference, 2011; + http://arxiv.org/pdf/1202.1056 + + Michael McKerns and Michael Aivazis, + "pathos: a framework for heterogeneous computing", 2010- ; + https://uqfoundation.github.io/project/pathos + +Please see https://uqfoundation.github.io/project/pathos or +http://arxiv.org/pdf/1202.1056 for further information. + diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill-0.3.8.dist-info/RECORD b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill-0.3.8.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..04b058e2534dc4ddd5398b6d08d133aea130d091 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill-0.3.8.dist-info/RECORD @@ -0,0 +1,97 @@ +../../../bin/get_gprof,sha256=nW1i0ag0wM2lcIDa5fYOwpojD6X0W7bB9GRFI2tjmdY,2519 +../../../bin/get_objgraph,sha256=7Wmp6PX4aKI2yo8mM1o-elUEnSwjR1Rr38Q05p-x__c,1713 +../../../bin/undill,sha256=Hy4HhuvBvu6TH6sm6o7Fih70V16kmkN95G9ELONz7Fk,649 +dill-0.3.8.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +dill-0.3.8.dist-info/LICENSE,sha256=UeiKI-eId86r1yfCGcel4z9l2pugOsT9KFupBKoc4is,1790 +dill-0.3.8.dist-info/METADATA,sha256=UxkSs2cU8JyrJsV5kS0QR9crJ07hrUJS2RiIMQaC4ss,10106 +dill-0.3.8.dist-info/RECORD,, +dill-0.3.8.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92 +dill-0.3.8.dist-info/top_level.txt,sha256=HLSIyYIjQzJiBvs3_-16ntezE3j6mWGTW0DT1xDd7X0,5 +dill/__diff.py,sha256=kirMxzB7E8lfjo21M5oIf7if95ny0aWhYB790KMpN08,7143 +dill/__info__.py,sha256=Kmel_yLTyH-hwNC5cVfzN-LV08AbS_AvSa2uwMeIQdk,10756 +dill/__init__.py,sha256=j-Jxl3H6bxatS0h2f8ywWs7DChwk7B9ozuZQBVcjYGU,3798 +dill/__pycache__/__diff.cpython-312.pyc,, +dill/__pycache__/__info__.cpython-312.pyc,, +dill/__pycache__/__init__.cpython-312.pyc,, +dill/__pycache__/_dill.cpython-312.pyc,, +dill/__pycache__/_objects.cpython-312.pyc,, +dill/__pycache__/_shims.cpython-312.pyc,, +dill/__pycache__/detect.cpython-312.pyc,, +dill/__pycache__/logger.cpython-312.pyc,, +dill/__pycache__/objtypes.cpython-312.pyc,, +dill/__pycache__/pointers.cpython-312.pyc,, +dill/__pycache__/session.cpython-312.pyc,, +dill/__pycache__/settings.cpython-312.pyc,, +dill/__pycache__/source.cpython-312.pyc,, +dill/__pycache__/temp.cpython-312.pyc,, +dill/_dill.py,sha256=3Eo6gKj1sODJjgPgYNT8TU-YL6QNQ7rIeWPUVnRzyqQ,88548 +dill/_objects.py,sha256=dPlUXzQIh8CA0fMy9NMbwwLGUPmXe5H8MdQtRWB1b_M,19605 +dill/_shims.py,sha256=IuzQcyPET5VWmWMoSGStieoedvNXlb5suDpa4bykTbQ,6635 +dill/detect.py,sha256=Mb-PfCxn1mg0l3TmHXyPNVEc4n3fuxc_nue6eL3-q_o,11114 +dill/logger.py,sha256=YS5ZloAOKjJRZaOBRCaMUDWmWVQZcicvbXVSrz8L8XU,11134 +dill/objtypes.py,sha256=BamGH3BEM6lLlxisuvXcGjsCRLNeoLs4_rFZrM5r2yM,736 +dill/pointers.py,sha256=vnQzjwGtKMGnmbdYRXRWNLMyceNPSw4f7UpvwCXLYbE,4467 +dill/session.py,sha256=NvCWpoP9r_rGBL2pOwwxOri8mFly5KlIWG3GwkBFnc0,23525 +dill/settings.py,sha256=7I3yvSpPKstOqpoW2gv3X77kXK-hZlqCnF7nJUGhxTY,630 +dill/source.py,sha256=DWfIxcBjpjbbKYz2DstV9kRdjajBdZLOcLXfsZsPo9U,45121 +dill/temp.py,sha256=KJUry4t0UjQCh5t4LXcxNyMF_uOGHwcjTuNYTJD9qdA,8027 +dill/tests/__init__.py,sha256=Gx-chVB-l-e7ncsGp2zF4BimTjbUyO7BY7RkrO835vY,479 +dill/tests/__main__.py,sha256=fHhioQwcOvTPlf1RM_wVQ0Y3ndETWJOuXJQ2rVtqliA,899 +dill/tests/__pycache__/__init__.cpython-312.pyc,, +dill/tests/__pycache__/__main__.cpython-312.pyc,, +dill/tests/__pycache__/test_abc.cpython-312.pyc,, +dill/tests/__pycache__/test_check.cpython-312.pyc,, +dill/tests/__pycache__/test_classdef.cpython-312.pyc,, +dill/tests/__pycache__/test_dataclasses.cpython-312.pyc,, +dill/tests/__pycache__/test_detect.cpython-312.pyc,, +dill/tests/__pycache__/test_dictviews.cpython-312.pyc,, +dill/tests/__pycache__/test_diff.cpython-312.pyc,, +dill/tests/__pycache__/test_extendpickle.cpython-312.pyc,, +dill/tests/__pycache__/test_fglobals.cpython-312.pyc,, +dill/tests/__pycache__/test_file.cpython-312.pyc,, +dill/tests/__pycache__/test_functions.cpython-312.pyc,, +dill/tests/__pycache__/test_functors.cpython-312.pyc,, +dill/tests/__pycache__/test_logger.cpython-312.pyc,, +dill/tests/__pycache__/test_mixins.cpython-312.pyc,, +dill/tests/__pycache__/test_module.cpython-312.pyc,, +dill/tests/__pycache__/test_moduledict.cpython-312.pyc,, +dill/tests/__pycache__/test_nested.cpython-312.pyc,, +dill/tests/__pycache__/test_objects.cpython-312.pyc,, +dill/tests/__pycache__/test_properties.cpython-312.pyc,, +dill/tests/__pycache__/test_pycapsule.cpython-312.pyc,, +dill/tests/__pycache__/test_recursive.cpython-312.pyc,, +dill/tests/__pycache__/test_registered.cpython-312.pyc,, +dill/tests/__pycache__/test_restricted.cpython-312.pyc,, +dill/tests/__pycache__/test_selected.cpython-312.pyc,, +dill/tests/__pycache__/test_session.cpython-312.pyc,, +dill/tests/__pycache__/test_source.cpython-312.pyc,, +dill/tests/__pycache__/test_temp.cpython-312.pyc,, +dill/tests/__pycache__/test_weakref.cpython-312.pyc,, +dill/tests/test_abc.py,sha256=BSjSKKCQ5_iPfFxAd0yBq4KSAJxelrlC3IzoAhjd1C4,4227 +dill/tests/test_check.py,sha256=4F5gkX6zxY7C5sD2_0Tkqf3T3jmQl0K15FOxYUTZQl0,1396 +dill/tests/test_classdef.py,sha256=fI3fVk4SlsjNMMs5RfU6DUCaxpP7YYRjvLZ2nhXMHuc,8600 +dill/tests/test_dataclasses.py,sha256=yKjFuG24ymLtjk-sZZdhvNY7aDqerTDpMcfi_eV4ft0,890 +dill/tests/test_detect.py,sha256=sE9THufHXCDysBPQ4QkN5DHn6DaIldVRAEciseIRH08,4083 +dill/tests/test_dictviews.py,sha256=Jhol0cQWPwoQrp7OPxGhU8FNRX2GgfFp9fTahCvQEPA,1337 +dill/tests/test_diff.py,sha256=5VIWf2fpV6auLHNfzkHLTrgx6AJBlE2xe5Wanfmq8TM,2667 +dill/tests/test_extendpickle.py,sha256=gONrMBHO94Edhnqm1wo49hgzwmaxHs7L-86Hs-7albY,1315 +dill/tests/test_fglobals.py,sha256=DCvdojmKcLN_X9vX4Qe1FbsqjeoJK-wsY2uJwBfNFro,1676 +dill/tests/test_file.py,sha256=jUU2h8qaDOIe1mn_Ng7wqCZcd7Ucx3TAaI-K_90_Tbk,13578 +dill/tests/test_functions.py,sha256=-mqTpUbzRu8GynjBGD25dRDm8qInIe07sRZmCcA_iXY,4267 +dill/tests/test_functors.py,sha256=7rx9wLmrgFwF0gUm_-SGOISPYSok0XjmrQ-jFMRt6gs,930 +dill/tests/test_logger.py,sha256=D9zGRaA-CEadG13orPS_D4gPVZlkqXf9Zu8wn2oMiYc,2385 +dill/tests/test_mixins.py,sha256=YtB24BjodooLj85ijFbAxiM7LlFQZAUL8RQVx9vIAwY,4007 +dill/tests/test_module.py,sha256=KLl_gZJJqDY7S_bD5wCqKL8JQCS0MDMoipVQSDfASlo,1943 +dill/tests/test_moduledict.py,sha256=faXG6-5AcmCfP3xe2FYGOUdSosU-9TWnKU_ZVqPDaxY,1182 +dill/tests/test_nested.py,sha256=ViWiOrChLZktS0z6qyKqMxDdTuy9kAX4qMgH_OreMcc,3146 +dill/tests/test_objects.py,sha256=pPAth0toC_UWztuKHC7NZlsRBb0g_gSAt70UbUtXEXo,1931 +dill/tests/test_properties.py,sha256=h35c-lYir1JG6oLPtrA0eYE0xoSohIimsA3yIfRw6yA,1346 +dill/tests/test_pycapsule.py,sha256=EXFyB6g1Wx9O9LM6StIeUKhrhln4_hou1xrtGwkt4Cw,1417 +dill/tests/test_recursive.py,sha256=bfr-BsK1Xu0PU7l2srHsDXdY2l1LeM3L3w7NraXO0cc,4182 +dill/tests/test_registered.py,sha256=J3oku053VfdJgYh4Z5_kyFRf-C52JglIzjcyxEaYOhk,1573 +dill/tests/test_restricted.py,sha256=xLMIae8sYJksAj9hKKyHFHIL8vtbGpFeOULz59snYM4,783 +dill/tests/test_selected.py,sha256=Hp-AAd6Qp5FJZ-vY_Bbejo5Rg6xFstec5QkSg5D7Aac,3218 +dill/tests/test_session.py,sha256=KoSPvs4c4VJ8mFMF7EUlD_3GwcOhhipt9fqHr--Go-4,10161 +dill/tests/test_source.py,sha256=wZTYBbpzUwj3Mz5OjrHQKfskaVVwuy2UQDg5p2wLbT4,6036 +dill/tests/test_temp.py,sha256=F_7nJkSetLIBSAYMw1-hYh03iVrEYwGs-4GIUzoBOfY,2619 +dill/tests/test_weakref.py,sha256=mrjZP5aPtUP1wBD6ibPsDsfI9ffmq_Ykt7ltoodi5Lg,1602 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill-0.3.8.dist-info/WHEEL b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill-0.3.8.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..becc9a66ea739ba941d48a749e248761cc6e658a --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill-0.3.8.dist-info/WHEEL @@ -0,0 +1,5 @@ +Wheel-Version: 1.0 +Generator: bdist_wheel (0.37.1) +Root-Is-Purelib: true +Tag: py3-none-any + diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill-0.3.8.dist-info/top_level.txt b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill-0.3.8.dist-info/top_level.txt new file mode 100644 index 0000000000000000000000000000000000000000..85eea7018a40c657c08ef73fcf3a39024b2df2cb --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill-0.3.8.dist-info/top_level.txt @@ -0,0 +1 @@ +dill diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill/__diff.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill/__diff.py new file mode 100644 index 0000000000000000000000000000000000000000..8a8937534fac7f4791a2883cbc65eee9d45b56e5 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill/__diff.py @@ -0,0 +1,234 @@ +#!/usr/bin/env python +# +# Author: Mike McKerns (mmckerns @caltech and @uqfoundation) +# Copyright (c) 2008-2016 California Institute of Technology. +# Copyright (c) 2016-2024 The Uncertainty Quantification Foundation. +# License: 3-clause BSD. The full license text is available at: +# - https://github.com/uqfoundation/dill/blob/master/LICENSE + +""" +Module to show if an object has changed since it was memorised +""" + +import builtins +import os +import sys +import types +try: + import numpy + HAS_NUMPY = True +except ImportError: + HAS_NUMPY = False + +# pypy doesn't use reference counting +getrefcount = getattr(sys, 'getrefcount', lambda x:0) + +# memo of objects indexed by id to a tuple (attributes, sequence items) +# attributes is a dict indexed by attribute name to attribute id +# sequence items is either a list of ids, of a dictionary of keys to ids +memo = {} +id_to_obj = {} +# types that cannot have changing attributes +builtins_types = set((str, list, dict, set, frozenset, int)) +dont_memo = set(id(i) for i in (memo, sys.modules, sys.path_importer_cache, + os.environ, id_to_obj)) + + +def get_attrs(obj): + """ + Gets all the attributes of an object though its __dict__ or return None + """ + if type(obj) in builtins_types \ + or type(obj) is type and obj in builtins_types: + return + return getattr(obj, '__dict__', None) + + +def get_seq(obj, cache={str: False, frozenset: False, list: True, set: True, + dict: True, tuple: True, type: False, + types.ModuleType: False, types.FunctionType: False, + types.BuiltinFunctionType: False}): + """ + Gets all the items in a sequence or return None + """ + try: + o_type = obj.__class__ + except AttributeError: + o_type = type(obj) + hsattr = hasattr + if o_type in cache: + if cache[o_type]: + if hsattr(obj, "copy"): + return obj.copy() + return obj + elif HAS_NUMPY and o_type in (numpy.ndarray, numpy.ma.core.MaskedConstant): + if obj.shape and obj.size: + return obj + else: + return [] + elif hsattr(obj, "__contains__") and hsattr(obj, "__iter__") \ + and hsattr(obj, "__len__") and hsattr(o_type, "__contains__") \ + and hsattr(o_type, "__iter__") and hsattr(o_type, "__len__"): + cache[o_type] = True + if hsattr(obj, "copy"): + return obj.copy() + return obj + else: + cache[o_type] = False + return None + + +def memorise(obj, force=False): + """ + Adds an object to the memo, and recursively adds all the objects + attributes, and if it is a container, its items. Use force=True to update + an object already in the memo. Updating is not recursively done. + """ + obj_id = id(obj) + if obj_id in memo and not force or obj_id in dont_memo: + return + id_ = id + g = get_attrs(obj) + if g is None: + attrs_id = None + else: + attrs_id = dict((key,id_(value)) for key, value in g.items()) + + s = get_seq(obj) + if s is None: + seq_id = None + elif hasattr(s, "items"): + seq_id = dict((id_(key),id_(value)) for key, value in s.items()) + elif not hasattr(s, "__len__"): #XXX: avoid TypeError from unexpected case + seq_id = None + else: + seq_id = [id_(i) for i in s] + + memo[obj_id] = attrs_id, seq_id + id_to_obj[obj_id] = obj + mem = memorise + if g is not None: + [mem(value) for key, value in g.items()] + + if s is not None: + if hasattr(s, "items"): + [(mem(key), mem(item)) + for key, item in s.items()] + else: + if hasattr(s, '__len__'): + [mem(item) for item in s] + else: mem(s) + + +def release_gone(): + itop, mp, src = id_to_obj.pop, memo.pop, getrefcount + [(itop(id_), mp(id_)) for id_, obj in list(id_to_obj.items()) + if src(obj) < 4] #XXX: correct for pypy? + + +def whats_changed(obj, seen=None, simple=False, first=True): + """ + Check an object against the memo. Returns a list in the form + (attribute changes, container changed). Attribute changes is a dict of + attribute name to attribute value. container changed is a boolean. + If simple is true, just returns a boolean. None for either item means + that it has not been checked yet + """ + # Special cases + if first: + # ignore the _ variable, which only appears in interactive sessions + if "_" in builtins.__dict__: + del builtins._ + if seen is None: + seen = {} + + obj_id = id(obj) + + if obj_id in seen: + if simple: + return any(seen[obj_id]) + return seen[obj_id] + + # Safety checks + if obj_id in dont_memo: + seen[obj_id] = [{}, False] + if simple: + return False + return seen[obj_id] + elif obj_id not in memo: + if simple: + return True + else: + raise RuntimeError("Object not memorised " + str(obj)) + + seen[obj_id] = ({}, False) + + chngd = whats_changed + id_ = id + + # compare attributes + attrs = get_attrs(obj) + if attrs is None: + changed = {} + else: + obj_attrs = memo[obj_id][0] + obj_get = obj_attrs.get + changed = dict((key,None) for key in obj_attrs if key not in attrs) + for key, o in attrs.items(): + if id_(o) != obj_get(key, None) or chngd(o, seen, True, False): + changed[key] = o + + # compare sequence + items = get_seq(obj) + seq_diff = False + if (items is not None) and (hasattr(items, '__len__')): + obj_seq = memo[obj_id][1] + if (len(items) != len(obj_seq)): + seq_diff = True + elif hasattr(obj, "items"): # dict type obj + obj_get = obj_seq.get + for key, item in items.items(): + if id_(item) != obj_get(id_(key)) \ + or chngd(key, seen, True, False) \ + or chngd(item, seen, True, False): + seq_diff = True + break + else: + for i, j in zip(items, obj_seq): # list type obj + if id_(i) != j or chngd(i, seen, True, False): + seq_diff = True + break + seen[obj_id] = changed, seq_diff + if simple: + return changed or seq_diff + return changed, seq_diff + + +def has_changed(*args, **kwds): + kwds['simple'] = True # ignore simple if passed in + return whats_changed(*args, **kwds) + +__import__ = __import__ + + +def _imp(*args, **kwds): + """ + Replaces the default __import__, to allow a module to be memorised + before the user can change it + """ + before = set(sys.modules.keys()) + mod = __import__(*args, **kwds) + after = set(sys.modules.keys()).difference(before) + for m in after: + memorise(sys.modules[m]) + return mod + +builtins.__import__ = _imp +if hasattr(builtins, "_"): + del builtins._ + +# memorise all already imported modules. This implies that this must be +# imported first for any changes to be recorded +for mod in list(sys.modules.values()): + memorise(mod) +release_gone() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill/__info__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill/__info__.py new file mode 100644 index 0000000000000000000000000000000000000000..a4592c4eff9961d8b9a6c14af7f1cfa45bc4d9ab --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill/__info__.py @@ -0,0 +1,291 @@ +#!/usr/bin/env python +# +# Author: Mike McKerns (mmckerns @caltech and @uqfoundation) +# Copyright (c) 2024 The Uncertainty Quantification Foundation. +# License: 3-clause BSD. The full license text is available at: +# - https://github.com/uqfoundation/dill/blob/master/LICENSE +''' +----------------------------- +dill: serialize all of Python +----------------------------- + +About Dill +========== + +``dill`` extends Python's ``pickle`` module for serializing and de-serializing +Python objects to the majority of the built-in Python types. Serialization +is the process of converting an object to a byte stream, and the inverse +of which is converting a byte stream back to a Python object hierarchy. + +``dill`` provides the user the same interface as the ``pickle`` module, and +also includes some additional features. In addition to pickling Python +objects, ``dill`` provides the ability to save the state of an interpreter +session in a single command. Hence, it would be feasible to save an +interpreter session, close the interpreter, ship the pickled file to +another computer, open a new interpreter, unpickle the session and +thus continue from the 'saved' state of the original interpreter +session. + +``dill`` can be used to store Python objects to a file, but the primary +usage is to send Python objects across the network as a byte stream. +``dill`` is quite flexible, and allows arbitrary user defined classes +and functions to be serialized. Thus ``dill`` is not intended to be +secure against erroneously or maliciously constructed data. It is +left to the user to decide whether the data they unpickle is from +a trustworthy source. + +``dill`` is part of ``pathos``, a Python framework for heterogeneous computing. +``dill`` is in active development, so any user feedback, bug reports, comments, +or suggestions are highly appreciated. A list of issues is located at +https://github.com/uqfoundation/dill/issues, with a legacy list maintained at +https://uqfoundation.github.io/project/pathos/query. + + +Major Features +============== + +``dill`` can pickle the following standard types: + + - none, type, bool, int, float, complex, bytes, str, + - tuple, list, dict, file, buffer, builtin, + - Python classes, namedtuples, dataclasses, metaclasses, + - instances of classes, + - set, frozenset, array, functions, exceptions + +``dill`` can also pickle more 'exotic' standard types: + + - functions with yields, nested functions, lambdas, + - cell, method, unboundmethod, module, code, methodwrapper, + - methoddescriptor, getsetdescriptor, memberdescriptor, wrapperdescriptor, + - dictproxy, slice, notimplemented, ellipsis, quit + +``dill`` cannot yet pickle these standard types: + + - frame, generator, traceback + +``dill`` also provides the capability to: + + - save and load Python interpreter sessions + - save and extract the source code from functions and classes + - interactively diagnose pickling errors + + +Current Release +=============== + +The latest released version of ``dill`` is available from: + + https://pypi.org/project/dill + +``dill`` is distributed under a 3-clause BSD license. + + +Development Version +=================== + +You can get the latest development version with all the shiny new features at: + + https://github.com/uqfoundation + +If you have a new contribution, please submit a pull request. + + +Installation +============ + +``dill`` can be installed with ``pip``:: + + $ pip install dill + +To optionally include the ``objgraph`` diagnostic tool in the install:: + + $ pip install dill[graph] + +To optionally include the ``gprof2dot`` diagnostic tool in the install:: + + $ pip install dill[profile] + +For windows users, to optionally install session history tools:: + + $ pip install dill[readline] + + +Requirements +============ + +``dill`` requires: + + - ``python`` (or ``pypy``), **>=3.8** + - ``setuptools``, **>=42** + +Optional requirements: + + - ``objgraph``, **>=1.7.2** + - ``gprof2dot``, **>=2022.7.29** + - ``pyreadline``, **>=1.7.1** (on windows) + + +Basic Usage +=========== + +``dill`` is a drop-in replacement for ``pickle``. Existing code can be +updated to allow complete pickling using:: + + >>> import dill as pickle + +or:: + + >>> from dill import dumps, loads + +``dumps`` converts the object to a unique byte string, and ``loads`` performs +the inverse operation:: + + >>> squared = lambda x: x**2 + >>> loads(dumps(squared))(3) + 9 + +There are a number of options to control serialization which are provided +as keyword arguments to several ``dill`` functions: + +* with *protocol*, the pickle protocol level can be set. This uses the + same value as the ``pickle`` module, *DEFAULT_PROTOCOL*. +* with *byref=True*, ``dill`` to behave a lot more like pickle with + certain objects (like modules) pickled by reference as opposed to + attempting to pickle the object itself. +* with *recurse=True*, objects referred to in the global dictionary are + recursively traced and pickled, instead of the default behavior of + attempting to store the entire global dictionary. +* with *fmode*, the contents of the file can be pickled along with the file + handle, which is useful if the object is being sent over the wire to a + remote system which does not have the original file on disk. Options are + *HANDLE_FMODE* for just the handle, *CONTENTS_FMODE* for the file content + and *FILE_FMODE* for content and handle. +* with *ignore=False*, objects reconstructed with types defined in the + top-level script environment use the existing type in the environment + rather than a possibly different reconstructed type. + +The default serialization can also be set globally in *dill.settings*. +Thus, we can modify how ``dill`` handles references to the global dictionary +locally or globally:: + + >>> import dill.settings + >>> dumps(absolute) == dumps(absolute, recurse=True) + False + >>> dill.settings['recurse'] = True + >>> dumps(absolute) == dumps(absolute, recurse=True) + True + +``dill`` also includes source code inspection, as an alternate to pickling:: + + >>> import dill.source + >>> print(dill.source.getsource(squared)) + squared = lambda x:x**2 + +To aid in debugging pickling issues, use *dill.detect* which provides +tools like pickle tracing:: + + >>> import dill.detect + >>> with dill.detect.trace(): + >>> dumps(squared) + ┬ F1: at 0x7fe074f8c280> + ├┬ F2: + │└ # F2 [34 B] + ├┬ Co: at 0x7fe07501eb30, file "", line 1> + │├┬ F2: + ││└ # F2 [19 B] + │└ # Co [87 B] + ├┬ D1: + │└ # D1 [22 B] + ├┬ D2: + │└ # D2 [2 B] + ├┬ D2: + │├┬ D2: + ││└ # D2 [2 B] + │└ # D2 [23 B] + └ # F1 [180 B] + +With trace, we see how ``dill`` stored the lambda (``F1``) by first storing +``_create_function``, the underlying code object (``Co``) and ``_create_code`` +(which is used to handle code objects), then we handle the reference to +the global dict (``D2``) plus other dictionaries (``D1`` and ``D2``) that +save the lambda object's state. A ``#`` marks when the object is actually stored. + + +More Information +================ + +Probably the best way to get started is to look at the documentation at +http://dill.rtfd.io. Also see ``dill.tests`` for a set of scripts that +demonstrate how ``dill`` can serialize different Python objects. You can +run the test suite with ``python -m dill.tests``. The contents of any +pickle file can be examined with ``undill``. As ``dill`` conforms to +the ``pickle`` interface, the examples and documentation found at +http://docs.python.org/library/pickle.html also apply to ``dill`` +if one will ``import dill as pickle``. The source code is also generally +well documented, so further questions may be resolved by inspecting the +code itself. Please feel free to submit a ticket on github, or ask a +question on stackoverflow (**@Mike McKerns**). +If you would like to share how you use ``dill`` in your work, please send +an email (to **mmckerns at uqfoundation dot org**). + + +Citation +======== + +If you use ``dill`` to do research that leads to publication, we ask that you +acknowledge use of ``dill`` by citing the following in your publication:: + + M.M. McKerns, L. Strand, T. Sullivan, A. Fang, M.A.G. Aivazis, + "Building a framework for predictive science", Proceedings of + the 10th Python in Science Conference, 2011; + http://arxiv.org/pdf/1202.1056 + + Michael McKerns and Michael Aivazis, + "pathos: a framework for heterogeneous computing", 2010- ; + https://uqfoundation.github.io/project/pathos + +Please see https://uqfoundation.github.io/project/pathos or +http://arxiv.org/pdf/1202.1056 for further information. + +''' + +__version__ = '0.3.8' +__author__ = 'Mike McKerns' + +__license__ = ''' +Copyright (c) 2004-2016 California Institute of Technology. +Copyright (c) 2016-2024 The Uncertainty Quantification Foundation. +All rights reserved. + +This software is available subject to the conditions and terms laid +out below. By downloading and using this software you are agreeing +to the following conditions. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + - Neither the names of the copyright holders nor the names of any of + the contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; +OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +''' diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..549048a438d42547473768f914d13dba3a4a7b71 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill/__init__.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python +# +# Author: Mike McKerns (mmckerns @caltech and @uqfoundation) +# Copyright (c) 2008-2016 California Institute of Technology. +# Copyright (c) 2016-2024 The Uncertainty Quantification Foundation. +# License: 3-clause BSD. The full license text is available at: +# - https://github.com/uqfoundation/dill/blob/master/LICENSE + +# author, version, license, and long description +try: # the package is installed + from .__info__ import __version__, __author__, __doc__, __license__ +except: # pragma: no cover + import os + import sys + parent = os.path.dirname(os.path.abspath(os.path.dirname(__file__))) + sys.path.append(parent) + # get distribution meta info + from version import (__version__, __author__, + get_license_text, get_readme_as_rst) + __license__ = get_license_text(os.path.join(parent, 'LICENSE')) + __license__ = "\n%s" % __license__ + __doc__ = get_readme_as_rst(os.path.join(parent, 'README.md')) + del os, sys, parent, get_license_text, get_readme_as_rst + + +from ._dill import ( + dump, dumps, load, loads, copy, + Pickler, Unpickler, register, pickle, pickles, check, + DEFAULT_PROTOCOL, HIGHEST_PROTOCOL, HANDLE_FMODE, CONTENTS_FMODE, FILE_FMODE, + PickleError, PickleWarning, PicklingError, PicklingWarning, UnpicklingError, + UnpicklingWarning, +) +from .session import ( + dump_module, load_module, load_module_asdict, + dump_session, load_session # backward compatibility +) +from . import detect, logger, session, source, temp + +# get global settings +from .settings import settings + +# make sure "trace" is turned off +logger.trace(False) + +objects = {} +# local import of dill._objects +#from . import _objects +#objects.update(_objects.succeeds) +#del _objects + +# local import of dill.objtypes +from . import objtypes as types + +def load_types(pickleable=True, unpickleable=True): + """load pickleable and/or unpickleable types to ``dill.types`` + + ``dill.types`` is meant to mimic the ``types`` module, providing a + registry of object types. By default, the module is empty (for import + speed purposes). Use the ``load_types`` function to load selected object + types to the ``dill.types`` module. + + Args: + pickleable (bool, default=True): if True, load pickleable types. + unpickleable (bool, default=True): if True, load unpickleable types. + + Returns: + None + """ + from importlib import reload + # local import of dill.objects + from . import _objects + if pickleable: + objects.update(_objects.succeeds) + else: + [objects.pop(obj,None) for obj in _objects.succeeds] + if unpickleable: + objects.update(_objects.failures) + else: + [objects.pop(obj,None) for obj in _objects.failures] + objects.update(_objects.registered) + del _objects + # reset contents of types to 'empty' + [types.__dict__.pop(obj) for obj in list(types.__dict__.keys()) \ + if obj.find('Type') != -1] + # add corresponding types from objects to types + reload(types) + +def extend(use_dill=True): + '''add (or remove) dill types to/from the pickle registry + + by default, ``dill`` populates its types to ``pickle.Pickler.dispatch``. + Thus, all ``dill`` types are available upon calling ``'import pickle'``. + To drop all ``dill`` types from the ``pickle`` dispatch, *use_dill=False*. + + Args: + use_dill (bool, default=True): if True, extend the dispatch table. + + Returns: + None + ''' + from ._dill import _revert_extension, _extend + if use_dill: _extend() + else: _revert_extension() + return + +extend() + + +def license(): + """print license""" + print (__license__) + return + +def citation(): + """print citation""" + print (__doc__[-491:-118]) + return + +# end of file diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill/_dill.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill/_dill.py new file mode 100644 index 0000000000000000000000000000000000000000..d42432ff3703f3f49fd760340b77bf84ce3e3a95 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill/_dill.py @@ -0,0 +1,2198 @@ +# -*- coding: utf-8 -*- +# +# Author: Mike McKerns (mmckerns @caltech and @uqfoundation) +# Copyright (c) 2008-2015 California Institute of Technology. +# Copyright (c) 2016-2024 The Uncertainty Quantification Foundation. +# License: 3-clause BSD. The full license text is available at: +# - https://github.com/uqfoundation/dill/blob/master/LICENSE +""" +dill: a utility for serialization of python objects + +The primary functions in `dill` are :func:`dump` and +:func:`dumps` for serialization ("pickling") to a +file or to a string, respectively, and :func:`load` +and :func:`loads` for deserialization ("unpickling"), +similarly, from a file or from a string. Other notable +functions are :func:`~dill.dump_module` and +:func:`~dill.load_module`, which are used to save and +restore module objects, including an intepreter session. + +Based on code written by Oren Tirosh and Armin Ronacher. +Extended to a (near) full set of the builtin types (in types module), +and coded to the pickle interface, by . +Initial port to python3 by Jonathan Dobson, continued by mmckerns. +Tested against "all" python types (Std. Lib. CH 1-15 @ 2.7) by mmckerns. +Tested against CH16+ Std. Lib. ... TBD. +""" + +from __future__ import annotations + +__all__ = [ + 'dump','dumps','load','loads','copy', + 'Pickler','Unpickler','register','pickle','pickles','check', + 'DEFAULT_PROTOCOL','HIGHEST_PROTOCOL','HANDLE_FMODE','CONTENTS_FMODE','FILE_FMODE', + 'PickleError','PickleWarning','PicklingError','PicklingWarning','UnpicklingError', + 'UnpicklingWarning', +] + +__module__ = 'dill' + +import warnings +from .logger import adapter as logger +from .logger import trace as _trace +log = logger # backward compatibility (see issue #582) + +import os +import sys +diff = None +_use_diff = False +OLD38 = (sys.hexversion < 0x3080000) +OLD39 = (sys.hexversion < 0x3090000) +OLD310 = (sys.hexversion < 0x30a0000) +OLD312a7 = (sys.hexversion < 0x30c00a7) +#XXX: get types from .objtypes ? +import builtins as __builtin__ +from pickle import _Pickler as StockPickler, Unpickler as StockUnpickler +from pickle import GLOBAL, POP +from _thread import LockType +from _thread import RLock as RLockType +#from io import IOBase +from types import CodeType, FunctionType, MethodType, GeneratorType, \ + TracebackType, FrameType, ModuleType, BuiltinMethodType +BufferType = memoryview #XXX: unregistered +ClassType = type # no 'old-style' classes +EllipsisType = type(Ellipsis) +#FileType = IOBase +NotImplementedType = type(NotImplemented) +SliceType = slice +TypeType = type # 'new-style' classes #XXX: unregistered +XRangeType = range +from types import MappingProxyType as DictProxyType, new_class +from pickle import DEFAULT_PROTOCOL, HIGHEST_PROTOCOL, PickleError, PicklingError, UnpicklingError +import __main__ as _main_module +import marshal +import gc +# import zlib +import abc +import dataclasses +from weakref import ReferenceType, ProxyType, CallableProxyType +from collections import OrderedDict +from enum import Enum, EnumMeta +from functools import partial +from operator import itemgetter, attrgetter +GENERATOR_FAIL = False +import importlib.machinery +EXTENSION_SUFFIXES = tuple(importlib.machinery.EXTENSION_SUFFIXES) +try: + import ctypes + HAS_CTYPES = True + # if using `pypy`, pythonapi is not found + IS_PYPY = not hasattr(ctypes, 'pythonapi') +except ImportError: + HAS_CTYPES = False + IS_PYPY = False +NumpyUfuncType = None +NumpyDType = None +NumpyArrayType = None +try: + if not importlib.machinery.PathFinder().find_spec('numpy'): + raise ImportError("No module named 'numpy'") + NumpyUfuncType = True + NumpyDType = True + NumpyArrayType = True +except ImportError: + pass +def __hook__(): + global NumpyArrayType, NumpyDType, NumpyUfuncType + from numpy import ufunc as NumpyUfuncType + from numpy import ndarray as NumpyArrayType + from numpy import dtype as NumpyDType + return True +if NumpyArrayType: # then has numpy + def ndarraysubclassinstance(obj_type): + if all((c.__module__, c.__name__) != ('numpy', 'ndarray') for c in obj_type.__mro__): + return False + # anything below here is a numpy array (or subclass) instance + __hook__() # import numpy (so the following works!!!) + # verify that __reduce__ has not been overridden + if obj_type.__reduce_ex__ is not NumpyArrayType.__reduce_ex__ \ + or obj_type.__reduce__ is not NumpyArrayType.__reduce__: + return False + return True + def numpyufunc(obj_type): + return any((c.__module__, c.__name__) == ('numpy', 'ufunc') for c in obj_type.__mro__) + def numpydtype(obj_type): + if all((c.__module__, c.__name__) != ('numpy', 'dtype') for c in obj_type.__mro__): + return False + # anything below here is a numpy dtype + __hook__() # import numpy (so the following works!!!) + return obj_type is type(NumpyDType) # handles subclasses +else: + def ndarraysubclassinstance(obj): return False + def numpyufunc(obj): return False + def numpydtype(obj): return False + +from types import GetSetDescriptorType, ClassMethodDescriptorType, \ + WrapperDescriptorType, MethodDescriptorType, MemberDescriptorType, \ + MethodWrapperType #XXX: unused + +# make sure to add these 'hand-built' types to _typemap +CellType = type((lambda x: lambda y: x)(0).__closure__[0]) +PartialType = type(partial(int, base=2)) +SuperType = type(super(Exception, TypeError())) +ItemGetterType = type(itemgetter(0)) +AttrGetterType = type(attrgetter('__repr__')) + +try: + from functools import _lru_cache_wrapper as LRUCacheType +except ImportError: + LRUCacheType = None + +if not isinstance(LRUCacheType, type): + LRUCacheType = None + +def get_file_type(*args, **kwargs): + open = kwargs.pop("open", __builtin__.open) + f = open(os.devnull, *args, **kwargs) + t = type(f) + f.close() + return t + +IS_PYODIDE = sys.platform == 'emscripten' + +FileType = get_file_type('rb', buffering=0) +TextWrapperType = get_file_type('r', buffering=-1) +BufferedRandomType = None if IS_PYODIDE else get_file_type('r+b', buffering=-1) +BufferedReaderType = get_file_type('rb', buffering=-1) +BufferedWriterType = get_file_type('wb', buffering=-1) +try: + from _pyio import open as _open + PyTextWrapperType = get_file_type('r', buffering=-1, open=_open) + PyBufferedRandomType = None if IS_PYODIDE else get_file_type('r+b', buffering=-1, open=_open) + PyBufferedReaderType = get_file_type('rb', buffering=-1, open=_open) + PyBufferedWriterType = get_file_type('wb', buffering=-1, open=_open) +except ImportError: + PyTextWrapperType = PyBufferedRandomType = PyBufferedReaderType = PyBufferedWriterType = None +from io import BytesIO as StringIO +InputType = OutputType = None +from socket import socket as SocketType +#FIXME: additionally calls ForkingPickler.register several times +from multiprocessing.reduction import _reduce_socket as reduce_socket +try: #pragma: no cover + IS_IPYTHON = __IPYTHON__ # is True + ExitType = None # IPython.core.autocall.ExitAutocall + IPYTHON_SINGLETONS = ('exit', 'quit', 'get_ipython') +except NameError: + IS_IPYTHON = False + try: ExitType = type(exit) # apparently 'exit' can be removed + except NameError: ExitType = None + IPYTHON_SINGLETONS = () + +import inspect +import typing + + +### Shims for different versions of Python and dill +class Sentinel(object): + """ + Create a unique sentinel object that is pickled as a constant. + """ + def __init__(self, name, module_name=None): + self.name = name + if module_name is None: + # Use the calling frame's module + self.__module__ = inspect.currentframe().f_back.f_globals['__name__'] + else: + self.__module__ = module_name # pragma: no cover + def __repr__(self): + return self.__module__ + '.' + self.name # pragma: no cover + def __copy__(self): + return self # pragma: no cover + def __deepcopy__(self, memo): + return self # pragma: no cover + def __reduce__(self): + return self.name + def __reduce_ex__(self, protocol): + return self.name + +from . import _shims +from ._shims import Reduce, Getattr + +### File modes +#: Pickles the file handle, preserving mode. The position of the unpickled +#: object is as for a new file handle. +HANDLE_FMODE = 0 +#: Pickles the file contents, creating a new file if on load the file does +#: not exist. The position = min(pickled position, EOF) and mode is chosen +#: as such that "best" preserves behavior of the original file. +CONTENTS_FMODE = 1 +#: Pickles the entire file (handle and contents), preserving mode and position. +FILE_FMODE = 2 + +### Shorthands (modified from python2.5/lib/pickle.py) +def copy(obj, *args, **kwds): + """ + Use pickling to 'copy' an object (i.e. `loads(dumps(obj))`). + + See :func:`dumps` and :func:`loads` for keyword arguments. + """ + ignore = kwds.pop('ignore', Unpickler.settings['ignore']) + return loads(dumps(obj, *args, **kwds), ignore=ignore) + +def dump(obj, file, protocol=None, byref=None, fmode=None, recurse=None, **kwds):#, strictio=None): + """ + Pickle an object to a file. + + See :func:`dumps` for keyword arguments. + """ + from .settings import settings + protocol = settings['protocol'] if protocol is None else int(protocol) + _kwds = kwds.copy() + _kwds.update(dict(byref=byref, fmode=fmode, recurse=recurse)) + Pickler(file, protocol, **_kwds).dump(obj) + return + +def dumps(obj, protocol=None, byref=None, fmode=None, recurse=None, **kwds):#, strictio=None): + """ + Pickle an object to a string. + + *protocol* is the pickler protocol, as defined for Python *pickle*. + + If *byref=True*, then dill behaves a lot more like pickle as certain + objects (like modules) are pickled by reference as opposed to attempting + to pickle the object itself. + + If *recurse=True*, then objects referred to in the global dictionary + are recursively traced and pickled, instead of the default behavior + of attempting to store the entire global dictionary. This is needed for + functions defined via *exec()*. + + *fmode* (:const:`HANDLE_FMODE`, :const:`CONTENTS_FMODE`, + or :const:`FILE_FMODE`) indicates how file handles will be pickled. + For example, when pickling a data file handle for transfer to a remote + compute service, *FILE_FMODE* will include the file contents in the + pickle and cursor position so that a remote method can operate + transparently on an object with an open file handle. + + Default values for keyword arguments can be set in :mod:`dill.settings`. + """ + file = StringIO() + dump(obj, file, protocol, byref, fmode, recurse, **kwds)#, strictio) + return file.getvalue() + +def load(file, ignore=None, **kwds): + """ + Unpickle an object from a file. + + See :func:`loads` for keyword arguments. + """ + return Unpickler(file, ignore=ignore, **kwds).load() + +def loads(str, ignore=None, **kwds): + """ + Unpickle an object from a string. + + If *ignore=False* then objects whose class is defined in the module + *__main__* are updated to reference the existing class in *__main__*, + otherwise they are left to refer to the reconstructed type, which may + be different. + + Default values for keyword arguments can be set in :mod:`dill.settings`. + """ + file = StringIO(str) + return load(file, ignore, **kwds) + +# def dumpzs(obj, protocol=None): +# """pickle an object to a compressed string""" +# return zlib.compress(dumps(obj, protocol)) + +# def loadzs(str): +# """unpickle an object from a compressed string""" +# return loads(zlib.decompress(str)) + +### End: Shorthands ### + +class MetaCatchingDict(dict): + def get(self, key, default=None): + try: + return self[key] + except KeyError: + return default + + def __missing__(self, key): + if issubclass(key, type): + return save_type + else: + raise KeyError() + +class PickleWarning(Warning, PickleError): + pass + +class PicklingWarning(PickleWarning, PicklingError): + pass + +class UnpicklingWarning(PickleWarning, UnpicklingError): + pass + +### Extend the Picklers +class Pickler(StockPickler): + """python's Pickler extended to interpreter sessions""" + dispatch: typing.Dict[type, typing.Callable[[Pickler, typing.Any], None]] \ + = MetaCatchingDict(StockPickler.dispatch.copy()) + """The dispatch table, a dictionary of serializing functions used + by Pickler to save objects of specific types. Use :func:`pickle` + or :func:`register` to associate types to custom functions. + + :meta hide-value: + """ + _session = False + from .settings import settings + + def __init__(self, file, *args, **kwds): + settings = Pickler.settings + _byref = kwds.pop('byref', None) + #_strictio = kwds.pop('strictio', None) + _fmode = kwds.pop('fmode', None) + _recurse = kwds.pop('recurse', None) + StockPickler.__init__(self, file, *args, **kwds) + self._main = _main_module + self._diff_cache = {} + self._byref = settings['byref'] if _byref is None else _byref + self._strictio = False #_strictio + self._fmode = settings['fmode'] if _fmode is None else _fmode + self._recurse = settings['recurse'] if _recurse is None else _recurse + self._postproc = OrderedDict() + self._file = file + + def save(self, obj, save_persistent_id=True): + # numpy hack + obj_type = type(obj) + if NumpyArrayType and not (obj_type is type or obj_type in Pickler.dispatch): + # register if the object is a numpy ufunc + # thanks to Paul Kienzle for pointing out ufuncs didn't pickle + if numpyufunc(obj_type): + @register(obj_type) + def save_numpy_ufunc(pickler, obj): + logger.trace(pickler, "Nu: %s", obj) + name = getattr(obj, '__qualname__', getattr(obj, '__name__', None)) + StockPickler.save_global(pickler, obj, name=name) + logger.trace(pickler, "# Nu") + return + # NOTE: the above 'save' performs like: + # import copy_reg + # def udump(f): return f.__name__ + # def uload(name): return getattr(numpy, name) + # copy_reg.pickle(NumpyUfuncType, udump, uload) + # register if the object is a numpy dtype + if numpydtype(obj_type): + @register(obj_type) + def save_numpy_dtype(pickler, obj): + logger.trace(pickler, "Dt: %s", obj) + pickler.save_reduce(_create_dtypemeta, (obj.type,), obj=obj) + logger.trace(pickler, "# Dt") + return + # NOTE: the above 'save' performs like: + # import copy_reg + # def uload(name): return type(NumpyDType(name)) + # def udump(f): return uload, (f.type,) + # copy_reg.pickle(NumpyDTypeType, udump, uload) + # register if the object is a subclassed numpy array instance + if ndarraysubclassinstance(obj_type): + @register(obj_type) + def save_numpy_array(pickler, obj): + logger.trace(pickler, "Nu: (%s, %s)", obj.shape, obj.dtype) + npdict = getattr(obj, '__dict__', None) + f, args, state = obj.__reduce__() + pickler.save_reduce(_create_array, (f,args,state,npdict), obj=obj) + logger.trace(pickler, "# Nu") + return + # end numpy hack + + if GENERATOR_FAIL and obj_type is GeneratorType: + msg = "Can't pickle %s: attribute lookup builtins.generator failed" % GeneratorType + raise PicklingError(msg) + StockPickler.save(self, obj, save_persistent_id) + + save.__doc__ = StockPickler.save.__doc__ + + def dump(self, obj): #NOTE: if settings change, need to update attributes + logger.trace_setup(self) + StockPickler.dump(self, obj) + dump.__doc__ = StockPickler.dump.__doc__ + +class Unpickler(StockUnpickler): + """python's Unpickler extended to interpreter sessions and more types""" + from .settings import settings + _session = False + + def find_class(self, module, name): + if (module, name) == ('__builtin__', '__main__'): + return self._main.__dict__ #XXX: above set w/save_module_dict + elif (module, name) == ('__builtin__', 'NoneType'): + return type(None) #XXX: special case: NoneType missing + if module == 'dill.dill': module = 'dill._dill' + return StockUnpickler.find_class(self, module, name) + + def __init__(self, *args, **kwds): + settings = Pickler.settings + _ignore = kwds.pop('ignore', None) + StockUnpickler.__init__(self, *args, **kwds) + self._main = _main_module + self._ignore = settings['ignore'] if _ignore is None else _ignore + + def load(self): #NOTE: if settings change, need to update attributes + obj = StockUnpickler.load(self) + if type(obj).__module__ == getattr(_main_module, '__name__', '__main__'): + if not self._ignore: + # point obj class to main + try: obj.__class__ = getattr(self._main, type(obj).__name__) + except (AttributeError,TypeError): pass # defined in a file + #_main_module.__dict__.update(obj.__dict__) #XXX: should update globals ? + return obj + load.__doc__ = StockUnpickler.load.__doc__ + pass + +''' +def dispatch_table(): + """get the dispatch table of registered types""" + return Pickler.dispatch +''' + +pickle_dispatch_copy = StockPickler.dispatch.copy() + +def pickle(t, func): + """expose :attr:`~Pickler.dispatch` table for user-created extensions""" + Pickler.dispatch[t] = func + return + +def register(t): + """decorator to register types to Pickler's :attr:`~Pickler.dispatch` table""" + def proxy(func): + Pickler.dispatch[t] = func + return func + return proxy + +def _revert_extension(): + """drop dill-registered types from pickle's dispatch table""" + for type, func in list(StockPickler.dispatch.items()): + if func.__module__ == __name__: + del StockPickler.dispatch[type] + if type in pickle_dispatch_copy: + StockPickler.dispatch[type] = pickle_dispatch_copy[type] + +def use_diff(on=True): + """ + Reduces size of pickles by only including object which have changed. + + Decreases pickle size but increases CPU time needed. + Also helps avoid some unpickleable objects. + MUST be called at start of script, otherwise changes will not be recorded. + """ + global _use_diff, diff + _use_diff = on + if _use_diff and diff is None: + try: + from . import diff as d + except ImportError: + import diff as d + diff = d + +def _create_typemap(): + import types + d = dict(list(__builtin__.__dict__.items()) + \ + list(types.__dict__.items())).items() + for key, value in d: + if getattr(value, '__module__', None) == 'builtins' \ + and type(value) is type: + yield key, value + return +_reverse_typemap = dict(_create_typemap()) +_reverse_typemap.update({ + 'PartialType': PartialType, + 'SuperType': SuperType, + 'ItemGetterType': ItemGetterType, + 'AttrGetterType': AttrGetterType, +}) +if sys.hexversion < 0x30800a2: + _reverse_typemap.update({ + 'CellType': CellType, + }) + +# "Incidental" implementation specific types. Unpickling these types in another +# implementation of Python (PyPy -> CPython) is not guaranteed to work + +# This dictionary should contain all types that appear in Python implementations +# but are not defined in https://docs.python.org/3/library/types.html#standard-interpreter-types +x=OrderedDict() +_incedental_reverse_typemap = { + 'FileType': FileType, + 'BufferedRandomType': BufferedRandomType, + 'BufferedReaderType': BufferedReaderType, + 'BufferedWriterType': BufferedWriterType, + 'TextWrapperType': TextWrapperType, + 'PyBufferedRandomType': PyBufferedRandomType, + 'PyBufferedReaderType': PyBufferedReaderType, + 'PyBufferedWriterType': PyBufferedWriterType, + 'PyTextWrapperType': PyTextWrapperType, +} + +_incedental_reverse_typemap.update({ + "DictKeysType": type({}.keys()), + "DictValuesType": type({}.values()), + "DictItemsType": type({}.items()), + + "OdictKeysType": type(x.keys()), + "OdictValuesType": type(x.values()), + "OdictItemsType": type(x.items()), +}) + +if ExitType: + _incedental_reverse_typemap['ExitType'] = ExitType +if InputType: + _incedental_reverse_typemap['InputType'] = InputType + _incedental_reverse_typemap['OutputType'] = OutputType + +''' +try: + import symtable + _incedental_reverse_typemap["SymtableEntryType"] = type(symtable.symtable("", "string", "exec")._table) +except: #FIXME: fails to pickle + pass + +if sys.hexversion >= 0x30a00a0: + _incedental_reverse_typemap['LineIteratorType'] = type(compile('3', '', 'eval').co_lines()) +''' + +if sys.hexversion >= 0x30b00b0: + from types import GenericAlias + _incedental_reverse_typemap["GenericAliasIteratorType"] = type(iter(GenericAlias(list, (int,)))) + ''' + _incedental_reverse_typemap['PositionsIteratorType'] = type(compile('3', '', 'eval').co_positions()) + ''' + +try: + import winreg + _incedental_reverse_typemap["HKEYType"] = winreg.HKEYType +except ImportError: + pass + +_reverse_typemap.update(_incedental_reverse_typemap) +_incedental_types = set(_incedental_reverse_typemap.values()) + +del x + +_typemap = dict((v, k) for k, v in _reverse_typemap.items()) + +def _unmarshal(string): + return marshal.loads(string) + +def _load_type(name): + return _reverse_typemap[name] + +def _create_type(typeobj, *args): + return typeobj(*args) + +def _create_function(fcode, fglobals, fname=None, fdefaults=None, + fclosure=None, fdict=None, fkwdefaults=None): + # same as FunctionType, but enable passing __dict__ to new function, + # __dict__ is the storehouse for attributes added after function creation + func = FunctionType(fcode, fglobals or dict(), fname, fdefaults, fclosure) + if fdict is not None: + func.__dict__.update(fdict) #XXX: better copy? option to copy? + if fkwdefaults is not None: + func.__kwdefaults__ = fkwdefaults + # 'recurse' only stores referenced modules/objects in fglobals, + # thus we need to make sure that we have __builtins__ as well + if "__builtins__" not in func.__globals__: + func.__globals__["__builtins__"] = globals()["__builtins__"] + # assert id(fglobals) == id(func.__globals__) + return func + +class match: + """ + Make avaialable a limited structural pattern matching-like syntax for Python < 3.10 + + Patterns can be only tuples (without types) currently. + Inspired by the package pattern-matching-PEP634. + + Usage: + >>> with match(args) as m: + >>> if m.case(('x', 'y')): + >>> # use m.x and m.y + >>> elif m.case(('x', 'y', 'z')): + >>> # use m.x, m.y and m.z + + Equivalent native code for Python >= 3.10: + >>> match args: + >>> case (x, y): + >>> # use x and y + >>> case (x, y, z): + >>> # use x, y and z + """ + def __init__(self, value): + self.value = value + self._fields = None + def __enter__(self): + return self + def __exit__(self, *exc_info): + return False + def case(self, args): # *args, **kwargs): + """just handles tuple patterns""" + if len(self.value) != len(args): # + len(kwargs): + return False + #if not all(isinstance(arg, pat) for arg, pat in zip(self.value[len(args):], kwargs.values())): + # return False + self.args = args # (*args, *kwargs) + return True + @property + def fields(self): + # Only bind names to values if necessary. + if self._fields is None: + self._fields = dict(zip(self.args, self.value)) + return self._fields + def __getattr__(self, item): + return self.fields[item] + +ALL_CODE_PARAMS = [ + # Version New attribute CodeType parameters + ((3,11,'a'), 'co_endlinetable', 'argcount posonlyargcount kwonlyargcount nlocals stacksize flags code consts names varnames filename name qualname firstlineno linetable endlinetable columntable exceptiontable freevars cellvars'), + ((3,11), 'co_exceptiontable', 'argcount posonlyargcount kwonlyargcount nlocals stacksize flags code consts names varnames filename name qualname firstlineno linetable exceptiontable freevars cellvars'), + ((3,10), 'co_linetable', 'argcount posonlyargcount kwonlyargcount nlocals stacksize flags code consts names varnames filename name firstlineno linetable freevars cellvars'), + ((3,8), 'co_posonlyargcount', 'argcount posonlyargcount kwonlyargcount nlocals stacksize flags code consts names varnames filename name firstlineno lnotab freevars cellvars'), + ((3,7), 'co_kwonlyargcount', 'argcount kwonlyargcount nlocals stacksize flags code consts names varnames filename name firstlineno lnotab freevars cellvars'), + ] +for version, new_attr, params in ALL_CODE_PARAMS: + if hasattr(CodeType, new_attr): + CODE_VERSION = version + CODE_PARAMS = params.split() + break +ENCODE_PARAMS = set(CODE_PARAMS).intersection( + ['code', 'lnotab', 'linetable', 'endlinetable', 'columntable', 'exceptiontable']) + +def _create_code(*args): + if not isinstance(args[0], int): # co_lnotab stored from >= 3.10 + LNOTAB, *args = args + else: # from < 3.10 (or pre-LNOTAB storage) + LNOTAB = b'' + + with match(args) as m: + # Python 3.11/3.12a (18 members) + if m.case(( + 'argcount', 'posonlyargcount', 'kwonlyargcount', 'nlocals', 'stacksize', 'flags', # args[0:6] + 'code', 'consts', 'names', 'varnames', 'filename', 'name', 'qualname', 'firstlineno', # args[6:14] + 'linetable', 'exceptiontable', 'freevars', 'cellvars' # args[14:] + )): + if CODE_VERSION == (3,11): + return CodeType( + *args[:6], + args[6].encode() if hasattr(args[6], 'encode') else args[6], # code + *args[7:14], + args[14].encode() if hasattr(args[14], 'encode') else args[14], # linetable + args[15].encode() if hasattr(args[15], 'encode') else args[15], # exceptiontable + args[16], + args[17], + ) + fields = m.fields + # Python 3.10 or 3.8/3.9 (16 members) + elif m.case(( + 'argcount', 'posonlyargcount', 'kwonlyargcount', 'nlocals', 'stacksize', 'flags', # args[0:6] + 'code', 'consts', 'names', 'varnames', 'filename', 'name', 'firstlineno', # args[6:13] + 'LNOTAB_OR_LINETABLE', 'freevars', 'cellvars' # args[13:] + )): + if CODE_VERSION == (3,10) or CODE_VERSION == (3,8): + return CodeType( + *args[:6], + args[6].encode() if hasattr(args[6], 'encode') else args[6], # code + *args[7:13], + args[13].encode() if hasattr(args[13], 'encode') else args[13], # lnotab/linetable + args[14], + args[15], + ) + fields = m.fields + if CODE_VERSION >= (3,10): + fields['linetable'] = m.LNOTAB_OR_LINETABLE + else: + fields['lnotab'] = LNOTAB if LNOTAB else m.LNOTAB_OR_LINETABLE + # Python 3.7 (15 args) + elif m.case(( + 'argcount', 'kwonlyargcount', 'nlocals', 'stacksize', 'flags', # args[0:5] + 'code', 'consts', 'names', 'varnames', 'filename', 'name', 'firstlineno', # args[5:12] + 'lnotab', 'freevars', 'cellvars' # args[12:] + )): + if CODE_VERSION == (3,7): + return CodeType( + *args[:5], + args[5].encode() if hasattr(args[5], 'encode') else args[5], # code + *args[6:12], + args[12].encode() if hasattr(args[12], 'encode') else args[12], # lnotab + args[13], + args[14], + ) + fields = m.fields + # Python 3.11a (20 members) + elif m.case(( + 'argcount', 'posonlyargcount', 'kwonlyargcount', 'nlocals', 'stacksize', 'flags', # args[0:6] + 'code', 'consts', 'names', 'varnames', 'filename', 'name', 'qualname', 'firstlineno', # args[6:14] + 'linetable', 'endlinetable', 'columntable', 'exceptiontable', 'freevars', 'cellvars' # args[14:] + )): + if CODE_VERSION == (3,11,'a'): + return CodeType( + *args[:6], + args[6].encode() if hasattr(args[6], 'encode') else args[6], # code + *args[7:14], + *(a.encode() if hasattr(a, 'encode') else a for a in args[14:18]), # linetable-exceptiontable + args[18], + args[19], + ) + fields = m.fields + else: + raise UnpicklingError("pattern match for code object failed") + + # The args format doesn't match this version. + fields.setdefault('posonlyargcount', 0) # from python <= 3.7 + fields.setdefault('lnotab', LNOTAB) # from python >= 3.10 + fields.setdefault('linetable', b'') # from python <= 3.9 + fields.setdefault('qualname', fields['name']) # from python <= 3.10 + fields.setdefault('exceptiontable', b'') # from python <= 3.10 + fields.setdefault('endlinetable', None) # from python != 3.11a + fields.setdefault('columntable', None) # from python != 3.11a + + args = (fields[k].encode() if k in ENCODE_PARAMS and hasattr(fields[k], 'encode') else fields[k] + for k in CODE_PARAMS) + return CodeType(*args) + +def _create_ftype(ftypeobj, func, args, kwds): + if kwds is None: + kwds = {} + if args is None: + args = () + return ftypeobj(func, *args, **kwds) + +def _create_typing_tuple(argz, *args): #NOTE: workaround python/cpython#94245 + if not argz: + return typing.Tuple[()].copy_with(()) + if argz == ((),): + return typing.Tuple[()] + return typing.Tuple[argz] + +def _create_lock(locked, *args): #XXX: ignores 'blocking' + from threading import Lock + lock = Lock() + if locked: + if not lock.acquire(False): + raise UnpicklingError("Cannot acquire lock") + return lock + +def _create_rlock(count, owner, *args): #XXX: ignores 'blocking' + lock = RLockType() + if owner is not None: + lock._acquire_restore((count, owner)) + if owner and not lock._is_owned(): + raise UnpicklingError("Cannot acquire lock") + return lock + +# thanks to matsjoyce for adding all the different file modes +def _create_filehandle(name, mode, position, closed, open, strictio, fmode, fdata): # buffering=0 + # only pickles the handle, not the file contents... good? or StringIO(data)? + # (for file contents see: http://effbot.org/librarybook/copy-reg.htm) + # NOTE: handle special cases first (are there more special cases?) + names = {'':sys.__stdin__, '':sys.__stdout__, + '':sys.__stderr__} #XXX: better fileno=(0,1,2) ? + if name in list(names.keys()): + f = names[name] #XXX: safer "f=sys.stdin" + elif name == '': + f = os.tmpfile() + elif name == '': + import tempfile + f = tempfile.TemporaryFile(mode) + else: + try: + exists = os.path.exists(name) + except Exception: + exists = False + if not exists: + if strictio: + raise FileNotFoundError("[Errno 2] No such file or directory: '%s'" % name) + elif "r" in mode and fmode != FILE_FMODE: + name = '' # or os.devnull? + current_size = 0 # or maintain position? + else: + current_size = os.path.getsize(name) + + if position > current_size: + if strictio: + raise ValueError("invalid buffer size") + elif fmode == CONTENTS_FMODE: + position = current_size + # try to open the file by name + # NOTE: has different fileno + try: + #FIXME: missing: *buffering*, encoding, softspace + if fmode == FILE_FMODE: + f = open(name, mode if "w" in mode else "w") + f.write(fdata) + if "w" not in mode: + f.close() + f = open(name, mode) + elif name == '': # file did not exist + import tempfile + f = tempfile.TemporaryFile(mode) + # treat x mode as w mode + elif fmode == CONTENTS_FMODE \ + and ("w" in mode or "x" in mode): + # stop truncation when opening + flags = os.O_CREAT + if "+" in mode: + flags |= os.O_RDWR + else: + flags |= os.O_WRONLY + f = os.fdopen(os.open(name, flags), mode) + # set name to the correct value + r = getattr(f, "buffer", f) + r = getattr(r, "raw", r) + r.name = name + assert f.name == name + else: + f = open(name, mode) + except (IOError, FileNotFoundError): + err = sys.exc_info()[1] + raise UnpicklingError(err) + if closed: + f.close() + elif position >= 0 and fmode != HANDLE_FMODE: + f.seek(position) + return f + +def _create_stringi(value, position, closed): + f = StringIO(value) + if closed: f.close() + else: f.seek(position) + return f + +def _create_stringo(value, position, closed): + f = StringIO() + if closed: f.close() + else: + f.write(value) + f.seek(position) + return f + +class _itemgetter_helper(object): + def __init__(self): + self.items = [] + def __getitem__(self, item): + self.items.append(item) + return + +class _attrgetter_helper(object): + def __init__(self, attrs, index=None): + self.attrs = attrs + self.index = index + def __getattribute__(self, attr): + attrs = object.__getattribute__(self, "attrs") + index = object.__getattribute__(self, "index") + if index is None: + index = len(attrs) + attrs.append(attr) + else: + attrs[index] = ".".join([attrs[index], attr]) + return type(self)(attrs, index) + +class _dictproxy_helper(dict): + def __ror__(self, a): + return a + +_dictproxy_helper_instance = _dictproxy_helper() + +__d = {} +try: + # In CPython 3.9 and later, this trick can be used to exploit the + # implementation of the __or__ function of MappingProxyType to get the true + # mapping referenced by the proxy. It may work for other implementations, + # but is not guaranteed. + MAPPING_PROXY_TRICK = __d is (DictProxyType(__d) | _dictproxy_helper_instance) +except Exception: + MAPPING_PROXY_TRICK = False +del __d + +# _CELL_REF and _CELL_EMPTY are used to stay compatible with versions of dill +# whose _create_cell functions do not have a default value. +# _CELL_REF can be safely removed entirely (replaced by empty tuples for calls +# to _create_cell) once breaking changes are allowed. +_CELL_REF = None +_CELL_EMPTY = Sentinel('_CELL_EMPTY') + +def _create_cell(contents=None): + if contents is not _CELL_EMPTY: + value = contents + return (lambda: value).__closure__[0] + +def _create_weakref(obj, *args): + from weakref import ref + if obj is None: # it's dead + from collections import UserDict + return ref(UserDict(), *args) + return ref(obj, *args) + +def _create_weakproxy(obj, callable=False, *args): + from weakref import proxy + if obj is None: # it's dead + if callable: return proxy(lambda x:x, *args) + from collections import UserDict + return proxy(UserDict(), *args) + return proxy(obj, *args) + +def _eval_repr(repr_str): + return eval(repr_str) + +def _create_array(f, args, state, npdict=None): + #array = numpy.core.multiarray._reconstruct(*args) + array = f(*args) + array.__setstate__(state) + if npdict is not None: # we also have saved state in __dict__ + array.__dict__.update(npdict) + return array + +def _create_dtypemeta(scalar_type): + if NumpyDType is True: __hook__() # a bit hacky I think + if scalar_type is None: + return NumpyDType + return type(NumpyDType(scalar_type)) + +def _create_namedtuple(name, fieldnames, modulename, defaults=None): + class_ = _import_module(modulename + '.' + name, safe=True) + if class_ is not None: + return class_ + import collections + t = collections.namedtuple(name, fieldnames, defaults=defaults, module=modulename) + return t + +def _create_capsule(pointer, name, context, destructor): + attr_found = False + try: + # based on https://github.com/python/cpython/blob/f4095e53ab708d95e019c909d5928502775ba68f/Objects/capsule.c#L209-L231 + uname = name.decode('utf8') + for i in range(1, uname.count('.')+1): + names = uname.rsplit('.', i) + try: + module = __import__(names[0]) + except ImportError: + pass + obj = module + for attr in names[1:]: + obj = getattr(obj, attr) + capsule = obj + attr_found = True + break + except Exception: + pass + + if attr_found: + if _PyCapsule_IsValid(capsule, name): + return capsule + raise UnpicklingError("%s object exists at %s but a PyCapsule object was expected." % (type(capsule), name)) + else: + #warnings.warn('Creating a new PyCapsule %s for a C data structure that may not be present in memory. Segmentation faults or other memory errors are possible.' % (name,), UnpicklingWarning) + capsule = _PyCapsule_New(pointer, name, destructor) + _PyCapsule_SetContext(capsule, context) + return capsule + +def _getattr(objclass, name, repr_str): + # hack to grab the reference directly + try: #XXX: works only for __builtin__ ? + attr = repr_str.split("'")[3] + return eval(attr+'.__dict__["'+name+'"]') + except Exception: + try: + attr = objclass.__dict__ + if type(attr) is DictProxyType: + attr = attr[name] + else: + attr = getattr(objclass,name) + except (AttributeError, KeyError): + attr = getattr(objclass,name) + return attr + +def _get_attr(self, name): + # stop recursive pickling + return getattr(self, name, None) or getattr(__builtin__, name) + +def _import_module(import_name, safe=False): + try: + if import_name.startswith('__runtime__.'): + return sys.modules[import_name] + elif '.' in import_name: + items = import_name.split('.') + module = '.'.join(items[:-1]) + obj = items[-1] + submodule = getattr(__import__(module, None, None, [obj]), obj) + if isinstance(submodule, (ModuleType, type)): + return submodule + return __import__(import_name, None, None, [obj]) + else: + return __import__(import_name) + except (ImportError, AttributeError, KeyError): + if safe: + return None + raise + +# https://github.com/python/cpython/blob/a8912a0f8d9eba6d502c37d522221f9933e976db/Lib/pickle.py#L322-L333 +def _getattribute(obj, name): + for subpath in name.split('.'): + if subpath == '': + raise AttributeError("Can't get local attribute {!r} on {!r}" + .format(name, obj)) + try: + parent = obj + obj = getattr(obj, subpath) + except AttributeError: + raise AttributeError("Can't get attribute {!r} on {!r}" + .format(name, obj)) + return obj, parent + +def _locate_function(obj, pickler=None): + module_name = getattr(obj, '__module__', None) + if module_name in ['__main__', None] or \ + pickler and is_dill(pickler, child=False) and pickler._session and module_name == pickler._main.__name__: + return False + if hasattr(obj, '__qualname__'): + module = _import_module(module_name, safe=True) + try: + found, _ = _getattribute(module, obj.__qualname__) + return found is obj + except AttributeError: + return False + else: + found = _import_module(module_name + '.' + obj.__name__, safe=True) + return found is obj + + +def _setitems(dest, source): + for k, v in source.items(): + dest[k] = v + + +def _save_with_postproc(pickler, reduction, is_pickler_dill=None, obj=Getattr.NO_DEFAULT, postproc_list=None): + if obj is Getattr.NO_DEFAULT: + obj = Reduce(reduction) # pragma: no cover + + if is_pickler_dill is None: + is_pickler_dill = is_dill(pickler, child=True) + if is_pickler_dill: + # assert id(obj) not in pickler._postproc, str(obj) + ' already pushed on stack!' + # if not hasattr(pickler, 'x'): pickler.x = 0 + # print(pickler.x*' ', 'push', obj, id(obj), pickler._recurse) + # pickler.x += 1 + if postproc_list is None: + postproc_list = [] + + # Recursive object not supported. Default to a global instead. + if id(obj) in pickler._postproc: + name = '%s.%s ' % (obj.__module__, getattr(obj, '__qualname__', obj.__name__)) if hasattr(obj, '__module__') else '' + warnings.warn('Cannot pickle %r: %shas recursive self-references that trigger a RecursionError.' % (obj, name), PicklingWarning) + pickler.save_global(obj) + return + pickler._postproc[id(obj)] = postproc_list + + # TODO: Use state_setter in Python 3.8 to allow for faster cPickle implementations + pickler.save_reduce(*reduction, obj=obj) + + if is_pickler_dill: + # pickler.x -= 1 + # print(pickler.x*' ', 'pop', obj, id(obj)) + postproc = pickler._postproc.pop(id(obj)) + # assert postproc_list == postproc, 'Stack tampered!' + for reduction in reversed(postproc): + if reduction[0] is _setitems: + # use the internal machinery of pickle.py to speedup when + # updating a dictionary in postproc + dest, source = reduction[1] + if source: + pickler.write(pickler.get(pickler.memo[id(dest)][0])) + pickler._batch_setitems(iter(source.items())) + else: + # Updating with an empty dictionary. Same as doing nothing. + continue + else: + pickler.save_reduce(*reduction) + # pop None created by calling preprocessing step off stack + pickler.write(POP) + +#@register(CodeType) +#def save_code(pickler, obj): +# logger.trace(pickler, "Co: %s", obj) +# pickler.save_reduce(_unmarshal, (marshal.dumps(obj),), obj=obj) +# logger.trace(pickler, "# Co") +# return + +# The following function is based on 'save_codeobject' from 'cloudpickle' +# Copyright (c) 2012, Regents of the University of California. +# Copyright (c) 2009 `PiCloud, Inc. `_. +# License: https://github.com/cloudpipe/cloudpickle/blob/master/LICENSE +@register(CodeType) +def save_code(pickler, obj): + logger.trace(pickler, "Co: %s", obj) + if hasattr(obj, "co_endlinetable"): # python 3.11a (20 args) + args = ( + obj.co_lnotab, # for < python 3.10 [not counted in args] + obj.co_argcount, obj.co_posonlyargcount, + obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, + obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, + obj.co_varnames, obj.co_filename, obj.co_name, obj.co_qualname, + obj.co_firstlineno, obj.co_linetable, obj.co_endlinetable, + obj.co_columntable, obj.co_exceptiontable, obj.co_freevars, + obj.co_cellvars + ) + elif hasattr(obj, "co_exceptiontable"): # python 3.11 (18 args) + with warnings.catch_warnings(): + if not OLD312a7: # issue 597 + warnings.filterwarnings('ignore', category=DeprecationWarning) + args = ( + obj.co_lnotab, # for < python 3.10 [not counted in args] + obj.co_argcount, obj.co_posonlyargcount, + obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, + obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, + obj.co_varnames, obj.co_filename, obj.co_name, obj.co_qualname, + obj.co_firstlineno, obj.co_linetable, obj.co_exceptiontable, + obj.co_freevars, obj.co_cellvars + ) + elif hasattr(obj, "co_linetable"): # python 3.10 (16 args) + args = ( + obj.co_lnotab, # for < python 3.10 [not counted in args] + obj.co_argcount, obj.co_posonlyargcount, + obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, + obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, + obj.co_varnames, obj.co_filename, obj.co_name, + obj.co_firstlineno, obj.co_linetable, obj.co_freevars, + obj.co_cellvars + ) + elif hasattr(obj, "co_posonlyargcount"): # python 3.8 (16 args) + args = ( + obj.co_argcount, obj.co_posonlyargcount, + obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, + obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, + obj.co_varnames, obj.co_filename, obj.co_name, + obj.co_firstlineno, obj.co_lnotab, obj.co_freevars, + obj.co_cellvars + ) + else: # python 3.7 (15 args) + args = ( + obj.co_argcount, obj.co_kwonlyargcount, obj.co_nlocals, + obj.co_stacksize, obj.co_flags, obj.co_code, obj.co_consts, + obj.co_names, obj.co_varnames, obj.co_filename, + obj.co_name, obj.co_firstlineno, obj.co_lnotab, + obj.co_freevars, obj.co_cellvars + ) + + pickler.save_reduce(_create_code, args, obj=obj) + logger.trace(pickler, "# Co") + return + +def _repr_dict(obj): + """Make a short string representation of a dictionary.""" + return "<%s object at %#012x>" % (type(obj).__name__, id(obj)) + +@register(dict) +def save_module_dict(pickler, obj): + if is_dill(pickler, child=False) and obj == pickler._main.__dict__ and \ + not (pickler._session and pickler._first_pass): + logger.trace(pickler, "D1: %s", _repr_dict(obj)) # obj + pickler.write(bytes('c__builtin__\n__main__\n', 'UTF-8')) + logger.trace(pickler, "# D1") + elif (not is_dill(pickler, child=False)) and (obj == _main_module.__dict__): + logger.trace(pickler, "D3: %s", _repr_dict(obj)) # obj + pickler.write(bytes('c__main__\n__dict__\n', 'UTF-8')) #XXX: works in general? + logger.trace(pickler, "# D3") + elif '__name__' in obj and obj != _main_module.__dict__ \ + and type(obj['__name__']) is str \ + and obj is getattr(_import_module(obj['__name__'],True), '__dict__', None): + logger.trace(pickler, "D4: %s", _repr_dict(obj)) # obj + pickler.write(bytes('c%s\n__dict__\n' % obj['__name__'], 'UTF-8')) + logger.trace(pickler, "# D4") + else: + logger.trace(pickler, "D2: %s", _repr_dict(obj)) # obj + if is_dill(pickler, child=False) and pickler._session: + # we only care about session the first pass thru + pickler._first_pass = False + StockPickler.save_dict(pickler, obj) + logger.trace(pickler, "# D2") + return + + +if not OLD310 and MAPPING_PROXY_TRICK: + def save_dict_view(dicttype): + def save_dict_view_for_function(func): + def _save_dict_view(pickler, obj): + logger.trace(pickler, "Dkvi: <%s>", obj) + mapping = obj.mapping | _dictproxy_helper_instance + pickler.save_reduce(func, (mapping,), obj=obj) + logger.trace(pickler, "# Dkvi") + return _save_dict_view + return [ + (funcname, save_dict_view_for_function(getattr(dicttype, funcname))) + for funcname in ('keys', 'values', 'items') + ] +else: + # The following functions are based on 'cloudpickle' + # https://github.com/cloudpipe/cloudpickle/blob/5d89947288a18029672596a4d719093cc6d5a412/cloudpickle/cloudpickle.py#L922-L940 + # Copyright (c) 2012, Regents of the University of California. + # Copyright (c) 2009 `PiCloud, Inc. `_. + # License: https://github.com/cloudpipe/cloudpickle/blob/master/LICENSE + def save_dict_view(dicttype): + def save_dict_keys(pickler, obj): + logger.trace(pickler, "Dk: <%s>", obj) + dict_constructor = _shims.Reduce(dicttype.fromkeys, (list(obj),)) + pickler.save_reduce(dicttype.keys, (dict_constructor,), obj=obj) + logger.trace(pickler, "# Dk") + + def save_dict_values(pickler, obj): + logger.trace(pickler, "Dv: <%s>", obj) + dict_constructor = _shims.Reduce(dicttype, (enumerate(obj),)) + pickler.save_reduce(dicttype.values, (dict_constructor,), obj=obj) + logger.trace(pickler, "# Dv") + + def save_dict_items(pickler, obj): + logger.trace(pickler, "Di: <%s>", obj) + pickler.save_reduce(dicttype.items, (dicttype(obj),), obj=obj) + logger.trace(pickler, "# Di") + + return ( + ('keys', save_dict_keys), + ('values', save_dict_values), + ('items', save_dict_items) + ) + +for __dicttype in ( + dict, + OrderedDict +): + __obj = __dicttype() + for __funcname, __savefunc in save_dict_view(__dicttype): + __tview = type(getattr(__obj, __funcname)()) + if __tview not in Pickler.dispatch: + Pickler.dispatch[__tview] = __savefunc +del __dicttype, __obj, __funcname, __tview, __savefunc + + +@register(ClassType) +def save_classobj(pickler, obj): #FIXME: enable pickler._byref + if not _locate_function(obj, pickler): + logger.trace(pickler, "C1: %s", obj) + pickler.save_reduce(ClassType, (obj.__name__, obj.__bases__, + obj.__dict__), obj=obj) + #XXX: or obj.__dict__.copy()), obj=obj) ? + logger.trace(pickler, "# C1") + else: + logger.trace(pickler, "C2: %s", obj) + name = getattr(obj, '__qualname__', getattr(obj, '__name__', None)) + StockPickler.save_global(pickler, obj, name=name) + logger.trace(pickler, "# C2") + return + +@register(typing._GenericAlias) +def save_generic_alias(pickler, obj): + args = obj.__args__ + if type(obj.__reduce__()) is str: + logger.trace(pickler, "Ga0: %s", obj) + StockPickler.save_global(pickler, obj, name=obj.__reduce__()) + logger.trace(pickler, "# Ga0") + elif obj.__origin__ is tuple and (not args or args == ((),)): + logger.trace(pickler, "Ga1: %s", obj) + pickler.save_reduce(_create_typing_tuple, (args,), obj=obj) + logger.trace(pickler, "# Ga1") + else: + logger.trace(pickler, "Ga2: %s", obj) + StockPickler.save_reduce(pickler, *obj.__reduce__(), obj=obj) + logger.trace(pickler, "# Ga2") + return + +@register(LockType) +def save_lock(pickler, obj): + logger.trace(pickler, "Lo: %s", obj) + pickler.save_reduce(_create_lock, (obj.locked(),), obj=obj) + logger.trace(pickler, "# Lo") + return + +@register(RLockType) +def save_rlock(pickler, obj): + logger.trace(pickler, "RL: %s", obj) + r = obj.__repr__() # don't use _release_save as it unlocks the lock + count = int(r.split('count=')[1].split()[0].rstrip('>')) + owner = int(r.split('owner=')[1].split()[0]) + pickler.save_reduce(_create_rlock, (count,owner,), obj=obj) + logger.trace(pickler, "# RL") + return + +#@register(SocketType) #FIXME: causes multiprocess test_pickling FAIL +def save_socket(pickler, obj): + logger.trace(pickler, "So: %s", obj) + pickler.save_reduce(*reduce_socket(obj)) + logger.trace(pickler, "# So") + return + +def _save_file(pickler, obj, open_): + if obj.closed: + position = 0 + else: + obj.flush() + if obj in (sys.__stdout__, sys.__stderr__, sys.__stdin__): + position = -1 + else: + position = obj.tell() + if is_dill(pickler, child=True) and pickler._fmode == FILE_FMODE: + f = open_(obj.name, "r") + fdata = f.read() + f.close() + else: + fdata = "" + if is_dill(pickler, child=True): + strictio = pickler._strictio + fmode = pickler._fmode + else: + strictio = False + fmode = 0 # HANDLE_FMODE + pickler.save_reduce(_create_filehandle, (obj.name, obj.mode, position, + obj.closed, open_, strictio, + fmode, fdata), obj=obj) + return + + +@register(FileType) #XXX: in 3.x has buffer=0, needs different _create? +@register(BufferedReaderType) +@register(BufferedWriterType) +@register(TextWrapperType) +def save_file(pickler, obj): + logger.trace(pickler, "Fi: %s", obj) + f = _save_file(pickler, obj, open) + logger.trace(pickler, "# Fi") + return f + +if BufferedRandomType: + @register(BufferedRandomType) + def save_file(pickler, obj): + logger.trace(pickler, "Fi: %s", obj) + f = _save_file(pickler, obj, open) + logger.trace(pickler, "# Fi") + return f + +if PyTextWrapperType: + @register(PyBufferedReaderType) + @register(PyBufferedWriterType) + @register(PyTextWrapperType) + def save_file(pickler, obj): + logger.trace(pickler, "Fi: %s", obj) + f = _save_file(pickler, obj, _open) + logger.trace(pickler, "# Fi") + return f + + if PyBufferedRandomType: + @register(PyBufferedRandomType) + def save_file(pickler, obj): + logger.trace(pickler, "Fi: %s", obj) + f = _save_file(pickler, obj, _open) + logger.trace(pickler, "# Fi") + return f + + +# The following two functions are based on 'saveCStringIoInput' +# and 'saveCStringIoOutput' from spickle +# Copyright (c) 2011 by science+computing ag +# License: http://www.apache.org/licenses/LICENSE-2.0 +if InputType: + @register(InputType) + def save_stringi(pickler, obj): + logger.trace(pickler, "Io: %s", obj) + if obj.closed: + value = ''; position = 0 + else: + value = obj.getvalue(); position = obj.tell() + pickler.save_reduce(_create_stringi, (value, position, \ + obj.closed), obj=obj) + logger.trace(pickler, "# Io") + return + + @register(OutputType) + def save_stringo(pickler, obj): + logger.trace(pickler, "Io: %s", obj) + if obj.closed: + value = ''; position = 0 + else: + value = obj.getvalue(); position = obj.tell() + pickler.save_reduce(_create_stringo, (value, position, \ + obj.closed), obj=obj) + logger.trace(pickler, "# Io") + return + +if LRUCacheType is not None: + from functools import lru_cache + @register(LRUCacheType) + def save_lru_cache(pickler, obj): + logger.trace(pickler, "LRU: %s", obj) + if OLD39: + kwargs = obj.cache_info() + args = (kwargs.maxsize,) + else: + kwargs = obj.cache_parameters() + args = (kwargs['maxsize'], kwargs['typed']) + if args != lru_cache.__defaults__: + wrapper = Reduce(lru_cache, args, is_callable=True) + else: + wrapper = lru_cache + pickler.save_reduce(wrapper, (obj.__wrapped__,), obj=obj) + logger.trace(pickler, "# LRU") + return + +@register(SuperType) +def save_super(pickler, obj): + logger.trace(pickler, "Su: %s", obj) + pickler.save_reduce(super, (obj.__thisclass__, obj.__self__), obj=obj) + logger.trace(pickler, "# Su") + return + +if IS_PYPY: + @register(MethodType) + def save_instancemethod0(pickler, obj): + code = getattr(obj.__func__, '__code__', None) + if code is not None and type(code) is not CodeType \ + and getattr(obj.__self__, obj.__name__) == obj: + # Some PyPy builtin functions have no module name + logger.trace(pickler, "Me2: %s", obj) + # TODO: verify that this works for all PyPy builtin methods + pickler.save_reduce(getattr, (obj.__self__, obj.__name__), obj=obj) + logger.trace(pickler, "# Me2") + return + + logger.trace(pickler, "Me1: %s", obj) + pickler.save_reduce(MethodType, (obj.__func__, obj.__self__), obj=obj) + logger.trace(pickler, "# Me1") + return +else: + @register(MethodType) + def save_instancemethod0(pickler, obj): + logger.trace(pickler, "Me1: %s", obj) + pickler.save_reduce(MethodType, (obj.__func__, obj.__self__), obj=obj) + logger.trace(pickler, "# Me1") + return + +if not IS_PYPY: + @register(MemberDescriptorType) + @register(GetSetDescriptorType) + @register(MethodDescriptorType) + @register(WrapperDescriptorType) + @register(ClassMethodDescriptorType) + def save_wrapper_descriptor(pickler, obj): + logger.trace(pickler, "Wr: %s", obj) + pickler.save_reduce(_getattr, (obj.__objclass__, obj.__name__, + obj.__repr__()), obj=obj) + logger.trace(pickler, "# Wr") + return +else: + @register(MemberDescriptorType) + @register(GetSetDescriptorType) + def save_wrapper_descriptor(pickler, obj): + logger.trace(pickler, "Wr: %s", obj) + pickler.save_reduce(_getattr, (obj.__objclass__, obj.__name__, + obj.__repr__()), obj=obj) + logger.trace(pickler, "# Wr") + return + +@register(CellType) +def save_cell(pickler, obj): + try: + f = obj.cell_contents + except ValueError: # cell is empty + logger.trace(pickler, "Ce3: %s", obj) + # _shims._CELL_EMPTY is defined in _shims.py to support PyPy 2.7. + # It unpickles to a sentinel object _dill._CELL_EMPTY, also created in + # _shims.py. This object is not present in Python 3 because the cell's + # contents can be deleted in newer versions of Python. The reduce object + # will instead unpickle to None if unpickled in Python 3. + + # When breaking changes are made to dill, (_shims._CELL_EMPTY,) can + # be replaced by () OR the delattr function can be removed repending on + # whichever is more convienient. + pickler.save_reduce(_create_cell, (_shims._CELL_EMPTY,), obj=obj) + # Call the function _delattr on the cell's cell_contents attribute + # The result of this function call will be None + pickler.save_reduce(_shims._delattr, (obj, 'cell_contents')) + # pop None created by calling _delattr off stack + pickler.write(POP) + logger.trace(pickler, "# Ce3") + return + if is_dill(pickler, child=True): + if id(f) in pickler._postproc: + # Already seen. Add to its postprocessing. + postproc = pickler._postproc[id(f)] + else: + # Haven't seen it. Add to the highest possible object and set its + # value as late as possible to prevent cycle. + postproc = next(iter(pickler._postproc.values()), None) + if postproc is not None: + logger.trace(pickler, "Ce2: %s", obj) + # _CELL_REF is defined in _shims.py to support older versions of + # dill. When breaking changes are made to dill, (_CELL_REF,) can + # be replaced by () + pickler.save_reduce(_create_cell, (_CELL_REF,), obj=obj) + postproc.append((_shims._setattr, (obj, 'cell_contents', f))) + logger.trace(pickler, "# Ce2") + return + logger.trace(pickler, "Ce1: %s", obj) + pickler.save_reduce(_create_cell, (f,), obj=obj) + logger.trace(pickler, "# Ce1") + return + +if MAPPING_PROXY_TRICK: + @register(DictProxyType) + def save_dictproxy(pickler, obj): + logger.trace(pickler, "Mp: %s", _repr_dict(obj)) # obj + mapping = obj | _dictproxy_helper_instance + pickler.save_reduce(DictProxyType, (mapping,), obj=obj) + logger.trace(pickler, "# Mp") + return +else: + @register(DictProxyType) + def save_dictproxy(pickler, obj): + logger.trace(pickler, "Mp: %s", _repr_dict(obj)) # obj + pickler.save_reduce(DictProxyType, (obj.copy(),), obj=obj) + logger.trace(pickler, "# Mp") + return + +@register(SliceType) +def save_slice(pickler, obj): + logger.trace(pickler, "Sl: %s", obj) + pickler.save_reduce(slice, (obj.start, obj.stop, obj.step), obj=obj) + logger.trace(pickler, "# Sl") + return + +@register(XRangeType) +@register(EllipsisType) +@register(NotImplementedType) +def save_singleton(pickler, obj): + logger.trace(pickler, "Si: %s", obj) + pickler.save_reduce(_eval_repr, (obj.__repr__(),), obj=obj) + logger.trace(pickler, "# Si") + return + +def _proxy_helper(obj): # a dead proxy returns a reference to None + """get memory address of proxy's reference object""" + _repr = repr(obj) + try: _str = str(obj) + except ReferenceError: # it's a dead proxy + return id(None) + if _str == _repr: return id(obj) # it's a repr + try: # either way, it's a proxy from here + address = int(_str.rstrip('>').split(' at ')[-1], base=16) + except ValueError: # special case: proxy of a 'type' + if not IS_PYPY: + address = int(_repr.rstrip('>').split(' at ')[-1], base=16) + else: + objects = iter(gc.get_objects()) + for _obj in objects: + if repr(_obj) == _str: return id(_obj) + # all bad below... nothing found so throw ReferenceError + msg = "Cannot reference object for proxy at '%s'" % id(obj) + raise ReferenceError(msg) + return address + +def _locate_object(address, module=None): + """get object located at the given memory address (inverse of id(obj))""" + special = [None, True, False] #XXX: more...? + for obj in special: + if address == id(obj): return obj + if module: + objects = iter(module.__dict__.values()) + else: objects = iter(gc.get_objects()) + for obj in objects: + if address == id(obj): return obj + # all bad below... nothing found so throw ReferenceError or TypeError + try: address = hex(address) + except TypeError: + raise TypeError("'%s' is not a valid memory address" % str(address)) + raise ReferenceError("Cannot reference object at '%s'" % address) + +@register(ReferenceType) +def save_weakref(pickler, obj): + refobj = obj() + logger.trace(pickler, "R1: %s", obj) + #refobj = ctypes.pythonapi.PyWeakref_GetObject(obj) # dead returns "None" + pickler.save_reduce(_create_weakref, (refobj,), obj=obj) + logger.trace(pickler, "# R1") + return + +@register(ProxyType) +@register(CallableProxyType) +def save_weakproxy(pickler, obj): + # Must do string substitution here and use %r to avoid ReferenceError. + logger.trace(pickler, "R2: %r" % obj) + refobj = _locate_object(_proxy_helper(obj)) + pickler.save_reduce(_create_weakproxy, (refobj, callable(obj)), obj=obj) + logger.trace(pickler, "# R2") + return + +def _is_builtin_module(module): + if not hasattr(module, "__file__"): return True + if module.__file__ is None: return False + # If a module file name starts with prefix, it should be a builtin + # module, so should always be pickled as a reference. + names = ["base_prefix", "base_exec_prefix", "exec_prefix", "prefix", "real_prefix"] + rp = os.path.realpath + # See https://github.com/uqfoundation/dill/issues/566 + return ( + any( + module.__file__.startswith(getattr(sys, name)) + or rp(module.__file__).startswith(rp(getattr(sys, name))) + for name in names + if hasattr(sys, name) + ) + or module.__file__.endswith(EXTENSION_SUFFIXES) + or 'site-packages' in module.__file__ + ) + +def _is_imported_module(module): + return getattr(module, '__loader__', None) is not None or module in sys.modules.values() + +@register(ModuleType) +def save_module(pickler, obj): + if False: #_use_diff: + if obj.__name__.split('.', 1)[0] != "dill": + try: + changed = diff.whats_changed(obj, seen=pickler._diff_cache)[0] + except RuntimeError: # not memorised module, probably part of dill + pass + else: + logger.trace(pickler, "M2: %s with diff", obj) + logger.info("Diff: %s", changed.keys()) + pickler.save_reduce(_import_module, (obj.__name__,), obj=obj, + state=changed) + logger.trace(pickler, "# M2") + return + + logger.trace(pickler, "M1: %s", obj) + pickler.save_reduce(_import_module, (obj.__name__,), obj=obj) + logger.trace(pickler, "# M1") + else: + builtin_mod = _is_builtin_module(obj) + is_session_main = is_dill(pickler, child=True) and obj is pickler._main + if (obj.__name__ not in ("builtins", "dill", "dill._dill") and not builtin_mod + or is_session_main): + logger.trace(pickler, "M1: %s", obj) + # Hack for handling module-type objects in load_module(). + mod_name = obj.__name__ if _is_imported_module(obj) else '__runtime__.%s' % obj.__name__ + # Second references are saved as __builtin__.__main__ in save_module_dict(). + main_dict = obj.__dict__.copy() + for item in ('__builtins__', '__loader__'): + main_dict.pop(item, None) + for item in IPYTHON_SINGLETONS: #pragma: no cover + if getattr(main_dict.get(item), '__module__', '').startswith('IPython'): + del main_dict[item] + pickler.save_reduce(_import_module, (mod_name,), obj=obj, state=main_dict) + logger.trace(pickler, "# M1") + elif obj.__name__ == "dill._dill": + logger.trace(pickler, "M2: %s", obj) + pickler.save_global(obj, name="_dill") + logger.trace(pickler, "# M2") + else: + logger.trace(pickler, "M2: %s", obj) + pickler.save_reduce(_import_module, (obj.__name__,), obj=obj) + logger.trace(pickler, "# M2") + return + +# The following function is based on '_extract_class_dict' from 'cloudpickle' +# Copyright (c) 2012, Regents of the University of California. +# Copyright (c) 2009 `PiCloud, Inc. `_. +# License: https://github.com/cloudpipe/cloudpickle/blob/master/LICENSE +def _get_typedict_type(cls, clsdict, attrs, postproc_list): + """Retrieve a copy of the dict of a class without the inherited methods""" + if len(cls.__bases__) == 1: + inherited_dict = cls.__bases__[0].__dict__ + else: + inherited_dict = {} + for base in reversed(cls.__bases__): + inherited_dict.update(base.__dict__) + to_remove = [] + for name, value in dict.items(clsdict): + try: + base_value = inherited_dict[name] + if value is base_value and hasattr(value, '__qualname__'): + to_remove.append(name) + except KeyError: + pass + for name in to_remove: + dict.pop(clsdict, name) + + if issubclass(type(cls), type): + clsdict.pop('__dict__', None) + clsdict.pop('__weakref__', None) + # clsdict.pop('__prepare__', None) + return clsdict, attrs + +def _get_typedict_abc(obj, _dict, attrs, postproc_list): + if hasattr(abc, '_get_dump'): + (registry, _, _, _) = abc._get_dump(obj) + register = obj.register + postproc_list.extend((register, (reg(),)) for reg in registry) + elif hasattr(obj, '_abc_registry'): + registry = obj._abc_registry + register = obj.register + postproc_list.extend((register, (reg,)) for reg in registry) + else: + raise PicklingError("Cannot find registry of ABC %s", obj) + + if '_abc_registry' in _dict: + _dict.pop('_abc_registry', None) + _dict.pop('_abc_cache', None) + _dict.pop('_abc_negative_cache', None) + # _dict.pop('_abc_negative_cache_version', None) + else: + _dict.pop('_abc_impl', None) + return _dict, attrs + +@register(TypeType) +def save_type(pickler, obj, postproc_list=None): + if obj in _typemap: + logger.trace(pickler, "T1: %s", obj) + # if obj in _incedental_types: + # warnings.warn('Type %r may only exist on this implementation of Python and cannot be unpickled in other implementations.' % (obj,), PicklingWarning) + pickler.save_reduce(_load_type, (_typemap[obj],), obj=obj) + logger.trace(pickler, "# T1") + elif obj.__bases__ == (tuple,) and all([hasattr(obj, attr) for attr in ('_fields','_asdict','_make','_replace')]): + # special case: namedtuples + logger.trace(pickler, "T6: %s", obj) + + obj_name = getattr(obj, '__qualname__', getattr(obj, '__name__', None)) + if obj.__name__ != obj_name: + if postproc_list is None: + postproc_list = [] + postproc_list.append((setattr, (obj, '__qualname__', obj_name))) + + if not obj._field_defaults: + _save_with_postproc(pickler, (_create_namedtuple, (obj.__name__, obj._fields, obj.__module__)), obj=obj, postproc_list=postproc_list) + else: + defaults = [obj._field_defaults[field] for field in obj._fields if field in obj._field_defaults] + _save_with_postproc(pickler, (_create_namedtuple, (obj.__name__, obj._fields, obj.__module__, defaults)), obj=obj, postproc_list=postproc_list) + logger.trace(pickler, "# T6") + return + + # special cases: NoneType, NotImplementedType, EllipsisType, EnumMeta + elif obj is type(None): + logger.trace(pickler, "T7: %s", obj) + #XXX: pickler.save_reduce(type, (None,), obj=obj) + pickler.write(GLOBAL + b'__builtin__\nNoneType\n') + logger.trace(pickler, "# T7") + elif obj is NotImplementedType: + logger.trace(pickler, "T7: %s", obj) + pickler.save_reduce(type, (NotImplemented,), obj=obj) + logger.trace(pickler, "# T7") + elif obj is EllipsisType: + logger.trace(pickler, "T7: %s", obj) + pickler.save_reduce(type, (Ellipsis,), obj=obj) + logger.trace(pickler, "# T7") + elif obj is EnumMeta: + logger.trace(pickler, "T7: %s", obj) + pickler.write(GLOBAL + b'enum\nEnumMeta\n') + logger.trace(pickler, "# T7") + + else: + _byref = getattr(pickler, '_byref', None) + obj_recursive = id(obj) in getattr(pickler, '_postproc', ()) + incorrectly_named = not _locate_function(obj, pickler) + if not _byref and not obj_recursive and incorrectly_named: # not a function, but the name was held over + if postproc_list is None: + postproc_list = [] + + # thanks to Tom Stepleton pointing out pickler._session unneeded + logger.trace(pickler, "T2: %s", obj) + _dict, attrs = _get_typedict_type(obj, obj.__dict__.copy(), None, postproc_list) # copy dict proxy to a dict + + #print (_dict) + #print ("%s\n%s" % (type(obj), obj.__name__)) + #print ("%s\n%s" % (obj.__bases__, obj.__dict__)) + slots = _dict.get('__slots__', ()) + if type(slots) == str: + # __slots__ accepts a single string + slots = (slots,) + + for name in slots: + _dict.pop(name, None) + + if isinstance(obj, abc.ABCMeta): + logger.trace(pickler, "ABC: %s", obj) + _dict, attrs = _get_typedict_abc(obj, _dict, attrs, postproc_list) + logger.trace(pickler, "# ABC") + + qualname = getattr(obj, '__qualname__', None) + if attrs is not None: + for k, v in attrs.items(): + postproc_list.append((setattr, (obj, k, v))) + # TODO: Consider using the state argument to save_reduce? + if qualname is not None: + postproc_list.append((setattr, (obj, '__qualname__', qualname))) + + if not hasattr(obj, '__orig_bases__'): + _save_with_postproc(pickler, (_create_type, ( + type(obj), obj.__name__, obj.__bases__, _dict + )), obj=obj, postproc_list=postproc_list) + else: + # This case will always work, but might be overkill. + _metadict = { + 'metaclass': type(obj) + } + + if _dict: + _dict_update = PartialType(_setitems, source=_dict) + else: + _dict_update = None + + _save_with_postproc(pickler, (new_class, ( + obj.__name__, obj.__orig_bases__, _metadict, _dict_update + )), obj=obj, postproc_list=postproc_list) + logger.trace(pickler, "# T2") + else: + obj_name = getattr(obj, '__qualname__', getattr(obj, '__name__', None)) + logger.trace(pickler, "T4: %s", obj) + if incorrectly_named: + warnings.warn( + "Cannot locate reference to %r." % (obj,), + PicklingWarning, + stacklevel=3, + ) + if obj_recursive: + warnings.warn( + "Cannot pickle %r: %s.%s has recursive self-references that " + "trigger a RecursionError." % (obj, obj.__module__, obj_name), + PicklingWarning, + stacklevel=3, + ) + #print (obj.__dict__) + #print ("%s\n%s" % (type(obj), obj.__name__)) + #print ("%s\n%s" % (obj.__bases__, obj.__dict__)) + StockPickler.save_global(pickler, obj, name=obj_name) + logger.trace(pickler, "# T4") + return + +@register(property) +@register(abc.abstractproperty) +def save_property(pickler, obj): + logger.trace(pickler, "Pr: %s", obj) + pickler.save_reduce(type(obj), (obj.fget, obj.fset, obj.fdel, obj.__doc__), + obj=obj) + logger.trace(pickler, "# Pr") + +@register(staticmethod) +@register(classmethod) +@register(abc.abstractstaticmethod) +@register(abc.abstractclassmethod) +def save_classmethod(pickler, obj): + logger.trace(pickler, "Cm: %s", obj) + orig_func = obj.__func__ + + # if type(obj.__dict__) is dict: + # if obj.__dict__: + # state = obj.__dict__ + # else: + # state = None + # else: + # state = (None, {'__dict__', obj.__dict__}) + + pickler.save_reduce(type(obj), (orig_func,), obj=obj) + logger.trace(pickler, "# Cm") + +@register(FunctionType) +def save_function(pickler, obj): + if not _locate_function(obj, pickler): + if type(obj.__code__) is not CodeType: + # Some PyPy builtin functions have no module name, and thus are not + # able to be located + module_name = getattr(obj, '__module__', None) + if module_name is None: + module_name = __builtin__.__name__ + module = _import_module(module_name, safe=True) + _pypy_builtin = False + try: + found, _ = _getattribute(module, obj.__qualname__) + if getattr(found, '__func__', None) is obj: + _pypy_builtin = True + except AttributeError: + pass + + if _pypy_builtin: + logger.trace(pickler, "F3: %s", obj) + pickler.save_reduce(getattr, (found, '__func__'), obj=obj) + logger.trace(pickler, "# F3") + return + + logger.trace(pickler, "F1: %s", obj) + _recurse = getattr(pickler, '_recurse', None) + _postproc = getattr(pickler, '_postproc', None) + _main_modified = getattr(pickler, '_main_modified', None) + _original_main = getattr(pickler, '_original_main', __builtin__)#'None' + postproc_list = [] + if _recurse: + # recurse to get all globals referred to by obj + from .detect import globalvars + globs_copy = globalvars(obj, recurse=True, builtin=True) + + # Add the name of the module to the globs dictionary to prevent + # the duplication of the dictionary. Pickle the unpopulated + # globals dictionary and set the remaining items after the function + # is created to correctly handle recursion. + globs = {'__name__': obj.__module__} + else: + globs_copy = obj.__globals__ + + # If the globals is the __dict__ from the module being saved as a + # session, substitute it by the dictionary being actually saved. + if _main_modified and globs_copy is _original_main.__dict__: + globs_copy = getattr(pickler, '_main', _original_main).__dict__ + globs = globs_copy + # If the globals is a module __dict__, do not save it in the pickle. + elif globs_copy is not None and obj.__module__ is not None and \ + getattr(_import_module(obj.__module__, True), '__dict__', None) is globs_copy: + globs = globs_copy + else: + globs = {'__name__': obj.__module__} + + if globs_copy is not None and globs is not globs_copy: + # In the case that the globals are copied, we need to ensure that + # the globals dictionary is updated when all objects in the + # dictionary are already created. + glob_ids = {id(g) for g in globs_copy.values()} + for stack_element in _postproc: + if stack_element in glob_ids: + _postproc[stack_element].append((_setitems, (globs, globs_copy))) + break + else: + postproc_list.append((_setitems, (globs, globs_copy))) + + closure = obj.__closure__ + state_dict = {} + for fattrname in ('__doc__', '__kwdefaults__', '__annotations__'): + fattr = getattr(obj, fattrname, None) + if fattr is not None: + state_dict[fattrname] = fattr + if obj.__qualname__ != obj.__name__: + state_dict['__qualname__'] = obj.__qualname__ + if '__name__' not in globs or obj.__module__ != globs['__name__']: + state_dict['__module__'] = obj.__module__ + + state = obj.__dict__ + if type(state) is not dict: + state_dict['__dict__'] = state + state = None + if state_dict: + state = state, state_dict + + _save_with_postproc(pickler, (_create_function, ( + obj.__code__, globs, obj.__name__, obj.__defaults__, + closure + ), state), obj=obj, postproc_list=postproc_list) + + # Lift closure cell update to earliest function (#458) + if _postproc: + topmost_postproc = next(iter(_postproc.values()), None) + if closure and topmost_postproc: + for cell in closure: + possible_postproc = (setattr, (cell, 'cell_contents', obj)) + try: + topmost_postproc.remove(possible_postproc) + except ValueError: + continue + + # Change the value of the cell + pickler.save_reduce(*possible_postproc) + # pop None created by calling preprocessing step off stack + pickler.write(POP) + + logger.trace(pickler, "# F1") + else: + logger.trace(pickler, "F2: %s", obj) + name = getattr(obj, '__qualname__', getattr(obj, '__name__', None)) + StockPickler.save_global(pickler, obj, name=name) + logger.trace(pickler, "# F2") + return + +if HAS_CTYPES and hasattr(ctypes, 'pythonapi'): + _PyCapsule_New = ctypes.pythonapi.PyCapsule_New + _PyCapsule_New.argtypes = (ctypes.c_void_p, ctypes.c_char_p, ctypes.c_void_p) + _PyCapsule_New.restype = ctypes.py_object + _PyCapsule_GetPointer = ctypes.pythonapi.PyCapsule_GetPointer + _PyCapsule_GetPointer.argtypes = (ctypes.py_object, ctypes.c_char_p) + _PyCapsule_GetPointer.restype = ctypes.c_void_p + _PyCapsule_GetDestructor = ctypes.pythonapi.PyCapsule_GetDestructor + _PyCapsule_GetDestructor.argtypes = (ctypes.py_object,) + _PyCapsule_GetDestructor.restype = ctypes.c_void_p + _PyCapsule_GetContext = ctypes.pythonapi.PyCapsule_GetContext + _PyCapsule_GetContext.argtypes = (ctypes.py_object,) + _PyCapsule_GetContext.restype = ctypes.c_void_p + _PyCapsule_GetName = ctypes.pythonapi.PyCapsule_GetName + _PyCapsule_GetName.argtypes = (ctypes.py_object,) + _PyCapsule_GetName.restype = ctypes.c_char_p + _PyCapsule_IsValid = ctypes.pythonapi.PyCapsule_IsValid + _PyCapsule_IsValid.argtypes = (ctypes.py_object, ctypes.c_char_p) + _PyCapsule_IsValid.restype = ctypes.c_bool + _PyCapsule_SetContext = ctypes.pythonapi.PyCapsule_SetContext + _PyCapsule_SetContext.argtypes = (ctypes.py_object, ctypes.c_void_p) + _PyCapsule_SetDestructor = ctypes.pythonapi.PyCapsule_SetDestructor + _PyCapsule_SetDestructor.argtypes = (ctypes.py_object, ctypes.c_void_p) + _PyCapsule_SetName = ctypes.pythonapi.PyCapsule_SetName + _PyCapsule_SetName.argtypes = (ctypes.py_object, ctypes.c_char_p) + _PyCapsule_SetPointer = ctypes.pythonapi.PyCapsule_SetPointer + _PyCapsule_SetPointer.argtypes = (ctypes.py_object, ctypes.c_void_p) + #from _socket import CAPI as _testcapsule + _testcapsule_name = b'dill._dill._testcapsule' + _testcapsule = _PyCapsule_New( + ctypes.cast(_PyCapsule_New, ctypes.c_void_p), + ctypes.c_char_p(_testcapsule_name), + None + ) + PyCapsuleType = type(_testcapsule) + @register(PyCapsuleType) + def save_capsule(pickler, obj): + logger.trace(pickler, "Cap: %s", obj) + name = _PyCapsule_GetName(obj) + #warnings.warn('Pickling a PyCapsule (%s) does not pickle any C data structures and could cause segmentation faults or other memory errors when unpickling.' % (name,), PicklingWarning) + pointer = _PyCapsule_GetPointer(obj, name) + context = _PyCapsule_GetContext(obj) + destructor = _PyCapsule_GetDestructor(obj) + pickler.save_reduce(_create_capsule, (pointer, name, context, destructor), obj=obj) + logger.trace(pickler, "# Cap") + _incedental_reverse_typemap['PyCapsuleType'] = PyCapsuleType + _reverse_typemap['PyCapsuleType'] = PyCapsuleType + _incedental_types.add(PyCapsuleType) +else: + _testcapsule = None + + +############################# +# A quick fix for issue #500 +# This should be removed when a better solution is found. + +if hasattr(dataclasses, "_HAS_DEFAULT_FACTORY_CLASS"): + @register(dataclasses._HAS_DEFAULT_FACTORY_CLASS) + def save_dataclasses_HAS_DEFAULT_FACTORY_CLASS(pickler, obj): + logger.trace(pickler, "DcHDF: %s", obj) + pickler.write(GLOBAL + b"dataclasses\n_HAS_DEFAULT_FACTORY\n") + logger.trace(pickler, "# DcHDF") + +if hasattr(dataclasses, "MISSING"): + @register(type(dataclasses.MISSING)) + def save_dataclasses_MISSING_TYPE(pickler, obj): + logger.trace(pickler, "DcM: %s", obj) + pickler.write(GLOBAL + b"dataclasses\nMISSING\n") + logger.trace(pickler, "# DcM") + +if hasattr(dataclasses, "KW_ONLY"): + @register(type(dataclasses.KW_ONLY)) + def save_dataclasses_KW_ONLY_TYPE(pickler, obj): + logger.trace(pickler, "DcKWO: %s", obj) + pickler.write(GLOBAL + b"dataclasses\nKW_ONLY\n") + logger.trace(pickler, "# DcKWO") + +if hasattr(dataclasses, "_FIELD_BASE"): + @register(dataclasses._FIELD_BASE) + def save_dataclasses_FIELD_BASE(pickler, obj): + logger.trace(pickler, "DcFB: %s", obj) + pickler.write(GLOBAL + b"dataclasses\n" + obj.name.encode() + b"\n") + logger.trace(pickler, "# DcFB") + +############################# + +# quick sanity checking +def pickles(obj,exact=False,safe=False,**kwds): + """ + Quick check if object pickles with dill. + + If *exact=True* then an equality test is done to check if the reconstructed + object matches the original object. + + If *safe=True* then any exception will raised in copy signal that the + object is not picklable, otherwise only pickling errors will be trapped. + + Additional keyword arguments are as :func:`dumps` and :func:`loads`. + """ + if safe: exceptions = (Exception,) # RuntimeError, ValueError + else: + exceptions = (TypeError, AssertionError, NotImplementedError, PicklingError, UnpicklingError) + try: + pik = copy(obj, **kwds) + #FIXME: should check types match first, then check content if "exact" + try: + #FIXME: should be "(pik == obj).all()" for numpy comparison, though that'll fail if shapes differ + result = bool(pik.all() == obj.all()) + except (AttributeError, TypeError): + warnings.filterwarnings('ignore') #FIXME: be specific + result = pik == obj + if warnings.filters: del warnings.filters[0] + if hasattr(result, 'toarray'): # for unusual types like sparse matrix + result = result.toarray().all() + if result: return True + if not exact: + result = type(pik) == type(obj) + if result: return result + # class instances might have been dumped with byref=False + return repr(type(pik)) == repr(type(obj)) #XXX: InstanceType? + return False + except exceptions: + return False + +def check(obj, *args, **kwds): + """ + Check pickling of an object across another process. + + *python* is the path to the python interpreter (defaults to sys.executable) + + Set *verbose=True* to print the unpickled object in the other process. + + Additional keyword arguments are as :func:`dumps` and :func:`loads`. + """ + # == undocumented == + # python -- the string path or executable name of the selected python + # verbose -- if True, be verbose about printing warning messages + # all other args and kwds are passed to dill.dumps #FIXME: ignore on load + verbose = kwds.pop('verbose', False) + python = kwds.pop('python', None) + if python is None: + import sys + python = sys.executable + # type check + isinstance(python, str) + import subprocess + fail = True + try: + _obj = dumps(obj, *args, **kwds) + fail = False + finally: + if fail and verbose: + print("DUMP FAILED") + #FIXME: fails if python interpreter path contains spaces + # Use the following instead (which also processes the 'ignore' keyword): + # ignore = kwds.pop('ignore', None) + # unpickle = "dill.loads(%s, ignore=%s)"%(repr(_obj), repr(ignore)) + # cmd = [python, "-c", "import dill; print(%s)"%unpickle] + # msg = "SUCCESS" if not subprocess.call(cmd) else "LOAD FAILED" + msg = "%s -c import dill; print(dill.loads(%s))" % (python, repr(_obj)) + msg = "SUCCESS" if not subprocess.call(msg.split(None,2)) else "LOAD FAILED" + if verbose: + print(msg) + return + +# use to protect against missing attributes +def is_dill(pickler, child=None): + "check the dill-ness of your pickler" + if child is False or not hasattr(pickler.__class__, 'mro'): + return 'dill' in pickler.__module__ + return Pickler in pickler.__class__.mro() + +def _extend(): + """extend pickle with all of dill's registered types""" + # need to have pickle not choke on _main_module? use is_dill(pickler) + for t,func in Pickler.dispatch.items(): + try: + StockPickler.dispatch[t] = func + except Exception: #TypeError, PicklingError, UnpicklingError + logger.trace(pickler, "skip: %s", t) + return + +del diff, _use_diff, use_diff + +# EOF diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill/_objects.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill/_objects.py new file mode 100644 index 0000000000000000000000000000000000000000..b2c02d3bbcf025144c9296c5d3ef30d5318a1870 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill/_objects.py @@ -0,0 +1,537 @@ +#!/usr/bin/env python +# +# Author: Mike McKerns (mmckerns @caltech and @uqfoundation) +# Copyright (c) 2008-2016 California Institute of Technology. +# Copyright (c) 2016-2024 The Uncertainty Quantification Foundation. +# License: 3-clause BSD. The full license text is available at: +# - https://github.com/uqfoundation/dill/blob/master/LICENSE +""" +all Python Standard Library objects (currently: CH 1-15 @ 2.7) +and some other common objects (i.e. numpy.ndarray) +""" + +__all__ = ['registered','failures','succeeds'] + +# helper imports +import warnings; warnings.filterwarnings("ignore", category=DeprecationWarning) +import sys +import queue as Queue +import dbm as anydbm +from io import BytesIO as StringIO +import re +import array +import collections +import codecs +import struct +import dataclasses +import datetime +import calendar +import weakref +import pprint +import decimal +import numbers +import functools +import itertools +import operator +import tempfile +import shelve +import zlib +import gzip +import zipfile +import tarfile +import csv +import hashlib +import hmac +import os +import logging +import logging.handlers +import optparse +#import __hello__ +import threading +import socket +import contextlib +try: + import bz2 + import sqlite3 + import dbm.ndbm as dbm + HAS_ALL = True +except ImportError: # Ubuntu + HAS_ALL = False +try: + #import curses + #from curses import textpad, panel + HAS_CURSES = True +except ImportError: # Windows + HAS_CURSES = False +try: + import ctypes + HAS_CTYPES = True + # if using `pypy`, pythonapi is not found + IS_PYPY = not hasattr(ctypes, 'pythonapi') +except ImportError: # MacPorts + HAS_CTYPES = False + IS_PYPY = False + +# helper objects +class _class: + def _method(self): + pass +# @classmethod +# def _clsmethod(cls): #XXX: test me +# pass +# @staticmethod +# def _static(self): #XXX: test me +# pass +class _class2: + def __call__(self): + pass +_instance2 = _class2() +class _newclass(object): + def _method(self): + pass +# @classmethod +# def _clsmethod(cls): #XXX: test me +# pass +# @staticmethod +# def _static(self): #XXX: test me +# pass +class _newclass2(object): + __slots__ = ['descriptor'] +def _function(x): yield x +def _function2(): + try: raise + except Exception: + from sys import exc_info + e, er, tb = exc_info() + return er, tb +if HAS_CTYPES: + class _Struct(ctypes.Structure): + pass + _Struct._fields_ = [("_field", ctypes.c_int),("next", ctypes.POINTER(_Struct))] +_filedescrip, _tempfile = tempfile.mkstemp('r') # deleted in cleanup +if sys.hexversion < 0x30d00a1: + _tmpf = tempfile.TemporaryFile('w') # emits OSError 9 in python 3.13 +else: + _tmpf = tempfile.NamedTemporaryFile('w').file # for > python 3.9 + +# objects used by dill for type declaration +registered = d = {} +# objects dill fails to pickle +failures = x = {} +# all other type objects +succeeds = a = {} + +# types module (part of CH 8) +a['BooleanType'] = bool(1) +a['BuiltinFunctionType'] = len +a['BuiltinMethodType'] = a['BuiltinFunctionType'] +a['BytesType'] = _bytes = codecs.latin_1_encode('\x00')[0] # bytes(1) +a['ClassType'] = _class +a['ComplexType'] = complex(1) +a['DictType'] = _dict = {} +a['DictionaryType'] = a['DictType'] +a['FloatType'] = float(1) +a['FunctionType'] = _function +a['InstanceType'] = _instance = _class() +a['IntType'] = _int = int(1) +a['ListType'] = _list = [] +a['NoneType'] = None +a['ObjectType'] = object() +a['StringType'] = _str = str(1) +a['TupleType'] = _tuple = () +a['TypeType'] = type +a['LongType'] = _int +a['UnicodeType'] = _str +# built-in constants (CH 4) +a['CopyrightType'] = copyright +# built-in types (CH 5) +a['ClassObjectType'] = _newclass # +a['ClassInstanceType'] = _newclass() # +a['SetType'] = _set = set() +a['FrozenSetType'] = frozenset() +# built-in exceptions (CH 6) +a['ExceptionType'] = _exception = _function2()[0] +# string services (CH 7) +a['SREPatternType'] = _srepattern = re.compile('') +# data types (CH 8) +a['ArrayType'] = array.array("f") +a['DequeType'] = collections.deque([0]) +a['DefaultDictType'] = collections.defaultdict(_function, _dict) +a['TZInfoType'] = datetime.tzinfo() +a['DateTimeType'] = datetime.datetime.today() +a['CalendarType'] = calendar.Calendar() +# numeric and mathematical types (CH 9) +a['DecimalType'] = decimal.Decimal(1) +a['CountType'] = itertools.count(0) +# data compression and archiving (CH 12) +a['TarInfoType'] = tarfile.TarInfo() +# generic operating system services (CH 15) +a['LoggerType'] = _logger = logging.getLogger() +a['FormatterType'] = logging.Formatter() # pickle ok +a['FilterType'] = logging.Filter() # pickle ok +a['LogRecordType'] = logging.makeLogRecord(_dict) # pickle ok +a['OptionParserType'] = _oparser = optparse.OptionParser() # pickle ok +a['OptionGroupType'] = optparse.OptionGroup(_oparser,"foo") # pickle ok +a['OptionType'] = optparse.Option('--foo') # pickle ok +if HAS_CTYPES: + z = x if IS_PYPY else a + z['CCharType'] = _cchar = ctypes.c_char() + z['CWCharType'] = ctypes.c_wchar() # fail == 2.6 + z['CByteType'] = ctypes.c_byte() + z['CUByteType'] = ctypes.c_ubyte() + z['CShortType'] = ctypes.c_short() + z['CUShortType'] = ctypes.c_ushort() + z['CIntType'] = ctypes.c_int() + z['CUIntType'] = ctypes.c_uint() + z['CLongType'] = ctypes.c_long() + z['CULongType'] = ctypes.c_ulong() + z['CLongLongType'] = ctypes.c_longlong() + z['CULongLongType'] = ctypes.c_ulonglong() + z['CFloatType'] = ctypes.c_float() + z['CDoubleType'] = ctypes.c_double() + z['CSizeTType'] = ctypes.c_size_t() + del z + a['CLibraryLoaderType'] = ctypes.cdll + a['StructureType'] = _Struct + # if not IS_PYPY: + # a['BigEndianStructureType'] = ctypes.BigEndianStructure() +#NOTE: also LittleEndianStructureType and UnionType... abstract classes +#NOTE: remember for ctypesobj.contents creates a new python object +#NOTE: ctypes.c_int._objects is memberdescriptor for object's __dict__ +#NOTE: base class of all ctypes data types is non-public _CData + +import fractions +import io +from io import StringIO as TextIO +# built-in functions (CH 2) +a['ByteArrayType'] = bytearray([1]) +# numeric and mathematical types (CH 9) +a['FractionType'] = fractions.Fraction() +a['NumberType'] = numbers.Number() +# generic operating system services (CH 15) +a['IOBaseType'] = io.IOBase() +a['RawIOBaseType'] = io.RawIOBase() +a['TextIOBaseType'] = io.TextIOBase() +a['BufferedIOBaseType'] = io.BufferedIOBase() +a['UnicodeIOType'] = TextIO() # the new StringIO +a['LoggerAdapterType'] = logging.LoggerAdapter(_logger,_dict) # pickle ok +if HAS_CTYPES: + z = x if IS_PYPY else a + z['CBoolType'] = ctypes.c_bool(1) + z['CLongDoubleType'] = ctypes.c_longdouble() + del z +import argparse +# data types (CH 8) +a['OrderedDictType'] = collections.OrderedDict(_dict) +a['CounterType'] = collections.Counter(_dict) +if HAS_CTYPES: + z = x if IS_PYPY else a + z['CSSizeTType'] = ctypes.c_ssize_t() + del z +# generic operating system services (CH 15) +a['NullHandlerType'] = logging.NullHandler() # pickle ok # new 2.7 +a['ArgParseFileType'] = argparse.FileType() # pickle ok + +# -- pickle fails on all below here ----------------------------------------- +# types module (part of CH 8) +a['CodeType'] = compile('','','exec') +a['DictProxyType'] = type.__dict__ +a['DictProxyType2'] = _newclass.__dict__ +a['EllipsisType'] = Ellipsis +a['ClosedFileType'] = open(os.devnull, 'wb', buffering=0).close() +a['GetSetDescriptorType'] = array.array.typecode +a['LambdaType'] = _lambda = lambda x: lambda y: x #XXX: works when not imported! +a['MemberDescriptorType'] = _newclass2.descriptor +if not IS_PYPY: + a['MemberDescriptorType2'] = datetime.timedelta.days +a['MethodType'] = _method = _class()._method #XXX: works when not imported! +a['ModuleType'] = datetime +a['NotImplementedType'] = NotImplemented +a['SliceType'] = slice(1) +a['UnboundMethodType'] = _class._method #XXX: works when not imported! +d['TextWrapperType'] = open(os.devnull, 'r') # same as mode='w','w+','r+' +d['BufferedRandomType'] = open(os.devnull, 'r+b') # same as mode='w+b' +d['BufferedReaderType'] = open(os.devnull, 'rb') # (default: buffering=-1) +d['BufferedWriterType'] = open(os.devnull, 'wb') +try: # oddities: deprecated + from _pyio import open as _open + d['PyTextWrapperType'] = _open(os.devnull, 'r', buffering=-1) + d['PyBufferedRandomType'] = _open(os.devnull, 'r+b', buffering=-1) + d['PyBufferedReaderType'] = _open(os.devnull, 'rb', buffering=-1) + d['PyBufferedWriterType'] = _open(os.devnull, 'wb', buffering=-1) +except ImportError: + pass +# other (concrete) object types +z = d if sys.hexversion < 0x30800a2 else a +z['CellType'] = (_lambda)(0).__closure__[0] +del z +a['XRangeType'] = _xrange = range(1) +a['MethodDescriptorType'] = type.__dict__['mro'] +a['WrapperDescriptorType'] = type.__repr__ +#a['WrapperDescriptorType2'] = type.__dict__['__module__']#XXX: GetSetDescriptor +a['ClassMethodDescriptorType'] = type.__dict__['__prepare__'] +# built-in functions (CH 2) +_methodwrap = (1).__lt__ +a['MethodWrapperType'] = _methodwrap +a['StaticMethodType'] = staticmethod(_method) +a['ClassMethodType'] = classmethod(_method) +a['PropertyType'] = property() +d['SuperType'] = super(Exception, _exception) +# string services (CH 7) +_in = _bytes +a['InputType'] = _cstrI = StringIO(_in) +a['OutputType'] = _cstrO = StringIO() +# data types (CH 8) +a['WeakKeyDictionaryType'] = weakref.WeakKeyDictionary() +a['WeakValueDictionaryType'] = weakref.WeakValueDictionary() +a['ReferenceType'] = weakref.ref(_instance) +a['DeadReferenceType'] = weakref.ref(_class()) +a['ProxyType'] = weakref.proxy(_instance) +a['DeadProxyType'] = weakref.proxy(_class()) +a['CallableProxyType'] = weakref.proxy(_instance2) +a['DeadCallableProxyType'] = weakref.proxy(_class2()) +a['QueueType'] = Queue.Queue() +# numeric and mathematical types (CH 9) +d['PartialType'] = functools.partial(int,base=2) +a['IzipType'] = zip('0','1') +a['ChainType'] = itertools.chain('0','1') +d['ItemGetterType'] = operator.itemgetter(0) +d['AttrGetterType'] = operator.attrgetter('__repr__') +# file and directory access (CH 10) +_fileW = _cstrO +# data persistence (CH 11) +if HAS_ALL: + x['ConnectionType'] = _conn = sqlite3.connect(':memory:') + x['CursorType'] = _conn.cursor() +a['ShelveType'] = shelve.Shelf({}) +# data compression and archiving (CH 12) +if HAS_ALL: + x['BZ2FileType'] = bz2.BZ2File(os.devnull) + x['BZ2CompressorType'] = bz2.BZ2Compressor() + x['BZ2DecompressorType'] = bz2.BZ2Decompressor() +#x['ZipFileType'] = _zip = zipfile.ZipFile(os.devnull,'w') +#_zip.write(_tempfile,'x') [causes annoying warning/error printed on import] +#a['ZipInfoType'] = _zip.getinfo('x') +a['TarFileType'] = tarfile.open(fileobj=_fileW,mode='w') +# file formats (CH 13) +x['DialectType'] = csv.get_dialect('excel') +if sys.hexversion < 0x30d00a1: + import xdrlib + a['PackerType'] = xdrlib.Packer() +# optional operating system services (CH 16) +a['LockType'] = threading.Lock() +a['RLockType'] = threading.RLock() +# generic operating system services (CH 15) # also closed/open and r/w/etc... +a['NamedLoggerType'] = _logger = logging.getLogger(__name__) +#a['FrozenModuleType'] = __hello__ #FIXME: prints "Hello world..." +# interprocess communication (CH 17) +x['SocketType'] = _socket = socket.socket() +x['SocketPairType'] = socket.socketpair()[0] +# python runtime services (CH 27) +a['GeneratorContextManagerType'] = contextlib.contextmanager(max)([1]) + +try: # ipython + __IPYTHON__ is True # is ipython +except NameError: + # built-in constants (CH 4) + a['QuitterType'] = quit + d['ExitType'] = a['QuitterType'] +try: # numpy #FIXME: slow... 0.05 to 0.1 sec to import numpy + from numpy import ufunc as _numpy_ufunc + from numpy import array as _numpy_array + from numpy import int32 as _numpy_int32 + a['NumpyUfuncType'] = _numpy_ufunc + a['NumpyArrayType'] = _numpy_array + a['NumpyInt32Type'] = _numpy_int32 +except ImportError: + pass +# numeric and mathematical types (CH 9) +a['ProductType'] = itertools.product('0','1') +# generic operating system services (CH 15) +a['FileHandlerType'] = logging.FileHandler(os.devnull) +a['RotatingFileHandlerType'] = logging.handlers.RotatingFileHandler(os.devnull) +a['SocketHandlerType'] = logging.handlers.SocketHandler('localhost',514) +a['MemoryHandlerType'] = logging.handlers.MemoryHandler(1) +# data types (CH 8) +a['WeakSetType'] = weakref.WeakSet() # 2.7 +# generic operating system services (CH 15) [errors when dill is imported] +#a['ArgumentParserType'] = _parser = argparse.ArgumentParser('PROG') +#a['NamespaceType'] = _parser.parse_args() # pickle ok +#a['SubParsersActionType'] = _parser.add_subparsers() +#a['MutuallyExclusiveGroupType'] = _parser.add_mutually_exclusive_group() +#a['ArgumentGroupType'] = _parser.add_argument_group() + +# -- dill fails in some versions below here --------------------------------- +# types module (part of CH 8) +d['FileType'] = open(os.devnull, 'rb', buffering=0) # same 'wb','wb+','rb+' +# built-in functions (CH 2) +# Iterators: +a['ListIteratorType'] = iter(_list) # empty vs non-empty +a['SetIteratorType'] = iter(_set) #XXX: empty vs non-empty #FIXME: list_iterator +a['TupleIteratorType']= iter(_tuple) # empty vs non-empty +a['XRangeIteratorType'] = iter(_xrange) # empty vs non-empty +a["BytesIteratorType"] = iter(b'') +a["BytearrayIteratorType"] = iter(bytearray(b'')) +z = x if IS_PYPY else a +z["CallableIteratorType"] = iter(iter, None) +del z +x["MemoryIteratorType"] = iter(memoryview(b'')) +a["ListReverseiteratorType"] = reversed([]) +X = a['OrderedDictType'] +d["OdictKeysType"] = X.keys() +d["OdictValuesType"] = X.values() +d["OdictItemsType"] = X.items() +a["OdictIteratorType"] = iter(X.keys()) #FIXME: list_iterator +del X +#FIXME: list_iterator +a['DictionaryItemIteratorType'] = iter(type.__dict__.items()) +a['DictionaryKeyIteratorType'] = iter(type.__dict__.keys()) +a['DictionaryValueIteratorType'] = iter(type.__dict__.values()) +if sys.hexversion >= 0x30800a0: + a["DictReversekeyiteratorType"] = reversed({}.keys()) + a["DictReversevalueiteratorType"] = reversed({}.values()) + a["DictReverseitemiteratorType"] = reversed({}.items()) + +try: + import symtable + #FIXME: fails to pickle + x["SymtableEntryType"] = symtable.symtable("", "string", "exec")._table +except ImportError: + pass + +if sys.hexversion >= 0x30a00a0 and not IS_PYPY: + x['LineIteratorType'] = compile('3', '', 'eval').co_lines() + +if sys.hexversion >= 0x30b00b0: + from types import GenericAlias + d["GenericAliasIteratorType"] = iter(GenericAlias(list, (int,))) + x['PositionsIteratorType'] = compile('3', '', 'eval').co_positions() + +# data types (CH 8) +a['PrettyPrinterType'] = pprint.PrettyPrinter() +# numeric and mathematical types (CH 9) +a['CycleType'] = itertools.cycle('0') +# file and directory access (CH 10) +a['TemporaryFileType'] = _tmpf +# data compression and archiving (CH 12) +x['GzipFileType'] = gzip.GzipFile(fileobj=_fileW) +# generic operating system services (CH 15) +a['StreamHandlerType'] = logging.StreamHandler() +# numeric and mathematical types (CH 9) +a['PermutationsType'] = itertools.permutations('0') +a['CombinationsType'] = itertools.combinations('0',1) +a['RepeatType'] = itertools.repeat(0) +a['CompressType'] = itertools.compress('0',[1]) +#XXX: ...and etc + +# -- dill fails on all below here ------------------------------------------- +# types module (part of CH 8) +x['GeneratorType'] = _generator = _function(1) #XXX: priority +x['FrameType'] = _generator.gi_frame #XXX: inspect.currentframe() +x['TracebackType'] = _function2()[1] #(see: inspect.getouterframes,getframeinfo) +# other (concrete) object types +# (also: Capsule / CObject ?) +# built-in functions (CH 2) +# built-in types (CH 5) +# string services (CH 7) +x['StructType'] = struct.Struct('c') +x['CallableIteratorType'] = _srepattern.finditer('') +x['SREMatchType'] = _srepattern.match('') +x['SREScannerType'] = _srepattern.scanner('') +x['StreamReader'] = codecs.StreamReader(_cstrI) #XXX: ... and etc +# python object persistence (CH 11) +# x['DbShelveType'] = shelve.open('foo','n')#,protocol=2) #XXX: delete foo +if HAS_ALL: + z = a if IS_PYPY else x + z['DbmType'] = dbm.open(_tempfile,'n') + del z +# x['DbCursorType'] = _dbcursor = anydbm.open('foo','n') #XXX: delete foo +# x['DbType'] = _dbcursor.db +# data compression and archiving (CH 12) +x['ZlibCompressType'] = zlib.compressobj() +x['ZlibDecompressType'] = zlib.decompressobj() +# file formats (CH 13) +x['CSVReaderType'] = csv.reader(_cstrI) +x['CSVWriterType'] = csv.writer(_cstrO) +x['CSVDictReaderType'] = csv.DictReader(_cstrI) +x['CSVDictWriterType'] = csv.DictWriter(_cstrO,{}) +# cryptographic services (CH 14) +x['HashType'] = hashlib.md5() +if (sys.hexversion < 0x30800a1): + x['HMACType'] = hmac.new(_in) +else: + x['HMACType'] = hmac.new(_in, digestmod='md5') +# generic operating system services (CH 15) +if HAS_CURSES: pass + #x['CursesWindowType'] = _curwin = curses.initscr() #FIXME: messes up tty + #x['CursesTextPadType'] = textpad.Textbox(_curwin) + #x['CursesPanelType'] = panel.new_panel(_curwin) +if HAS_CTYPES: + x['CCharPType'] = ctypes.c_char_p() + x['CWCharPType'] = ctypes.c_wchar_p() + x['CVoidPType'] = ctypes.c_void_p() + if sys.platform[:3] == 'win': + x['CDLLType'] = _cdll = ctypes.cdll.msvcrt + else: + x['CDLLType'] = _cdll = ctypes.CDLL(None) + if not IS_PYPY: + x['PyDLLType'] = _pydll = ctypes.pythonapi + x['FuncPtrType'] = _cdll._FuncPtr() + x['CCharArrayType'] = ctypes.create_string_buffer(1) + x['CWCharArrayType'] = ctypes.create_unicode_buffer(1) + x['CParamType'] = ctypes.byref(_cchar) + x['LPCCharType'] = ctypes.pointer(_cchar) + x['LPCCharObjType'] = _lpchar = ctypes.POINTER(ctypes.c_char) + x['NullPtrType'] = _lpchar() + x['NullPyObjectType'] = ctypes.py_object() + x['PyObjectType'] = ctypes.py_object(lambda :None) + z = a if IS_PYPY else x + z['FieldType'] = _field = _Struct._field + z['CFUNCTYPEType'] = _cfunc = ctypes.CFUNCTYPE(ctypes.c_char) + if sys.hexversion < 0x30c00b3: + x['CFunctionType'] = _cfunc(str) + del z +# numeric and mathematical types (CH 9) +a['MethodCallerType'] = operator.methodcaller('mro') # 2.6 +# built-in types (CH 5) +x['MemoryType'] = memoryview(_in) # 2.7 +x['MemoryType2'] = memoryview(bytearray(_in)) # 2.7 +d['DictItemsType'] = _dict.items() # 2.7 +d['DictKeysType'] = _dict.keys() # 2.7 +d['DictValuesType'] = _dict.values() # 2.7 +# generic operating system services (CH 15) +a['RawTextHelpFormatterType'] = argparse.RawTextHelpFormatter('PROG') +a['RawDescriptionHelpFormatterType'] = argparse.RawDescriptionHelpFormatter('PROG') +a['ArgDefaultsHelpFormatterType'] = argparse.ArgumentDefaultsHelpFormatter('PROG') +z = a if IS_PYPY else x +z['CmpKeyType'] = _cmpkey = functools.cmp_to_key(_methodwrap) # 2.7, >=3.2 +z['CmpKeyObjType'] = _cmpkey('0') #2.7, >=3.2 +del z +# oddities: removed, etc +x['BufferType'] = x['MemoryType'] + +from dill._dill import _testcapsule +if _testcapsule is not None: + d['PyCapsuleType'] = _testcapsule +del _testcapsule + +if hasattr(dataclasses, '_HAS_DEFAULT_FACTORY'): + a['DataclassesHasDefaultFactoryType'] = dataclasses._HAS_DEFAULT_FACTORY + +if hasattr(dataclasses, 'MISSING'): + a['DataclassesMissingType'] = dataclasses.MISSING + +if hasattr(dataclasses, 'KW_ONLY'): + a['DataclassesKWOnlyType'] = dataclasses.KW_ONLY + +if hasattr(dataclasses, '_FIELD_BASE'): + a['DataclassesFieldBaseType'] = dataclasses._FIELD + +# -- cleanup ---------------------------------------------------------------- +a.update(d) # registered also succeed +if sys.platform[:3] == 'win': + os.close(_filedescrip) # required on win32 +os.remove(_tempfile) + + +# EOF diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill/_shims.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill/_shims.py new file mode 100644 index 0000000000000000000000000000000000000000..da1abbecc834859271d5e096f041fe299e04e072 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill/_shims.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python +# +# Author: Mike McKerns (mmckerns @caltech and @uqfoundation) +# Author: Anirudh Vegesana (avegesan@cs.stanford.edu) +# Copyright (c) 2021-2024 The Uncertainty Quantification Foundation. +# License: 3-clause BSD. The full license text is available at: +# - https://github.com/uqfoundation/dill/blob/master/LICENSE +""" +Provides shims for compatibility between versions of dill and Python. + +Compatibility shims should be provided in this file. Here are two simple example +use cases. + +Deprecation of constructor function: +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Assume that we were transitioning _import_module in _dill.py to +the builtin function importlib.import_module when present. + +@move_to(_dill) +def _import_module(import_name): + ... # code already in _dill.py + +_import_module = Getattr(importlib, 'import_module', Getattr(_dill, '_import_module', None)) + +The code will attempt to find import_module in the importlib module. If not +present, it will use the _import_module function in _dill. + +Emulate new Python behavior in older Python versions: +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +CellType.cell_contents behaves differently in Python 3.6 and 3.7. It is +read-only in Python 3.6 and writable and deletable in 3.7. + +if _dill.OLD37 and _dill.HAS_CTYPES and ...: + @move_to(_dill) + def _setattr(object, name, value): + if type(object) is _dill.CellType and name == 'cell_contents': + _PyCell_Set.argtypes = (ctypes.py_object, ctypes.py_object) + _PyCell_Set(object, value) + else: + setattr(object, name, value) +... # more cases below + +_setattr = Getattr(_dill, '_setattr', setattr) + +_dill._setattr will be used when present to emulate Python 3.7 functionality in +older versions of Python while defaulting to the standard setattr in 3.7+. + +See this PR for the discussion that lead to this system: +https://github.com/uqfoundation/dill/pull/443 +""" + +import inspect +import sys + +_dill = sys.modules['dill._dill'] + + +class Reduce(object): + """ + Reduce objects are wrappers used for compatibility enforcement during + unpickle-time. They should only be used in calls to pickler.save and + other Reduce objects. They are only evaluated within unpickler.load. + + Pickling a Reduce object makes the two implementations equivalent: + + pickler.save(Reduce(*reduction)) + + pickler.save_reduce(*reduction, obj=reduction) + """ + __slots__ = ['reduction'] + def __new__(cls, *reduction, **kwargs): + """ + Args: + *reduction: a tuple that matches the format given here: + https://docs.python.org/3/library/pickle.html#object.__reduce__ + is_callable: a bool to indicate that the object created by + unpickling `reduction` is callable. If true, the current Reduce + is allowed to be used as the function in further save_reduce calls + or Reduce objects. + """ + is_callable = kwargs.get('is_callable', False) # Pleases Py2. Can be removed later + if is_callable: + self = object.__new__(_CallableReduce) + else: + self = object.__new__(Reduce) + self.reduction = reduction + return self + def __repr__(self): + return 'Reduce%s' % (self.reduction,) + def __copy__(self): + return self # pragma: no cover + def __deepcopy__(self, memo): + return self # pragma: no cover + def __reduce__(self): + return self.reduction + def __reduce_ex__(self, protocol): + return self.__reduce__() + +class _CallableReduce(Reduce): + # A version of Reduce for functions. Used to trick pickler.save_reduce into + # thinking that Reduce objects of functions are themselves meaningful functions. + def __call__(self, *args, **kwargs): + reduction = self.__reduce__() + func = reduction[0] + f_args = reduction[1] + obj = func(*f_args) + return obj(*args, **kwargs) + +__NO_DEFAULT = _dill.Sentinel('Getattr.NO_DEFAULT') + +def Getattr(object, name, default=__NO_DEFAULT): + """ + A Reduce object that represents the getattr operation. When unpickled, the + Getattr will access an attribute 'name' of 'object' and return the value + stored there. If the attribute doesn't exist, the default value will be + returned if present. + + The following statements are equivalent: + + Getattr(collections, 'OrderedDict') + Getattr(collections, 'spam', None) + Getattr(*args) + + Reduce(getattr, (collections, 'OrderedDict')) + Reduce(getattr, (collections, 'spam', None)) + Reduce(getattr, args) + + During unpickling, the first two will result in collections.OrderedDict and + None respectively because the first attribute exists and the second one does + not, forcing it to use the default value given in the third argument. + """ + + if default is Getattr.NO_DEFAULT: + reduction = (getattr, (object, name)) + else: + reduction = (getattr, (object, name, default)) + + return Reduce(*reduction, is_callable=callable(default)) + +Getattr.NO_DEFAULT = __NO_DEFAULT +del __NO_DEFAULT + +def move_to(module, name=None): + def decorator(func): + if name is None: + fname = func.__name__ + else: + fname = name + module.__dict__[fname] = func + func.__module__ = module.__name__ + return func + return decorator + +def register_shim(name, default): + """ + A easier to understand and more compact way of "softly" defining a function. + These two pieces of code are equivalent: + + if _dill.OLD3X: + def _create_class(): + ... + _create_class = register_shim('_create_class', types.new_class) + + if _dill.OLD3X: + @move_to(_dill) + def _create_class(): + ... + _create_class = Getattr(_dill, '_create_class', types.new_class) + + Intuitively, it creates a function or object in the versions of dill/python + that require special reimplementations, and use a core library or default + implementation if that function or object does not exist. + """ + func = globals().get(name) + if func is not None: + _dill.__dict__[name] = func + func.__module__ = _dill.__name__ + + if default is Getattr.NO_DEFAULT: + reduction = (getattr, (_dill, name)) + else: + reduction = (getattr, (_dill, name, default)) + + return Reduce(*reduction, is_callable=callable(default)) + +###################### +## Compatibility Shims are defined below +###################### + +_CELL_EMPTY = register_shim('_CELL_EMPTY', None) + +_setattr = register_shim('_setattr', setattr) +_delattr = register_shim('_delattr', delattr) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill/detect.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill/detect.py new file mode 100644 index 0000000000000000000000000000000000000000..6f76e729d2469fa3f028a189314977c11397edd7 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill/detect.py @@ -0,0 +1,284 @@ +#!/usr/bin/env python +# +# Author: Mike McKerns (mmckerns @caltech and @uqfoundation) +# Copyright (c) 2008-2016 California Institute of Technology. +# Copyright (c) 2016-2024 The Uncertainty Quantification Foundation. +# License: 3-clause BSD. The full license text is available at: +# - https://github.com/uqfoundation/dill/blob/master/LICENSE +""" +Methods for detecting objects leading to pickling failures. +""" + +import dis +from inspect import ismethod, isfunction, istraceback, isframe, iscode + +from .pointers import parent, reference, at, parents, children +from .logger import trace + +__all__ = ['baditems','badobjects','badtypes','code','errors','freevars', + 'getmodule','globalvars','nestedcode','nestedglobals','outermost', + 'referredglobals','referrednested','trace','varnames'] + +def getmodule(object, _filename=None, force=False): + """get the module of the object""" + from inspect import getmodule as getmod + module = getmod(object, _filename) + if module or not force: return module + import builtins + from .source import getname + name = getname(object, force=True) + return builtins if name in vars(builtins).keys() else None + +def outermost(func): # is analogous to getsource(func,enclosing=True) + """get outermost enclosing object (i.e. the outer function in a closure) + + NOTE: this is the object-equivalent of getsource(func, enclosing=True) + """ + if ismethod(func): + _globals = func.__func__.__globals__ or {} + elif isfunction(func): + _globals = func.__globals__ or {} + else: + return #XXX: or raise? no matches + _globals = _globals.items() + # get the enclosing source + from .source import getsourcelines + try: lines,lnum = getsourcelines(func, enclosing=True) + except Exception: #TypeError, IOError + lines,lnum = [],None + code = ''.join(lines) + # get all possible names,objects that are named in the enclosing source + _locals = ((name,obj) for (name,obj) in _globals if name in code) + # now only save the objects that generate the enclosing block + for name,obj in _locals: #XXX: don't really need 'name' + try: + if getsourcelines(obj) == (lines,lnum): return obj + except Exception: #TypeError, IOError + pass + return #XXX: or raise? no matches + +def nestedcode(func, recurse=True): #XXX: or return dict of {co_name: co} ? + """get the code objects for any nested functions (e.g. in a closure)""" + func = code(func) + if not iscode(func): return [] #XXX: or raise? no matches + nested = set() + for co in func.co_consts: + if co is None: continue + co = code(co) + if co: + nested.add(co) + if recurse: nested |= set(nestedcode(co, recurse=True)) + return list(nested) + +def code(func): + """get the code object for the given function or method + + NOTE: use dill.source.getsource(CODEOBJ) to get the source code + """ + if ismethod(func): func = func.__func__ + if isfunction(func): func = func.__code__ + if istraceback(func): func = func.tb_frame + if isframe(func): func = func.f_code + if iscode(func): return func + return + +#XXX: ugly: parse dis.dis for name after " len(referrednested(func)), try calling func(). + If possible, python builds code objects, but delays building functions + until func() is called. + """ + import gc + funcs = set() + # get the code objects, and try to track down by referrence + for co in nestedcode(func, recurse): + # look for function objects that refer to the code object + for obj in gc.get_referrers(co): + # get methods + _ = getattr(obj, '__func__', None) # ismethod + if getattr(_, '__code__', None) is co: funcs.add(obj) + # get functions + elif getattr(obj, '__code__', None) is co: funcs.add(obj) + # get frame objects + elif getattr(obj, 'f_code', None) is co: funcs.add(obj) + # get code objects + elif hasattr(obj, 'co_code') and obj is co: funcs.add(obj) +# frameobjs => func.__code__.co_varnames not in func.__code__.co_cellvars +# funcobjs => func.__code__.co_cellvars not in func.__code__.co_varnames +# frameobjs are not found, however funcobjs are... +# (see: test_mixins.quad ... and test_mixins.wtf) +# after execution, code objects get compiled, and then may be found by gc + return list(funcs) + + +def freevars(func): + """get objects defined in enclosing code that are referred to by func + + returns a dict of {name:object}""" + if ismethod(func): func = func.__func__ + if isfunction(func): + closures = func.__closure__ or () + func = func.__code__.co_freevars # get freevars + else: + return {} + + def get_cell_contents(): + for name, c in zip(func, closures): + try: + cell_contents = c.cell_contents + except ValueError: # cell is empty + continue + yield name, c.cell_contents + + return dict(get_cell_contents()) + +# thanks to Davies Liu for recursion of globals +def nestedglobals(func, recurse=True): + """get the names of any globals found within func""" + func = code(func) + if func is None: return list() + import sys + from .temp import capture + CAN_NULL = sys.hexversion >= 0x30b00a7 # NULL may be prepended >= 3.11a7 + names = set() + with capture('stdout') as out: + dis.dis(func) #XXX: dis.dis(None) disassembles last traceback + for line in out.getvalue().splitlines(): + if '_GLOBAL' in line: + name = line.split('(')[-1].split(')')[0] + if CAN_NULL: + names.add(name.replace('NULL + ', '').replace(' + NULL', '')) + else: + names.add(name) + for co in getattr(func, 'co_consts', tuple()): + if co and recurse and iscode(co): + names.update(nestedglobals(co, recurse=True)) + return list(names) + +def referredglobals(func, recurse=True, builtin=False): + """get the names of objects in the global scope referred to by func""" + return globalvars(func, recurse, builtin).keys() + +def globalvars(func, recurse=True, builtin=False): + """get objects defined in global scope that are referred to by func + + return a dict of {name:object}""" + if ismethod(func): func = func.__func__ + if isfunction(func): + globs = vars(getmodule(sum)).copy() if builtin else {} + # get references from within closure + orig_func, func = func, set() + for obj in orig_func.__closure__ or {}: + try: + cell_contents = obj.cell_contents + except ValueError: # cell is empty + pass + else: + _vars = globalvars(cell_contents, recurse, builtin) or {} + func.update(_vars) #XXX: (above) be wary of infinte recursion? + globs.update(_vars) + # get globals + globs.update(orig_func.__globals__ or {}) + # get names of references + if not recurse: + func.update(orig_func.__code__.co_names) + else: + func.update(nestedglobals(orig_func.__code__)) + # find globals for all entries of func + for key in func.copy(): #XXX: unnecessary...? + nested_func = globs.get(key) + if nested_func is orig_func: + #func.remove(key) if key in func else None + continue #XXX: globalvars(func, False)? + func.update(globalvars(nested_func, True, builtin)) + elif iscode(func): + globs = vars(getmodule(sum)).copy() if builtin else {} + #globs.update(globals()) + if not recurse: + func = func.co_names # get names + else: + orig_func = func.co_name # to stop infinite recursion + func = set(nestedglobals(func)) + # find globals for all entries of func + for key in func.copy(): #XXX: unnecessary...? + if key is orig_func: + #func.remove(key) if key in func else None + continue #XXX: globalvars(func, False)? + nested_func = globs.get(key) + func.update(globalvars(nested_func, True, builtin)) + else: + return {} + #NOTE: if name not in __globals__, then we skip it... + return dict((name,globs[name]) for name in func if name in globs) + + +def varnames(func): + """get names of variables defined by func + + returns a tuple (local vars, local vars referrenced by nested functions)""" + func = code(func) + if not iscode(func): + return () #XXX: better ((),())? or None? + return func.co_varnames, func.co_cellvars + + +def baditems(obj, exact=False, safe=False): #XXX: obj=globals() ? + """get items in object that fail to pickle""" + if not hasattr(obj,'__iter__'): # is not iterable + return [j for j in (badobjects(obj,0,exact,safe),) if j is not None] + obj = obj.values() if getattr(obj,'values',None) else obj + _obj = [] # can't use a set, as items may be unhashable + [_obj.append(badobjects(i,0,exact,safe)) for i in obj if i not in _obj] + return [j for j in _obj if j is not None] + + +def badobjects(obj, depth=0, exact=False, safe=False): + """get objects that fail to pickle""" + from dill import pickles + if not depth: + if pickles(obj,exact,safe): return None + return obj + return dict(((attr, badobjects(getattr(obj,attr),depth-1,exact,safe)) \ + for attr in dir(obj) if not pickles(getattr(obj,attr),exact,safe))) + +def badtypes(obj, depth=0, exact=False, safe=False): + """get types for objects that fail to pickle""" + from dill import pickles + if not depth: + if pickles(obj,exact,safe): return None + return type(obj) + return dict(((attr, badtypes(getattr(obj,attr),depth-1,exact,safe)) \ + for attr in dir(obj) if not pickles(getattr(obj,attr),exact,safe))) + +def errors(obj, depth=0, exact=False, safe=False): + """get errors for objects that fail to pickle""" + from dill import pickles, copy + if not depth: + try: + pik = copy(obj) + if exact: + assert pik == obj, \ + "Unpickling produces %s instead of %s" % (pik,obj) + assert type(pik) == type(obj), \ + "Unpickling produces %s instead of %s" % (type(pik),type(obj)) + return None + except Exception: + import sys + return sys.exc_info()[1] + _dict = {} + for attr in dir(obj): + try: + _attr = getattr(obj,attr) + except Exception: + import sys + _dict[attr] = sys.exc_info()[1] + continue + if not pickles(_attr,exact,safe): + _dict[attr] = errors(_attr,depth-1,exact,safe) + return _dict + + +# EOF diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill/logger.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill/logger.py new file mode 100644 index 0000000000000000000000000000000000000000..d975e8efb53efbea714616cf8ad49c3020c9d9d3 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill/logger.py @@ -0,0 +1,285 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Author: Leonardo Gama (@leogama) +# Copyright (c) 2022-2024 The Uncertainty Quantification Foundation. +# License: 3-clause BSD. The full license text is available at: +# - https://github.com/uqfoundation/dill/blob/master/LICENSE +""" +Logging utilities for dill. + +The 'logger' object is dill's top-level logger. + +The 'adapter' object wraps the logger and implements a 'trace()' method that +generates a detailed tree-style trace for the pickling call at log level INFO. + +The 'trace()' function sets and resets dill's logger log level, enabling and +disabling the pickling trace. + +The trace shows a tree structure depicting the depth of each object serialized +*with dill save functions*, but not the ones that use save functions from +'pickle._Pickler.dispatch'. If the information is available, it also displays +the size in bytes that the object contributed to the pickle stream (including +its child objects). Sample trace output: + + >>> import dill, dill.tests + >>> dill.detect.trace(True) + >>> dill.dump_session(main=dill.tests) + ┬ M1: + ├┬ F2: + │└ # F2 [32 B] + ├┬ D2: + │├┬ T4: + ││└ # T4 [35 B] + │├┬ D2: + ││├┬ T4: + │││└ # T4 [50 B] + ││├┬ D2: + │││└ # D2 [84 B] + ││└ # D2 [413 B] + │└ # D2 [763 B] + └ # M1 [813 B] +""" + +__all__ = ['adapter', 'logger', 'trace'] + +import codecs +import contextlib +import locale +import logging +import math +import os +from functools import partial +from typing import TextIO, Union + +import dill + +# Tree drawing characters: Unicode to ASCII map. +ASCII_MAP = str.maketrans({"│": "|", "├": "|", "┬": "+", "└": "`"}) + +## Notes about the design choices ## + +# Here is some domumentation of the Standard Library's logging internals that +# can't be found completely in the official documentation. dill's logger is +# obtained by calling logging.getLogger('dill') and therefore is an instance of +# logging.getLoggerClass() at the call time. As this is controlled by the user, +# in order to add some functionality to it it's necessary to use a LoggerAdapter +# to wrap it, overriding some of the adapter's methods and creating new ones. +# +# Basic calling sequence +# ====================== +# +# Python's logging functionality can be conceptually divided into five steps: +# 0. Check logging level -> abort if call level is greater than logger level +# 1. Gather information -> construct a LogRecord from passed arguments and context +# 2. Filter (optional) -> discard message if the record matches a filter +# 3. Format -> format message with args, then format output string with message plus record +# 4. Handle -> write the formatted string to output as defined in the handler +# +# dill.logging.logger.log -> # or logger.info, etc. +# Logger.log -> \ +# Logger._log -> }- accept 'extra' parameter for custom record entries +# Logger.makeRecord -> / +# LogRecord.__init__ +# Logger.handle -> +# Logger.callHandlers -> +# Handler.handle -> +# Filterer.filter -> +# Filter.filter +# StreamHandler.emit -> +# Handler.format -> +# Formatter.format -> +# LogRecord.getMessage # does: record.message = msg % args +# Formatter.formatMessage -> +# PercentStyle.format # does: self._fmt % vars(record) +# +# NOTE: All methods from the second line on are from logging.__init__.py + +class TraceAdapter(logging.LoggerAdapter): + """ + Tracks object tree depth and calculates pickled object size. + + A single instance of this wraps the module's logger, as the logging API + doesn't allow setting it directly with a custom Logger subclass. The added + 'trace()' method receives a pickle instance as the first argument and + creates extra values to be added in the LogRecord from it, then calls + 'info()'. + + Usage of logger with 'trace()' method: + + >>> from dill.logger import adapter as logger #NOTE: not dill.logger.logger + >>> ... + >>> def save_atype(pickler, obj): + >>> logger.trace(pickler, "Message with %s and %r etc. placeholders", 'text', obj) + >>> ... + """ + def __init__(self, logger): + self.logger = logger + def addHandler(self, handler): + formatter = TraceFormatter("%(prefix)s%(message)s%(suffix)s", handler=handler) + handler.setFormatter(formatter) + self.logger.addHandler(handler) + def removeHandler(self, handler): + self.logger.removeHandler(handler) + def process(self, msg, kwargs): + # A no-op override, as we don't have self.extra. + return msg, kwargs + def trace_setup(self, pickler): + # Called by Pickler.dump(). + if not dill._dill.is_dill(pickler, child=False): + return + if self.isEnabledFor(logging.INFO): + pickler._trace_depth = 1 + pickler._size_stack = [] + else: + pickler._trace_depth = None + def trace(self, pickler, msg, *args, **kwargs): + if not hasattr(pickler, '_trace_depth'): + logger.info(msg, *args, **kwargs) + return + if pickler._trace_depth is None: + return + extra = kwargs.get('extra', {}) + pushed_obj = msg.startswith('#') + size = None + try: + # Streams are not required to be tellable. + size = pickler._file.tell() + frame = pickler.framer.current_frame + try: + size += frame.tell() + except AttributeError: + # PyPy may use a BytesBuilder as frame + size += len(frame) + except (AttributeError, TypeError): + pass + if size is not None: + if not pushed_obj: + pickler._size_stack.append(size) + else: + size -= pickler._size_stack.pop() + extra['size'] = size + if pushed_obj: + pickler._trace_depth -= 1 + extra['depth'] = pickler._trace_depth + kwargs['extra'] = extra + self.info(msg, *args, **kwargs) + if not pushed_obj: + pickler._trace_depth += 1 + +class TraceFormatter(logging.Formatter): + """ + Generates message prefix and suffix from record. + + This Formatter adds prefix and suffix strings to the log message in trace + mode (an also provides empty string defaults for normal logs). + """ + def __init__(self, *args, handler=None, **kwargs): + super().__init__(*args, **kwargs) + try: + encoding = handler.stream.encoding + if encoding is None: + raise AttributeError + except AttributeError: + encoding = locale.getpreferredencoding() + try: + encoding = codecs.lookup(encoding).name + except LookupError: + self.is_utf8 = False + else: + self.is_utf8 = (encoding == codecs.lookup('utf-8').name) + def format(self, record): + fields = {'prefix': "", 'suffix': ""} + if getattr(record, 'depth', 0) > 0: + if record.msg.startswith("#"): + prefix = (record.depth - 1)*"│" + "└" + elif record.depth == 1: + prefix = "┬" + else: + prefix = (record.depth - 2)*"│" + "├┬" + if not self.is_utf8: + prefix = prefix.translate(ASCII_MAP) + "-" + fields['prefix'] = prefix + " " + if hasattr(record, 'size') and record.size is not None and record.size >= 1: + # Show object size in human-readable form. + power = int(math.log(record.size, 2)) // 10 + size = record.size >> power*10 + fields['suffix'] = " [%d %sB]" % (size, "KMGTP"[power] + "i" if power else "") + vars(record).update(fields) + return super().format(record) + +logger = logging.getLogger('dill') +logger.propagate = False +adapter = TraceAdapter(logger) +stderr_handler = logging._StderrHandler() +adapter.addHandler(stderr_handler) + +def trace(arg: Union[bool, TextIO, str, os.PathLike] = None, *, mode: str = 'a') -> None: + """print a trace through the stack when pickling; useful for debugging + + With a single boolean argument, enable or disable the tracing. + + Example usage: + + >>> import dill + >>> dill.detect.trace(True) + >>> dill.dump_session() + + Alternatively, ``trace()`` can be used as a context manager. With no + arguments, it just takes care of restoring the tracing state on exit. + Either a file handle, or a file name and (optionally) a file mode may be + specitfied to redirect the tracing output in the ``with`` block context. A + log function is yielded by the manager so the user can write extra + information to the file. + + Example usage: + + >>> from dill import detect + >>> D = {'a': 42, 'b': {'x': None}} + >>> with detect.trace(): + >>> dumps(D) + ┬ D2: + ├┬ D2: + │└ # D2 [8 B] + └ # D2 [22 B] + >>> squared = lambda x: x**2 + >>> with detect.trace('output.txt', mode='w') as log: + >>> log("> D = %r", D) + >>> dumps(D) + >>> log("> squared = %r", squared) + >>> dumps(squared) + + Arguments: + arg: a boolean value, or an optional file-like or path-like object for the context manager + mode: mode string for ``open()`` if a file name is passed as the first argument + """ + if not isinstance(arg, bool): + return TraceManager(file=arg, mode=mode) + logger.setLevel(logging.INFO if arg else logging.WARNING) + +class TraceManager(contextlib.AbstractContextManager): + """context manager version of trace(); can redirect the trace to a file""" + def __init__(self, file, mode): + self.file = file + self.mode = mode + self.redirect = file is not None + self.file_is_stream = hasattr(file, 'write') + def __enter__(self): + if self.redirect: + stderr_handler.flush() + if self.file_is_stream: + self.handler = logging.StreamHandler(self.file) + else: + self.handler = logging.FileHandler(self.file, self.mode) + adapter.removeHandler(stderr_handler) + adapter.addHandler(self.handler) + self.old_level = adapter.getEffectiveLevel() + adapter.setLevel(logging.INFO) + return adapter.info + def __exit__(self, *exc_info): + adapter.setLevel(self.old_level) + if self.redirect: + adapter.removeHandler(self.handler) + adapter.addHandler(stderr_handler) + if not self.file_is_stream: + self.handler.close() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill/objtypes.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill/objtypes.py new file mode 100644 index 0000000000000000000000000000000000000000..526b5835ea213fe08cf81a1d389368bd95a86cc8 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill/objtypes.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python +# +# Author: Mike McKerns (mmckerns @caltech and @uqfoundation) +# Copyright (c) 2008-2016 California Institute of Technology. +# Copyright (c) 2016-2024 The Uncertainty Quantification Foundation. +# License: 3-clause BSD. The full license text is available at: +# - https://github.com/uqfoundation/dill/blob/master/LICENSE +""" +all Python Standard Library object types (currently: CH 1-15 @ 2.7) +and some other common object types (i.e. numpy.ndarray) + +to load more objects and types, use dill.load_types() +""" + +# non-local import of dill.objects +from dill import objects +for _type in objects.keys(): + exec("%s = type(objects['%s'])" % (_type,_type)) + +del objects +try: + del _type +except NameError: + pass diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill/pointers.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill/pointers.py new file mode 100644 index 0000000000000000000000000000000000000000..c3b48cae00b87830dc15dcd8fc047ca74aae5b66 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill/pointers.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python +# +# Author: Mike McKerns (mmckerns @caltech and @uqfoundation) +# Copyright (c) 2008-2016 California Institute of Technology. +# Copyright (c) 2016-2024 The Uncertainty Quantification Foundation. +# License: 3-clause BSD. The full license text is available at: +# - https://github.com/uqfoundation/dill/blob/master/LICENSE + +__all__ = ['parent', 'reference', 'at', 'parents', 'children'] + +import gc +import sys + +from ._dill import _proxy_helper as reference +from ._dill import _locate_object as at + +def parent(obj, objtype, ignore=()): + """ +>>> listiter = iter([4,5,6,7]) +>>> obj = parent(listiter, list) +>>> obj == [4,5,6,7] # actually 'is', but don't have handle any longer +True + +NOTE: objtype can be a single type (e.g. int or list) or a tuple of types. + +WARNING: if obj is a sequence (e.g. list), may produce unexpected results. +Parent finds *one* parent (e.g. the last member of the sequence). + """ + depth = 1 #XXX: always looking for the parent (only, right?) + chain = parents(obj, objtype, depth, ignore) + parent = chain.pop() + if parent is obj: + return None + return parent + + +def parents(obj, objtype, depth=1, ignore=()): #XXX: objtype=object ? + """Find the chain of referents for obj. Chain will end with obj. + + objtype: an object type or tuple of types to search for + depth: search depth (e.g. depth=2 is 'grandparents') + ignore: an object or tuple of objects to ignore in the search + """ + edge_func = gc.get_referents # looking for refs, not back_refs + predicate = lambda x: isinstance(x, objtype) # looking for parent type + #if objtype is None: predicate = lambda x: True #XXX: in obj.mro() ? + ignore = (ignore,) if not hasattr(ignore, '__len__') else ignore + ignore = (id(obj) for obj in ignore) + chain = find_chain(obj, predicate, edge_func, depth)[::-1] + #XXX: should pop off obj... ? + return chain + + +def children(obj, objtype, depth=1, ignore=()): #XXX: objtype=object ? + """Find the chain of referrers for obj. Chain will start with obj. + + objtype: an object type or tuple of types to search for + depth: search depth (e.g. depth=2 is 'grandchildren') + ignore: an object or tuple of objects to ignore in the search + + NOTE: a common thing to ignore is all globals, 'ignore=(globals(),)' + + NOTE: repeated calls may yield different results, as python stores + the last value in the special variable '_'; thus, it is often good + to execute something to replace '_' (e.g. >>> 1+1). + """ + edge_func = gc.get_referrers # looking for back_refs, not refs + predicate = lambda x: isinstance(x, objtype) # looking for child type + #if objtype is None: predicate = lambda x: True #XXX: in obj.mro() ? + ignore = (ignore,) if not hasattr(ignore, '__len__') else ignore + ignore = (id(obj) for obj in ignore) + chain = find_chain(obj, predicate, edge_func, depth, ignore) + #XXX: should pop off obj... ? + return chain + + +# more generic helper function (cut-n-paste from objgraph) +# Source at http://mg.pov.lt/objgraph/ +# Copyright (c) 2008-2010 Marius Gedminas +# Copyright (c) 2010 Stefano Rivera +# Released under the MIT licence (see objgraph/objgrah.py) + +def find_chain(obj, predicate, edge_func, max_depth=20, extra_ignore=()): + queue = [obj] + depth = {id(obj): 0} + parent = {id(obj): None} + ignore = set(extra_ignore) + ignore.add(id(extra_ignore)) + ignore.add(id(queue)) + ignore.add(id(depth)) + ignore.add(id(parent)) + ignore.add(id(ignore)) + ignore.add(id(sys._getframe())) # this function + ignore.add(id(sys._getframe(1))) # find_chain/find_backref_chain, likely + gc.collect() + while queue: + target = queue.pop(0) + if predicate(target): + chain = [target] + while parent[id(target)] is not None: + target = parent[id(target)] + chain.append(target) + return chain + tdepth = depth[id(target)] + if tdepth < max_depth: + referrers = edge_func(target) + ignore.add(id(referrers)) + for source in referrers: + if id(source) in ignore: + continue + if id(source) not in depth: + depth[id(source)] = tdepth + 1 + parent[id(source)] = target + queue.append(source) + return [obj] # not found + + +# backward compatibility +refobject = at + + +# EOF diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill/session.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill/session.py new file mode 100644 index 0000000000000000000000000000000000000000..e91068afc5b77dffe9646f0167ab2af5e97523fb --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill/session.py @@ -0,0 +1,613 @@ +#!/usr/bin/env python +# +# Author: Mike McKerns (mmckerns @caltech and @uqfoundation) +# Author: Leonardo Gama (@leogama) +# Copyright (c) 2008-2015 California Institute of Technology. +# Copyright (c) 2016-2024 The Uncertainty Quantification Foundation. +# License: 3-clause BSD. The full license text is available at: +# - https://github.com/uqfoundation/dill/blob/master/LICENSE +""" +Pickle and restore the intepreter session. +""" + +__all__ = [ + 'dump_module', 'load_module', 'load_module_asdict', + 'dump_session', 'load_session' # backward compatibility +] + +import re +import os +import sys +import warnings + +from dill import _dill, Pickler, Unpickler +from ._dill import ( + BuiltinMethodType, FunctionType, MethodType, ModuleType, TypeType, + _import_module, _is_builtin_module, _is_imported_module, _main_module, + _reverse_typemap, __builtin__, +) + +# Type hints. +from typing import Optional, Union + +import pathlib +import tempfile + +TEMPDIR = pathlib.PurePath(tempfile.gettempdir()) + +def _module_map(): + """get map of imported modules""" + from collections import defaultdict + from types import SimpleNamespace + modmap = SimpleNamespace( + by_name=defaultdict(list), + by_id=defaultdict(list), + top_level={}, + ) + for modname, module in sys.modules.items(): + if modname in ('__main__', '__mp_main__') or not isinstance(module, ModuleType): + continue + if '.' not in modname: + modmap.top_level[id(module)] = modname + for objname, modobj in module.__dict__.items(): + modmap.by_name[objname].append((modobj, modname)) + modmap.by_id[id(modobj)].append((modobj, objname, modname)) + return modmap + +IMPORTED_AS_TYPES = (ModuleType, TypeType, FunctionType, MethodType, BuiltinMethodType) +if 'PyCapsuleType' in _reverse_typemap: + IMPORTED_AS_TYPES += (_reverse_typemap['PyCapsuleType'],) +IMPORTED_AS_MODULES = ('ctypes', 'typing', 'subprocess', 'threading', + r'concurrent\.futures(\.\w+)?', r'multiprocessing(\.\w+)?') +IMPORTED_AS_MODULES = tuple(re.compile(x) for x in IMPORTED_AS_MODULES) + +def _lookup_module(modmap, name, obj, main_module): + """lookup name or id of obj if module is imported""" + for modobj, modname in modmap.by_name[name]: + if modobj is obj and sys.modules[modname] is not main_module: + return modname, name + __module__ = getattr(obj, '__module__', None) + if isinstance(obj, IMPORTED_AS_TYPES) or (__module__ is not None + and any(regex.fullmatch(__module__) for regex in IMPORTED_AS_MODULES)): + for modobj, objname, modname in modmap.by_id[id(obj)]: + if sys.modules[modname] is not main_module: + return modname, objname + return None, None + +def _stash_modules(main_module): + modmap = _module_map() + newmod = ModuleType(main_module.__name__) + + imported = [] + imported_as = [] + imported_top_level = [] # keep separated for backward compatibility + original = {} + for name, obj in main_module.__dict__.items(): + if obj is main_module: + original[name] = newmod # self-reference + elif obj is main_module.__dict__: + original[name] = newmod.__dict__ + # Avoid incorrectly matching a singleton value in another package (ex.: __doc__). + elif any(obj is singleton for singleton in (None, False, True)) \ + or isinstance(obj, ModuleType) and _is_builtin_module(obj): # always saved by ref + original[name] = obj + else: + source_module, objname = _lookup_module(modmap, name, obj, main_module) + if source_module is not None: + if objname == name: + imported.append((source_module, name)) + else: + imported_as.append((source_module, objname, name)) + else: + try: + imported_top_level.append((modmap.top_level[id(obj)], name)) + except KeyError: + original[name] = obj + + if len(original) < len(main_module.__dict__): + newmod.__dict__.update(original) + newmod.__dill_imported = imported + newmod.__dill_imported_as = imported_as + newmod.__dill_imported_top_level = imported_top_level + if getattr(newmod, '__loader__', None) is None and _is_imported_module(main_module): + # Trick _is_imported_module() to force saving as an imported module. + newmod.__loader__ = True # will be discarded by save_module() + return newmod + else: + return main_module + +def _restore_modules(unpickler, main_module): + try: + for modname, name in main_module.__dict__.pop('__dill_imported'): + main_module.__dict__[name] = unpickler.find_class(modname, name) + for modname, objname, name in main_module.__dict__.pop('__dill_imported_as'): + main_module.__dict__[name] = unpickler.find_class(modname, objname) + for modname, name in main_module.__dict__.pop('__dill_imported_top_level'): + main_module.__dict__[name] = __import__(modname) + except KeyError: + pass + +#NOTE: 06/03/15 renamed main_module to main +def dump_module( + filename: Union[str, os.PathLike] = None, + module: Optional[Union[ModuleType, str]] = None, + refimported: bool = False, + **kwds +) -> None: + """Pickle the current state of :py:mod:`__main__` or another module to a file. + + Save the contents of :py:mod:`__main__` (e.g. from an interactive + interpreter session), an imported module, or a module-type object (e.g. + built with :py:class:`~types.ModuleType`), to a file. The pickled + module can then be restored with the function :py:func:`load_module`. + + Args: + filename: a path-like object or a writable stream. If `None` + (the default), write to a named file in a temporary directory. + module: a module object or the name of an importable module. If `None` + (the default), :py:mod:`__main__` is saved. + refimported: if `True`, all objects identified as having been imported + into the module's namespace are saved by reference. *Note:* this is + similar but independent from ``dill.settings[`byref`]``, as + ``refimported`` refers to virtually all imported objects, while + ``byref`` only affects select objects. + **kwds: extra keyword arguments passed to :py:class:`Pickler()`. + + Raises: + :py:exc:`PicklingError`: if pickling fails. + + Examples: + + - Save current interpreter session state: + + >>> import dill + >>> squared = lambda x: x*x + >>> dill.dump_module() # save state of __main__ to /tmp/session.pkl + + - Save the state of an imported/importable module: + + >>> import dill + >>> import pox + >>> pox.plus_one = lambda x: x+1 + >>> dill.dump_module('pox_session.pkl', module=pox) + + - Save the state of a non-importable, module-type object: + + >>> import dill + >>> from types import ModuleType + >>> foo = ModuleType('foo') + >>> foo.values = [1,2,3] + >>> import math + >>> foo.sin = math.sin + >>> dill.dump_module('foo_session.pkl', module=foo, refimported=True) + + - Restore the state of the saved modules: + + >>> import dill + >>> dill.load_module() + >>> squared(2) + 4 + >>> pox = dill.load_module('pox_session.pkl') + >>> pox.plus_one(1) + 2 + >>> foo = dill.load_module('foo_session.pkl') + >>> [foo.sin(x) for x in foo.values] + [0.8414709848078965, 0.9092974268256817, 0.1411200080598672] + + - Use `refimported` to save imported objects by reference: + + >>> import dill + >>> from html.entities import html5 + >>> type(html5), len(html5) + (dict, 2231) + >>> import io + >>> buf = io.BytesIO() + >>> dill.dump_module(buf) # saves __main__, with html5 saved by value + >>> len(buf.getvalue()) # pickle size in bytes + 71665 + >>> buf = io.BytesIO() + >>> dill.dump_module(buf, refimported=True) # html5 saved by reference + >>> len(buf.getvalue()) + 438 + + *Changed in version 0.3.6:* Function ``dump_session()`` was renamed to + ``dump_module()``. Parameters ``main`` and ``byref`` were renamed to + ``module`` and ``refimported``, respectively. + + Note: + Currently, ``dill.settings['byref']`` and ``dill.settings['recurse']`` + don't apply to this function. + """ + for old_par, par in [('main', 'module'), ('byref', 'refimported')]: + if old_par in kwds: + message = "The argument %r has been renamed %r" % (old_par, par) + if old_par == 'byref': + message += " to distinguish it from dill.settings['byref']" + warnings.warn(message + ".", PendingDeprecationWarning) + if locals()[par]: # the defaults are None and False + raise TypeError("both %r and %r arguments were used" % (par, old_par)) + refimported = kwds.pop('byref', refimported) + module = kwds.pop('main', module) + + from .settings import settings + protocol = settings['protocol'] + main = module + if main is None: + main = _main_module + elif isinstance(main, str): + main = _import_module(main) + if not isinstance(main, ModuleType): + raise TypeError("%r is not a module" % main) + if hasattr(filename, 'write'): + file = filename + else: + if filename is None: + filename = str(TEMPDIR/'session.pkl') + file = open(filename, 'wb') + try: + pickler = Pickler(file, protocol, **kwds) + pickler._original_main = main + if refimported: + main = _stash_modules(main) + pickler._main = main #FIXME: dill.settings are disabled + pickler._byref = False # disable pickling by name reference + pickler._recurse = False # disable pickling recursion for globals + pickler._session = True # is best indicator of when pickling a session + pickler._first_pass = True + pickler._main_modified = main is not pickler._original_main + pickler.dump(main) + finally: + if file is not filename: # if newly opened file + file.close() + return + +# Backward compatibility. +def dump_session(filename=None, main=None, byref=False, **kwds): + warnings.warn("dump_session() has been renamed dump_module()", PendingDeprecationWarning) + dump_module(filename, module=main, refimported=byref, **kwds) +dump_session.__doc__ = dump_module.__doc__ + +class _PeekableReader: + """lightweight stream wrapper that implements peek()""" + def __init__(self, stream): + self.stream = stream + def read(self, n): + return self.stream.read(n) + def readline(self): + return self.stream.readline() + def tell(self): + return self.stream.tell() + def close(self): + return self.stream.close() + def peek(self, n): + stream = self.stream + try: + if hasattr(stream, 'flush'): stream.flush() + position = stream.tell() + stream.seek(position) # assert seek() works before reading + chunk = stream.read(n) + stream.seek(position) + return chunk + except (AttributeError, OSError): + raise NotImplementedError("stream is not peekable: %r", stream) from None + +def _make_peekable(stream): + """return stream as an object with a peek() method""" + import io + if hasattr(stream, 'peek'): + return stream + if not (hasattr(stream, 'tell') and hasattr(stream, 'seek')): + try: + return io.BufferedReader(stream) + except Exception: + pass + return _PeekableReader(stream) + +def _identify_module(file, main=None): + """identify the name of the module stored in the given file-type object""" + from pickletools import genops + UNICODE = {'UNICODE', 'BINUNICODE', 'SHORT_BINUNICODE'} + found_import = False + try: + for opcode, arg, pos in genops(file.peek(256)): + if not found_import: + if opcode.name in ('GLOBAL', 'SHORT_BINUNICODE') and \ + arg.endswith('_import_module'): + found_import = True + else: + if opcode.name in UNICODE: + return arg + else: + raise UnpicklingError("reached STOP without finding main module") + except (NotImplementedError, ValueError) as error: + # ValueError occours when the end of the chunk is reached (without a STOP). + if isinstance(error, NotImplementedError) and main is not None: + # file is not peekable, but we have main. + return None + raise UnpicklingError("unable to identify main module") from error + +def load_module( + filename: Union[str, os.PathLike] = None, + module: Optional[Union[ModuleType, str]] = None, + **kwds +) -> Optional[ModuleType]: + """Update the selected module (default is :py:mod:`__main__`) with + the state saved at ``filename``. + + Restore a module to the state saved with :py:func:`dump_module`. The + saved module can be :py:mod:`__main__` (e.g. an interpreter session), + an imported module, or a module-type object (e.g. created with + :py:class:`~types.ModuleType`). + + When restoring the state of a non-importable module-type object, the + current instance of this module may be passed as the argument ``main``. + Otherwise, a new instance is created with :py:class:`~types.ModuleType` + and returned. + + Args: + filename: a path-like object or a readable stream. If `None` + (the default), read from a named file in a temporary directory. + module: a module object or the name of an importable module; + the module name and kind (i.e. imported or non-imported) must + match the name and kind of the module stored at ``filename``. + **kwds: extra keyword arguments passed to :py:class:`Unpickler()`. + + Raises: + :py:exc:`UnpicklingError`: if unpickling fails. + :py:exc:`ValueError`: if the argument ``main`` and module saved + at ``filename`` are incompatible. + + Returns: + A module object, if the saved module is not :py:mod:`__main__` or + a module instance wasn't provided with the argument ``main``. + + Examples: + + - Save the state of some modules: + + >>> import dill + >>> squared = lambda x: x*x + >>> dill.dump_module() # save state of __main__ to /tmp/session.pkl + >>> + >>> import pox # an imported module + >>> pox.plus_one = lambda x: x+1 + >>> dill.dump_module('pox_session.pkl', module=pox) + >>> + >>> from types import ModuleType + >>> foo = ModuleType('foo') # a module-type object + >>> foo.values = [1,2,3] + >>> import math + >>> foo.sin = math.sin + >>> dill.dump_module('foo_session.pkl', module=foo, refimported=True) + + - Restore the state of the interpreter: + + >>> import dill + >>> dill.load_module() # updates __main__ from /tmp/session.pkl + >>> squared(2) + 4 + + - Load the saved state of an importable module: + + >>> import dill + >>> pox = dill.load_module('pox_session.pkl') + >>> pox.plus_one(1) + 2 + >>> import sys + >>> pox in sys.modules.values() + True + + - Load the saved state of a non-importable module-type object: + + >>> import dill + >>> foo = dill.load_module('foo_session.pkl') + >>> [foo.sin(x) for x in foo.values] + [0.8414709848078965, 0.9092974268256817, 0.1411200080598672] + >>> import math + >>> foo.sin is math.sin # foo.sin was saved by reference + True + >>> import sys + >>> foo in sys.modules.values() + False + + - Update the state of a non-importable module-type object: + + >>> import dill + >>> from types import ModuleType + >>> foo = ModuleType('foo') + >>> foo.values = ['a','b'] + >>> foo.sin = lambda x: x*x + >>> dill.load_module('foo_session.pkl', module=foo) + >>> [foo.sin(x) for x in foo.values] + [0.8414709848078965, 0.9092974268256817, 0.1411200080598672] + + *Changed in version 0.3.6:* Function ``load_session()`` was renamed to + ``load_module()``. Parameter ``main`` was renamed to ``module``. + + See also: + :py:func:`load_module_asdict` to load the contents of module saved + with :py:func:`dump_module` into a dictionary. + """ + if 'main' in kwds: + warnings.warn( + "The argument 'main' has been renamed 'module'.", + PendingDeprecationWarning + ) + if module is not None: + raise TypeError("both 'module' and 'main' arguments were used") + module = kwds.pop('main') + main = module + if hasattr(filename, 'read'): + file = filename + else: + if filename is None: + filename = str(TEMPDIR/'session.pkl') + file = open(filename, 'rb') + try: + file = _make_peekable(file) + #FIXME: dill.settings are disabled + unpickler = Unpickler(file, **kwds) + unpickler._session = True + + # Resolve unpickler._main + pickle_main = _identify_module(file, main) + if main is None and pickle_main is not None: + main = pickle_main + if isinstance(main, str): + if main.startswith('__runtime__.'): + # Create runtime module to load the session into. + main = ModuleType(main.partition('.')[-1]) + else: + main = _import_module(main) + if main is not None: + if not isinstance(main, ModuleType): + raise TypeError("%r is not a module" % main) + unpickler._main = main + else: + main = unpickler._main + + # Check against the pickle's main. + is_main_imported = _is_imported_module(main) + if pickle_main is not None: + is_runtime_mod = pickle_main.startswith('__runtime__.') + if is_runtime_mod: + pickle_main = pickle_main.partition('.')[-1] + error_msg = "can't update{} module{} %r with the saved state of{} module{} %r" + if is_runtime_mod and is_main_imported: + raise ValueError( + error_msg.format(" imported", "", "", "-type object") + % (main.__name__, pickle_main) + ) + if not is_runtime_mod and not is_main_imported: + raise ValueError( + error_msg.format("", "-type object", " imported", "") + % (pickle_main, main.__name__) + ) + if main.__name__ != pickle_main: + raise ValueError(error_msg.format("", "", "", "") % (main.__name__, pickle_main)) + + # This is for find_class() to be able to locate it. + if not is_main_imported: + runtime_main = '__runtime__.%s' % main.__name__ + sys.modules[runtime_main] = main + + loaded = unpickler.load() + finally: + if not hasattr(filename, 'read'): # if newly opened file + file.close() + try: + del sys.modules[runtime_main] + except (KeyError, NameError): + pass + assert loaded is main + _restore_modules(unpickler, main) + if main is _main_module or main is module: + return None + else: + return main + +# Backward compatibility. +def load_session(filename=None, main=None, **kwds): + warnings.warn("load_session() has been renamed load_module().", PendingDeprecationWarning) + load_module(filename, module=main, **kwds) +load_session.__doc__ = load_module.__doc__ + +def load_module_asdict( + filename: Union[str, os.PathLike] = None, + update: bool = False, + **kwds +) -> dict: + """ + Load the contents of a saved module into a dictionary. + + ``load_module_asdict()`` is the near-equivalent of:: + + lambda filename: vars(dill.load_module(filename)).copy() + + however, does not alter the original module. Also, the path of + the loaded module is stored in the ``__session__`` attribute. + + Args: + filename: a path-like object or a readable stream. If `None` + (the default), read from a named file in a temporary directory. + update: if `True`, initialize the dictionary with the current state + of the module prior to loading the state stored at filename. + **kwds: extra keyword arguments passed to :py:class:`Unpickler()` + + Raises: + :py:exc:`UnpicklingError`: if unpickling fails + + Returns: + A copy of the restored module's dictionary. + + Note: + If ``update`` is True, the corresponding module may first be imported + into the current namespace before the saved state is loaded from + filename to the dictionary. Note that any module that is imported into + the current namespace as a side-effect of using ``update`` will not be + modified by loading the saved module in filename to a dictionary. + + Example: + >>> import dill + >>> alist = [1, 2, 3] + >>> anum = 42 + >>> dill.dump_module() + >>> anum = 0 + >>> new_var = 'spam' + >>> main = dill.load_module_asdict() + >>> main['__name__'], main['__session__'] + ('__main__', '/tmp/session.pkl') + >>> main is globals() # loaded objects don't reference globals + False + >>> main['alist'] == alist + True + >>> main['alist'] is alist # was saved by value + False + >>> main['anum'] == anum # changed after the session was saved + False + >>> new_var in main # would be True if the option 'update' was set + False + """ + if 'module' in kwds: + raise TypeError("'module' is an invalid keyword argument for load_module_asdict()") + if hasattr(filename, 'read'): + file = filename + else: + if filename is None: + filename = str(TEMPDIR/'session.pkl') + file = open(filename, 'rb') + try: + file = _make_peekable(file) + main_name = _identify_module(file) + old_main = sys.modules.get(main_name) + main = ModuleType(main_name) + if update: + if old_main is None: + old_main = _import_module(main_name) + main.__dict__.update(old_main.__dict__) + else: + main.__builtins__ = __builtin__ + sys.modules[main_name] = main + load_module(file, **kwds) + finally: + if not hasattr(filename, 'read'): # if newly opened file + file.close() + try: + if old_main is None: + del sys.modules[main_name] + else: + sys.modules[main_name] = old_main + except NameError: # failed before setting old_main + pass + main.__session__ = str(filename) + return main.__dict__ + + +# Internal exports for backward compatibility with dill v0.3.5.1 +# Can't be placed in dill._dill because of circular import problems. +for name in ( + '_lookup_module', '_module_map', '_restore_modules', '_stash_modules', + 'dump_session', 'load_session' # backward compatibility functions +): + setattr(_dill, name, globals()[name]) +del name diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill/settings.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill/settings.py new file mode 100644 index 0000000000000000000000000000000000000000..19c18fc5f5159c99ef80759bde406bf15dfecc63 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill/settings.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python +# +# Author: Mike McKerns (mmckerns @caltech and @uqfoundation) +# Copyright (c) 2008-2016 California Institute of Technology. +# Copyright (c) 2016-2024 The Uncertainty Quantification Foundation. +# License: 3-clause BSD. The full license text is available at: +# - https://github.com/uqfoundation/dill/blob/master/LICENSE +""" +global settings for Pickler +""" + +from pickle import DEFAULT_PROTOCOL + +settings = { + #'main' : None, + 'protocol' : DEFAULT_PROTOCOL, + 'byref' : False, + #'strictio' : False, + 'fmode' : 0, #HANDLE_FMODE + 'recurse' : False, + 'ignore' : False, +} + +del DEFAULT_PROTOCOL + diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill/source.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill/source.py new file mode 100644 index 0000000000000000000000000000000000000000..1d7bd0d3733c5a76847c4066fe56713145c8a438 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill/source.py @@ -0,0 +1,1017 @@ +#!/usr/bin/env python +# +# Author: Mike McKerns (mmckerns @caltech and @uqfoundation) +# Copyright (c) 2008-2016 California Institute of Technology. +# Copyright (c) 2016-2024 The Uncertainty Quantification Foundation. +# License: 3-clause BSD. The full license text is available at: +# - https://github.com/uqfoundation/dill/blob/master/LICENSE +# +# inspired by inspect.py from Python-2.7.6 +# inspect.py author: 'Ka-Ping Yee ' +# inspect.py merged into original dill.source by Mike McKerns 4/13/14 +""" +Extensions to python's 'inspect' module, which can be used +to retrieve information from live python objects. The methods +defined in this module are augmented to facilitate access to +source code of interactively defined functions and classes, +as well as provide access to source code for objects defined +in a file. +""" + +__all__ = ['findsource', 'getsourcelines', 'getsource', 'indent', 'outdent', \ + '_wrap', 'dumpsource', 'getname', '_namespace', 'getimport', \ + '_importable', 'importable','isdynamic', 'isfrommain'] + +import linecache +import re +from inspect import (getblock, getfile, getmodule, getsourcefile, indentsize, + isbuiltin, isclass, iscode, isframe, isfunction, ismethod, + ismodule, istraceback) +from tokenize import TokenError + +from ._dill import IS_IPYTHON + + +def isfrommain(obj): + "check if object was built in __main__" + module = getmodule(obj) + if module and module.__name__ == '__main__': + return True + return False + + +def isdynamic(obj): + "check if object was built in the interpreter" + try: file = getfile(obj) + except TypeError: file = None + if file == '' and isfrommain(obj): + return True + return False + + +def _matchlambda(func, line): + """check if lambda object 'func' matches raw line of code 'line'""" + from .detect import code as getcode + from .detect import freevars, globalvars, varnames + dummy = lambda : '__this_is_a_big_dummy_function__' + # process the line (removing leading whitespace, etc) + lhs,rhs = line.split('lambda ',1)[-1].split(":", 1) #FIXME: if !1 inputs + try: #FIXME: unsafe + _ = eval("lambda %s : %s" % (lhs,rhs), globals(),locals()) + except Exception: _ = dummy + # get code objects, for comparison + _, code = getcode(_).co_code, getcode(func).co_code + # check if func is in closure + _f = [line.count(i) for i in freevars(func).keys()] + if not _f: # not in closure + # check if code matches + if _ == code: return True + return False + # weak check on freevars + if not all(_f): return False #XXX: VERY WEAK + # weak check on varnames and globalvars + _f = varnames(func) + _f = [line.count(i) for i in _f[0]+_f[1]] + if _f and not all(_f): return False #XXX: VERY WEAK + _f = [line.count(i) for i in globalvars(func).keys()] + if _f and not all(_f): return False #XXX: VERY WEAK + # check if func is a double lambda + if (line.count('lambda ') > 1) and (lhs in freevars(func).keys()): + _lhs,_rhs = rhs.split('lambda ',1)[-1].split(":",1) #FIXME: if !1 inputs + try: #FIXME: unsafe + _f = eval("lambda %s : %s" % (_lhs,_rhs), globals(),locals()) + except Exception: _f = dummy + # get code objects, for comparison + _, code = getcode(_f).co_code, getcode(func).co_code + if len(_) != len(code): return False + #NOTE: should be same code same order, but except for 't' and '\x88' + _ = set((i,j) for (i,j) in zip(_,code) if i != j) + if len(_) != 1: return False #('t','\x88') + return True + # check indentsize + if not indentsize(line): return False #FIXME: is this a good check??? + # check if code 'pattern' matches + #XXX: or pattern match against dis.dis(code)? (or use uncompyle2?) + _ = _.split(_[0]) # 't' #XXX: remove matching values if starts the same? + _f = code.split(code[0]) # '\x88' + #NOTE: should be same code different order, with different first element + _ = dict(re.match(r'([\W\D\S])(.*)', _[i]).groups() for i in range(1,len(_))) + _f = dict(re.match(r'([\W\D\S])(.*)', _f[i]).groups() for i in range(1,len(_f))) + if (_.keys() == _f.keys()) and (sorted(_.values()) == sorted(_f.values())): + return True + return False + + +def findsource(object): + """Return the entire source file and starting line number for an object. + For interactively-defined objects, the 'file' is the interpreter's history. + + The argument may be a module, class, method, function, traceback, frame, + or code object. The source code is returned as a list of all the lines + in the file and the line number indexes a line in that list. An IOError + is raised if the source code cannot be retrieved, while a TypeError is + raised for objects where the source code is unavailable (e.g. builtins).""" + + module = getmodule(object) + try: file = getfile(module) + except TypeError: file = None + is_module_main = (module and module.__name__ == '__main__' and not file) + if IS_IPYTHON and is_module_main: + #FIXME: quick fix for functions and classes in IPython interpreter + try: + file = getfile(object) + sourcefile = getsourcefile(object) + except TypeError: + if isclass(object): + for object_method in filter(isfunction, object.__dict__.values()): + # look for a method of the class + file_candidate = getfile(object_method) + if not file_candidate.startswith('': pat1 = r'(.*(?': + pat1 = r'(.*(?' + if stdin: + lnum = len(lines) - 1 # can't get lnum easily, so leverage pat + if not pat1: pat1 = r'^(\s*def\s)|(.*(? 0: #XXX: won't find decorators in ? + line = lines[lnum] + if pat1.match(line): + if not stdin: break # co_firstlineno does the job + if name == '': # hackery needed to confirm a match + if _matchlambda(obj, line): break + else: # not a lambda, just look for the name + if name in line: # need to check for decorator... + hats = 0 + for _lnum in range(lnum-1,-1,-1): + if pat2.match(lines[_lnum]): hats += 1 + else: break + lnum = lnum - hats + break + lnum = lnum - 1 + return lines, lnum + + try: # turn instances into classes + if not isclass(object) and isclass(type(object)): # __class__ + object = object.__class__ #XXX: sometimes type(class) is better? + #XXX: we don't find how the instance was built + except AttributeError: pass + if isclass(object): + name = object.__name__ + pat = re.compile(r'^(\s*)class\s*' + name + r'\b') + # make some effort to find the best matching class definition: + # use the one with the least indentation, which is the one + # that's most probably not inside a function definition. + candidates = [] + for i in range(len(lines)-1,-1,-1): + match = pat.match(lines[i]) + if match: + # if it's at toplevel, it's already the best one + if lines[i][0] == 'c': + return lines, i + # else add whitespace to candidate list + candidates.append((match.group(1), i)) + if candidates: + # this will sort by whitespace, and by line number, + # less whitespace first #XXX: should sort high lnum before low + candidates.sort() + return lines, candidates[0][1] + else: + raise IOError('could not find class definition') + raise IOError('could not find code object') + + +def getblocks(object, lstrip=False, enclosing=False, locate=False): + """Return a list of source lines and starting line number for an object. + Interactively-defined objects refer to lines in the interpreter's history. + + If enclosing=True, then also return any enclosing code. + If lstrip=True, ensure there is no indentation in the first line of code. + If locate=True, then also return the line number for the block of code. + + DEPRECATED: use 'getsourcelines' instead + """ + lines, lnum = findsource(object) + + if ismodule(object): + if lstrip: lines = _outdent(lines) + return ([lines], [0]) if locate is True else [lines] + + #XXX: 'enclosing' means: closures only? or classes and files? + indent = indentsize(lines[lnum]) + block = getblock(lines[lnum:]) #XXX: catch any TokenError here? + + if not enclosing or not indent: + if lstrip: block = _outdent(block) + return ([block], [lnum]) if locate is True else [block] + + pat1 = r'^(\s*def\s)|(.*(? indent: #XXX: should be >= ? + line += len(code) - skip + elif target in ''.join(code): + blocks.append(code) # save code block as the potential winner + _lnum.append(line - skip) # save the line number for the match + line += len(code) - skip + else: + line += 1 + skip = 0 + # find skip: the number of consecutive decorators + elif pat2.match(lines[line]): + try: code = getblock(lines[line:]) + except TokenError: code = [lines[line]] + skip = 1 + for _line in code[1:]: # skip lines that are decorators + if not pat2.match(_line): break + skip += 1 + line += skip + # no match: reset skip and go to the next line + else: + line +=1 + skip = 0 + + if not blocks: + blocks = [block] + _lnum = [lnum] + if lstrip: blocks = [_outdent(block) for block in blocks] + # return last match + return (blocks, _lnum) if locate is True else blocks + + +def getsourcelines(object, lstrip=False, enclosing=False): + """Return a list of source lines and starting line number for an object. + Interactively-defined objects refer to lines in the interpreter's history. + + The argument may be a module, class, method, function, traceback, frame, + or code object. The source code is returned as a list of the lines + corresponding to the object and the line number indicates where in the + original source file the first line of code was found. An IOError is + raised if the source code cannot be retrieved, while a TypeError is + raised for objects where the source code is unavailable (e.g. builtins). + + If lstrip=True, ensure there is no indentation in the first line of code. + If enclosing=True, then also return any enclosing code.""" + code, n = getblocks(object, lstrip=lstrip, enclosing=enclosing, locate=True) + return code[-1], n[-1] + + +#NOTE: broke backward compatibility 4/16/14 (was lstrip=True, force=True) +def getsource(object, alias='', lstrip=False, enclosing=False, \ + force=False, builtin=False): + """Return the text of the source code for an object. The source code for + interactively-defined objects are extracted from the interpreter's history. + + The argument may be a module, class, method, function, traceback, frame, + or code object. The source code is returned as a single string. An + IOError is raised if the source code cannot be retrieved, while a + TypeError is raised for objects where the source code is unavailable + (e.g. builtins). + + If alias is provided, then add a line of code that renames the object. + If lstrip=True, ensure there is no indentation in the first line of code. + If enclosing=True, then also return any enclosing code. + If force=True, catch (TypeError,IOError) and try to use import hooks. + If builtin=True, force an import for any builtins + """ + # hascode denotes a callable + hascode = _hascode(object) + # is a class instance type (and not in builtins) + instance = _isinstance(object) + + # get source lines; if fail, try to 'force' an import + try: # fails for builtins, and other assorted object types + lines, lnum = getsourcelines(object, enclosing=enclosing) + except (TypeError, IOError): # failed to get source, resort to import hooks + if not force: # don't try to get types that findsource can't get + raise + if not getmodule(object): # get things like 'None' and '1' + if not instance: return getimport(object, alias, builtin=builtin) + # special handling (numpy arrays, ...) + _import = getimport(object, builtin=builtin) + name = getname(object, force=True) + _alias = "%s = " % alias if alias else "" + if alias == name: _alias = "" + return _import+_alias+"%s\n" % name + else: #FIXME: could use a good bit of cleanup, since using getimport... + if not instance: return getimport(object, alias, builtin=builtin) + # now we are dealing with an instance... + name = object.__class__.__name__ + module = object.__module__ + if module in ['builtins','__builtin__']: + return getimport(object, alias, builtin=builtin) + else: #FIXME: leverage getimport? use 'from module import name'? + lines, lnum = ["%s = __import__('%s', fromlist=['%s']).%s\n" % (name,module,name,name)], 0 + obj = eval(lines[0].lstrip(name + ' = ')) + lines, lnum = getsourcelines(obj, enclosing=enclosing) + + # strip leading indent (helps ensure can be imported) + if lstrip or alias: + lines = _outdent(lines) + + # instantiate, if there's a nice repr #XXX: BAD IDEA??? + if instance: #and force: #XXX: move into findsource or getsourcelines ? + if '(' in repr(object): lines.append('%r\n' % object) + #else: #XXX: better to somehow to leverage __reduce__ ? + # reconstructor,args = object.__reduce__() + # _ = reconstructor(*args) + else: # fall back to serialization #XXX: bad idea? + #XXX: better not duplicate work? #XXX: better new/enclose=True? + lines = dumpsource(object, alias='', new=force, enclose=False) + lines, lnum = [line+'\n' for line in lines.split('\n')][:-1], 0 + #else: object.__code__ # raise AttributeError + + # add an alias to the source code + if alias: + if hascode: + skip = 0 + for line in lines: # skip lines that are decorators + if not line.startswith('@'): break + skip += 1 + #XXX: use regex from findsource / getsourcelines ? + if lines[skip].lstrip().startswith('def '): # we have a function + if alias != object.__name__: + lines.append('\n%s = %s\n' % (alias, object.__name__)) + elif 'lambda ' in lines[skip]: # we have a lambda + if alias != lines[skip].split('=')[0].strip(): + lines[skip] = '%s = %s' % (alias, lines[skip]) + else: # ...try to use the object's name + if alias != object.__name__: + lines.append('\n%s = %s\n' % (alias, object.__name__)) + else: # class or class instance + if instance: + if alias != lines[-1].split('=')[0].strip(): + lines[-1] = ('%s = ' % alias) + lines[-1] + else: + name = getname(object, force=True) or object.__name__ + if alias != name: + lines.append('\n%s = %s\n' % (alias, name)) + return ''.join(lines) + + +def _hascode(object): + '''True if object has an attribute that stores it's __code__''' + return getattr(object,'__code__',None) or getattr(object,'func_code',None) + +def _isinstance(object): + '''True if object is a class instance type (and is not a builtin)''' + if _hascode(object) or isclass(object) or ismodule(object): + return False + if istraceback(object) or isframe(object) or iscode(object): + return False + # special handling (numpy arrays, ...) + if not getmodule(object) and getmodule(type(object)).__name__ in ['numpy']: + return True +# # check if is instance of a builtin +# if not getmodule(object) and getmodule(type(object)).__name__ in ['__builtin__','builtins']: +# return False + _types = ('") + if not repr(type(object)).startswith(_types): #FIXME: weak hack + return False + if not getmodule(object) or object.__module__ in ['builtins','__builtin__'] or getname(object, force=True) in ['array']: + return False + return True # by process of elimination... it's what we want + + +def _intypes(object): + '''check if object is in the 'types' module''' + import types + # allow user to pass in object or object.__name__ + if type(object) is not type(''): + object = getname(object, force=True) + if object == 'ellipsis': object = 'EllipsisType' + return True if hasattr(types, object) else False + + +def _isstring(object): #XXX: isstringlike better? + '''check if object is a string-like type''' + return isinstance(object, (str, bytes)) + + +def indent(code, spaces=4): + '''indent a block of code with whitespace (default is 4 spaces)''' + indent = indentsize(code) + if type(spaces) is int: spaces = ' '*spaces + # if '\t' is provided, will indent with a tab + nspaces = indentsize(spaces) + # blank lines (etc) need to be ignored + lines = code.split('\n') +## stq = "'''"; dtq = '"""' +## in_stq = in_dtq = False + for i in range(len(lines)): + #FIXME: works... but shouldn't indent 2nd+ lines of multiline doc + _indent = indentsize(lines[i]) + if indent > _indent: continue + lines[i] = spaces+lines[i] +## #FIXME: may fail when stq and dtq in same line (depends on ordering) +## nstq, ndtq = lines[i].count(stq), lines[i].count(dtq) +## if not in_dtq and not in_stq: +## lines[i] = spaces+lines[i] # we indent +## # entering a comment block +## if nstq%2: in_stq = not in_stq +## if ndtq%2: in_dtq = not in_dtq +## # leaving a comment block +## elif in_dtq and ndtq%2: in_dtq = not in_dtq +## elif in_stq and nstq%2: in_stq = not in_stq +## else: pass + if lines[-1].strip() == '': lines[-1] = '' + return '\n'.join(lines) + + +def _outdent(lines, spaces=None, all=True): + '''outdent lines of code, accounting for docs and line continuations''' + indent = indentsize(lines[0]) + if spaces is None or spaces > indent or spaces < 0: spaces = indent + for i in range(len(lines) if all else 1): + #FIXME: works... but shouldn't outdent 2nd+ lines of multiline doc + _indent = indentsize(lines[i]) + if spaces > _indent: _spaces = _indent + else: _spaces = spaces + lines[i] = lines[i][_spaces:] + return lines + +def outdent(code, spaces=None, all=True): + '''outdent a block of code (default is to strip all leading whitespace)''' + indent = indentsize(code) + if spaces is None or spaces > indent or spaces < 0: spaces = indent + #XXX: will this delete '\n' in some cases? + if not all: return code[spaces:] + return '\n'.join(_outdent(code.split('\n'), spaces=spaces, all=all)) + + +#XXX: not sure what the point of _wrap is... +__globals__ = globals() +__locals__ = locals() +def _wrap(f): + """ encapsulate a function and it's __import__ """ + def func(*args, **kwds): + try: + # _ = eval(getsource(f, force=True)) #XXX: safer but less robust + exec(getimportable(f, alias='_'), __globals__, __locals__) + except Exception: + raise ImportError('cannot import name ' + f.__name__) + return _(*args, **kwds) + func.__name__ = f.__name__ + func.__doc__ = f.__doc__ + return func + + +def _enclose(object, alias=''): #FIXME: needs alias to hold returned object + """create a function enclosure around the source of some object""" + #XXX: dummy and stub should append a random string + dummy = '__this_is_a_big_dummy_enclosing_function__' + stub = '__this_is_a_stub_variable__' + code = 'def %s():\n' % dummy + code += indent(getsource(object, alias=stub, lstrip=True, force=True)) + code += indent('return %s\n' % stub) + if alias: code += '%s = ' % alias + code += '%s(); del %s\n' % (dummy, dummy) + #code += "globals().pop('%s',lambda :None)()\n" % dummy + return code + + +def dumpsource(object, alias='', new=False, enclose=True): + """'dump to source', where the code includes a pickled object. + + If new=True and object is a class instance, then create a new + instance using the unpacked class source code. If enclose, then + create the object inside a function enclosure (thus minimizing + any global namespace pollution). + """ + from dill import dumps + pik = repr(dumps(object)) + code = 'import dill\n' + if enclose: + stub = '__this_is_a_stub_variable__' #XXX: *must* be same _enclose.stub + pre = '%s = ' % stub + new = False #FIXME: new=True doesn't work with enclose=True + else: + stub = alias + pre = '%s = ' % stub if alias else alias + + # if a 'new' instance is not needed, then just dump and load + if not new or not _isinstance(object): + code += pre + 'dill.loads(%s)\n' % pik + else: #XXX: other cases where source code is needed??? + code += getsource(object.__class__, alias='', lstrip=True, force=True) + mod = repr(object.__module__) # should have a module (no builtins here) + code += pre + 'dill.loads(%s.replace(b%s,bytes(__name__,"UTF-8")))\n' % (pik,mod) + #code += 'del %s' % object.__class__.__name__ #NOTE: kills any existing! + + if enclose: + # generation of the 'enclosure' + dummy = '__this_is_a_big_dummy_object__' + dummy = _enclose(dummy, alias=alias) + # hack to replace the 'dummy' with the 'real' code + dummy = dummy.split('\n') + code = dummy[0]+'\n' + indent(code) + '\n'.join(dummy[-3:]) + + return code #XXX: better 'dumpsourcelines', returning list of lines? + + +def getname(obj, force=False, fqn=False): #XXX: throw(?) to raise error on fail? + """get the name of the object. for lambdas, get the name of the pointer """ + if fqn: return '.'.join(_namespace(obj)) + module = getmodule(obj) + if not module: # things like "None" and "1" + if not force: return None + return repr(obj) + try: + #XXX: 'wrong' for decorators and curried functions ? + # if obj.func_closure: ...use logic from getimportable, etc ? + name = obj.__name__ + if name == '': + return getsource(obj).split('=',1)[0].strip() + # handle some special cases + if module.__name__ in ['builtins','__builtin__']: + if name == 'ellipsis': name = 'EllipsisType' + return name + except AttributeError: #XXX: better to just throw AttributeError ? + if not force: return None + name = repr(obj) + if name.startswith('<'): # or name.split('('): + return None + return name + + +def _namespace(obj): + """_namespace(obj); return namespace hierarchy (as a list of names) + for the given object. For an instance, find the class hierarchy. + + For example: + + >>> from functools import partial + >>> p = partial(int, base=2) + >>> _namespace(p) + [\'functools\', \'partial\'] + """ + # mostly for functions and modules and such + #FIXME: 'wrong' for decorators and curried functions + try: #XXX: needs some work and testing on different types + module = qual = str(getmodule(obj)).split()[1].strip('>').strip('"').strip("'") + qual = qual.split('.') + if ismodule(obj): + return qual + # get name of a lambda, function, etc + name = getname(obj) or obj.__name__ # failing, raise AttributeError + # check special cases (NoneType, ...) + if module in ['builtins','__builtin__']: # BuiltinFunctionType + if _intypes(name): return ['types'] + [name] + return qual + [name] #XXX: can be wrong for some aliased objects + except Exception: pass + # special case: numpy.inf and numpy.nan (we don't want them as floats) + if str(obj) in ['inf','nan','Inf','NaN']: # is more, but are they needed? + return ['numpy'] + [str(obj)] + # mostly for classes and class instances and such + module = getattr(obj.__class__, '__module__', None) + qual = str(obj.__class__) + try: qual = qual[qual.index("'")+1:-2] + except ValueError: pass # str(obj.__class__) made the 'try' unnecessary + qual = qual.split(".") + if module in ['builtins','__builtin__']: + # check special cases (NoneType, Ellipsis, ...) + if qual[-1] == 'ellipsis': qual[-1] = 'EllipsisType' + if _intypes(qual[-1]): module = 'types' #XXX: BuiltinFunctionType + qual = [module] + qual + return qual + + +#NOTE: 05/25/14 broke backward compatibility: added 'alias' as 3rd argument +def _getimport(head, tail, alias='', verify=True, builtin=False): + """helper to build a likely import string from head and tail of namespace. + ('head','tail') are used in the following context: "from head import tail" + + If verify=True, then test the import string before returning it. + If builtin=True, then force an import for builtins where possible. + If alias is provided, then rename the object on import. + """ + # special handling for a few common types + if tail in ['Ellipsis', 'NotImplemented'] and head in ['types']: + head = len.__module__ + elif tail in ['None'] and head in ['types']: + _alias = '%s = ' % alias if alias else '' + if alias == tail: _alias = '' + return _alias+'%s\n' % tail + # we don't need to import from builtins, so return '' +# elif tail in ['NoneType','int','float','long','complex']: return '' #XXX: ? + if head in ['builtins','__builtin__']: + # special cases (NoneType, Ellipsis, ...) #XXX: BuiltinFunctionType + if tail == 'ellipsis': tail = 'EllipsisType' + if _intypes(tail): head = 'types' + elif not builtin: + _alias = '%s = ' % alias if alias else '' + if alias == tail: _alias = '' + return _alias+'%s\n' % tail + else: pass # handle builtins below + # get likely import string + if not head: _str = "import %s" % tail + else: _str = "from %s import %s" % (head, tail) + _alias = " as %s\n" % alias if alias else "\n" + if alias == tail: _alias = "\n" + _str += _alias + # FIXME: fails on most decorators, currying, and such... + # (could look for magic __wrapped__ or __func__ attr) + # (could fix in 'namespace' to check obj for closure) + if verify and not head.startswith('dill.'):# weird behavior for dill + #print(_str) + try: exec(_str) #XXX: check if == obj? (name collision) + except ImportError: #XXX: better top-down or bottom-up recursion? + _head = head.rsplit(".",1)[0] #(or get all, then compare == obj?) + if not _head: raise + if _head != head: + _str = _getimport(_head, tail, alias, verify) + return _str + + +#XXX: rename builtin to force? vice versa? verify to force? (as in getsource) +#NOTE: 05/25/14 broke backward compatibility: added 'alias' as 2nd argument +def getimport(obj, alias='', verify=True, builtin=False, enclosing=False): + """get the likely import string for the given object + + obj is the object to inspect + If verify=True, then test the import string before returning it. + If builtin=True, then force an import for builtins where possible. + If enclosing=True, get the import for the outermost enclosing callable. + If alias is provided, then rename the object on import. + """ + if enclosing: + from .detect import outermost + _obj = outermost(obj) + obj = _obj if _obj else obj + # get the namespace + qual = _namespace(obj) + head = '.'.join(qual[:-1]) + tail = qual[-1] + # for named things... with a nice repr #XXX: move into _namespace? + try: # look for '<...>' and be mindful it might be in lists, dicts, etc... + name = repr(obj).split('<',1)[1].split('>',1)[1] + name = None # we have a 'object'-style repr + except Exception: # it's probably something 'importable' + if head in ['builtins','__builtin__']: + name = repr(obj) #XXX: catch [1,2], (1,2), set([1,2])... others? + else: + name = repr(obj).split('(')[0] + #if not repr(obj).startswith('<'): name = repr(obj).split('(')[0] + #else: name = None + if name: # try using name instead of tail + try: return _getimport(head, name, alias, verify, builtin) + except ImportError: pass + except SyntaxError: + if head in ['builtins','__builtin__']: + _alias = '%s = ' % alias if alias else '' + if alias == name: _alias = '' + return _alias+'%s\n' % name + else: pass + try: + #if type(obj) is type(abs): _builtin = builtin # BuiltinFunctionType + #else: _builtin = False + return _getimport(head, tail, alias, verify, builtin) + except ImportError: + raise # could do some checking against obj + except SyntaxError: + if head in ['builtins','__builtin__']: + _alias = '%s = ' % alias if alias else '' + if alias == tail: _alias = '' + return _alias+'%s\n' % tail + raise # could do some checking against obj + + +def _importable(obj, alias='', source=None, enclosing=False, force=True, \ + builtin=True, lstrip=True): + """get an import string (or the source code) for the given object + + This function will attempt to discover the name of the object, or the repr + of the object, or the source code for the object. To attempt to force + discovery of the source code, use source=True, to attempt to force the + use of an import, use source=False; otherwise an import will be sought + for objects not defined in __main__. The intent is to build a string + that can be imported from a python file. obj is the object to inspect. + If alias is provided, then rename the object with the given alias. + + If source=True, use these options: + If enclosing=True, then also return any enclosing code. + If force=True, catch (TypeError,IOError) and try to use import hooks. + If lstrip=True, ensure there is no indentation in the first line of code. + + If source=False, use these options: + If enclosing=True, get the import for the outermost enclosing callable. + If force=True, then don't test the import string before returning it. + If builtin=True, then force an import for builtins where possible. + """ + if source is None: + source = True if isfrommain(obj) else False + if source: # first try to get the source + try: + return getsource(obj, alias, enclosing=enclosing, \ + force=force, lstrip=lstrip, builtin=builtin) + except Exception: pass + try: + if not _isinstance(obj): + return getimport(obj, alias, enclosing=enclosing, \ + verify=(not force), builtin=builtin) + # first 'get the import', then 'get the instance' + _import = getimport(obj, enclosing=enclosing, \ + verify=(not force), builtin=builtin) + name = getname(obj, force=True) + if not name: + raise AttributeError("object has no atribute '__name__'") + _alias = "%s = " % alias if alias else "" + if alias == name: _alias = "" + return _import+_alias+"%s\n" % name + + except Exception: pass + if not source: # try getsource, only if it hasn't been tried yet + try: + return getsource(obj, alias, enclosing=enclosing, \ + force=force, lstrip=lstrip, builtin=builtin) + except Exception: pass + # get the name (of functions, lambdas, and classes) + # or hope that obj can be built from the __repr__ + #XXX: what to do about class instances and such? + obj = getname(obj, force=force) + # we either have __repr__ or __name__ (or None) + if not obj or obj.startswith('<'): + raise AttributeError("object has no atribute '__name__'") + _alias = '%s = ' % alias if alias else '' + if alias == obj: _alias = '' + return _alias+'%s\n' % obj + #XXX: possible failsafe... (for example, for instances when source=False) + # "import dill; result = dill.loads(); # repr()" + +def _closuredimport(func, alias='', builtin=False): + """get import for closured objects; return a dict of 'name' and 'import'""" + import re + from .detect import freevars, outermost + free_vars = freevars(func) + func_vars = {} + # split into 'funcs' and 'non-funcs' + for name,obj in list(free_vars.items()): + if not isfunction(obj): continue + # get import for 'funcs' + fobj = free_vars.pop(name) + src = getsource(fobj) + if src.lstrip().startswith('@'): # we have a decorator + src = getimport(fobj, alias=alias, builtin=builtin) + else: # we have to "hack" a bit... and maybe be lucky + encl = outermost(func) + # pattern: 'func = enclosing(fobj' + pat = r'.*[\w\s]=\s*'+getname(encl)+r'\('+getname(fobj) + mod = getname(getmodule(encl)) + #HACK: get file containing 'outer' function; is func there? + lines,_ = findsource(encl) + candidate = [line for line in lines if getname(encl) in line and \ + re.match(pat, line)] + if not candidate: + mod = getname(getmodule(fobj)) + #HACK: get file containing 'inner' function; is func there? + lines,_ = findsource(fobj) + candidate = [line for line in lines \ + if getname(fobj) in line and re.match(pat, line)] + if not len(candidate): raise TypeError('import could not be found') + candidate = candidate[-1] + name = candidate.split('=',1)[0].split()[-1].strip() + src = _getimport(mod, name, alias=alias, builtin=builtin) + func_vars[name] = src + if not func_vars: + name = outermost(func) + mod = getname(getmodule(name)) + if not mod or name is func: # then it can be handled by getimport + name = getname(func, force=True) #XXX: better key? + src = getimport(func, alias=alias, builtin=builtin) + else: + lines,_ = findsource(name) + # pattern: 'func = enclosing(' + candidate = [line for line in lines if getname(name) in line and \ + re.match(r'.*[\w\s]=\s*'+getname(name)+r'\(', line)] + if not len(candidate): raise TypeError('import could not be found') + candidate = candidate[-1] + name = candidate.split('=',1)[0].split()[-1].strip() + src = _getimport(mod, name, alias=alias, builtin=builtin) + func_vars[name] = src + return func_vars + +#XXX: should be able to use __qualname__ +def _closuredsource(func, alias=''): + """get source code for closured objects; return a dict of 'name' + and 'code blocks'""" + #FIXME: this entire function is a messy messy HACK + # - pollutes global namespace + # - fails if name of freevars are reused + # - can unnecessarily duplicate function code + from .detect import freevars + free_vars = freevars(func) + func_vars = {} + # split into 'funcs' and 'non-funcs' + for name,obj in list(free_vars.items()): + if not isfunction(obj): + # get source for 'non-funcs' + free_vars[name] = getsource(obj, force=True, alias=name) + continue + # get source for 'funcs' + fobj = free_vars.pop(name) + src = getsource(fobj, alias) # DO NOT include dependencies + # if source doesn't start with '@', use name as the alias + if not src.lstrip().startswith('@'): #FIXME: 'enclose' in dummy; + src = importable(fobj,alias=name)# wrong ref 'name' + org = getsource(func, alias, enclosing=False, lstrip=True) + src = (src, org) # undecorated first, then target + else: #NOTE: reproduces the code! + org = getsource(func, enclosing=True, lstrip=False) + src = importable(fobj, alias, source=True) # include dependencies + src = (org, src) # target first, then decorated + func_vars[name] = src + src = ''.join(free_vars.values()) + if not func_vars: #FIXME: 'enclose' in dummy; wrong ref 'name' + org = getsource(func, alias, force=True, enclosing=False, lstrip=True) + src = (src, org) # variables first, then target + else: + src = (src, None) # just variables (better '' instead of None?) + func_vars[None] = src + # FIXME: remove duplicates (however, order is important...) + return func_vars + +def importable(obj, alias='', source=None, builtin=True): + """get an importable string (i.e. source code or the import string) + for the given object, including any required objects from the enclosing + and global scope + + This function will attempt to discover the name of the object, or the repr + of the object, or the source code for the object. To attempt to force + discovery of the source code, use source=True, to attempt to force the + use of an import, use source=False; otherwise an import will be sought + for objects not defined in __main__. The intent is to build a string + that can be imported from a python file. + + obj is the object to inspect. If alias is provided, then rename the + object with the given alias. If builtin=True, then force an import for + builtins where possible. + """ + #NOTE: we always 'force', and 'lstrip' as necessary + #NOTE: for 'enclosing', use importable(outermost(obj)) + if source is None: + source = True if isfrommain(obj) else False + elif builtin and isbuiltin(obj): + source = False + tried_source = tried_import = False + while True: + if not source: # we want an import + try: + if _isinstance(obj): # for instances, punt to _importable + return _importable(obj, alias, source=False, builtin=builtin) + src = _closuredimport(obj, alias=alias, builtin=builtin) + if len(src) == 0: + raise NotImplementedError('not implemented') + if len(src) > 1: + raise NotImplementedError('not implemented') + return list(src.values())[0] + except Exception: + if tried_source: raise + tried_import = True + # we want the source + try: + src = _closuredsource(obj, alias=alias) + if len(src) == 0: + raise NotImplementedError('not implemented') + # groan... an inline code stitcher + def _code_stitcher(block): + "stitch together the strings in tuple 'block'" + if block[0] and block[-1]: block = '\n'.join(block) + elif block[0]: block = block[0] + elif block[-1]: block = block[-1] + else: block = '' + return block + # get free_vars first + _src = _code_stitcher(src.pop(None)) + _src = [_src] if _src else [] + # get func_vars + for xxx in src.values(): + xxx = _code_stitcher(xxx) + if xxx: _src.append(xxx) + # make a single source string + if not len(_src): + src = '' + elif len(_src) == 1: + src = _src[0] + else: + src = '\n'.join(_src) + # get source code of objects referred to by obj in global scope + from .detect import globalvars + obj = globalvars(obj) #XXX: don't worry about alias? recurse? etc? + obj = list(getsource(_obj,name,force=True) for (name,_obj) in obj.items() if not isbuiltin(_obj)) + obj = '\n'.join(obj) if obj else '' + # combine all referred-to source (global then enclosing) + if not obj: return src + if not src: return obj + return obj + src + except Exception: + if tried_import: raise + tried_source = True + source = not source + # should never get here + return + + +# backward compatibility +def getimportable(obj, alias='', byname=True, explicit=False): + return importable(obj,alias,source=(not byname),builtin=explicit) + #return outdent(_importable(obj,alias,source=(not byname),builtin=explicit)) +def likely_import(obj, passive=False, explicit=False): + return getimport(obj, verify=(not passive), builtin=explicit) +def _likely_import(first, last, passive=False, explicit=True): + return _getimport(first, last, verify=(not passive), builtin=explicit) +_get_name = getname +getblocks_from_history = getblocks + + + +# EOF diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill/temp.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill/temp.py new file mode 100644 index 0000000000000000000000000000000000000000..c4d7165e1d5867bcf1d3cc6191435583b61672ed --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/dill/temp.py @@ -0,0 +1,252 @@ +#!/usr/bin/env python +# +# Author: Mike McKerns (mmckerns @caltech and @uqfoundation) +# Copyright (c) 2008-2016 California Institute of Technology. +# Copyright (c) 2016-2024 The Uncertainty Quantification Foundation. +# License: 3-clause BSD. The full license text is available at: +# - https://github.com/uqfoundation/dill/blob/master/LICENSE +""" +Methods for serialized objects (or source code) stored in temporary files +and file-like objects. +""" +#XXX: better instead to have functions write to any given file-like object ? +#XXX: currently, all file-like objects are created by the function... + +__all__ = ['dump_source', 'dump', 'dumpIO_source', 'dumpIO',\ + 'load_source', 'load', 'loadIO_source', 'loadIO',\ + 'capture'] + +import contextlib + + +@contextlib.contextmanager +def capture(stream='stdout'): + """builds a context that temporarily replaces the given stream name + + >>> with capture('stdout') as out: + ... print ("foo!") + ... + >>> print (out.getvalue()) + foo! + + """ + import sys + from io import StringIO + orig = getattr(sys, stream) + setattr(sys, stream, StringIO()) + try: + yield getattr(sys, stream) + finally: + setattr(sys, stream, orig) + + +def b(x): # deal with b'foo' versus 'foo' + import codecs + return codecs.latin_1_encode(x)[0] + +def load_source(file, **kwds): + """load an object that was stored with dill.temp.dump_source + + file: filehandle + alias: string name of stored object + mode: mode to open the file, one of: {'r', 'rb'} + + >>> f = lambda x: x**2 + >>> pyfile = dill.temp.dump_source(f, alias='_f') + >>> _f = dill.temp.load_source(pyfile) + >>> _f(4) + 16 + """ + alias = kwds.pop('alias', None) + mode = kwds.pop('mode', 'r') + fname = getattr(file, 'name', file) # fname=file.name or fname=file (if str) + source = open(fname, mode=mode, **kwds).read() + if not alias: + tag = source.strip().splitlines()[-1].split() + if tag[0] != '#NAME:': + stub = source.splitlines()[0] + raise IOError("unknown name for code: %s" % stub) + alias = tag[-1] + local = {} + exec(source, local) + _ = eval("%s" % alias, local) + return _ + +def dump_source(object, **kwds): + """write object source to a NamedTemporaryFile (instead of dill.dump) +Loads with "import" or "dill.temp.load_source". Returns the filehandle. + + >>> f = lambda x: x**2 + >>> pyfile = dill.temp.dump_source(f, alias='_f') + >>> _f = dill.temp.load_source(pyfile) + >>> _f(4) + 16 + + >>> f = lambda x: x**2 + >>> pyfile = dill.temp.dump_source(f, dir='.') + >>> modulename = os.path.basename(pyfile.name).split('.py')[0] + >>> exec('from %s import f as _f' % modulename) + >>> _f(4) + 16 + +Optional kwds: + If 'alias' is specified, the object will be renamed to the given string. + + If 'prefix' is specified, the file name will begin with that prefix, + otherwise a default prefix is used. + + If 'dir' is specified, the file will be created in that directory, + otherwise a default directory is used. + + If 'text' is specified and true, the file is opened in text + mode. Else (the default) the file is opened in binary mode. On + some operating systems, this makes no difference. + +NOTE: Keep the return value for as long as you want your file to exist ! + """ #XXX: write a "load_source"? + from .source import importable, getname + import tempfile + kwds.setdefault('delete', True) + kwds.pop('suffix', '') # this is *always* '.py' + alias = kwds.pop('alias', '') #XXX: include an alias so a name is known + name = str(alias) or getname(object) + name = "\n#NAME: %s\n" % name + #XXX: assumes kwds['dir'] is writable and on $PYTHONPATH + file = tempfile.NamedTemporaryFile(suffix='.py', **kwds) + file.write(b(''.join([importable(object, alias=alias),name]))) + file.flush() + return file + +def load(file, **kwds): + """load an object that was stored with dill.temp.dump + + file: filehandle + mode: mode to open the file, one of: {'r', 'rb'} + + >>> dumpfile = dill.temp.dump([1, 2, 3, 4, 5]) + >>> dill.temp.load(dumpfile) + [1, 2, 3, 4, 5] + """ + import dill as pickle + mode = kwds.pop('mode', 'rb') + name = getattr(file, 'name', file) # name=file.name or name=file (if str) + return pickle.load(open(name, mode=mode, **kwds)) + +def dump(object, **kwds): + """dill.dump of object to a NamedTemporaryFile. +Loads with "dill.temp.load". Returns the filehandle. + + >>> dumpfile = dill.temp.dump([1, 2, 3, 4, 5]) + >>> dill.temp.load(dumpfile) + [1, 2, 3, 4, 5] + +Optional kwds: + If 'suffix' is specified, the file name will end with that suffix, + otherwise there will be no suffix. + + If 'prefix' is specified, the file name will begin with that prefix, + otherwise a default prefix is used. + + If 'dir' is specified, the file will be created in that directory, + otherwise a default directory is used. + + If 'text' is specified and true, the file is opened in text + mode. Else (the default) the file is opened in binary mode. On + some operating systems, this makes no difference. + +NOTE: Keep the return value for as long as you want your file to exist ! + """ + import dill as pickle + import tempfile + kwds.setdefault('delete', True) + file = tempfile.NamedTemporaryFile(**kwds) + pickle.dump(object, file) + file.flush() + return file + +def loadIO(buffer, **kwds): + """load an object that was stored with dill.temp.dumpIO + + buffer: buffer object + + >>> dumpfile = dill.temp.dumpIO([1, 2, 3, 4, 5]) + >>> dill.temp.loadIO(dumpfile) + [1, 2, 3, 4, 5] + """ + import dill as pickle + from io import BytesIO as StringIO + value = getattr(buffer, 'getvalue', buffer) # value or buffer.getvalue + if value != buffer: value = value() # buffer.getvalue() + return pickle.load(StringIO(value)) + +def dumpIO(object, **kwds): + """dill.dump of object to a buffer. +Loads with "dill.temp.loadIO". Returns the buffer object. + + >>> dumpfile = dill.temp.dumpIO([1, 2, 3, 4, 5]) + >>> dill.temp.loadIO(dumpfile) + [1, 2, 3, 4, 5] + """ + import dill as pickle + from io import BytesIO as StringIO + file = StringIO() + pickle.dump(object, file) + file.flush() + return file + +def loadIO_source(buffer, **kwds): + """load an object that was stored with dill.temp.dumpIO_source + + buffer: buffer object + alias: string name of stored object + + >>> f = lambda x:x**2 + >>> pyfile = dill.temp.dumpIO_source(f, alias='_f') + >>> _f = dill.temp.loadIO_source(pyfile) + >>> _f(4) + 16 + """ + alias = kwds.pop('alias', None) + source = getattr(buffer, 'getvalue', buffer) # source or buffer.getvalue + if source != buffer: source = source() # buffer.getvalue() + source = source.decode() # buffer to string + if not alias: + tag = source.strip().splitlines()[-1].split() + if tag[0] != '#NAME:': + stub = source.splitlines()[0] + raise IOError("unknown name for code: %s" % stub) + alias = tag[-1] + local = {} + exec(source, local) + _ = eval("%s" % alias, local) + return _ + +def dumpIO_source(object, **kwds): + """write object source to a buffer (instead of dill.dump) +Loads by with dill.temp.loadIO_source. Returns the buffer object. + + >>> f = lambda x:x**2 + >>> pyfile = dill.temp.dumpIO_source(f, alias='_f') + >>> _f = dill.temp.loadIO_source(pyfile) + >>> _f(4) + 16 + +Optional kwds: + If 'alias' is specified, the object will be renamed to the given string. + """ + from .source import importable, getname + from io import BytesIO as StringIO + alias = kwds.pop('alias', '') #XXX: include an alias so a name is known + name = str(alias) or getname(object) + name = "\n#NAME: %s\n" % name + #XXX: assumes kwds['dir'] is writable and on $PYTHONPATH + file = StringIO() + file.write(b(''.join([importable(object, alias=alias),name]))) + file.flush() + return file + + +del contextlib + + +# EOF diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/evaluate/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/evaluate/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d3d7fc99ea1325805857b10f5e800e07c0691043 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/evaluate/__init__.py @@ -0,0 +1,51 @@ +# flake8: noqa +# Copyright 2020 The HuggingFace Evaluate Authors and the TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +# pylint: enable=line-too-long +# pylint: disable=g-import-not-at-top,g-bad-import-order,wrong-import-position + +__version__ = "0.4.6" + +from packaging import version + + +SCRIPTS_VERSION = "main" if version.parse(__version__).is_devrelease else __version__ + +del version + +from .evaluation_suite import EvaluationSuite +from .evaluator import ( + AudioClassificationEvaluator, + AutomaticSpeechRecognitionEvaluator, + Evaluator, + ImageClassificationEvaluator, + QuestionAnsweringEvaluator, + SummarizationEvaluator, + Text2TextGenerationEvaluator, + TextClassificationEvaluator, + TextGenerationEvaluator, + TokenClassificationEvaluator, + TranslationEvaluator, + evaluator, +) +from .hub import push_to_hub +from .info import ComparisonInfo, EvaluationModuleInfo, MeasurementInfo, MetricInfo +from .inspect import inspect_evaluation_module, list_evaluation_modules +from .loading import load +from .module import CombinedEvaluations, Comparison, EvaluationModule, Measurement, Metric, combine +from .saving import save +from .utils import * +from .utils import gradio, logging diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/evaluate/config.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/evaluate/config.py new file mode 100644 index 0000000000000000000000000000000000000000..4909fa251ff82893d7c3c536bb111ae947735a8a --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/evaluate/config.py @@ -0,0 +1,192 @@ +import importlib +import os +import platform +from pathlib import Path + +from packaging import version + +from .utils.logging import get_logger + + +logger = get_logger(__name__) + + +# Metrics +S3_METRICS_BUCKET_PREFIX = "https://s3.amazonaws.com/datasets.huggingface.co/datasets/metrics" +CLOUDFRONT_METRICS_DISTRIB_PREFIX = "https://cdn-datasets.huggingface.co/datasets/metric" +REPO_METRICS_URL = "https://raw.githubusercontent.com/huggingface/evaluate/{revision}/metrics/{path}/{name}" +REPO_MEASUREMENTS_URL = "https://raw.githubusercontent.com/huggingface/evaluate/{revision}/measurements/{path}/{name}" +REPO_COMPARISONS_URL = "https://raw.githubusercontent.com/huggingface/evaluate/{revision}/comparisons/{path}/{name}" + +# Evaluation module types +EVALUATION_MODULE_TYPES = ["metric", "comparison", "measurement"] + +# Hub +HF_ENDPOINT = os.environ.get("HF_ENDPOINT", "https://huggingface.co") +HF_LIST_ENDPOINT = HF_ENDPOINT + "/api/spaces?filter={type}" +HUB_EVALUATE_URL = HF_ENDPOINT + "/spaces/{path}/resolve/{revision}/{name}" +HUB_DEFAULT_VERSION = "main" + +PY_VERSION = version.parse(platform.python_version()) + +if PY_VERSION < version.parse("3.8"): + import importlib_metadata +else: + import importlib.metadata as importlib_metadata + +# General environment variables accepted values for booleans +ENV_VARS_TRUE_VALUES = {"1", "ON", "YES", "TRUE"} +ENV_VARS_TRUE_AND_AUTO_VALUES = ENV_VARS_TRUE_VALUES.union({"AUTO"}) + + +# Imports +PANDAS_VERSION = version.parse(importlib_metadata.version("pandas")) +PYARROW_VERSION = version.parse(importlib_metadata.version("pyarrow")) + +USE_TF = os.environ.get("USE_TF", "AUTO").upper() +USE_TORCH = os.environ.get("USE_TORCH", "AUTO").upper() +USE_JAX = os.environ.get("USE_JAX", "AUTO").upper() + +TORCH_VERSION = "N/A" +TORCH_AVAILABLE = False + +if USE_TORCH in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TF not in ENV_VARS_TRUE_VALUES: + TORCH_AVAILABLE = importlib.util.find_spec("torch") is not None + if TORCH_AVAILABLE: + try: + TORCH_VERSION = version.parse(importlib_metadata.version("torch")) + logger.info(f"PyTorch version {TORCH_VERSION} available.") + except importlib_metadata.PackageNotFoundError: + pass +else: + logger.info("Disabling PyTorch because USE_TF is set") + +TF_VERSION = "N/A" +TF_AVAILABLE = False + +if USE_TF in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TORCH not in ENV_VARS_TRUE_VALUES: + TF_AVAILABLE = importlib.util.find_spec("tensorflow") is not None + if TF_AVAILABLE: + # For the metadata, we have to look for both tensorflow and tensorflow-cpu + for package in [ + "tensorflow", + "tensorflow-cpu", + "tensorflow-gpu", + "tf-nightly", + "tf-nightly-cpu", + "tf-nightly-gpu", + "intel-tensorflow", + "tensorflow-rocm", + "tensorflow-macos", + ]: + try: + TF_VERSION = version.parse(importlib_metadata.version(package)) + except importlib_metadata.PackageNotFoundError: + continue + else: + break + else: + TF_AVAILABLE = False + if TF_AVAILABLE: + if TF_VERSION.major < 2: + logger.info(f"TensorFlow found but with version {TF_VERSION}. `datasets` requires version 2 minimum.") + TF_AVAILABLE = False + else: + logger.info(f"TensorFlow version {TF_VERSION} available.") +else: + logger.info("Disabling Tensorflow because USE_TORCH is set") + + +JAX_VERSION = "N/A" +JAX_AVAILABLE = False + +if USE_JAX in ENV_VARS_TRUE_AND_AUTO_VALUES: + JAX_AVAILABLE = importlib.util.find_spec("jax") is not None + if JAX_AVAILABLE: + try: + JAX_VERSION = version.parse(importlib_metadata.version("jax")) + logger.info(f"JAX version {JAX_VERSION} available.") + except importlib_metadata.PackageNotFoundError: + pass +else: + logger.info("Disabling JAX because USE_JAX is set to False") + + +# Cache location +DEFAULT_XDG_CACHE_HOME = "~/.cache" +XDG_CACHE_HOME = os.getenv("XDG_CACHE_HOME", DEFAULT_XDG_CACHE_HOME) +DEFAULT_HF_CACHE_HOME = os.path.join(XDG_CACHE_HOME, "huggingface") +HF_CACHE_HOME = os.path.expanduser(os.getenv("HF_HOME", DEFAULT_HF_CACHE_HOME)) + +DEFAULT_HF_EVALUATE_CACHE = os.path.join(HF_CACHE_HOME, "evaluate") +HF_EVALUATE_CACHE = Path(os.getenv("HF_EVALUATE_CACHE", DEFAULT_HF_EVALUATE_CACHE)) + +DEFAULT_HF_METRICS_CACHE = os.path.join(HF_CACHE_HOME, "metrics") +HF_METRICS_CACHE = Path(os.getenv("HF_METRICS_CACHE", DEFAULT_HF_METRICS_CACHE)) + +DEFAULT_HF_MODULES_CACHE = os.path.join(HF_CACHE_HOME, "modules") +HF_MODULES_CACHE = Path(os.getenv("HF_MODULES_CACHE", DEFAULT_HF_MODULES_CACHE)) + +DOWNLOADED_DATASETS_DIR = "downloads" +DEFAULT_DOWNLOADED_EVALUATE_PATH = os.path.join(HF_EVALUATE_CACHE, DOWNLOADED_DATASETS_DIR) +DOWNLOADED_EVALUATE_PATH = Path(os.getenv("HF_DATASETS_DOWNLOADED_EVALUATE_PATH", DEFAULT_DOWNLOADED_EVALUATE_PATH)) + +EXTRACTED_EVALUATE_DIR = "extracted" +DEFAULT_EXTRACTED_EVALUATE_PATH = os.path.join(DEFAULT_DOWNLOADED_EVALUATE_PATH, EXTRACTED_EVALUATE_DIR) +EXTRACTED_EVALUATE_PATH = Path(os.getenv("HF_DATASETS_EXTRACTED_EVALUATE_PATH", DEFAULT_EXTRACTED_EVALUATE_PATH)) + +# Download count for the website +HF_UPDATE_DOWNLOAD_COUNTS = ( + os.environ.get("HF_UPDATE_DOWNLOAD_COUNTS", "AUTO").upper() in ENV_VARS_TRUE_AND_AUTO_VALUES +) + +# Offline mode +HF_EVALUATE_OFFLINE = os.environ.get("HF_EVALUATE_OFFLINE", "AUTO").upper() in ENV_VARS_TRUE_VALUES + + +# File names +LICENSE_FILENAME = "LICENSE" +METRIC_INFO_FILENAME = "metric_info.json" +DATASETDICT_JSON_FILENAME = "dataset_dict.json" + +MODULE_NAME_FOR_DYNAMIC_MODULES = "evaluate_modules" + +HF_HUB_ALLOWED_TASKS = [ + "image-classification", + "translation", + "image-segmentation", + "fill-mask", + "automatic-speech-recognition", + "token-classification", + "sentence-similarity", + "audio-classification", + "question-answering", + "summarization", + "zero-shot-classification", + "table-to-text", + "feature-extraction", + "other", + "multiple-choice", + "text-classification", + "text-to-image", + "text2text-generation", + "zero-shot-image-classification", + "tabular-classification", + "tabular-regression", + "image-to-image", + "tabular-to-text", + "unconditional-image-generation", + "text-retrieval", + "text-to-speech", + "object-detection", + "audio-to-audio", + "text-generation", + "conversational", + "table-question-answering", + "visual-question-answering", + "image-to-text", + "reinforcement-learning", + "voice-activity-detection", + "time-series-forecasting", + "document-question-answering", +] diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/evaluate/hub.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/evaluate/hub.py new file mode 100644 index 0000000000000000000000000000000000000000..86118332c6d2293f475e84e80726364dcc63e292 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/evaluate/hub.py @@ -0,0 +1,133 @@ +from typing import Dict + +import requests +from huggingface_hub import dataset_info, model_info +from huggingface_hub.repocard import metadata_update + +from .config import HF_HUB_ALLOWED_TASKS +from .utils.logging import get_logger + + +logger = get_logger(__name__) + + +def push_to_hub( + model_id: str, + task_type: str, + dataset_type: str, + dataset_name: str, + metric_type: str, + metric_name: str, + metric_value: float, + task_name: str = None, + dataset_config: str = None, + dataset_split: str = None, + dataset_revision: str = None, + dataset_args: Dict[str, int] = None, + metric_config: str = None, + metric_args: Dict[str, int] = None, + overwrite: bool = False, +): + r""" + Pushes the result of a metric to the metadata of a model repository in the Hub. + + Args: + model_id (`str`): + Model id from https://hf.co/models. + task_type (`str`): + Task id, refer to the [Hub allowed tasks](https://github.com/huggingface/evaluate/blob/main/src/evaluate/config.py#L154) for allowed values. + dataset_type (`str`): + Dataset id from https://hf.co/datasets. + dataset_name (`str`): + Pretty name for the dataset. + metric_type (`str`): + Metric id from https://hf.co/metrics. + metric_name (`str`): + Pretty name for the metric. + metric_value (`float`): + Computed metric value. + task_name (`str`, *optional*): + Pretty name for the task. + dataset_config (`str`, *optional*): + Dataset configuration used in [`~datasets.load_dataset`]. + See [`~datasets.load_dataset`] for more info. + dataset_split (`str`, *optional*): + Name of split used for metric computation. + dataset_revision (`str`, *optional*): + Git hash for the specific version of the dataset. + dataset_args (`dict[str, int]`, *optional*): + Additional arguments passed to [`~datasets.load_dataset`]. + metric_config (`str`, *optional*): + Configuration for the metric (e.g. the GLUE metric has a configuration for each subset). + metric_args (`dict[str, int]`, *optional*): + Arguments passed during [`~evaluate.EvaluationModule.compute`]. + overwrite (`bool`, *optional*, defaults to `False`): + If set to `True` an existing metric field can be overwritten, otherwise + attempting to overwrite any existing fields will cause an error. + + Example: + + ```python + >>> push_to_hub( + ... model_id="huggingface/gpt2-wikitext2", + ... metric_value=0.5 + ... metric_type="bleu", + ... metric_name="BLEU", + ... dataset_name="WikiText", + ... dataset_type="wikitext", + ... dataset_split="test", + ... task_type="text-generation", + ... task_name="Text Generation" + ... ) + ```""" + if task_type not in HF_HUB_ALLOWED_TASKS: + raise ValueError(f"Task type not supported. Task has to be one of {HF_HUB_ALLOWED_TASKS}") + + try: + dataset_info(dataset_type) + except requests.exceptions.HTTPError: + logger.warning(f"Dataset {dataset_type} not found on the Hub at hf.co/datasets/{dataset_type}") + + try: + model_info(model_id) + except requests.exceptions.HTTPError: + raise ValueError(f"Model {model_id} not found on the Hub at hf.co/{model_id}") + + result = { + "task": { + "type": task_type, + }, + "dataset": { + "type": dataset_type, + "name": dataset_name, + }, + "metrics": [ + { + "type": metric_type, + "value": metric_value, + }, + ], + } + + if dataset_config is not None: + result["dataset"]["config"] = dataset_config + if dataset_split is not None: + result["dataset"]["split"] = dataset_split + if dataset_revision is not None: + result["dataset"]["revision"] = dataset_revision + if dataset_args is not None: + result["dataset"]["args"] = dataset_args + + if task_name is not None: + result["task"]["name"] = task_name + + if metric_name is not None: + result["metrics"][0]["name"] = metric_name + if metric_config is not None: + result["metrics"][0]["config"] = metric_config + if metric_args is not None: + result["metrics"][0]["args"] = metric_args + + metadata = {"model-index": [{"results": [result]}]} + + return metadata_update(repo_id=model_id, metadata=metadata, overwrite=overwrite) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/evaluate/info.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/evaluate/info.py new file mode 100644 index 0000000000000000000000000000000000000000..cc095784e4f1c1f473dd85955447d97d5fdc4e65 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/evaluate/info.py @@ -0,0 +1,157 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +""" EvaluationModuleInfo records information we know about a dataset and a metric. +""" + +import dataclasses +import json +import os +from dataclasses import asdict, dataclass, field +from typing import List, Optional, Union + +from datasets.features import Features, Value + +from . import config +from .utils.logging import get_logger + + +logger = get_logger(__name__) + + +@dataclass +class EvaluationModuleInfo: + """Base class to store information about an evaluation used for `MetricInfo`, `ComparisonInfo`, + and `MeasurementInfo`. + + `EvaluationModuleInfo` documents an evaluation, including its name, version, and features. + See the constructor arguments and properties for a full list. + + Note: Not all fields are known on construction and may be updated later. + """ + + # Set in the dataset scripts + description: str + citation: str + features: Union[Features, List[Features]] + inputs_description: str = field(default_factory=str) + homepage: str = field(default_factory=str) + license: str = field(default_factory=str) + codebase_urls: List[str] = field(default_factory=list) + reference_urls: List[str] = field(default_factory=list) + streamable: bool = False + format: Optional[str] = None + module_type: str = "metric" # deprecate this in the future + + # Set later by the builder + module_name: Optional[str] = None + config_name: Optional[str] = None + experiment_id: Optional[str] = None + + def __post_init__(self): + if self.format is not None: + for key, value in self.features.items(): + if not isinstance(value, Value): + raise ValueError( + f"When using 'numpy' format, all features should be a `datasets.Value` feature. " + f"Here {key} is an instance of {value.__class__.__name__}" + ) + + def write_to_directory(self, metric_info_dir): + """Write `EvaluationModuleInfo` as JSON to `metric_info_dir`. + Also save the license separately in LICENSE. + + Args: + metric_info_dir (`str`): + The directory to save `metric_info_dir` to. + + Example: + + ```py + >>> my_metric.info.write_to_directory("/path/to/directory/") + ``` + """ + with open(os.path.join(metric_info_dir, config.METRIC_INFO_FILENAME), "w", encoding="utf-8") as f: + json.dump(asdict(self), f) + + with open(os.path.join(metric_info_dir, config.LICENSE_FILENAME), "w", encoding="utf-8") as f: + f.write(self.license) + + @classmethod + def from_directory(cls, metric_info_dir) -> "EvaluationModuleInfo": + """Create `EvaluationModuleInfo` from the JSON file in `metric_info_dir`. + + Args: + metric_info_dir (`str`): + The directory containing the `metric_info` JSON file. This + should be the root directory of a specific metric version. + + Example: + + ```py + >>> my_metric = EvaluationModuleInfo.from_directory("/path/to/directory/") + ``` + """ + logger.info(f"Loading Metric info from {metric_info_dir}") + if not metric_info_dir: + raise ValueError("Calling EvaluationModuleInfo.from_directory() with undefined metric_info_dir.") + + with open(os.path.join(metric_info_dir, config.METRIC_INFO_FILENAME), encoding="utf-8") as f: + metric_info_dict = json.load(f) + return cls.from_dict(metric_info_dict) + + @classmethod + def from_dict(cls, metric_info_dict: dict) -> "EvaluationModuleInfo": + field_names = {f.name for f in dataclasses.fields(cls)} + return cls(**{k: v for k, v in metric_info_dict.items() if k in field_names}) + + +@dataclass +class MetricInfo(EvaluationModuleInfo): + """Information about a metric. + + `EvaluationModuleInfo` documents a metric, including its name, version, and features. + See the constructor arguments and properties for a full list. + + Note: Not all fields are known on construction and may be updated later. + """ + + module_type: str = "metric" + + +@dataclass +class ComparisonInfo(EvaluationModuleInfo): + """Information about a comparison. + + `EvaluationModuleInfo` documents a comparison, including its name, version, and features. + See the constructor arguments and properties for a full list. + + Note: Not all fields are known on construction and may be updated later. + """ + + module_type: str = "comparison" + + +@dataclass +class MeasurementInfo(EvaluationModuleInfo): + """Information about a measurement. + + `EvaluationModuleInfo` documents a measurement, including its name, version, and features. + See the constructor arguments and properties for a full list. + + Note: Not all fields are known on construction and may be updated later. + """ + + module_type: str = "measurement" diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/evaluate/loading.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/evaluate/loading.py new file mode 100644 index 0000000000000000000000000000000000000000..a9cbbe60de07fed296cd27fd1954ef14cf0de032 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/evaluate/loading.py @@ -0,0 +1,771 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +"""Access datasets.""" +import filecmp +import importlib +import inspect +import json +import os +import re +import shutil +import time +from dataclasses import dataclass +from pathlib import Path +from typing import List, Optional, Tuple, Type, Union +from urllib.parse import urlparse + +from datasets import DownloadConfig, DownloadMode +from datasets.builder import DatasetBuilder +from datasets.packaged_modules import _EXTENSION_TO_MODULE, _hash_python_lines +from datasets.utils.filelock import FileLock +from datasets.utils.version import Version + +from . import SCRIPTS_VERSION, config +from .module import EvaluationModule +from .utils.file_utils import ( + cached_path, + head_hf_s3, + hf_hub_url, + init_hf_modules, + is_relative_path, + relative_to_absolute_path, + url_or_path_join, +) +from .utils.logging import get_logger + + +logger = get_logger(__name__) + + +ALL_ALLOWED_EXTENSIONS = list(_EXTENSION_TO_MODULE.keys()) + ["zip"] + + +def init_dynamic_modules( + name: str = config.MODULE_NAME_FOR_DYNAMIC_MODULES, hf_modules_cache: Optional[Union[Path, str]] = None +): + """ + Create a module with name `name` in which you can add dynamic modules + such as metrics or datasets. The module can be imported using its name. + The module is created in the HF_MODULE_CACHE directory by default (~/.cache/huggingface/modules) but it can + be overriden by specifying a path to another directory in `hf_modules_cache`. + """ + hf_modules_cache = init_hf_modules(hf_modules_cache) + dynamic_modules_path = os.path.join(hf_modules_cache, name) + os.makedirs(dynamic_modules_path, exist_ok=True) + if not os.path.exists(os.path.join(dynamic_modules_path, "__init__.py")): + with open(os.path.join(dynamic_modules_path, "__init__.py"), "w"): + pass + return dynamic_modules_path + + +def import_main_class(module_path) -> Optional[Union[Type[DatasetBuilder], Type[EvaluationModule]]]: + """Import a module at module_path and return its main class, a Metric by default""" + module = importlib.import_module(module_path) + main_cls_type = EvaluationModule + + # Find the main class in our imported module + module_main_cls = None + for name, obj in module.__dict__.items(): + if isinstance(obj, type) and issubclass(obj, main_cls_type): + if inspect.isabstract(obj): + continue + module_main_cls = obj + break + + return module_main_cls + + +def files_to_hash(file_paths: List[str]) -> str: + """ + Convert a list of scripts or text files provided in file_paths into a hashed filename in a repeatable way. + """ + # List all python files in directories if directories are supplied as part of external imports + to_use_files: List[Union[Path, str]] = [] + for file_path in file_paths: + if os.path.isdir(file_path): + to_use_files.extend(list(Path(file_path).rglob("*.[pP][yY]"))) + else: + to_use_files.append(file_path) + + # Get the code from all these files + lines = [] + for file_path in to_use_files: + with open(file_path, encoding="utf-8") as f: + lines.extend(f.readlines()) + return _hash_python_lines(lines) + + +def convert_github_url(url_path: str) -> Tuple[str, Optional[str]]: + """Convert a link to a file on a github repo in a link to the raw github object.""" + parsed = urlparse(url_path) + sub_directory = None + if parsed.scheme in ("http", "https", "s3") and parsed.netloc == "github.com": + if "blob" in url_path: + if not url_path.endswith(".py"): + raise ValueError(f"External import from github at {url_path} should point to a file ending with '.py'") + url_path = url_path.replace("blob", "raw") # Point to the raw file + else: + # Parse github url to point to zip + github_path = parsed.path[1:] + repo_info, branch = github_path.split("/tree/") if "/tree/" in github_path else (github_path, "master") + repo_owner, repo_name = repo_info.split("/") + url_path = f"https://github.com/{repo_owner}/{repo_name}/archive/{branch}.zip" + sub_directory = f"{repo_name}-{branch}" + return url_path, sub_directory + + +def increase_load_count(name: str, resource_type: str): + """Update the download count of a dataset or metric.""" + if not config.HF_EVALUATE_OFFLINE and config.HF_UPDATE_DOWNLOAD_COUNTS: + try: + head_hf_s3(name, filename=name + ".py", dataset=(resource_type == "dataset")) + except Exception: + pass + + +def get_imports(file_path: str) -> Tuple[str, str, str, str]: + """Find whether we should import or clone additional files for a given processing script. + And list the import. + + We allow: + - library dependencies, + - local dependencies and + - external dependencies whose url is specified with a comment starting from "# From:' followed by the raw url to a file, an archive or a github repository. + external dependencies will be downloaded (and extracted if needed in the dataset folder). + We also add an `__init__.py` to each sub-folder of a downloaded folder so the user can import from them in the script. + + Note that only direct import in the dataset processing script will be handled + We don't recursively explore the additional import to download further files. + + Example:: + + import tensorflow + import .c4_utils + import .clicr.dataset-code.build_json_dataset # From: https://raw.githubusercontent.com/clips/clicr/master/dataset-code/build_json_dataset + """ + lines = [] + with open(file_path, encoding="utf-8") as f: + lines.extend(f.readlines()) + + logger.debug(f"Checking {file_path} for additional imports.") + imports: List[Tuple[str, str, str, Optional[str]]] = [] + is_in_docstring = False + for line in lines: + docstr_start_match = re.findall(r'[\s\S]*?"""[\s\S]*?', line) + + if len(docstr_start_match) == 1: + # flip True <=> False only if doctstring + # starts at line without finishing + is_in_docstring = not is_in_docstring + + if is_in_docstring: + # import statements in doctstrings should + # not be added as required dependencies + continue + + match = re.match(r"^import\s+(\.?)([^\s\.]+)[^#\r\n]*(?:#\s+From:\s+)?([^\r\n]*)", line, flags=re.MULTILINE) + if match is None: + match = re.match( + r"^from\s+(\.?)([^\s\.]+)(?:[^\s]*)\s+import\s+[^#\r\n]*(?:#\s+From:\s+)?([^\r\n]*)", + line, + flags=re.MULTILINE, + ) + if match is None: + continue + if match.group(1): + # The import starts with a '.', we will download the relevant file + if any(imp[1] == match.group(2) for imp in imports): + # We already have this import + continue + if match.group(3): + # The import has a comment with 'From:', we'll retrieve it from the given url + url_path = match.group(3) + url_path, sub_directory = convert_github_url(url_path) + imports.append(("external", match.group(2), url_path, sub_directory)) + elif match.group(2): + # The import should be at the same place as the file + imports.append(("internal", match.group(2), match.group(2), None)) + else: + if match.group(3): + # The import has a comment with `From: git+https:...`, asks user to pip install from git. + url_path = match.group(3) + imports.append(("library", match.group(2), url_path, None)) + else: + imports.append(("library", match.group(2), match.group(2), None)) + + return imports + + +def _download_additional_modules( + name: str, base_path: str, imports: Tuple[str, str, str, str], download_config: Optional[DownloadConfig] +) -> List[Tuple[str, str]]: + """ + Download additional module for a module .py at URL (or local path) /.py + The imports must have been parsed first using ``get_imports``. + + If some modules need to be installed with pip, an error is raised showing how to install them. + This function return the list of downloaded modules as tuples (import_name, module_file_path). + + The downloaded modules can then be moved into an importable directory with ``_copy_script_and_other_resources_in_importable_dir``. + """ + local_imports = [] + library_imports = [] + download_config = download_config.copy() + if download_config.download_desc is None: + download_config.download_desc = "Downloading extra modules" + for import_type, import_name, import_path, sub_directory in imports: + if import_type == "library": + library_imports.append((import_name, import_path)) # Import from a library + continue + + if import_name == name: + raise ValueError( + f"Error in the {name} script, importing relative {import_name} module " + f"but {import_name} is the name of the script. " + f"Please change relative import {import_name} to another name and add a '# From: URL_OR_PATH' " + f"comment pointing to the original relative import file path." + ) + if import_type == "internal": + url_or_filename = url_or_path_join(base_path, import_path + ".py") + elif import_type == "external": + url_or_filename = import_path + else: + raise ValueError("Wrong import_type") + + local_import_path = cached_path( + url_or_filename, + download_config=download_config, + ) + if sub_directory is not None: + local_import_path = os.path.join(local_import_path, sub_directory) + local_imports.append((import_name, local_import_path)) + + # Check library imports + needs_to_be_installed = set() + for library_import_name, library_import_path in library_imports: + try: + lib = importlib.import_module(library_import_name) # noqa F841 + except ImportError: + library_import_name = "scikit-learn" if library_import_name == "sklearn" else library_import_name + needs_to_be_installed.add((library_import_name, library_import_path)) + if needs_to_be_installed: + raise ImportError( + f"To be able to use {name}, you need to install the following dependencies" + f"{[lib_name for lib_name, lib_path in needs_to_be_installed]} using 'pip install " + f"{' '.join([lib_path for lib_name, lib_path in needs_to_be_installed])}' for instance'" + ) + return local_imports + + +def _copy_script_and_other_resources_in_importable_dir( + name: str, + importable_directory_path: str, + subdirectory_name: str, + original_local_path: str, + local_imports: List[Tuple[str, str]], + additional_files: List[Tuple[str, str]], + download_mode: Optional[DownloadMode], +) -> str: + """Copy a script and its required imports to an importable directory + + Args: + name (str): name of the resource to load + importable_directory_path (str): path to the loadable folder in the dynamic modules directory + subdirectory_name (str): name of the subdirectory in importable_directory_path in which to place the script + original_local_path (str): local path to the resource script + local_imports (List[Tuple[str, str]]): list of (destination_filename, import_file_to_copy) + additional_files (List[Tuple[str, str]]): list of (destination_filename, additional_file_to_copy) + download_mode (Optional[DownloadMode]): download mode + + Return: + importable_local_file: path to an importable module with importlib.import_module + """ + + # Define a directory with a unique name in our dataset or metric folder + # path is: ./datasets|metrics/dataset|metric_name/hash_from_code/script.py + # we use a hash as subdirectory_name to be able to have multiple versions of a dataset/metric processing file together + importable_subdirectory = os.path.join(importable_directory_path, subdirectory_name) + importable_local_file = os.path.join(importable_subdirectory, name + ".py") + + # Prevent parallel disk operations + lock_path = importable_directory_path + ".lock" + with FileLock(lock_path): + # Create main dataset/metrics folder if needed + if download_mode == DownloadMode.FORCE_REDOWNLOAD and os.path.exists(importable_directory_path): + shutil.rmtree(importable_directory_path) + os.makedirs(importable_directory_path, exist_ok=True) + + # add an __init__ file to the main dataset folder if needed + init_file_path = os.path.join(importable_directory_path, "__init__.py") + if not os.path.exists(init_file_path): + with open(init_file_path, "w"): + pass + + # Create hash dataset folder if needed + os.makedirs(importable_subdirectory, exist_ok=True) + # add an __init__ file to the hash dataset folder if needed + init_file_path = os.path.join(importable_subdirectory, "__init__.py") + if not os.path.exists(init_file_path): + with open(init_file_path, "w"): + pass + + # Copy dataset.py file in hash folder if needed + if not os.path.exists(importable_local_file): + shutil.copyfile(original_local_path, importable_local_file) + + # Record metadata associating original dataset path with local unique folder + meta_path = importable_local_file.split(".py")[0] + ".json" + if not os.path.exists(meta_path): + meta = {"original file path": original_local_path, "local file path": importable_local_file} + # the filename is *.py in our case, so better rename to filenam.json instead of filename.py.json + with open(meta_path, "w", encoding="utf-8") as meta_file: + json.dump(meta, meta_file) + + # Copy all the additional imports + for import_name, import_path in local_imports: + if os.path.isfile(import_path): + full_path_local_import = os.path.join(importable_subdirectory, import_name + ".py") + if not os.path.exists(full_path_local_import): + shutil.copyfile(import_path, full_path_local_import) + elif os.path.isdir(import_path): + full_path_local_import = os.path.join(importable_subdirectory, import_name) + if not os.path.exists(full_path_local_import): + shutil.copytree(import_path, full_path_local_import) + else: + raise OSError(f"Error with local import at {import_path}") + + # Copy aditional files like dataset infos file if needed + for file_name, original_path in additional_files: + destination_additional_path = os.path.join(importable_subdirectory, file_name) + if not os.path.exists(destination_additional_path) or not filecmp.cmp( + original_path, destination_additional_path + ): + shutil.copyfile(original_path, destination_additional_path) + return importable_local_file + + +def _create_importable_file( + local_path: str, + local_imports: List[Tuple[str, str]], + additional_files: List[Tuple[str, str]], + dynamic_modules_path: str, + module_namespace: str, + name: str, + download_mode: DownloadMode, +) -> Tuple[str, str]: + importable_directory_path = os.path.join(dynamic_modules_path, module_namespace, name.replace("/", "--")) + Path(importable_directory_path).mkdir(parents=True, exist_ok=True) + (Path(importable_directory_path).parent / "__init__.py").touch(exist_ok=True) + hash = files_to_hash([local_path] + [loc[1] for loc in local_imports]) + importable_local_file = _copy_script_and_other_resources_in_importable_dir( + name=name.split("/")[-1], + importable_directory_path=importable_directory_path, + subdirectory_name=hash, + original_local_path=local_path, + local_imports=local_imports, + additional_files=additional_files, + download_mode=download_mode, + ) + logger.debug(f"Created importable dataset file at {importable_local_file}") + module_path = ".".join( + [os.path.basename(dynamic_modules_path), module_namespace, name.replace("/", "--"), hash, name.split("/")[-1]] + ) + return module_path, hash + + +@dataclass +class ImportableModule: + module_path: str + hash: str + + +class _EvaluationModuleFactory: + def get_module(self) -> ImportableModule: + raise NotImplementedError + + +class LocalEvaluationModuleFactory(_EvaluationModuleFactory): + """Get the module of a local metric. The metric script is loaded from a local script.""" + + def __init__( + self, + path: str, + module_type: str = "metrics", + download_config: Optional[DownloadConfig] = None, + download_mode: Optional[DownloadMode] = None, + dynamic_modules_path: Optional[str] = None, + ): + self.path = path + self.module_type = module_type + self.name = Path(path).stem + self.download_config = download_config or DownloadConfig() + self.download_mode = download_mode + self.dynamic_modules_path = dynamic_modules_path + + def get_module(self) -> ImportableModule: + # get script and other files + imports = get_imports(self.path) + local_imports = _download_additional_modules( + name=self.name, + base_path=str(Path(self.path).parent), + imports=imports, + download_config=self.download_config, + ) + # copy the script and the files in an importable directory + dynamic_modules_path = self.dynamic_modules_path if self.dynamic_modules_path else init_dynamic_modules() + module_path, hash = _create_importable_file( + local_path=self.path, + local_imports=local_imports, + additional_files=[], + dynamic_modules_path=dynamic_modules_path, + module_namespace=self.module_type, + name=self.name, + download_mode=self.download_mode, + ) + # make the new module to be noticed by the import system + importlib.invalidate_caches() + return ImportableModule(module_path, hash) + + +class HubEvaluationModuleFactory(_EvaluationModuleFactory): + """Get the module of a metric from a metric repository on the Hub.""" + + def __init__( + self, + name: str, + module_type: str = "metrics", + revision: Optional[Union[str, Version]] = None, + download_config: Optional[DownloadConfig] = None, + download_mode: Optional[DownloadMode] = None, + dynamic_modules_path: Optional[str] = None, + ): + self.name = name + self.module_type = module_type + self.revision = revision + self.download_config = download_config or DownloadConfig() + self.download_mode = download_mode + self.dynamic_modules_path = dynamic_modules_path + assert self.name.count("/") == 1 + increase_load_count(name, resource_type="metric") + + def download_loading_script(self, revision) -> str: + file_path = hf_hub_url(path=self.name, name=self.name.split("/")[1] + ".py", revision=revision) + download_config = self.download_config.copy() + if download_config.download_desc is None: + download_config.download_desc = "Downloading builder script" + return cached_path(file_path, download_config=download_config) + + def get_module(self) -> ImportableModule: + revision = self.revision or os.getenv("HF_SCRIPTS_VERSION", SCRIPTS_VERSION) + + if re.match(r"\d*\.\d*\.\d*", revision): # revision is version number (three digits separated by full stops) + revision = "v" + revision # tagging convention on evaluate repository starts with v + + # get script and other files + try: + local_path = self.download_loading_script(revision) + except FileNotFoundError as err: + # if there is no file found with current revision tag try to load main + if self.revision is None and os.getenv("HF_SCRIPTS_VERSION", SCRIPTS_VERSION) != "main": + revision = "main" + local_path = self.download_loading_script(revision) + else: + raise err + + imports = get_imports(local_path) + local_imports = _download_additional_modules( + name=self.name, + base_path=hf_hub_url(path=self.name, name="", revision=revision), + imports=imports, + download_config=self.download_config, + ) + # copy the script and the files in an importable directory + dynamic_modules_path = self.dynamic_modules_path if self.dynamic_modules_path else init_dynamic_modules() + module_path, hash = _create_importable_file( + local_path=local_path, + local_imports=local_imports, + additional_files=[], + dynamic_modules_path=dynamic_modules_path, + module_namespace=self.module_type, + name=self.name, + download_mode=self.download_mode, + ) + # make the new module to be noticed by the import system + importlib.invalidate_caches() + return ImportableModule(module_path, hash) + + +class CachedEvaluationModuleFactory(_EvaluationModuleFactory): + """ + Get the module of a metric that has been loaded once already and cached. + The script that is loaded from the cache is the most recent one with a matching name. + """ + + def __init__( + self, + name: str, + module_type: str = "metrics", + dynamic_modules_path: Optional[str] = None, + ): + self.name = name + self.module_type = module_type + self.dynamic_modules_path = dynamic_modules_path + assert self.name.count("/") == 0 + + def get_module(self) -> ImportableModule: + dynamic_modules_path = self.dynamic_modules_path if self.dynamic_modules_path else init_dynamic_modules() + importable_directory_path = os.path.join(dynamic_modules_path, self.module_type, self.name) + hashes = ( + [h for h in os.listdir(importable_directory_path) if len(h) == 64] + if os.path.isdir(importable_directory_path) + else None + ) + if not hashes: + raise FileNotFoundError(f"Metric {self.name} is not cached in {dynamic_modules_path}") + # get most recent + + def _get_modification_time(module_hash): + return ( + (Path(importable_directory_path) / module_hash / (self.name.split("--")[-1] + ".py")).stat().st_mtime + ) + + hash = sorted(hashes, key=_get_modification_time)[-1] + logger.warning( + f"Using the latest cached version of the module from {os.path.join(importable_directory_path, hash)} " + f"(last modified on {time.ctime(_get_modification_time(hash))}) since it " + f"couldn't be found locally at {self.name}, or remotely on the Hugging Face Hub." + ) + # make the new module to be noticed by the import system + module_path = ".".join( + [os.path.basename(dynamic_modules_path), self.module_type, self.name, hash, self.name.split("--")[-1]] + ) + importlib.invalidate_caches() + return ImportableModule(module_path, hash) + + +def evaluation_module_factory( + path: str, + module_type: Optional[str] = None, + revision: Optional[Union[str, Version]] = None, + download_config: Optional[DownloadConfig] = None, + download_mode: Optional[DownloadMode] = None, + force_local_path: Optional[str] = None, + dynamic_modules_path: Optional[str] = None, + **download_kwargs, +) -> ImportableModule: + """ + Download/extract/cache a metric module. + + Metrics codes are cached inside the the dynamic modules cache to allow easy import (avoid ugly sys.path tweaks). + + Args: + + path (str): Path or name of the metric script. + + - if ``path`` is a local metric script or a directory containing a local metric script (if the script has the same name as the directory): + -> load the module from the metric script + e.g. ``'./metrics/accuracy'`` or ``'./metrics/accuracy/accuracy.py'``. + - if ``path`` is a metric on the Hugging Face Hub (ex: `glue`, `squad`) + -> load the module from the metric script in the github repository at huggingface/datasets + e.g. ``'accuracy'`` or ``'rouge'``. + + revision (Optional ``Union[str, datasets.Version]``): + If specified, the module will be loaded from the datasets repository at this version. + By default: + - it is set to the local version of the lib. + - it will also try to load it from the master branch if it's not available at the local version of the lib. + Specifying a version that is different from your local version of the lib might cause compatibility issues. + download_config (:class:`DownloadConfig`, optional): Specific download configuration parameters. + download_mode (:class:`DownloadMode`, default ``REUSE_DATASET_IF_EXISTS``): Download/generate mode. + force_local_path (Optional str): Optional path to a local path to download and prepare the script to. + Used to inspect or modify the script folder. + dynamic_modules_path (Optional str, defaults to HF_MODULES_CACHE / "datasets_modules", i.e. ~/.cache/huggingface/modules/datasets_modules): + Optional path to the directory in which the dynamic modules are saved. It must have been initialized with :obj:`init_dynamic_modules`. + By default the datasets and metrics are stored inside the `datasets_modules` module. + download_kwargs: optional attributes for DownloadConfig() which will override the attributes in download_config if supplied. + + Returns: + ImportableModule + """ + if download_config is None: + download_config = DownloadConfig(**download_kwargs) + download_mode = DownloadMode(download_mode or DownloadMode.REUSE_DATASET_IF_EXISTS) + download_config.extract_compressed_file = True + download_config.force_extract = True + + filename = list(filter(lambda x: x, path.replace(os.sep, "/").split("/")))[-1] + if not filename.endswith(".py"): + filename = filename + ".py" + combined_path = os.path.join(path, filename) + # Try locally + if path.endswith(filename): + if os.path.isfile(path): + return LocalEvaluationModuleFactory( + path, download_mode=download_mode, dynamic_modules_path=dynamic_modules_path + ).get_module() + else: + raise FileNotFoundError(f"Couldn't find a metric script at {relative_to_absolute_path(path)}") + elif os.path.isfile(combined_path): + return LocalEvaluationModuleFactory( + combined_path, download_mode=download_mode, dynamic_modules_path=dynamic_modules_path + ).get_module() + elif is_relative_path(path) and path.count("/") <= 1 and not force_local_path: + try: + # load a canonical evaluation module from hub + if path.count("/") == 0: + # if no type provided look through all possible modules + if module_type is None: + for current_type in ["metric", "comparison", "measurement"]: + try: + return HubEvaluationModuleFactory( + f"evaluate-{current_type}/{path}", + revision=revision, + download_config=download_config, + download_mode=download_mode, + dynamic_modules_path=dynamic_modules_path, + ).get_module() + except ConnectionError: + pass + raise FileNotFoundError + # if module_type provided load specific module_type + else: + return HubEvaluationModuleFactory( + f"evaluate-{module_type}/{path}", + revision=revision, + download_config=download_config, + download_mode=download_mode, + dynamic_modules_path=dynamic_modules_path, + ).get_module() + # load community evaluation module from hub + elif path.count("/") == 1: + return HubEvaluationModuleFactory( + path, + revision=revision, + download_config=download_config, + download_mode=download_mode, + dynamic_modules_path=dynamic_modules_path, + ).get_module() + except Exception as e1: # noqa: all the attempts failed, before raising the error we should check if the module is already cached. + # if it's a canonical module we need to check if it's any of the types + if path.count("/") == 0: + for current_type in ["metric", "comparison", "measurement"]: + try: + return CachedEvaluationModuleFactory( + f"evaluate-{current_type}--{path}", dynamic_modules_path=dynamic_modules_path + ).get_module() + except Exception as e2: # noqa: if it's not in the cache, then it doesn't exist. + pass + # if it's a community module we just need to check on path + elif path.count("/") == 1: + try: + return CachedEvaluationModuleFactory( + path.replace("/", "--"), dynamic_modules_path=dynamic_modules_path + ).get_module() + except Exception as e2: # noqa: if it's not in the cache, then it doesn't exist. + pass + if not isinstance(e1, (ConnectionError, FileNotFoundError)): + raise e1 from None + raise FileNotFoundError( + f"Couldn't find a module script at {relative_to_absolute_path(combined_path)}. " + f"Module '{path}' doesn't exist on the Hugging Face Hub either." + ) from None + else: + raise FileNotFoundError(f"Couldn't find a module script at {relative_to_absolute_path(combined_path)}.") + + +def load( + path: str, + config_name: Optional[str] = None, + module_type: Optional[str] = None, + process_id: int = 0, + num_process: int = 1, + cache_dir: Optional[str] = None, + experiment_id: Optional[str] = None, + keep_in_memory: bool = False, + download_config: Optional[DownloadConfig] = None, + download_mode: Optional[DownloadMode] = None, + revision: Optional[Union[str, Version]] = None, + **init_kwargs, +) -> EvaluationModule: + """Load a [`~evaluate.EvaluationModule`]. + + Args: + + path (`str`): + Path to the evaluation processing script with the evaluation builder. Can be either: + - a local path to processing script or the directory containing the script (if the script has the same name as the directory), + e.g. `'./metrics/rouge'` or `'./metrics/rouge/rouge.py'` + - a evaluation module identifier on the HuggingFace evaluate repo e.g. `'rouge'` or `'bleu'` that are in either `'metrics/'`, + `'comparisons/'`, or `'measurements/'` depending on the provided `module_type` + config_name (`str`, *optional*): + Selecting a configuration for the metric (e.g. the GLUE metric has a configuration for each subset). + module_type (`str`, default `'metric'`): + Type of evaluation module, can be one of `'metric'`, `'comparison'`, or `'measurement'`. + process_id (`int`, *optional*): + For distributed evaluation: id of the process. + num_process (`int`, *optional*): + For distributed evaluation: total number of processes. + cache_dir (`str`, *optional*): + Path to store the temporary predictions and references (default to `~/.cache/huggingface/evaluate/`). + experiment_id (`str`): + A specific experiment id. This is used if several distributed evaluations share the same file system. + This is useful to compute metrics in distributed setups (in particular non-additive metrics like F1). + keep_in_memory (`bool`): + Whether to store the temporary results in memory (defaults to `False`). + download_config ([`~evaluate.DownloadConfig`], *optional*): + Specific download configuration parameters. + download_mode ([`DownloadMode`], defaults to `REUSE_DATASET_IF_EXISTS`): + Download/generate mode. + revision (`Union[str, evaluate.Version]`, *optional*): + If specified, the module will be loaded from the datasets repository + at this version. By default it is set to the local version of the lib. Specifying a version that is different from + your local version of the lib might cause compatibility issues. + + Returns: + [`evaluate.EvaluationModule`] + + Example: + + ```py + >>> from evaluate import load + >>> accuracy = load("accuracy") + ``` + """ + download_mode = DownloadMode(download_mode or DownloadMode.REUSE_DATASET_IF_EXISTS) + evaluation_module = evaluation_module_factory( + path, module_type=module_type, revision=revision, download_config=download_config, download_mode=download_mode + ) + evaluation_cls = import_main_class(evaluation_module.module_path) + evaluation_instance = evaluation_cls( + config_name=config_name, + process_id=process_id, + num_process=num_process, + cache_dir=cache_dir, + keep_in_memory=keep_in_memory, + experiment_id=experiment_id, + hash=evaluation_module.hash, + **init_kwargs, + ) + + if module_type and module_type != evaluation_instance.module_type: + raise TypeError( + f"No module of module type '{module_type}' not found for '{path}' locally, or on the Hugging Face Hub. Found module of module type '{evaluation_instance.module_type}' instead." + ) + + # Download and prepare resources for the metric + evaluation_instance.download_and_prepare(download_config=download_config) + + return evaluation_instance diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/evaluate/module.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/evaluate/module.py new file mode 100644 index 0000000000000000000000000000000000000000..ca38b9b1584690bfed2658a717eeb972576d0846 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/evaluate/module.py @@ -0,0 +1,1034 @@ +# Copyright 2020 The HuggingFace Datasets Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +""" EvaluationModule base class.""" +import collections +import itertools +import os +import types +import uuid +from typing import Any, Dict, List, Optional, Tuple, Union + +import numpy as np +import pyarrow as pa +from datasets import DatasetInfo, DownloadConfig, DownloadManager +from datasets.arrow_dataset import Dataset +from datasets.arrow_reader import ArrowReader +from datasets.arrow_writer import ArrowWriter +from datasets.features import Features, Sequence, Value +from datasets.features.features import _check_non_null_non_empty_recursive +from datasets.utils.filelock import BaseFileLock, FileLock, Timeout +from datasets.utils.py_utils import copyfunc, temp_seed, zip_dict + +from . import config +from .info import EvaluationModuleInfo +from .naming import camelcase_to_snakecase +from .utils.logging import get_logger + + +logger = get_logger(__name__) + + +class FileFreeLock(BaseFileLock): + """Thread lock until a file **cannot** be locked""" + + def __init__(self, lock_file, *args, **kwargs): + self.filelock = FileLock(lock_file) + super().__init__(lock_file, *args, **kwargs) + self._lock_file_fd = None + + def _acquire(self): + try: + self.filelock.acquire(timeout=0.01, poll_interval=0.02) # Try to lock once + except Timeout: + # We couldn't acquire the lock, the file is locked! + self._lock_file_fd = self.filelock.lock_file + else: + # We were able to acquire the lock, the file is not yet locked! + self.filelock.release() + self._lock_file_fd = None + + def _release(self): + self._lock_file_fd = None + + @property + def is_locked(self) -> bool: + return self._lock_file_fd is not None + + +# lists - summarize long lists similarly to NumPy +# arrays/tensors - let the frameworks control formatting +def summarize_if_long_list(obj): + if type(obj) is not list or len(obj) <= 6: + return f"{obj}" + + def format_chunk(chunk): + return ", ".join(repr(x) for x in chunk) + + return f"[{format_chunk(obj[:3])}, ..., {format_chunk(obj[-3:])}]" + + +class EvaluationModuleInfoMixin: + """This base class exposes some attributes of EvaluationModuleInfo + at the base level of the EvaluationModule for easy access. + """ + + def __init__(self, info: EvaluationModuleInfo): + self._module_info = info + + @property + def info(self): + """:class:`evaluate.EvaluationModuleInfo` object containing all the metadata in the evaluation module.""" + return self._module_info + + @property + def name(self) -> str: + return self._module_info.module_name + + @property + def experiment_id(self) -> Optional[str]: + return self._module_info.experiment_id + + @property + def description(self) -> str: + return self._module_info.description + + @property + def citation(self) -> str: + return self._module_info.citation + + @property + def features(self) -> Features: + return self._module_info.features + + @property + def inputs_description(self) -> str: + return self._module_info.inputs_description + + @property + def homepage(self) -> Optional[str]: + return self._module_info.homepage + + @property + def license(self) -> str: + return self._module_info.license + + @property + def codebase_urls(self) -> Optional[List[str]]: + return self._module_info.codebase_urls + + @property + def reference_urls(self) -> Optional[List[str]]: + return self._module_info.reference_urls + + @property + def streamable(self) -> bool: + return self._module_info.streamable + + @property + def format(self) -> Optional[str]: + return self._module_info.format + + @property + def module_type(self) -> str: + return self._module_info.module_type + + +class EvaluationModule(EvaluationModuleInfoMixin): + """A `EvaluationModule` is the base class and common API for metrics, comparisons, and measurements. + + Args: + config_name (`str`): + This is used to define a hash specific to a module computation script and prevents the module's data + to be overridden when the module loading script is modified. + keep_in_memory (`bool`): + Keep all predictions and references in memory. Not possible in distributed settings. + cache_dir (`str`): + Path to a directory in which temporary prediction/references data will be stored. + The data directory should be located on a shared file-system in distributed setups. + num_process (`int`): + Specify the total number of nodes in a distributed settings. + This is useful to compute module in distributed setups (in particular non-additive modules like F1). + process_id (`int`): + Specify the id of the current process in a distributed setup (between 0 and num_process-1) + This is useful to compute module in distributed setups (in particular non-additive metrics like F1). + seed (`int`, optional): + If specified, this will temporarily set numpy's random seed when [`~evaluate.EvaluationModule.compute`] is run. + experiment_id (`str`): + A specific experiment id. This is used if several distributed evaluations share the same file system. + This is useful to compute module in distributed setups (in particular non-additive metrics like F1). + hash (`str`): + Used to identify the evaluation module according to the hashed file contents. + max_concurrent_cache_files (`int`): + Max number of concurrent module cache files (default `10000`). + timeout (`Union[int, float]`): + Timeout in second for distributed setting synchronization. + """ + + def __init__( + self, + config_name: Optional[str] = None, + keep_in_memory: bool = False, + cache_dir: Optional[str] = None, + num_process: int = 1, + process_id: int = 0, + seed: Optional[int] = None, + experiment_id: Optional[str] = None, + hash: str = None, + max_concurrent_cache_files: int = 10000, + timeout: Union[int, float] = 100, + **kwargs, + ): + # prepare info + self.config_name = config_name or "default" + info = self._info() + info.module_name = camelcase_to_snakecase(self.__class__.__name__) + info.config_name = self.config_name + info.experiment_id = experiment_id or "default_experiment" + EvaluationModuleInfoMixin.__init__(self, info) # For easy access on low level + + # Safety checks on num_process and process_id + if not isinstance(process_id, int) or process_id < 0: + raise ValueError("'process_id' should be a number greater than 0") + if not isinstance(num_process, int) or num_process <= process_id: + raise ValueError("'num_process' should be a number greater than process_id") + if keep_in_memory and num_process != 1: + raise ValueError("Using 'keep_in_memory' is not possible in distributed setting (num_process > 1).") + + self.num_process = num_process + self.process_id = process_id + self.max_concurrent_cache_files = max_concurrent_cache_files + + self.keep_in_memory = keep_in_memory + self._data_dir_root = os.path.expanduser(cache_dir or config.HF_METRICS_CACHE) + self.data_dir = self._build_data_dir() + if seed is None: + _, seed, pos, *_ = np.random.get_state() + self.seed: int = seed[pos] if pos < 624 else seed[0] + else: + self.seed: int = seed + self.timeout: Union[int, float] = timeout + + # Update 'compute' and 'add' docstring + # methods need to be copied otherwise it changes the docstrings of every instance + self.compute = types.MethodType(copyfunc(self.compute), self) + self.add_batch = types.MethodType(copyfunc(self.add_batch), self) + self.add = types.MethodType(copyfunc(self.add), self) + self.compute.__func__.__doc__ += self.info.inputs_description + self.add_batch.__func__.__doc__ += self.info.inputs_description + self.add.__func__.__doc__ += self.info.inputs_description + + # self.arrow_schema = pa.schema(field for field in self.info.features.type) + self.selected_feature_format = None + self.buf_writer = None + self.writer = None + self.writer_batch_size = None + self.data = None + + # This is the cache file we store our predictions/references in + # Keep it None for now so we can (cloud)pickle the object + self.cache_file_name = None + self.filelock = None + self.rendez_vous_lock = None + + # This is all the cache files on which we have a lock when we are in a distributed setting + self.file_paths = None + self.filelocks = None + + # This fingerprints the evaluation module according to the hashed contents of the module code + self._hash = hash + + def __len__(self): + """Return the number of examples (predictions or predictions/references pair) + currently stored in the evaluation module's cache. + """ + return 0 if self.writer is None else len(self.writer) + + def __repr__(self): + return ( + f'EvaluationModule(name: "{self.name}", module_type: "{self.module_type}", ' + f'features: {self.features}, usage: """{self.inputs_description}""", ' + f"stored examples: {len(self)})" + ) + + def _build_data_dir(self): + """Path of this evaluation module in cache_dir: + Will be: + self._data_dir_root/self.name/self.config_name/self.hash (if not none)/ + If any of these element is missing or if ``with_version=False`` the corresponding subfolders are dropped. + """ + builder_data_dir = self._data_dir_root + builder_data_dir = os.path.join(builder_data_dir, self.name, self.config_name) + os.makedirs(builder_data_dir, exist_ok=True) + return builder_data_dir + + def _create_cache_file(self, timeout=1) -> Tuple[str, FileLock]: + """Create a new cache file. If the default cache file is used, we generated a new hash.""" + file_path = os.path.join(self.data_dir, f"{self.experiment_id}-{self.num_process}-{self.process_id}.arrow") + filelock = None + for i in range(self.max_concurrent_cache_files): + filelock = FileLock(file_path + ".lock") + try: + filelock.acquire(timeout=timeout) + except Timeout: + # If we have reached the max number of attempts or we are not allow to find a free name (distributed setup) + # We raise an error + if self.num_process != 1: + raise ValueError( + f"Error in _create_cache_file: another evaluation module instance is already using the local cache file at {file_path}. " + f"Please specify an experiment_id (currently: {self.experiment_id}) to avoid collision " + f"between distributed evaluation module instances." + ) from None + if i == self.max_concurrent_cache_files - 1: + raise ValueError( + f"Cannot acquire lock, too many evaluation module instance are operating concurrently on this file system." + f"You should set a larger value of max_concurrent_cache_files when creating the evaluation module " + f"(current value is {self.max_concurrent_cache_files})." + ) from None + # In other cases (allow to find new file name + not yet at max num of attempts) we can try to sample a new hashing name. + file_uuid = str(uuid.uuid4()) + file_path = os.path.join( + self.data_dir, f"{self.experiment_id}-{file_uuid}-{self.num_process}-{self.process_id}.arrow" + ) + else: + break + + return file_path, filelock + + def _get_all_cache_files(self) -> Tuple[List[str], List[FileLock]]: + """Get a lock on all the cache files in a distributed setup. + We wait for timeout second to let all the distributed node finish their tasks (default is 100 seconds). + """ + if self.num_process == 1: + if self.cache_file_name is None: + raise ValueError( + "Evaluation module cache file doesn't exist. Please make sure that you call `add` or `add_batch` " + "at least once before calling `compute`." + ) + file_paths = [self.cache_file_name] + else: + file_paths = [ + os.path.join(self.data_dir, f"{self.experiment_id}-{self.num_process}-{process_id}.arrow") + for process_id in range(self.num_process) + ] + + # Let's acquire a lock on each process files to be sure they are finished writing + filelocks = [] + for process_id, file_path in enumerate(file_paths): + if process_id == 0: # process 0 already has its lock file + filelocks.append(self.filelock) + else: + filelock = FileLock(file_path + ".lock") + try: + filelock.acquire(timeout=self.timeout) + except Timeout: + raise ValueError( + f"Cannot acquire lock on cached file {file_path} for process {process_id}." + ) from None + else: + filelocks.append(filelock) + + return file_paths, filelocks + + def _check_all_processes_locks(self): + expected_lock_file_names = [ + os.path.join(self.data_dir, f"{self.experiment_id}-{self.num_process}-{process_id}.arrow.lock") + for process_id in range(self.num_process) + ] + for expected_lock_file_name in expected_lock_file_names: + nofilelock = FileFreeLock(expected_lock_file_name) + try: + nofilelock.acquire(timeout=self.timeout) + except Timeout: + raise ValueError( + f"Expected to find locked file {expected_lock_file_name} from process {self.process_id} but it doesn't exist." + ) from None + else: + nofilelock.release() + + def _check_rendez_vous(self): + expected_lock_file_name = os.path.join(self.data_dir, f"{self.experiment_id}-{self.num_process}-0.arrow.lock") + nofilelock = FileFreeLock(expected_lock_file_name) + try: + nofilelock.acquire(timeout=self.timeout) + except Timeout: + raise ValueError( + f"Expected to find locked file {expected_lock_file_name} from process {self.process_id} but it doesn't exist." + ) from None + else: + nofilelock.release() + lock_file_name = os.path.join(self.data_dir, f"{self.experiment_id}-{self.num_process}-rdv.lock") + rendez_vous_lock = FileLock(lock_file_name) + try: + rendez_vous_lock.acquire(timeout=self.timeout) + except Timeout: + raise ValueError(f"Couldn't acquire lock on {lock_file_name} from process {self.process_id}.") from None + else: + rendez_vous_lock.release() + + def _finalize(self): + """Close all the writing process and load/gather the data + from all the nodes if main node or all_process is True. + """ + if self.writer is not None: + self.writer.finalize() + self.writer = None + # release the locks of the processes > 0 so that process 0 can lock them to read + delete the data + if self.filelock is not None and self.process_id > 0: + self.filelock.release() + + if self.keep_in_memory: + # Read the predictions and references + reader = ArrowReader(path=self.data_dir, info=DatasetInfo(features=self.selected_feature_format)) + self.data = Dataset.from_buffer(self.buf_writer.getvalue()) + + elif self.process_id == 0: + # Let's acquire a lock on each node files to be sure they are finished writing + file_paths, filelocks = self._get_all_cache_files() + + # Read the predictions and references + try: + reader = ArrowReader(path="", info=DatasetInfo(features=self.selected_feature_format)) + self.data = Dataset(**reader.read_files([{"filename": f} for f in file_paths])) + except FileNotFoundError: + raise ValueError( + "Error in finalize: another evaluation module instance is already using the local cache file. " + "Please specify an experiment_id to avoid collision between distributed evaluation module instances." + ) from None + + # Store file paths and locks and we will release/delete them after the computation. + self.file_paths = file_paths + self.filelocks = filelocks + + def compute(self, *, predictions=None, references=None, **kwargs) -> Optional[dict]: + """Compute the evaluation module. + + Usage of positional arguments is not allowed to prevent mistakes. + + Args: + predictions (`list/array/tensor`, *optional*): + Predictions. + references (`list/array/tensor`, *optional*): + References. + **kwargs (optional): + Keyword arguments that will be forwarded to the evaluation module [`~evaluate.EvaluationModule.compute`] + method (see details in the docstring). + + Return: + `dict` or `None` + + - Dictionary with the results if this evaluation module is run on the main process (`process_id == 0`). + - `None` if the evaluation module is not run on the main process (`process_id != 0`). + + ```py + >>> import evaluate + >>> accuracy = evaluate.load("accuracy") + >>> accuracy.compute(predictions=[0, 1, 1, 0], references=[0, 1, 0, 1]) + ``` + """ + all_kwargs = {"predictions": predictions, "references": references, **kwargs} + if predictions is None and references is None: + missing_kwargs = {k: None for k in self._feature_names() if k not in all_kwargs} + all_kwargs.update(missing_kwargs) + else: + missing_inputs = [k for k in self._feature_names() if k not in all_kwargs] + if missing_inputs: + raise ValueError( + f"Evaluation module inputs are missing: {missing_inputs}. All required inputs are {list(self._feature_names())}" + ) + inputs = {input_name: all_kwargs[input_name] for input_name in self._feature_names()} + compute_kwargs = {k: kwargs[k] for k in kwargs if k not in self._feature_names()} + + if any(v is not None for v in inputs.values()): + self.add_batch(**inputs) + self._finalize() + + self.cache_file_name = None + self.filelock = None + self.selected_feature_format = None + + if self.process_id == 0: + self.data.set_format(type=self.info.format) + + inputs = {input_name: self.data[input_name][:] for input_name in self._feature_names()} + with temp_seed(self.seed): + output = self._compute(**inputs, **compute_kwargs) + + if self.buf_writer is not None: + self.buf_writer = None + del self.data + self.data = None + else: + # Release locks and delete all the cache files. Process 0 is released last. + for filelock, file_path in reversed(list(zip(self.filelocks, self.file_paths))): + logger.info(f"Removing {file_path}") + del self.data + self.data = None + del self.writer + self.writer = None + os.remove(file_path) + filelock.release() + + return output + else: + return None + + def add_batch(self, *, predictions=None, references=None, **kwargs): + """Add a batch of predictions and references for the evaluation module's stack. + + Args: + predictions (`list/array/tensor`, *optional*): + Predictions. + references (`list/array/tensor`, *optional*): + References. + + Example: + + ```py + >>> import evaluate + >>> accuracy = evaluate.load("accuracy") + >>> for refs, preds in zip([[0,1],[0,1]], [[1,0],[0,1]]): + ... accuracy.add_batch(references=refs, predictions=preds) + ``` + """ + bad_inputs = [input_name for input_name in kwargs if input_name not in self._feature_names()] + if bad_inputs: + raise ValueError( + f"Bad inputs for evaluation module: {bad_inputs}. All required inputs are {list(self._feature_names())}" + ) + batch = {"predictions": predictions, "references": references, **kwargs} + batch = {input_name: batch[input_name] for input_name in self._feature_names()} + if self.writer is None: + self.selected_feature_format = self._infer_feature_from_batch(batch) + self._init_writer() + try: + for key, column in batch.items(): + if len(column) > 0: + self._enforce_nested_string_type(self.selected_feature_format[key], column[0]) + batch = self.selected_feature_format.encode_batch(batch) + self.writer.write_batch(batch) + except (pa.ArrowInvalid, TypeError): + if any(len(batch[c]) != len(next(iter(batch.values()))) for c in batch): + col0 = next(iter(batch)) + bad_col = [c for c in batch if len(batch[c]) != len(batch[col0])][0] + error_msg = ( + f"Mismatch in the number of {col0} ({len(batch[col0])}) and {bad_col} ({len(batch[bad_col])})" + ) + elif set(self.selected_feature_format) != {"references", "predictions"}: + error_msg = ( + f"Module inputs don't match the expected format.\n" + f"Expected format: {self.selected_feature_format },\n" + ) + error_msg_inputs = ",\n".join( + f"Input {input_name}: {summarize_if_long_list(batch[input_name])}" + for input_name in self.selected_feature_format + ) + error_msg += error_msg_inputs + else: + error_msg = ( + f"Predictions and/or references don't match the expected format.\n" + f"Expected format: {self.selected_feature_format },\n" + f"Input predictions: {summarize_if_long_list(predictions)},\n" + f"Input references: {summarize_if_long_list(references)}" + ) + raise ValueError(error_msg) from None + + def add(self, *, prediction=None, reference=None, **kwargs): + """Add one prediction and reference for the evaluation module's stack. + + Args: + prediction (`list/array/tensor`, *optional*): + Predictions. + reference (`list/array/tensor`, *optional*): + References. + + Example: + + ```py + >>> import evaluate + >>> accuracy = evaluate.load("accuracy") + >>> accuracy.add(references=[0,1], predictions=[1,0]) + ``` + """ + bad_inputs = [input_name for input_name in kwargs if input_name not in self._feature_names()] + if bad_inputs: + raise ValueError( + f"Bad inputs for evaluation module: {bad_inputs}. All required inputs are {list(self._feature_names())}" + ) + example = {"predictions": prediction, "references": reference, **kwargs} + example = {input_name: example[input_name] for input_name in self._feature_names()} + if self.writer is None: + self.selected_feature_format = self._infer_feature_from_example(example) + self._init_writer() + try: + self._enforce_nested_string_type(self.selected_feature_format, example) + example = self.selected_feature_format.encode_example(example) + self.writer.write(example) + except (pa.ArrowInvalid, TypeError): + error_msg = ( + f"Evaluation module inputs don't match the expected format.\n" + f"Expected format: {self.selected_feature_format},\n" + ) + error_msg_inputs = ",\n".join( + f"Input {input_name}: {summarize_if_long_list(example[input_name])}" + for input_name in self.selected_feature_format + ) + error_msg += error_msg_inputs + raise ValueError(error_msg) from None + + def _infer_feature_from_batch(self, batch): + if isinstance(self.features, Features): + return self.features + else: + example = dict([(k, v[0]) for k, v in batch.items()]) + return self._infer_feature_from_example(example) + + def _infer_feature_from_example(self, example): + if isinstance(self.features, Features): + return self.features + else: + for features in self.features: + try: + self._enforce_nested_string_type(features, example) + features.encode_example(example) + return features + except (ValueError, TypeError): + continue + feature_strings = "\n".join([f"Feature option {i}: {feature}" for i, feature in enumerate(self.features)]) + error_msg = ( + f"Predictions and/or references don't match the expected format.\n" + f"Expected format:\n{feature_strings},\n" + f"Input predictions: {summarize_if_long_list(example['predictions'])},\n" + f"Input references: {summarize_if_long_list(example['references'])}" + ) + raise ValueError(error_msg) from None + + def _feature_names(self): + if isinstance(self.features, list): + feature_names = list(self.features[0].keys()) + else: + feature_names = list(self.features.keys()) + return feature_names + + def _init_writer(self, timeout=1): + if self.num_process > 1: + if self.process_id == 0: + file_path = os.path.join(self.data_dir, f"{self.experiment_id}-{self.num_process}-rdv.lock") + self.rendez_vous_lock = FileLock(file_path) + try: + self.rendez_vous_lock.acquire(timeout=timeout) + except TimeoutError: + raise ValueError( + f"Error in _init_writer: another evalution module instance is already using the local cache file at {file_path}. " + f"Please specify an experiment_id (currently: {self.experiment_id}) to avoid collision " + f"between distributed evaluation module instances." + ) from None + + if self.keep_in_memory: + self.buf_writer = pa.BufferOutputStream() + self.writer = ArrowWriter( + features=self.selected_feature_format, stream=self.buf_writer, writer_batch_size=self.writer_batch_size + ) + else: + self.buf_writer = None + + # Get cache file name and lock it + if self.cache_file_name is None or self.filelock is None: + cache_file_name, filelock = self._create_cache_file() # get ready + self.cache_file_name = cache_file_name + self.filelock = filelock + + self.writer = ArrowWriter( + features=self.selected_feature_format, + path=self.cache_file_name, + writer_batch_size=self.writer_batch_size, + ) + # Setup rendez-vous here if + if self.num_process > 1: + if self.process_id == 0: + self._check_all_processes_locks() # wait for everyone to be ready + self.rendez_vous_lock.release() # let everyone go + else: + self._check_rendez_vous() # wait for master to be ready and to let everyone go + + def _info(self) -> EvaluationModuleInfo: + """Construct the EvaluationModuleInfo object. See `EvaluationModuleInfo` for details. + + Warning: This function is only called once and the result is cached for all + following .info() calls. + + Returns: + info: (EvaluationModuleInfo) The EvaluationModule information + """ + raise NotImplementedError + + def download_and_prepare( + self, + download_config: Optional[DownloadConfig] = None, + dl_manager: Optional[DownloadManager] = None, + ): + """Downloads and prepares evaluation module for reading. + + Args: + download_config ([`DownloadConfig`], *optional*): + Specific download configuration parameters. + dl_manager ([`DownloadManager`], *optional*): + Specific download manager to use. + + Example: + + ```py + >>> import evaluate + ``` + """ + if dl_manager is None: + if download_config is None: + download_config = DownloadConfig() + download_config.cache_dir = os.path.join(self.data_dir, "downloads") + download_config.force_download = False + + dl_manager = DownloadManager( + dataset_name=self.name, download_config=download_config, data_dir=self.data_dir + ) + + self._download_and_prepare(dl_manager) + + def _download_and_prepare(self, dl_manager): + """Downloads and prepares resources for the evaluation module. + + This is the internal implementation to overwrite called when user calls + `download_and_prepare`. It should download all required resources for the evaluation module. + + Args: + dl_manager (:class:`DownloadManager`): `DownloadManager` used to download and cache data. + """ + return None + + def _compute(self, *, predictions=None, references=None, **kwargs) -> Dict[str, Any]: + """This method defines the common API for all the evaluation module in the library""" + raise NotImplementedError + + def __del__(self): + if hasattr(self, "filelock") and self.filelock is not None: + self.filelock.release() + if hasattr(self, "rendez_vous_lock") and self.rendez_vous_lock is not None: + self.rendez_vous_lock.release() + if hasattr(self, "writer"): # in case it was already deleted + del self.writer + if hasattr(self, "data"): # in case it was already deleted + del self.data + + def _enforce_nested_string_type(self, schema, obj): + """ + Recursively checks if there is any Value feature of type string and throws TypeError if corresponding object is not a string. + Since any Python object can be cast to string this avoids implicitly casting wrong input types (e.g. lists) to string without error. + """ + # Nested structures: we allow dict, list, tuples, sequences + if isinstance(schema, dict): + return [self._enforce_nested_string_type(sub_schema, o) for k, (sub_schema, o) in zip_dict(schema, obj)] + + elif isinstance(schema, (list, tuple)): + sub_schema = schema[0] + return [self._enforce_nested_string_type(sub_schema, o) for o in obj] + elif isinstance(schema, Sequence): + # We allow to reverse list of dict => dict of list for compatiblity with tfds + if isinstance(schema.feature, dict): + if isinstance(obj, (list, tuple)): + # obj is a list of dict + for k, dict_tuples in zip_dict(schema.feature, *obj): + for sub_obj in dict_tuples[1:]: + if _check_non_null_non_empty_recursive(sub_obj, dict_tuples[0]): + self._enforce_nested_string_type(dict_tuples[0], sub_obj) + break + return None + else: + # obj is a single dict + for k, (sub_schema, sub_objs) in zip_dict(schema.feature, obj): + for sub_obj in sub_objs: + if _check_non_null_non_empty_recursive(sub_obj, sub_schema): + self._enforce_nested_string_type(sub_schema, sub_obj) + break + return None + # schema.feature is not a dict + if isinstance(obj, str): # don't interpret a string as a list + raise ValueError(f"Got a string but expected a list instead: '{obj}'") + if obj is None: + return None + else: + if len(obj) > 0: + for first_elmt in obj: + if _check_non_null_non_empty_recursive(first_elmt, schema.feature): + break + if not isinstance(first_elmt, list): + return self._enforce_nested_string_type(schema.feature, first_elmt) + + elif isinstance(schema, Value): + if pa.types.is_string(schema.pa_type) and not isinstance(obj, str): + raise TypeError(f"Expected type str but got {type(obj)}.") + + +class Metric(EvaluationModule): + """A Metric is the base class and common API for all metrics. + + Args: + config_name (`str`): + This is used to define a hash specific to a metric computation script and prevents the metric's data + to be overridden when the metric loading script is modified. + keep_in_memory (`bool`): + Keep all predictions and references in memory. Not possible in distributed settings. + cache_dir (`str`): + Path to a directory in which temporary prediction/references data will be stored. + The data directory should be located on a shared file-system in distributed setups. + num_process (`int`): + Specify the total number of nodes in a distributed settings. + This is useful to compute metrics in distributed setups (in particular non-additive metrics like F1). + process_id (`int`): + Specify the id of the current process in a distributed setup (between 0 and num_process-1) + This is useful to compute metrics in distributed setups (in particular non-additive metrics like F1). + seed (`int`, *optional*): + If specified, this will temporarily set numpy's random seed when [`~evaluate.Metric.compute`] is run. + experiment_id (`str`): + A specific experiment id. This is used if several distributed evaluations share the same file system. + This is useful to compute metrics in distributed setups (in particular non-additive metrics like F1). + max_concurrent_cache_files (`int`): + Max number of concurrent metric cache files (default `10000`). + timeout (`Union[int, float]`): + Timeout in second for distributed setting synchronization. + """ + + +class Comparison(EvaluationModule): + """A Comparison is the base class and common API for all comparisons. + + Args: + config_name (`str`): + This is used to define a hash specific to a comparison computation script and prevents the comparison's data + to be overridden when the comparison loading script is modified. + keep_in_memory (`bool`): + Keep all predictions and references in memory. Not possible in distributed settings. + cache_dir (`str`): + Path to a directory in which temporary prediction/references data will be stored. + The data directory should be located on a shared file-system in distributed setups. + num_process (`int`): + Specify the total number of nodes in a distributed settings. + This is useful to compute comparisons in distributed setups (in particular non-additive comparisons). + process_id (`int`): + Specify the id of the current process in a distributed setup (between 0 and num_process-1) + This is useful to compute comparisons in distributed setups (in particular non-additive comparisons). + seed (`int`, *optional*): + If specified, this will temporarily set numpy's random seed when [`~evaluate.Comparison.compute`] is run. + experiment_id (`str`): + A specific experiment id. This is used if several distributed evaluations share the same file system. + This is useful to compute comparisons in distributed setups (in particular non-additive comparisons). + max_concurrent_cache_files (`int`): + Max number of concurrent comparison cache files (default `10000`). + timeout (`Union[int, float]`): + Timeout in second for distributed setting synchronization. + """ + + +class Measurement(EvaluationModule): + """A Measurement is the base class and common API for all measurements. + + Args: + config_name (`str`): + This is used to define a hash specific to a measurement computation script and prevents the measurement's data + to be overridden when the measurement loading script is modified. + keep_in_memory (`bool`): + Keep all predictions and references in memory. Not possible in distributed settings. + cache_dir (`str`): + Path to a directory in which temporary prediction/references data will be stored. + The data directory should be located on a shared file-system in distributed setups. + num_process (`int`): + Specify the total number of nodes in a distributed settings. + This is useful to compute measurements in distributed setups (in particular non-additive measurements). + process_id (`int`): + Specify the id of the current process in a distributed setup (between 0 and num_process-1) + This is useful to compute measurements in distributed setups (in particular non-additive measurements). + seed (`int`, *optional*): + If specified, this will temporarily set numpy's random seed when [`~evaluate.Measurement.compute`] is run. + experiment_id (`str`): + A specific experiment id. This is used if several distributed evaluations share the same file system. + This is useful to compute measurements in distributed setups (in particular non-additive measurements). + max_concurrent_cache_files (`int`): + Max number of concurrent measurement cache files (default `10000`). + timeout (`Union[int, float]`): + Timeout in second for distributed setting synchronization. + """ + + +class CombinedEvaluations: + def __init__(self, evaluation_modules, force_prefix=False): + from .loading import load # avoid circular imports + + self.evaluation_module_names = None + if isinstance(evaluation_modules, list): + self.evaluation_modules = evaluation_modules + elif isinstance(evaluation_modules, dict): + self.evaluation_modules = list(evaluation_modules.values()) + self.evaluation_module_names = list(evaluation_modules.keys()) + loaded_modules = [] + + for module in self.evaluation_modules: + if isinstance(module, str): + module = load(module) + loaded_modules.append(module) + self.evaluation_modules = loaded_modules + + if self.evaluation_module_names is None: + self.evaluation_module_names = [module.name for module in self.evaluation_modules] + + self.force_prefix = force_prefix + + def add(self, prediction=None, reference=None, **kwargs): + """Add one prediction and reference for each evaluation module's stack. + + Args: + predictions (`list/array/tensor`, *optional*): + Predictions. + references (`list/array/tensor`, *optional*): + References. + + Example: + + ```py + >>> import evaluate + >>> accuracy = evaluate.load("accuracy") + >>> f1 = evaluate.load("f1") + >>> clf_metrics = combine(["accuracy", "f1"]) + >>> for ref, pred in zip([0,1,0,1], [1,0,0,1]): + ... clf_metrics.add(references=ref, predictions=pred) + ``` + """ + for evaluation_module in self.evaluation_modules: + batch = {"predictions": prediction, "references": reference, **kwargs} + batch = {input_name: batch[input_name] for input_name in evaluation_module._feature_names()} + evaluation_module.add(**batch) + + def add_batch(self, predictions=None, references=None, **kwargs): + """Add a batch of predictions and references for each evaluation module's stack. + + Args: + predictions (`list/array/tensor`, *optional*): + Predictions. + references (`list/array/tensor`, *optional*): + References. + + Example: + ```py + >>> import evaluate + >>> accuracy = evaluate.load("accuracy") + >>> f1 = evaluate.load("f1") + >>> clf_metrics = combine(["accuracy", "f1"]) + >>> for refs, preds in zip([[0,1],[0,1]], [[1,0],[0,1]]): + ... clf_metrics.add(references=refs, predictions=preds) + ``` + """ + for evaluation_module in self.evaluation_modules: + batch = {"predictions": predictions, "references": references, **kwargs} + batch = {input_name: batch[input_name] for input_name in evaluation_module._feature_names()} + evaluation_module.add_batch(**batch) + + def compute(self, predictions=None, references=None, **kwargs): + """Compute each evaluation module. + + Usage of positional arguments is not allowed to prevent mistakes. + + Args: + predictions (`list/array/tensor`, *optional*): + Predictions. + references (`list/array/tensor`, *optional*): + References. + **kwargs (*optional*): + Keyword arguments that will be forwarded to the evaluation module [`~evaluate.EvaluationModule.compute`] + method (see details in the docstring). + + Return: + `dict` or `None` + + - Dictionary with the results if this evaluation module is run on the main process (`process_id == 0`). + - `None` if the evaluation module is not run on the main process (`process_id != 0`). + + Example: + + ```py + >>> import evaluate + >>> accuracy = evaluate.load("accuracy") + >>> f1 = evaluate.load("f1") + >>> clf_metrics = combine(["accuracy", "f1"]) + >>> clf_metrics.compute(predictions=[0,1], references=[1,1]) + {'accuracy': 0.5, 'f1': 0.6666666666666666} + ``` + """ + results = [] + + for evaluation_module in self.evaluation_modules: + batch = {"predictions": predictions, "references": references, **kwargs} + results.append(evaluation_module.compute(**batch)) + + return self._merge_results(results) + + def _merge_results(self, results): + merged_results = {} + results_keys = list(itertools.chain.from_iterable([r.keys() for r in results])) + duplicate_keys = {item for item, count in collections.Counter(results_keys).items() if count > 1} + + duplicate_names = [ + item for item, count in collections.Counter(self.evaluation_module_names).items() if count > 1 + ] + duplicate_counter = {name: 0 for name in duplicate_names} + + for module_name, result in zip(self.evaluation_module_names, results): + for k, v in result.items(): + if k not in duplicate_keys and not self.force_prefix: + merged_results[f"{k}"] = v + elif module_name in duplicate_counter: + merged_results[f"{module_name}_{duplicate_counter[module_name]}_{k}"] = v + else: + merged_results[f"{module_name}_{k}"] = v + + if module_name in duplicate_counter: + duplicate_counter[module_name] += 1 + + return merged_results + + +def combine(evaluations, force_prefix=False): + """Combines several metrics, comparisons, or measurements into a single `CombinedEvaluations` object that + can be used like a single evaluation module. + + If two scores have the same name, then they are prefixed with their module names. + And if two modules have the same name, please use a dictionary to give them different names, otherwise an integer id is appended to the prefix. + + Args: + evaluations (`Union[list, dict]`): + A list or dictionary of evaluation modules. The modules can either be passed + as strings or loaded `EvaluationModule`s. If a dictionary is passed its keys are the names used and the values the modules. + The names are used as prefix in case there are name overlaps in the returned results of each module or if `force_prefix=True`. + force_prefix (`bool`, *optional*, defaults to `False`): + If `True` all scores from the modules are prefixed with their name. If + a dictionary is passed the keys are used as name otherwise the module's name. + + Examples: + + ```py + >>> import evaluate + >>> accuracy = evaluate.load("accuracy") + >>> f1 = evaluate.load("f1") + >>> clf_metrics = combine(["accuracy", "f1"]) + ``` + """ + + return CombinedEvaluations(evaluations, force_prefix=force_prefix) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/evaluate/saving.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/evaluate/saving.py new file mode 100644 index 0000000000000000000000000000000000000000..4eea1a6a0c63551f0db9302130c7f5a7acced4fa --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/evaluate/saving.py @@ -0,0 +1,73 @@ +import json +import os +import subprocess +import sys +from datetime import datetime +from pathlib import Path + +from datasets.utils.filelock import FileLock + +from . import __version__ + + +def save(path_or_file, **data): + """ + Saves results to a JSON file. Also saves system information such as current time, current commit + hash if inside a repository, and Python system information. + + Args: + path_or_file (`str`): + Path or file to store the file. If only a folder is provided + the results file will be saved in the format `"result-%Y_%m_%d-%H_%M_%S.json"`. + + Example: + ```py + >>> import evaluate + >>> result = {"bleu": 0.7} + >>> params = {"model": "gpt-2"} + >>> evaluate.save("./results/", **result, **params) + ``` + """ + current_time = datetime.now() + + file_path = _setup_path(path_or_file, current_time) + + data["_timestamp"] = current_time.isoformat() + data["_git_commit_hash"] = _git_commit_hash() + data["_evaluate_version"] = __version__ + data["_python_version"] = sys.version + data["_interpreter_path"] = sys.executable + + with FileLock(str(file_path) + ".lock"): + with open(file_path, "w") as f: + json.dump(data, f) + + # cleanup lock file + try: + os.remove(str(file_path) + ".lock") + except FileNotFoundError: + pass + + return file_path + + +def _setup_path(path_or_file, current_time): + path_or_file = Path(path_or_file) + is_file = len(path_or_file.suffix) > 0 + if is_file: + folder = path_or_file.parent + file_name = path_or_file.name + else: + folder = path_or_file + file_name = "result-" + current_time.strftime("%Y_%m_%d-%H_%M_%S") + ".json" + folder.mkdir(parents=True, exist_ok=True) + return folder / file_name + + +def _git_commit_hash(): + res = subprocess.run("git rev-parse --is-inside-work-tree".split(), cwd="./", stdout=subprocess.PIPE) + if res.stdout.decode().strip() == "true": + res = subprocess.run("git rev-parse HEAD".split(), cwd=os.getcwd(), stdout=subprocess.PIPE) + return res.stdout.decode().strip() + else: + return None diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/evaluate/visualization.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/evaluate/visualization.py new file mode 100644 index 0000000000000000000000000000000000000000..b8be8605805e4d11ef93a2911ab32afe934a78bb --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/evaluate/visualization.py @@ -0,0 +1,230 @@ +import textwrap + +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd + + +class ComplexRadar: + """Create a complex radar chart with different scales for each variable + Args: + fig (`matplotlib.figure`) : A matplotlib figure object to add the axes on. + variables (`list`) : a list of variables to. plot + ranges (`list` of `tuples`): A list of ranges (min, max) for each variable + n_ring_levels (`int): Number of ordinate or ring levels to draw. + Default: 5. + show_scales (`bool`): Indicates if we the ranges for each variable are plotted. + Default: True. + format_cfg (`dict`): A dictionary with formatting configurations. + Default: None. + Returns: + `matplotlib.figure.Figure`: a radar plot. + """ + + def __init__(self, fig, variables, ranges, n_ring_levels=5, show_scales=True, format_cfg=None): + + self.format_cfg = format_cfg + + # Calculate angles and create for each variable an axes + # Consider here the trick with having the first axes element twice (len+1) + angles = np.arange(0, 360, 360.0 / len(variables)) + axes = [ + fig.add_axes([0.1, 0.1, 0.9, 0.9], polar=True, label="axes{}".format(i), **self.format_cfg["axes_args"]) + for i in range(len(variables) + 1) + ] + + # Ensure clockwise rotation (first variable at the top N) + for ax in axes: + ax.set_theta_zero_location("N") + ax.set_theta_direction(-1) + ax.set_axisbelow(True) + + # Writing the ranges on each axes + for i, ax in enumerate(axes): + + # Here we do the trick by repeating the first iteration + j = 0 if (i == 0 or i == 1) else i - 1 + ax.set_ylim(*ranges[j]) + # Set endpoint to True if you like to have values right before the last circle + grid = np.linspace(*ranges[j], num=n_ring_levels, endpoint=self.format_cfg["incl_endpoint"]) + gridlabel = ["{}".format(round(x, 2)) for x in grid] + gridlabel[0] = "" # remove values from the center + lines, labels = ax.set_rgrids( + grid, labels=gridlabel, angle=angles[j], **self.format_cfg["rgrid_tick_lbls_args"] + ) + + ax.set_ylim(*ranges[j]) + ax.spines["polar"].set_visible(False) + ax.grid(visible=False) + + if show_scales is False: + ax.set_yticklabels([]) + + # Set all axes except the first one unvisible + for ax in axes[1:]: + ax.patch.set_visible(False) + ax.xaxis.set_visible(False) + + # Setting the attributes + self.angle = np.deg2rad(np.r_[angles, angles[0]]) + self.ranges = ranges + self.ax = axes[0] + self.ax1 = axes[1] + self.plot_counter = 0 + + # Draw (inner) circles and lines + self.ax.yaxis.grid(**self.format_cfg["rad_ln_args"]) + # Draw outer circle + self.ax.spines["polar"].set(**self.format_cfg["outer_ring"]) + # Draw angle lines + self.ax.xaxis.grid(**self.format_cfg["angle_ln_args"]) + + # ax1 is the duplicate of axes[0] (self.ax) + # Remove everything from ax1 except the plot itself + self.ax1.axis("off") + self.ax1.set_zorder(9) + + # Create the outer labels for each variable + l, text = self.ax.set_thetagrids(angles, labels=variables) + + # Beautify them + labels = [t.get_text() for t in self.ax.get_xticklabels()] + labels = [ + "\n".join( + textwrap.wrap( + label, + self.format_cfg["theta_tick_lbls_txt_wrap"], + break_long_words=self.format_cfg["theta_tick_lbls_brk_lng_wrds"], + ) + ) + for label in labels + ] + self.ax.set_xticklabels(labels, **self.format_cfg["theta_tick_lbls"]) + + for t, a in zip(self.ax.get_xticklabels(), angles): + if a == 0: + t.set_ha("center") + elif a > 0 and a < 180: + t.set_ha("left") + elif a == 180: + t.set_ha("center") + else: + t.set_ha("right") + + self.ax.tick_params(axis="both", pad=self.format_cfg["theta_tick_lbls_pad"]) + + def _scale_data(self, data, ranges): + """Scales data[1:] to ranges[0]""" + for d, (y1, y2) in zip(data[1:], ranges[1:]): + assert (y1 <= d <= y2) or (y2 <= d <= y1) + x1, x2 = ranges[0] + d = data[0] + sdata = [d] + for d, (y1, y2) in zip(data[1:], ranges[1:]): + sdata.append((d - y1) / (y2 - y1) * (x2 - x1) + x1) + return sdata + + def plot(self, data, *args, **kwargs): + """Plots a line""" + sdata = self._scale_data(data, self.ranges) + self.ax1.plot(self.angle, np.r_[sdata, sdata[0]], *args, **kwargs) + self.plot_counter = self.plot_counter + 1 + + def use_legend(self, *args, **kwargs): + """Shows a legend""" + self.ax1.legend(*args, **kwargs) + + +def radar_plot(data, model_names, invert_range=[], config=None, fig=None): + """Create a complex radar chart with different scales for each variable + Source: https://towardsdatascience.com/how-to-create-and-visualize-complex-radar-charts-f7764d0f3652 + + Args: + data (`List[dict]`): the results (list of metric + value pairs). + E.g. data = [{"accuracy": 0.9, "precision":0.8},{"accuracy": 0.7, "precision":0.6}] + names (`List[dict]`): model names. + E.g. names = ["model1", "model 2", ...] + invert_range (`List[dict]`, optional): the metrics to invert (in cases when smaller is better, e.g. speed) + E.g. invert_range=["latency_in_seconds"] + config (`dict`, optional) : a specification of the formatting configurations, namely: + + - rad_ln_args (`dict`, default `{"visible": True}`): The visibility of the radial (circle) lines. + + - outer_ring (`dict`, default `{"visible": True}`): The visibility of the outer ring. + + - angle_ln_args (`dict`, default `{"visible": True}`): The visibility of the angle lines. + + - rgrid_tick_lbls_args (`dict`, default `{"fontsize": 12}`): The font size of the tick labels on the scales. + + - theta_tick_lbls (`dict`, default `{"fontsize": 12}`): The font size of the variable labels on the plot. + + - theta_tick_lbls_pad (`int`, default `3`): The padding of the variable labels on the plot. + + - theta_tick_lbls_brk_lng_wrds (`bool`, default `True` ): Whether long words in the label are broken up or not. + + - theta_tick_lbls_txt_wrap (`int`, default `15`): Text wrap for tick labels + + - incl_endpoint (`bool`, default `False`): Include value endpoints on calse + + - marker (`str`, default `"o"`): the shape of the marker used in the radar plot. + + - markersize (`int`, default `3`): the shape of the marker used in the radar plot. + + - legend_loc (`str`, default `"upper right"`): the location of the legend in the radar plot. Must be one of: 'upper left', 'upper right', 'lower left', 'lower right'. + + - bbox_to_anchor (`tuple`, default `(2, 1)`: anchor for the legend. + fig (`matplotlib.figure.Figure`, optional): figure used to plot the radar plot. + + Returns: + `matplotlib.figure.Figure` + """ + data = pd.DataFrame(data) + data.index = model_names + variables = data.keys() + if all(x in variables for x in invert_range) is False: + raise ValueError("All of the metrics in `invert_range` should be in the data provided.") + min_max_per_variable = data.describe().T[["min", "max"]] + min_max_per_variable["min"] = min_max_per_variable["min"] - 0.1 * ( + min_max_per_variable["max"] - min_max_per_variable["min"] + ) + min_max_per_variable["max"] = min_max_per_variable["max"] + 0.1 * ( + min_max_per_variable["max"] - min_max_per_variable["min"] + ) + + ranges = list(min_max_per_variable.itertuples(index=False, name=None)) + ranges = [ + (max_value, min_value) if var in invert_range else (min_value, max_value) + for var, (min_value, max_value) in zip(variables, ranges) + ] + format_cfg = { + "axes_args": {}, + "rad_ln_args": {"visible": True}, + "outer_ring": {"visible": True}, + "angle_ln_args": {"visible": True}, + "rgrid_tick_lbls_args": {"fontsize": 12}, + "theta_tick_lbls": {"fontsize": 12}, + "theta_tick_lbls_pad": 3, + "theta_tick_lbls_brk_lng_wrds": True, + "theta_tick_lbls_txt_wrap": 15, + "incl_endpoint": False, + "marker": "o", + "markersize": 3, + "legend_loc": "upper right", + "bbox_to_anchor": (2, 1), + } + if config is not None: + format_cfg.update(config) + if fig is None: + fig = plt.figure() + radar = ComplexRadar( + fig, + variables, + ranges, + n_ring_levels=3, + show_scales=True, + format_cfg=format_cfg, + ) + for g in zip(data.index): + radar.plot(data.loc[g].values, label=g, marker=format_cfg["marker"], markersize=format_cfg["markersize"]) + radar.use_legend(**{"loc": format_cfg["legend_loc"], "bbox_to_anchor": format_cfg["bbox_to_anchor"]}) + return fig diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/jinja2/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/jinja2/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1a423a3eac16ac550e8e4008d7f5d79401b50e0f --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/jinja2/__init__.py @@ -0,0 +1,38 @@ +"""Jinja is a template engine written in pure Python. It provides a +non-XML syntax that supports inline expressions and an optional +sandboxed environment. +""" + +from .bccache import BytecodeCache as BytecodeCache +from .bccache import FileSystemBytecodeCache as FileSystemBytecodeCache +from .bccache import MemcachedBytecodeCache as MemcachedBytecodeCache +from .environment import Environment as Environment +from .environment import Template as Template +from .exceptions import TemplateAssertionError as TemplateAssertionError +from .exceptions import TemplateError as TemplateError +from .exceptions import TemplateNotFound as TemplateNotFound +from .exceptions import TemplateRuntimeError as TemplateRuntimeError +from .exceptions import TemplatesNotFound as TemplatesNotFound +from .exceptions import TemplateSyntaxError as TemplateSyntaxError +from .exceptions import UndefinedError as UndefinedError +from .loaders import BaseLoader as BaseLoader +from .loaders import ChoiceLoader as ChoiceLoader +from .loaders import DictLoader as DictLoader +from .loaders import FileSystemLoader as FileSystemLoader +from .loaders import FunctionLoader as FunctionLoader +from .loaders import ModuleLoader as ModuleLoader +from .loaders import PackageLoader as PackageLoader +from .loaders import PrefixLoader as PrefixLoader +from .runtime import ChainableUndefined as ChainableUndefined +from .runtime import DebugUndefined as DebugUndefined +from .runtime import make_logging_undefined as make_logging_undefined +from .runtime import StrictUndefined as StrictUndefined +from .runtime import Undefined as Undefined +from .utils import clear_caches as clear_caches +from .utils import is_undefined as is_undefined +from .utils import pass_context as pass_context +from .utils import pass_environment as pass_environment +from .utils import pass_eval_context as pass_eval_context +from .utils import select_autoescape as select_autoescape + +__version__ = "3.1.6" diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/jinja2/_identifier.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/jinja2/_identifier.py new file mode 100644 index 0000000000000000000000000000000000000000..928c1503c7d414a8a86bbf5a82c68d42cb089bd2 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/jinja2/_identifier.py @@ -0,0 +1,6 @@ +import re + +# generated by scripts/generate_identifier_pattern.py +pattern = re.compile( + r"[\w·̀-ͯ·҃-֑҇-ׇֽֿׁׂׅׄؐ-ًؚ-ٰٟۖ-ۜ۟-۪ۤۧۨ-ܑۭܰ-݊ަ-ް߫-߽߳ࠖ-࠙ࠛ-ࠣࠥ-ࠧࠩ-࡙࠭-࡛࣓-ࣣ࣡-ःऺ-़ा-ॏ॑-ॗॢॣঁ-ঃ়া-ৄেৈো-্ৗৢৣ৾ਁ-ਃ਼ਾ-ੂੇੈੋ-੍ੑੰੱੵઁ-ઃ઼ા-ૅે-ૉો-્ૢૣૺ-૿ଁ-ଃ଼ା-ୄେୈୋ-୍ୖୗୢୣஂா-ூெ-ைொ-்ௗఀ-ఄా-ౄె-ైొ-్ౕౖౢౣಁ-ಃ಼ಾ-ೄೆ-ೈೊ-್ೕೖೢೣഀ-ഃ഻഼ാ-ൄെ-ൈൊ-്ൗൢൣංඃ්ා-ුූෘ-ෟෲෳัิ-ฺ็-๎ັິ-ູົຼ່-ໍ༹༘༙༵༷༾༿ཱ-྄྆྇ྍ-ྗྙ-ྼ࿆ါ-ှၖ-ၙၞ-ၠၢ-ၤၧ-ၭၱ-ၴႂ-ႍႏႚ-ႝ፝-፟ᜒ-᜔ᜲ-᜴ᝒᝓᝲᝳ឴-៓៝᠋-᠍ᢅᢆᢩᤠ-ᤫᤰ-᤻ᨗ-ᨛᩕ-ᩞ᩠-᩿᩼᪰-᪽ᬀ-ᬄ᬴-᭄᭫-᭳ᮀ-ᮂᮡ-ᮭ᯦-᯳ᰤ-᰷᳐-᳔᳒-᳨᳭ᳲ-᳴᳷-᳹᷀-᷹᷻-᷿‿⁀⁔⃐-⃥⃜⃡-⃰℘℮⳯-⵿⳱ⷠ-〪ⷿ-゙゚〯꙯ꙴ-꙽ꚞꚟ꛰꛱ꠂ꠆ꠋꠣ-ꠧꢀꢁꢴ-ꣅ꣠-꣱ꣿꤦ-꤭ꥇ-꥓ꦀ-ꦃ꦳-꧀ꧥꨩ-ꨶꩃꩌꩍꩻ-ꩽꪰꪲ-ꪴꪷꪸꪾ꪿꫁ꫫ-ꫯꫵ꫶ꯣ-ꯪ꯬꯭ﬞ︀-️︠-︯︳︴﹍-﹏＿𐇽𐋠𐍶-𐍺𐨁-𐨃𐨅𐨆𐨌-𐨏𐨸-𐨿𐨺𐫦𐫥𐴤-𐽆𐴧-𐽐𑀀-𑀂𑀸-𑁆𑁿-𑂂𑂰-𑂺𑄀-𑄂𑄧-𑄴𑅅𑅆𑅳𑆀-𑆂𑆳-𑇀𑇉-𑇌𑈬-𑈷𑈾𑋟-𑋪𑌀-𑌃𑌻𑌼𑌾-𑍄𑍇𑍈𑍋-𑍍𑍗𑍢𑍣𑍦-𑍬𑍰-𑍴𑐵-𑑆𑑞𑒰-𑓃𑖯-𑖵𑖸-𑗀𑗜𑗝𑘰-𑙀𑚫-𑚷𑜝-𑜫𑠬-𑠺𑨁-𑨊𑨳-𑨹𑨻-𑨾𑩇𑩑-𑩛𑪊-𑪙𑰯-𑰶𑰸-𑰿𑲒-𑲧𑲩-𑲶𑴱-𑴶𑴺𑴼𑴽𑴿-𑵅𑵇𑶊-𑶎𑶐𑶑𑶓-𑶗𑻳-𑻶𖫰-𖫴𖬰-𖬶𖽑-𖽾𖾏-𖾒𛲝𛲞𝅥-𝅩𝅭-𝅲𝅻-𝆂𝆅-𝆋𝆪-𝆭𝉂-𝉄𝨀-𝨶𝨻-𝩬𝩵𝪄𝪛-𝪟𝪡-𝪯𞀀-𞀆𞀈-𞀘𞀛-𞀡𞀣𞀤𞀦-𞣐𞀪-𞣖𞥄-𞥊󠄀-󠇯]+" # noqa: B950 +) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/jinja2/async_utils.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/jinja2/async_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..f0c140205c50a3df9863ce1ab610b0c62a483f1b --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/jinja2/async_utils.py @@ -0,0 +1,99 @@ +import inspect +import typing as t +from functools import WRAPPER_ASSIGNMENTS +from functools import wraps + +from .utils import _PassArg +from .utils import pass_eval_context + +if t.TYPE_CHECKING: + import typing_extensions as te + +V = t.TypeVar("V") + + +def async_variant(normal_func): # type: ignore + def decorator(async_func): # type: ignore + pass_arg = _PassArg.from_obj(normal_func) + need_eval_context = pass_arg is None + + if pass_arg is _PassArg.environment: + + def is_async(args: t.Any) -> bool: + return t.cast(bool, args[0].is_async) + + else: + + def is_async(args: t.Any) -> bool: + return t.cast(bool, args[0].environment.is_async) + + # Take the doc and annotations from the sync function, but the + # name from the async function. Pallets-Sphinx-Themes + # build_function_directive expects __wrapped__ to point to the + # sync function. + async_func_attrs = ("__module__", "__name__", "__qualname__") + normal_func_attrs = tuple(set(WRAPPER_ASSIGNMENTS).difference(async_func_attrs)) + + @wraps(normal_func, assigned=normal_func_attrs) + @wraps(async_func, assigned=async_func_attrs, updated=()) + def wrapper(*args, **kwargs): # type: ignore + b = is_async(args) + + if need_eval_context: + args = args[1:] + + if b: + return async_func(*args, **kwargs) + + return normal_func(*args, **kwargs) + + if need_eval_context: + wrapper = pass_eval_context(wrapper) + + wrapper.jinja_async_variant = True # type: ignore[attr-defined] + return wrapper + + return decorator + + +_common_primitives = {int, float, bool, str, list, dict, tuple, type(None)} + + +async def auto_await(value: t.Union[t.Awaitable["V"], "V"]) -> "V": + # Avoid a costly call to isawaitable + if type(value) in _common_primitives: + return t.cast("V", value) + + if inspect.isawaitable(value): + return await t.cast("t.Awaitable[V]", value) + + return value + + +class _IteratorToAsyncIterator(t.Generic[V]): + def __init__(self, iterator: "t.Iterator[V]"): + self._iterator = iterator + + def __aiter__(self) -> "te.Self": + return self + + async def __anext__(self) -> V: + try: + return next(self._iterator) + except StopIteration as e: + raise StopAsyncIteration(e.value) from e + + +def auto_aiter( + iterable: "t.Union[t.AsyncIterable[V], t.Iterable[V]]", +) -> "t.AsyncIterator[V]": + if hasattr(iterable, "__aiter__"): + return iterable.__aiter__() + else: + return _IteratorToAsyncIterator(iter(iterable)) + + +async def auto_to_list( + value: "t.Union[t.AsyncIterable[V], t.Iterable[V]]", +) -> t.List["V"]: + return [x async for x in auto_aiter(value)] diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/jinja2/bccache.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/jinja2/bccache.py new file mode 100644 index 0000000000000000000000000000000000000000..ada8b099ff251ea9c6da4c42e1383f37e359f06a --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/jinja2/bccache.py @@ -0,0 +1,408 @@ +"""The optional bytecode cache system. This is useful if you have very +complex template situations and the compilation of all those templates +slows down your application too much. + +Situations where this is useful are often forking web applications that +are initialized on the first request. +""" + +import errno +import fnmatch +import marshal +import os +import pickle +import stat +import sys +import tempfile +import typing as t +from hashlib import sha1 +from io import BytesIO +from types import CodeType + +if t.TYPE_CHECKING: + import typing_extensions as te + + from .environment import Environment + + class _MemcachedClient(te.Protocol): + def get(self, key: str) -> bytes: ... + + def set( + self, key: str, value: bytes, timeout: t.Optional[int] = None + ) -> None: ... + + +bc_version = 5 +# Magic bytes to identify Jinja bytecode cache files. Contains the +# Python major and minor version to avoid loading incompatible bytecode +# if a project upgrades its Python version. +bc_magic = ( + b"j2" + + pickle.dumps(bc_version, 2) + + pickle.dumps((sys.version_info[0] << 24) | sys.version_info[1], 2) +) + + +class Bucket: + """Buckets are used to store the bytecode for one template. It's created + and initialized by the bytecode cache and passed to the loading functions. + + The buckets get an internal checksum from the cache assigned and use this + to automatically reject outdated cache material. Individual bytecode + cache subclasses don't have to care about cache invalidation. + """ + + def __init__(self, environment: "Environment", key: str, checksum: str) -> None: + self.environment = environment + self.key = key + self.checksum = checksum + self.reset() + + def reset(self) -> None: + """Resets the bucket (unloads the bytecode).""" + self.code: t.Optional[CodeType] = None + + def load_bytecode(self, f: t.BinaryIO) -> None: + """Loads bytecode from a file or file like object.""" + # make sure the magic header is correct + magic = f.read(len(bc_magic)) + if magic != bc_magic: + self.reset() + return + # the source code of the file changed, we need to reload + checksum = pickle.load(f) + if self.checksum != checksum: + self.reset() + return + # if marshal_load fails then we need to reload + try: + self.code = marshal.load(f) + except (EOFError, ValueError, TypeError): + self.reset() + return + + def write_bytecode(self, f: t.IO[bytes]) -> None: + """Dump the bytecode into the file or file like object passed.""" + if self.code is None: + raise TypeError("can't write empty bucket") + f.write(bc_magic) + pickle.dump(self.checksum, f, 2) + marshal.dump(self.code, f) + + def bytecode_from_string(self, string: bytes) -> None: + """Load bytecode from bytes.""" + self.load_bytecode(BytesIO(string)) + + def bytecode_to_string(self) -> bytes: + """Return the bytecode as bytes.""" + out = BytesIO() + self.write_bytecode(out) + return out.getvalue() + + +class BytecodeCache: + """To implement your own bytecode cache you have to subclass this class + and override :meth:`load_bytecode` and :meth:`dump_bytecode`. Both of + these methods are passed a :class:`~jinja2.bccache.Bucket`. + + A very basic bytecode cache that saves the bytecode on the file system:: + + from os import path + + class MyCache(BytecodeCache): + + def __init__(self, directory): + self.directory = directory + + def load_bytecode(self, bucket): + filename = path.join(self.directory, bucket.key) + if path.exists(filename): + with open(filename, 'rb') as f: + bucket.load_bytecode(f) + + def dump_bytecode(self, bucket): + filename = path.join(self.directory, bucket.key) + with open(filename, 'wb') as f: + bucket.write_bytecode(f) + + A more advanced version of a filesystem based bytecode cache is part of + Jinja. + """ + + def load_bytecode(self, bucket: Bucket) -> None: + """Subclasses have to override this method to load bytecode into a + bucket. If they are not able to find code in the cache for the + bucket, it must not do anything. + """ + raise NotImplementedError() + + def dump_bytecode(self, bucket: Bucket) -> None: + """Subclasses have to override this method to write the bytecode + from a bucket back to the cache. If it unable to do so it must not + fail silently but raise an exception. + """ + raise NotImplementedError() + + def clear(self) -> None: + """Clears the cache. This method is not used by Jinja but should be + implemented to allow applications to clear the bytecode cache used + by a particular environment. + """ + + def get_cache_key( + self, name: str, filename: t.Optional[t.Union[str]] = None + ) -> str: + """Returns the unique hash key for this template name.""" + hash = sha1(name.encode("utf-8")) + + if filename is not None: + hash.update(f"|{filename}".encode()) + + return hash.hexdigest() + + def get_source_checksum(self, source: str) -> str: + """Returns a checksum for the source.""" + return sha1(source.encode("utf-8")).hexdigest() + + def get_bucket( + self, + environment: "Environment", + name: str, + filename: t.Optional[str], + source: str, + ) -> Bucket: + """Return a cache bucket for the given template. All arguments are + mandatory but filename may be `None`. + """ + key = self.get_cache_key(name, filename) + checksum = self.get_source_checksum(source) + bucket = Bucket(environment, key, checksum) + self.load_bytecode(bucket) + return bucket + + def set_bucket(self, bucket: Bucket) -> None: + """Put the bucket into the cache.""" + self.dump_bytecode(bucket) + + +class FileSystemBytecodeCache(BytecodeCache): + """A bytecode cache that stores bytecode on the filesystem. It accepts + two arguments: The directory where the cache items are stored and a + pattern string that is used to build the filename. + + If no directory is specified a default cache directory is selected. On + Windows the user's temp directory is used, on UNIX systems a directory + is created for the user in the system temp directory. + + The pattern can be used to have multiple separate caches operate on the + same directory. The default pattern is ``'__jinja2_%s.cache'``. ``%s`` + is replaced with the cache key. + + >>> bcc = FileSystemBytecodeCache('/tmp/jinja_cache', '%s.cache') + + This bytecode cache supports clearing of the cache using the clear method. + """ + + def __init__( + self, directory: t.Optional[str] = None, pattern: str = "__jinja2_%s.cache" + ) -> None: + if directory is None: + directory = self._get_default_cache_dir() + self.directory = directory + self.pattern = pattern + + def _get_default_cache_dir(self) -> str: + def _unsafe_dir() -> "te.NoReturn": + raise RuntimeError( + "Cannot determine safe temp directory. You " + "need to explicitly provide one." + ) + + tmpdir = tempfile.gettempdir() + + # On windows the temporary directory is used specific unless + # explicitly forced otherwise. We can just use that. + if os.name == "nt": + return tmpdir + if not hasattr(os, "getuid"): + _unsafe_dir() + + dirname = f"_jinja2-cache-{os.getuid()}" + actual_dir = os.path.join(tmpdir, dirname) + + try: + os.mkdir(actual_dir, stat.S_IRWXU) + except OSError as e: + if e.errno != errno.EEXIST: + raise + try: + os.chmod(actual_dir, stat.S_IRWXU) + actual_dir_stat = os.lstat(actual_dir) + if ( + actual_dir_stat.st_uid != os.getuid() + or not stat.S_ISDIR(actual_dir_stat.st_mode) + or stat.S_IMODE(actual_dir_stat.st_mode) != stat.S_IRWXU + ): + _unsafe_dir() + except OSError as e: + if e.errno != errno.EEXIST: + raise + + actual_dir_stat = os.lstat(actual_dir) + if ( + actual_dir_stat.st_uid != os.getuid() + or not stat.S_ISDIR(actual_dir_stat.st_mode) + or stat.S_IMODE(actual_dir_stat.st_mode) != stat.S_IRWXU + ): + _unsafe_dir() + + return actual_dir + + def _get_cache_filename(self, bucket: Bucket) -> str: + return os.path.join(self.directory, self.pattern % (bucket.key,)) + + def load_bytecode(self, bucket: Bucket) -> None: + filename = self._get_cache_filename(bucket) + + # Don't test for existence before opening the file, since the + # file could disappear after the test before the open. + try: + f = open(filename, "rb") + except (FileNotFoundError, IsADirectoryError, PermissionError): + # PermissionError can occur on Windows when an operation is + # in progress, such as calling clear(). + return + + with f: + bucket.load_bytecode(f) + + def dump_bytecode(self, bucket: Bucket) -> None: + # Write to a temporary file, then rename to the real name after + # writing. This avoids another process reading the file before + # it is fully written. + name = self._get_cache_filename(bucket) + f = tempfile.NamedTemporaryFile( + mode="wb", + dir=os.path.dirname(name), + prefix=os.path.basename(name), + suffix=".tmp", + delete=False, + ) + + def remove_silent() -> None: + try: + os.remove(f.name) + except OSError: + # Another process may have called clear(). On Windows, + # another program may be holding the file open. + pass + + try: + with f: + bucket.write_bytecode(f) + except BaseException: + remove_silent() + raise + + try: + os.replace(f.name, name) + except OSError: + # Another process may have called clear(). On Windows, + # another program may be holding the file open. + remove_silent() + except BaseException: + remove_silent() + raise + + def clear(self) -> None: + # imported lazily here because google app-engine doesn't support + # write access on the file system and the function does not exist + # normally. + from os import remove + + files = fnmatch.filter(os.listdir(self.directory), self.pattern % ("*",)) + for filename in files: + try: + remove(os.path.join(self.directory, filename)) + except OSError: + pass + + +class MemcachedBytecodeCache(BytecodeCache): + """This class implements a bytecode cache that uses a memcache cache for + storing the information. It does not enforce a specific memcache library + (tummy's memcache or cmemcache) but will accept any class that provides + the minimal interface required. + + Libraries compatible with this class: + + - `cachelib `_ + - `python-memcached `_ + + (Unfortunately the django cache interface is not compatible because it + does not support storing binary data, only text. You can however pass + the underlying cache client to the bytecode cache which is available + as `django.core.cache.cache._client`.) + + The minimal interface for the client passed to the constructor is this: + + .. class:: MinimalClientInterface + + .. method:: set(key, value[, timeout]) + + Stores the bytecode in the cache. `value` is a string and + `timeout` the timeout of the key. If timeout is not provided + a default timeout or no timeout should be assumed, if it's + provided it's an integer with the number of seconds the cache + item should exist. + + .. method:: get(key) + + Returns the value for the cache key. If the item does not + exist in the cache the return value must be `None`. + + The other arguments to the constructor are the prefix for all keys that + is added before the actual cache key and the timeout for the bytecode in + the cache system. We recommend a high (or no) timeout. + + This bytecode cache does not support clearing of used items in the cache. + The clear method is a no-operation function. + + .. versionadded:: 2.7 + Added support for ignoring memcache errors through the + `ignore_memcache_errors` parameter. + """ + + def __init__( + self, + client: "_MemcachedClient", + prefix: str = "jinja2/bytecode/", + timeout: t.Optional[int] = None, + ignore_memcache_errors: bool = True, + ): + self.client = client + self.prefix = prefix + self.timeout = timeout + self.ignore_memcache_errors = ignore_memcache_errors + + def load_bytecode(self, bucket: Bucket) -> None: + try: + code = self.client.get(self.prefix + bucket.key) + except Exception: + if not self.ignore_memcache_errors: + raise + else: + bucket.bytecode_from_string(code) + + def dump_bytecode(self, bucket: Bucket) -> None: + key = self.prefix + bucket.key + value = bucket.bytecode_to_string() + + try: + if self.timeout is not None: + self.client.set(key, value, self.timeout) + else: + self.client.set(key, value) + except Exception: + if not self.ignore_memcache_errors: + raise diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/jinja2/compiler.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/jinja2/compiler.py new file mode 100644 index 0000000000000000000000000000000000000000..a4ff6a1b11af3e1a868d1a74c48d842390259b43 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/jinja2/compiler.py @@ -0,0 +1,1998 @@ +"""Compiles nodes from the parser into Python code.""" + +import typing as t +from contextlib import contextmanager +from functools import update_wrapper +from io import StringIO +from itertools import chain +from keyword import iskeyword as is_python_keyword + +from markupsafe import escape +from markupsafe import Markup + +from . import nodes +from .exceptions import TemplateAssertionError +from .idtracking import Symbols +from .idtracking import VAR_LOAD_ALIAS +from .idtracking import VAR_LOAD_PARAMETER +from .idtracking import VAR_LOAD_RESOLVE +from .idtracking import VAR_LOAD_UNDEFINED +from .nodes import EvalContext +from .optimizer import Optimizer +from .utils import _PassArg +from .utils import concat +from .visitor import NodeVisitor + +if t.TYPE_CHECKING: + import typing_extensions as te + + from .environment import Environment + +F = t.TypeVar("F", bound=t.Callable[..., t.Any]) + +operators = { + "eq": "==", + "ne": "!=", + "gt": ">", + "gteq": ">=", + "lt": "<", + "lteq": "<=", + "in": "in", + "notin": "not in", +} + + +def optimizeconst(f: F) -> F: + def new_func( + self: "CodeGenerator", node: nodes.Expr, frame: "Frame", **kwargs: t.Any + ) -> t.Any: + # Only optimize if the frame is not volatile + if self.optimizer is not None and not frame.eval_ctx.volatile: + new_node = self.optimizer.visit(node, frame.eval_ctx) + + if new_node != node: + return self.visit(new_node, frame) + + return f(self, node, frame, **kwargs) + + return update_wrapper(new_func, f) # type: ignore[return-value] + + +def _make_binop(op: str) -> t.Callable[["CodeGenerator", nodes.BinExpr, "Frame"], None]: + @optimizeconst + def visitor(self: "CodeGenerator", node: nodes.BinExpr, frame: Frame) -> None: + if ( + self.environment.sandboxed and op in self.environment.intercepted_binops # type: ignore + ): + self.write(f"environment.call_binop(context, {op!r}, ") + self.visit(node.left, frame) + self.write(", ") + self.visit(node.right, frame) + else: + self.write("(") + self.visit(node.left, frame) + self.write(f" {op} ") + self.visit(node.right, frame) + + self.write(")") + + return visitor + + +def _make_unop( + op: str, +) -> t.Callable[["CodeGenerator", nodes.UnaryExpr, "Frame"], None]: + @optimizeconst + def visitor(self: "CodeGenerator", node: nodes.UnaryExpr, frame: Frame) -> None: + if ( + self.environment.sandboxed and op in self.environment.intercepted_unops # type: ignore + ): + self.write(f"environment.call_unop(context, {op!r}, ") + self.visit(node.node, frame) + else: + self.write("(" + op) + self.visit(node.node, frame) + + self.write(")") + + return visitor + + +def generate( + node: nodes.Template, + environment: "Environment", + name: t.Optional[str], + filename: t.Optional[str], + stream: t.Optional[t.TextIO] = None, + defer_init: bool = False, + optimized: bool = True, +) -> t.Optional[str]: + """Generate the python source for a node tree.""" + if not isinstance(node, nodes.Template): + raise TypeError("Can't compile non template nodes") + + generator = environment.code_generator_class( + environment, name, filename, stream, defer_init, optimized + ) + generator.visit(node) + + if stream is None: + return generator.stream.getvalue() # type: ignore + + return None + + +def has_safe_repr(value: t.Any) -> bool: + """Does the node have a safe representation?""" + if value is None or value is NotImplemented or value is Ellipsis: + return True + + if type(value) in {bool, int, float, complex, range, str, Markup}: + return True + + if type(value) in {tuple, list, set, frozenset}: + return all(has_safe_repr(v) for v in value) + + if type(value) is dict: # noqa E721 + return all(has_safe_repr(k) and has_safe_repr(v) for k, v in value.items()) + + return False + + +def find_undeclared( + nodes: t.Iterable[nodes.Node], names: t.Iterable[str] +) -> t.Set[str]: + """Check if the names passed are accessed undeclared. The return value + is a set of all the undeclared names from the sequence of names found. + """ + visitor = UndeclaredNameVisitor(names) + try: + for node in nodes: + visitor.visit(node) + except VisitorExit: + pass + return visitor.undeclared + + +class MacroRef: + def __init__(self, node: t.Union[nodes.Macro, nodes.CallBlock]) -> None: + self.node = node + self.accesses_caller = False + self.accesses_kwargs = False + self.accesses_varargs = False + + +class Frame: + """Holds compile time information for us.""" + + def __init__( + self, + eval_ctx: EvalContext, + parent: t.Optional["Frame"] = None, + level: t.Optional[int] = None, + ) -> None: + self.eval_ctx = eval_ctx + + # the parent of this frame + self.parent = parent + + if parent is None: + self.symbols = Symbols(level=level) + + # in some dynamic inheritance situations the compiler needs to add + # write tests around output statements. + self.require_output_check = False + + # inside some tags we are using a buffer rather than yield statements. + # this for example affects {% filter %} or {% macro %}. If a frame + # is buffered this variable points to the name of the list used as + # buffer. + self.buffer: t.Optional[str] = None + + # the name of the block we're in, otherwise None. + self.block: t.Optional[str] = None + + else: + self.symbols = Symbols(parent.symbols, level=level) + self.require_output_check = parent.require_output_check + self.buffer = parent.buffer + self.block = parent.block + + # a toplevel frame is the root + soft frames such as if conditions. + self.toplevel = False + + # the root frame is basically just the outermost frame, so no if + # conditions. This information is used to optimize inheritance + # situations. + self.rootlevel = False + + # variables set inside of loops and blocks should not affect outer frames, + # but they still needs to be kept track of as part of the active context. + self.loop_frame = False + self.block_frame = False + + # track whether the frame is being used in an if-statement or conditional + # expression as it determines which errors should be raised during runtime + # or compile time. + self.soft_frame = False + + def copy(self) -> "te.Self": + """Create a copy of the current one.""" + rv = object.__new__(self.__class__) + rv.__dict__.update(self.__dict__) + rv.symbols = self.symbols.copy() + return rv + + def inner(self, isolated: bool = False) -> "Frame": + """Return an inner frame.""" + if isolated: + return Frame(self.eval_ctx, level=self.symbols.level + 1) + return Frame(self.eval_ctx, self) + + def soft(self) -> "te.Self": + """Return a soft frame. A soft frame may not be modified as + standalone thing as it shares the resources with the frame it + was created of, but it's not a rootlevel frame any longer. + + This is only used to implement if-statements and conditional + expressions. + """ + rv = self.copy() + rv.rootlevel = False + rv.soft_frame = True + return rv + + __copy__ = copy + + +class VisitorExit(RuntimeError): + """Exception used by the `UndeclaredNameVisitor` to signal a stop.""" + + +class DependencyFinderVisitor(NodeVisitor): + """A visitor that collects filter and test calls.""" + + def __init__(self) -> None: + self.filters: t.Set[str] = set() + self.tests: t.Set[str] = set() + + def visit_Filter(self, node: nodes.Filter) -> None: + self.generic_visit(node) + self.filters.add(node.name) + + def visit_Test(self, node: nodes.Test) -> None: + self.generic_visit(node) + self.tests.add(node.name) + + def visit_Block(self, node: nodes.Block) -> None: + """Stop visiting at blocks.""" + + +class UndeclaredNameVisitor(NodeVisitor): + """A visitor that checks if a name is accessed without being + declared. This is different from the frame visitor as it will + not stop at closure frames. + """ + + def __init__(self, names: t.Iterable[str]) -> None: + self.names = set(names) + self.undeclared: t.Set[str] = set() + + def visit_Name(self, node: nodes.Name) -> None: + if node.ctx == "load" and node.name in self.names: + self.undeclared.add(node.name) + if self.undeclared == self.names: + raise VisitorExit() + else: + self.names.discard(node.name) + + def visit_Block(self, node: nodes.Block) -> None: + """Stop visiting a blocks.""" + + +class CompilerExit(Exception): + """Raised if the compiler encountered a situation where it just + doesn't make sense to further process the code. Any block that + raises such an exception is not further processed. + """ + + +class CodeGenerator(NodeVisitor): + def __init__( + self, + environment: "Environment", + name: t.Optional[str], + filename: t.Optional[str], + stream: t.Optional[t.TextIO] = None, + defer_init: bool = False, + optimized: bool = True, + ) -> None: + if stream is None: + stream = StringIO() + self.environment = environment + self.name = name + self.filename = filename + self.stream = stream + self.created_block_context = False + self.defer_init = defer_init + self.optimizer: t.Optional[Optimizer] = None + + if optimized: + self.optimizer = Optimizer(environment) + + # aliases for imports + self.import_aliases: t.Dict[str, str] = {} + + # a registry for all blocks. Because blocks are moved out + # into the global python scope they are registered here + self.blocks: t.Dict[str, nodes.Block] = {} + + # the number of extends statements so far + self.extends_so_far = 0 + + # some templates have a rootlevel extends. In this case we + # can safely assume that we're a child template and do some + # more optimizations. + self.has_known_extends = False + + # the current line number + self.code_lineno = 1 + + # registry of all filters and tests (global, not block local) + self.tests: t.Dict[str, str] = {} + self.filters: t.Dict[str, str] = {} + + # the debug information + self.debug_info: t.List[t.Tuple[int, int]] = [] + self._write_debug_info: t.Optional[int] = None + + # the number of new lines before the next write() + self._new_lines = 0 + + # the line number of the last written statement + self._last_line = 0 + + # true if nothing was written so far. + self._first_write = True + + # used by the `temporary_identifier` method to get new + # unique, temporary identifier + self._last_identifier = 0 + + # the current indentation + self._indentation = 0 + + # Tracks toplevel assignments + self._assign_stack: t.List[t.Set[str]] = [] + + # Tracks parameter definition blocks + self._param_def_block: t.List[t.Set[str]] = [] + + # Tracks the current context. + self._context_reference_stack = ["context"] + + @property + def optimized(self) -> bool: + return self.optimizer is not None + + # -- Various compilation helpers + + def fail(self, msg: str, lineno: int) -> "te.NoReturn": + """Fail with a :exc:`TemplateAssertionError`.""" + raise TemplateAssertionError(msg, lineno, self.name, self.filename) + + def temporary_identifier(self) -> str: + """Get a new unique identifier.""" + self._last_identifier += 1 + return f"t_{self._last_identifier}" + + def buffer(self, frame: Frame) -> None: + """Enable buffering for the frame from that point onwards.""" + frame.buffer = self.temporary_identifier() + self.writeline(f"{frame.buffer} = []") + + def return_buffer_contents( + self, frame: Frame, force_unescaped: bool = False + ) -> None: + """Return the buffer contents of the frame.""" + if not force_unescaped: + if frame.eval_ctx.volatile: + self.writeline("if context.eval_ctx.autoescape:") + self.indent() + self.writeline(f"return Markup(concat({frame.buffer}))") + self.outdent() + self.writeline("else:") + self.indent() + self.writeline(f"return concat({frame.buffer})") + self.outdent() + return + elif frame.eval_ctx.autoescape: + self.writeline(f"return Markup(concat({frame.buffer}))") + return + self.writeline(f"return concat({frame.buffer})") + + def indent(self) -> None: + """Indent by one.""" + self._indentation += 1 + + def outdent(self, step: int = 1) -> None: + """Outdent by step.""" + self._indentation -= step + + def start_write(self, frame: Frame, node: t.Optional[nodes.Node] = None) -> None: + """Yield or write into the frame buffer.""" + if frame.buffer is None: + self.writeline("yield ", node) + else: + self.writeline(f"{frame.buffer}.append(", node) + + def end_write(self, frame: Frame) -> None: + """End the writing process started by `start_write`.""" + if frame.buffer is not None: + self.write(")") + + def simple_write( + self, s: str, frame: Frame, node: t.Optional[nodes.Node] = None + ) -> None: + """Simple shortcut for start_write + write + end_write.""" + self.start_write(frame, node) + self.write(s) + self.end_write(frame) + + def blockvisit(self, nodes: t.Iterable[nodes.Node], frame: Frame) -> None: + """Visit a list of nodes as block in a frame. If the current frame + is no buffer a dummy ``if 0: yield None`` is written automatically. + """ + try: + self.writeline("pass") + for node in nodes: + self.visit(node, frame) + except CompilerExit: + pass + + def write(self, x: str) -> None: + """Write a string into the output stream.""" + if self._new_lines: + if not self._first_write: + self.stream.write("\n" * self._new_lines) + self.code_lineno += self._new_lines + if self._write_debug_info is not None: + self.debug_info.append((self._write_debug_info, self.code_lineno)) + self._write_debug_info = None + self._first_write = False + self.stream.write(" " * self._indentation) + self._new_lines = 0 + self.stream.write(x) + + def writeline( + self, x: str, node: t.Optional[nodes.Node] = None, extra: int = 0 + ) -> None: + """Combination of newline and write.""" + self.newline(node, extra) + self.write(x) + + def newline(self, node: t.Optional[nodes.Node] = None, extra: int = 0) -> None: + """Add one or more newlines before the next write.""" + self._new_lines = max(self._new_lines, 1 + extra) + if node is not None and node.lineno != self._last_line: + self._write_debug_info = node.lineno + self._last_line = node.lineno + + def signature( + self, + node: t.Union[nodes.Call, nodes.Filter, nodes.Test], + frame: Frame, + extra_kwargs: t.Optional[t.Mapping[str, t.Any]] = None, + ) -> None: + """Writes a function call to the stream for the current node. + A leading comma is added automatically. The extra keyword + arguments may not include python keywords otherwise a syntax + error could occur. The extra keyword arguments should be given + as python dict. + """ + # if any of the given keyword arguments is a python keyword + # we have to make sure that no invalid call is created. + kwarg_workaround = any( + is_python_keyword(t.cast(str, k)) + for k in chain((x.key for x in node.kwargs), extra_kwargs or ()) + ) + + for arg in node.args: + self.write(", ") + self.visit(arg, frame) + + if not kwarg_workaround: + for kwarg in node.kwargs: + self.write(", ") + self.visit(kwarg, frame) + if extra_kwargs is not None: + for key, value in extra_kwargs.items(): + self.write(f", {key}={value}") + if node.dyn_args: + self.write(", *") + self.visit(node.dyn_args, frame) + + if kwarg_workaround: + if node.dyn_kwargs is not None: + self.write(", **dict({") + else: + self.write(", **{") + for kwarg in node.kwargs: + self.write(f"{kwarg.key!r}: ") + self.visit(kwarg.value, frame) + self.write(", ") + if extra_kwargs is not None: + for key, value in extra_kwargs.items(): + self.write(f"{key!r}: {value}, ") + if node.dyn_kwargs is not None: + self.write("}, **") + self.visit(node.dyn_kwargs, frame) + self.write(")") + else: + self.write("}") + + elif node.dyn_kwargs is not None: + self.write(", **") + self.visit(node.dyn_kwargs, frame) + + def pull_dependencies(self, nodes: t.Iterable[nodes.Node]) -> None: + """Find all filter and test names used in the template and + assign them to variables in the compiled namespace. Checking + that the names are registered with the environment is done when + compiling the Filter and Test nodes. If the node is in an If or + CondExpr node, the check is done at runtime instead. + + .. versionchanged:: 3.0 + Filters and tests in If and CondExpr nodes are checked at + runtime instead of compile time. + """ + visitor = DependencyFinderVisitor() + + for node in nodes: + visitor.visit(node) + + for id_map, names, dependency in ( + (self.filters, visitor.filters, "filters"), + ( + self.tests, + visitor.tests, + "tests", + ), + ): + for name in sorted(names): + if name not in id_map: + id_map[name] = self.temporary_identifier() + + # add check during runtime that dependencies used inside of executed + # blocks are defined, as this step may be skipped during compile time + self.writeline("try:") + self.indent() + self.writeline(f"{id_map[name]} = environment.{dependency}[{name!r}]") + self.outdent() + self.writeline("except KeyError:") + self.indent() + self.writeline("@internalcode") + self.writeline(f"def {id_map[name]}(*unused):") + self.indent() + self.writeline( + f'raise TemplateRuntimeError("No {dependency[:-1]}' + f' named {name!r} found.")' + ) + self.outdent() + self.outdent() + + def enter_frame(self, frame: Frame) -> None: + undefs = [] + for target, (action, param) in frame.symbols.loads.items(): + if action == VAR_LOAD_PARAMETER: + pass + elif action == VAR_LOAD_RESOLVE: + self.writeline(f"{target} = {self.get_resolve_func()}({param!r})") + elif action == VAR_LOAD_ALIAS: + self.writeline(f"{target} = {param}") + elif action == VAR_LOAD_UNDEFINED: + undefs.append(target) + else: + raise NotImplementedError("unknown load instruction") + if undefs: + self.writeline(f"{' = '.join(undefs)} = missing") + + def leave_frame(self, frame: Frame, with_python_scope: bool = False) -> None: + if not with_python_scope: + undefs = [] + for target in frame.symbols.loads: + undefs.append(target) + if undefs: + self.writeline(f"{' = '.join(undefs)} = missing") + + def choose_async(self, async_value: str = "async ", sync_value: str = "") -> str: + return async_value if self.environment.is_async else sync_value + + def func(self, name: str) -> str: + return f"{self.choose_async()}def {name}" + + def macro_body( + self, node: t.Union[nodes.Macro, nodes.CallBlock], frame: Frame + ) -> t.Tuple[Frame, MacroRef]: + """Dump the function def of a macro or call block.""" + frame = frame.inner() + frame.symbols.analyze_node(node) + macro_ref = MacroRef(node) + + explicit_caller = None + skip_special_params = set() + args = [] + + for idx, arg in enumerate(node.args): + if arg.name == "caller": + explicit_caller = idx + if arg.name in ("kwargs", "varargs"): + skip_special_params.add(arg.name) + args.append(frame.symbols.ref(arg.name)) + + undeclared = find_undeclared(node.body, ("caller", "kwargs", "varargs")) + + if "caller" in undeclared: + # In older Jinja versions there was a bug that allowed caller + # to retain the special behavior even if it was mentioned in + # the argument list. However thankfully this was only really + # working if it was the last argument. So we are explicitly + # checking this now and error out if it is anywhere else in + # the argument list. + if explicit_caller is not None: + try: + node.defaults[explicit_caller - len(node.args)] + except IndexError: + self.fail( + "When defining macros or call blocks the " + 'special "caller" argument must be omitted ' + "or be given a default.", + node.lineno, + ) + else: + args.append(frame.symbols.declare_parameter("caller")) + macro_ref.accesses_caller = True + if "kwargs" in undeclared and "kwargs" not in skip_special_params: + args.append(frame.symbols.declare_parameter("kwargs")) + macro_ref.accesses_kwargs = True + if "varargs" in undeclared and "varargs" not in skip_special_params: + args.append(frame.symbols.declare_parameter("varargs")) + macro_ref.accesses_varargs = True + + # macros are delayed, they never require output checks + frame.require_output_check = False + frame.symbols.analyze_node(node) + self.writeline(f"{self.func('macro')}({', '.join(args)}):", node) + self.indent() + + self.buffer(frame) + self.enter_frame(frame) + + self.push_parameter_definitions(frame) + for idx, arg in enumerate(node.args): + ref = frame.symbols.ref(arg.name) + self.writeline(f"if {ref} is missing:") + self.indent() + try: + default = node.defaults[idx - len(node.args)] + except IndexError: + self.writeline( + f'{ref} = undefined("parameter {arg.name!r} was not provided",' + f" name={arg.name!r})" + ) + else: + self.writeline(f"{ref} = ") + self.visit(default, frame) + self.mark_parameter_stored(ref) + self.outdent() + self.pop_parameter_definitions() + + self.blockvisit(node.body, frame) + self.return_buffer_contents(frame, force_unescaped=True) + self.leave_frame(frame, with_python_scope=True) + self.outdent() + + return frame, macro_ref + + def macro_def(self, macro_ref: MacroRef, frame: Frame) -> None: + """Dump the macro definition for the def created by macro_body.""" + arg_tuple = ", ".join(repr(x.name) for x in macro_ref.node.args) + name = getattr(macro_ref.node, "name", None) + if len(macro_ref.node.args) == 1: + arg_tuple += "," + self.write( + f"Macro(environment, macro, {name!r}, ({arg_tuple})," + f" {macro_ref.accesses_kwargs!r}, {macro_ref.accesses_varargs!r}," + f" {macro_ref.accesses_caller!r}, context.eval_ctx.autoescape)" + ) + + def position(self, node: nodes.Node) -> str: + """Return a human readable position for the node.""" + rv = f"line {node.lineno}" + if self.name is not None: + rv = f"{rv} in {self.name!r}" + return rv + + def dump_local_context(self, frame: Frame) -> str: + items_kv = ", ".join( + f"{name!r}: {target}" + for name, target in frame.symbols.dump_stores().items() + ) + return f"{{{items_kv}}}" + + def write_commons(self) -> None: + """Writes a common preamble that is used by root and block functions. + Primarily this sets up common local helpers and enforces a generator + through a dead branch. + """ + self.writeline("resolve = context.resolve_or_missing") + self.writeline("undefined = environment.undefined") + self.writeline("concat = environment.concat") + # always use the standard Undefined class for the implicit else of + # conditional expressions + self.writeline("cond_expr_undefined = Undefined") + self.writeline("if 0: yield None") + + def push_parameter_definitions(self, frame: Frame) -> None: + """Pushes all parameter targets from the given frame into a local + stack that permits tracking of yet to be assigned parameters. In + particular this enables the optimization from `visit_Name` to skip + undefined expressions for parameters in macros as macros can reference + otherwise unbound parameters. + """ + self._param_def_block.append(frame.symbols.dump_param_targets()) + + def pop_parameter_definitions(self) -> None: + """Pops the current parameter definitions set.""" + self._param_def_block.pop() + + def mark_parameter_stored(self, target: str) -> None: + """Marks a parameter in the current parameter definitions as stored. + This will skip the enforced undefined checks. + """ + if self._param_def_block: + self._param_def_block[-1].discard(target) + + def push_context_reference(self, target: str) -> None: + self._context_reference_stack.append(target) + + def pop_context_reference(self) -> None: + self._context_reference_stack.pop() + + def get_context_ref(self) -> str: + return self._context_reference_stack[-1] + + def get_resolve_func(self) -> str: + target = self._context_reference_stack[-1] + if target == "context": + return "resolve" + return f"{target}.resolve" + + def derive_context(self, frame: Frame) -> str: + return f"{self.get_context_ref()}.derived({self.dump_local_context(frame)})" + + def parameter_is_undeclared(self, target: str) -> bool: + """Checks if a given target is an undeclared parameter.""" + if not self._param_def_block: + return False + return target in self._param_def_block[-1] + + def push_assign_tracking(self) -> None: + """Pushes a new layer for assignment tracking.""" + self._assign_stack.append(set()) + + def pop_assign_tracking(self, frame: Frame) -> None: + """Pops the topmost level for assignment tracking and updates the + context variables if necessary. + """ + vars = self._assign_stack.pop() + if ( + not frame.block_frame + and not frame.loop_frame + and not frame.toplevel + or not vars + ): + return + public_names = [x for x in vars if x[:1] != "_"] + if len(vars) == 1: + name = next(iter(vars)) + ref = frame.symbols.ref(name) + if frame.loop_frame: + self.writeline(f"_loop_vars[{name!r}] = {ref}") + return + if frame.block_frame: + self.writeline(f"_block_vars[{name!r}] = {ref}") + return + self.writeline(f"context.vars[{name!r}] = {ref}") + else: + if frame.loop_frame: + self.writeline("_loop_vars.update({") + elif frame.block_frame: + self.writeline("_block_vars.update({") + else: + self.writeline("context.vars.update({") + for idx, name in enumerate(sorted(vars)): + if idx: + self.write(", ") + ref = frame.symbols.ref(name) + self.write(f"{name!r}: {ref}") + self.write("})") + if not frame.block_frame and not frame.loop_frame and public_names: + if len(public_names) == 1: + self.writeline(f"context.exported_vars.add({public_names[0]!r})") + else: + names_str = ", ".join(map(repr, sorted(public_names))) + self.writeline(f"context.exported_vars.update(({names_str}))") + + # -- Statement Visitors + + def visit_Template( + self, node: nodes.Template, frame: t.Optional[Frame] = None + ) -> None: + assert frame is None, "no root frame allowed" + eval_ctx = EvalContext(self.environment, self.name) + + from .runtime import async_exported + from .runtime import exported + + if self.environment.is_async: + exported_names = sorted(exported + async_exported) + else: + exported_names = sorted(exported) + + self.writeline("from jinja2.runtime import " + ", ".join(exported_names)) + + # if we want a deferred initialization we cannot move the + # environment into a local name + envenv = "" if self.defer_init else ", environment=environment" + + # do we have an extends tag at all? If not, we can save some + # overhead by just not processing any inheritance code. + have_extends = node.find(nodes.Extends) is not None + + # find all blocks + for block in node.find_all(nodes.Block): + if block.name in self.blocks: + self.fail(f"block {block.name!r} defined twice", block.lineno) + self.blocks[block.name] = block + + # find all imports and import them + for import_ in node.find_all(nodes.ImportedName): + if import_.importname not in self.import_aliases: + imp = import_.importname + self.import_aliases[imp] = alias = self.temporary_identifier() + if "." in imp: + module, obj = imp.rsplit(".", 1) + self.writeline(f"from {module} import {obj} as {alias}") + else: + self.writeline(f"import {imp} as {alias}") + + # add the load name + self.writeline(f"name = {self.name!r}") + + # generate the root render function. + self.writeline( + f"{self.func('root')}(context, missing=missing{envenv}):", extra=1 + ) + self.indent() + self.write_commons() + + # process the root + frame = Frame(eval_ctx) + if "self" in find_undeclared(node.body, ("self",)): + ref = frame.symbols.declare_parameter("self") + self.writeline(f"{ref} = TemplateReference(context)") + frame.symbols.analyze_node(node) + frame.toplevel = frame.rootlevel = True + frame.require_output_check = have_extends and not self.has_known_extends + if have_extends: + self.writeline("parent_template = None") + self.enter_frame(frame) + self.pull_dependencies(node.body) + self.blockvisit(node.body, frame) + self.leave_frame(frame, with_python_scope=True) + self.outdent() + + # make sure that the parent root is called. + if have_extends: + if not self.has_known_extends: + self.indent() + self.writeline("if parent_template is not None:") + self.indent() + if not self.environment.is_async: + self.writeline("yield from parent_template.root_render_func(context)") + else: + self.writeline("agen = parent_template.root_render_func(context)") + self.writeline("try:") + self.indent() + self.writeline("async for event in agen:") + self.indent() + self.writeline("yield event") + self.outdent() + self.outdent() + self.writeline("finally: await agen.aclose()") + self.outdent(1 + (not self.has_known_extends)) + + # at this point we now have the blocks collected and can visit them too. + for name, block in self.blocks.items(): + self.writeline( + f"{self.func('block_' + name)}(context, missing=missing{envenv}):", + block, + 1, + ) + self.indent() + self.write_commons() + # It's important that we do not make this frame a child of the + # toplevel template. This would cause a variety of + # interesting issues with identifier tracking. + block_frame = Frame(eval_ctx) + block_frame.block_frame = True + undeclared = find_undeclared(block.body, ("self", "super")) + if "self" in undeclared: + ref = block_frame.symbols.declare_parameter("self") + self.writeline(f"{ref} = TemplateReference(context)") + if "super" in undeclared: + ref = block_frame.symbols.declare_parameter("super") + self.writeline(f"{ref} = context.super({name!r}, block_{name})") + block_frame.symbols.analyze_node(block) + block_frame.block = name + self.writeline("_block_vars = {}") + self.enter_frame(block_frame) + self.pull_dependencies(block.body) + self.blockvisit(block.body, block_frame) + self.leave_frame(block_frame, with_python_scope=True) + self.outdent() + + blocks_kv_str = ", ".join(f"{x!r}: block_{x}" for x in self.blocks) + self.writeline(f"blocks = {{{blocks_kv_str}}}", extra=1) + debug_kv_str = "&".join(f"{k}={v}" for k, v in self.debug_info) + self.writeline(f"debug_info = {debug_kv_str!r}") + + def visit_Block(self, node: nodes.Block, frame: Frame) -> None: + """Call a block and register it for the template.""" + level = 0 + if frame.toplevel: + # if we know that we are a child template, there is no need to + # check if we are one + if self.has_known_extends: + return + if self.extends_so_far > 0: + self.writeline("if parent_template is None:") + self.indent() + level += 1 + + if node.scoped: + context = self.derive_context(frame) + else: + context = self.get_context_ref() + + if node.required: + self.writeline(f"if len(context.blocks[{node.name!r}]) <= 1:", node) + self.indent() + self.writeline( + f'raise TemplateRuntimeError("Required block {node.name!r} not found")', + node, + ) + self.outdent() + + if not self.environment.is_async and frame.buffer is None: + self.writeline( + f"yield from context.blocks[{node.name!r}][0]({context})", node + ) + else: + self.writeline(f"gen = context.blocks[{node.name!r}][0]({context})") + self.writeline("try:") + self.indent() + self.writeline( + f"{self.choose_async()}for event in gen:", + node, + ) + self.indent() + self.simple_write("event", frame) + self.outdent() + self.outdent() + self.writeline( + f"finally: {self.choose_async('await gen.aclose()', 'gen.close()')}" + ) + + self.outdent(level) + + def visit_Extends(self, node: nodes.Extends, frame: Frame) -> None: + """Calls the extender.""" + if not frame.toplevel: + self.fail("cannot use extend from a non top-level scope", node.lineno) + + # if the number of extends statements in general is zero so + # far, we don't have to add a check if something extended + # the template before this one. + if self.extends_so_far > 0: + # if we have a known extends we just add a template runtime + # error into the generated code. We could catch that at compile + # time too, but i welcome it not to confuse users by throwing the + # same error at different times just "because we can". + if not self.has_known_extends: + self.writeline("if parent_template is not None:") + self.indent() + self.writeline('raise TemplateRuntimeError("extended multiple times")') + + # if we have a known extends already we don't need that code here + # as we know that the template execution will end here. + if self.has_known_extends: + raise CompilerExit() + else: + self.outdent() + + self.writeline("parent_template = environment.get_template(", node) + self.visit(node.template, frame) + self.write(f", {self.name!r})") + self.writeline("for name, parent_block in parent_template.blocks.items():") + self.indent() + self.writeline("context.blocks.setdefault(name, []).append(parent_block)") + self.outdent() + + # if this extends statement was in the root level we can take + # advantage of that information and simplify the generated code + # in the top level from this point onwards + if frame.rootlevel: + self.has_known_extends = True + + # and now we have one more + self.extends_so_far += 1 + + def visit_Include(self, node: nodes.Include, frame: Frame) -> None: + """Handles includes.""" + if node.ignore_missing: + self.writeline("try:") + self.indent() + + func_name = "get_or_select_template" + if isinstance(node.template, nodes.Const): + if isinstance(node.template.value, str): + func_name = "get_template" + elif isinstance(node.template.value, (tuple, list)): + func_name = "select_template" + elif isinstance(node.template, (nodes.Tuple, nodes.List)): + func_name = "select_template" + + self.writeline(f"template = environment.{func_name}(", node) + self.visit(node.template, frame) + self.write(f", {self.name!r})") + if node.ignore_missing: + self.outdent() + self.writeline("except TemplateNotFound:") + self.indent() + self.writeline("pass") + self.outdent() + self.writeline("else:") + self.indent() + + def loop_body() -> None: + self.indent() + self.simple_write("event", frame) + self.outdent() + + if node.with_context: + self.writeline( + f"gen = template.root_render_func(" + "template.new_context(context.get_all(), True," + f" {self.dump_local_context(frame)}))" + ) + self.writeline("try:") + self.indent() + self.writeline(f"{self.choose_async()}for event in gen:") + loop_body() + self.outdent() + self.writeline( + f"finally: {self.choose_async('await gen.aclose()', 'gen.close()')}" + ) + elif self.environment.is_async: + self.writeline( + "for event in (await template._get_default_module_async())" + "._body_stream:" + ) + loop_body() + else: + self.writeline("yield from template._get_default_module()._body_stream") + + if node.ignore_missing: + self.outdent() + + def _import_common( + self, node: t.Union[nodes.Import, nodes.FromImport], frame: Frame + ) -> None: + self.write(f"{self.choose_async('await ')}environment.get_template(") + self.visit(node.template, frame) + self.write(f", {self.name!r}).") + + if node.with_context: + f_name = f"make_module{self.choose_async('_async')}" + self.write( + f"{f_name}(context.get_all(), True, {self.dump_local_context(frame)})" + ) + else: + self.write(f"_get_default_module{self.choose_async('_async')}(context)") + + def visit_Import(self, node: nodes.Import, frame: Frame) -> None: + """Visit regular imports.""" + self.writeline(f"{frame.symbols.ref(node.target)} = ", node) + if frame.toplevel: + self.write(f"context.vars[{node.target!r}] = ") + + self._import_common(node, frame) + + if frame.toplevel and not node.target.startswith("_"): + self.writeline(f"context.exported_vars.discard({node.target!r})") + + def visit_FromImport(self, node: nodes.FromImport, frame: Frame) -> None: + """Visit named imports.""" + self.newline(node) + self.write("included_template = ") + self._import_common(node, frame) + var_names = [] + discarded_names = [] + for name in node.names: + if isinstance(name, tuple): + name, alias = name + else: + alias = name + self.writeline( + f"{frame.symbols.ref(alias)} =" + f" getattr(included_template, {name!r}, missing)" + ) + self.writeline(f"if {frame.symbols.ref(alias)} is missing:") + self.indent() + # The position will contain the template name, and will be formatted + # into a string that will be compiled into an f-string. Curly braces + # in the name must be replaced with escapes so that they will not be + # executed as part of the f-string. + position = self.position(node).replace("{", "{{").replace("}", "}}") + message = ( + "the template {included_template.__name__!r}" + f" (imported on {position})" + f" does not export the requested name {name!r}" + ) + self.writeline( + f"{frame.symbols.ref(alias)} = undefined(f{message!r}, name={name!r})" + ) + self.outdent() + if frame.toplevel: + var_names.append(alias) + if not alias.startswith("_"): + discarded_names.append(alias) + + if var_names: + if len(var_names) == 1: + name = var_names[0] + self.writeline(f"context.vars[{name!r}] = {frame.symbols.ref(name)}") + else: + names_kv = ", ".join( + f"{name!r}: {frame.symbols.ref(name)}" for name in var_names + ) + self.writeline(f"context.vars.update({{{names_kv}}})") + if discarded_names: + if len(discarded_names) == 1: + self.writeline(f"context.exported_vars.discard({discarded_names[0]!r})") + else: + names_str = ", ".join(map(repr, discarded_names)) + self.writeline( + f"context.exported_vars.difference_update(({names_str}))" + ) + + def visit_For(self, node: nodes.For, frame: Frame) -> None: + loop_frame = frame.inner() + loop_frame.loop_frame = True + test_frame = frame.inner() + else_frame = frame.inner() + + # try to figure out if we have an extended loop. An extended loop + # is necessary if the loop is in recursive mode if the special loop + # variable is accessed in the body if the body is a scoped block. + extended_loop = ( + node.recursive + or "loop" + in find_undeclared(node.iter_child_nodes(only=("body",)), ("loop",)) + or any(block.scoped for block in node.find_all(nodes.Block)) + ) + + loop_ref = None + if extended_loop: + loop_ref = loop_frame.symbols.declare_parameter("loop") + + loop_frame.symbols.analyze_node(node, for_branch="body") + if node.else_: + else_frame.symbols.analyze_node(node, for_branch="else") + + if node.test: + loop_filter_func = self.temporary_identifier() + test_frame.symbols.analyze_node(node, for_branch="test") + self.writeline(f"{self.func(loop_filter_func)}(fiter):", node.test) + self.indent() + self.enter_frame(test_frame) + self.writeline(self.choose_async("async for ", "for ")) + self.visit(node.target, loop_frame) + self.write(" in ") + self.write(self.choose_async("auto_aiter(fiter)", "fiter")) + self.write(":") + self.indent() + self.writeline("if ", node.test) + self.visit(node.test, test_frame) + self.write(":") + self.indent() + self.writeline("yield ") + self.visit(node.target, loop_frame) + self.outdent(3) + self.leave_frame(test_frame, with_python_scope=True) + + # if we don't have an recursive loop we have to find the shadowed + # variables at that point. Because loops can be nested but the loop + # variable is a special one we have to enforce aliasing for it. + if node.recursive: + self.writeline( + f"{self.func('loop')}(reciter, loop_render_func, depth=0):", node + ) + self.indent() + self.buffer(loop_frame) + + # Use the same buffer for the else frame + else_frame.buffer = loop_frame.buffer + + # make sure the loop variable is a special one and raise a template + # assertion error if a loop tries to write to loop + if extended_loop: + self.writeline(f"{loop_ref} = missing") + + for name in node.find_all(nodes.Name): + if name.ctx == "store" and name.name == "loop": + self.fail( + "Can't assign to special loop variable in for-loop target", + name.lineno, + ) + + if node.else_: + iteration_indicator = self.temporary_identifier() + self.writeline(f"{iteration_indicator} = 1") + + self.writeline(self.choose_async("async for ", "for "), node) + self.visit(node.target, loop_frame) + if extended_loop: + self.write(f", {loop_ref} in {self.choose_async('Async')}LoopContext(") + else: + self.write(" in ") + + if node.test: + self.write(f"{loop_filter_func}(") + if node.recursive: + self.write("reciter") + else: + if self.environment.is_async and not extended_loop: + self.write("auto_aiter(") + self.visit(node.iter, frame) + if self.environment.is_async and not extended_loop: + self.write(")") + if node.test: + self.write(")") + + if node.recursive: + self.write(", undefined, loop_render_func, depth):") + else: + self.write(", undefined):" if extended_loop else ":") + + self.indent() + self.enter_frame(loop_frame) + + self.writeline("_loop_vars = {}") + self.blockvisit(node.body, loop_frame) + if node.else_: + self.writeline(f"{iteration_indicator} = 0") + self.outdent() + self.leave_frame( + loop_frame, with_python_scope=node.recursive and not node.else_ + ) + + if node.else_: + self.writeline(f"if {iteration_indicator}:") + self.indent() + self.enter_frame(else_frame) + self.blockvisit(node.else_, else_frame) + self.leave_frame(else_frame) + self.outdent() + + # if the node was recursive we have to return the buffer contents + # and start the iteration code + if node.recursive: + self.return_buffer_contents(loop_frame) + self.outdent() + self.start_write(frame, node) + self.write(f"{self.choose_async('await ')}loop(") + if self.environment.is_async: + self.write("auto_aiter(") + self.visit(node.iter, frame) + if self.environment.is_async: + self.write(")") + self.write(", loop)") + self.end_write(frame) + + # at the end of the iteration, clear any assignments made in the + # loop from the top level + if self._assign_stack: + self._assign_stack[-1].difference_update(loop_frame.symbols.stores) + + def visit_If(self, node: nodes.If, frame: Frame) -> None: + if_frame = frame.soft() + self.writeline("if ", node) + self.visit(node.test, if_frame) + self.write(":") + self.indent() + self.blockvisit(node.body, if_frame) + self.outdent() + for elif_ in node.elif_: + self.writeline("elif ", elif_) + self.visit(elif_.test, if_frame) + self.write(":") + self.indent() + self.blockvisit(elif_.body, if_frame) + self.outdent() + if node.else_: + self.writeline("else:") + self.indent() + self.blockvisit(node.else_, if_frame) + self.outdent() + + def visit_Macro(self, node: nodes.Macro, frame: Frame) -> None: + macro_frame, macro_ref = self.macro_body(node, frame) + self.newline() + if frame.toplevel: + if not node.name.startswith("_"): + self.write(f"context.exported_vars.add({node.name!r})") + self.writeline(f"context.vars[{node.name!r}] = ") + self.write(f"{frame.symbols.ref(node.name)} = ") + self.macro_def(macro_ref, macro_frame) + + def visit_CallBlock(self, node: nodes.CallBlock, frame: Frame) -> None: + call_frame, macro_ref = self.macro_body(node, frame) + self.writeline("caller = ") + self.macro_def(macro_ref, call_frame) + self.start_write(frame, node) + self.visit_Call(node.call, frame, forward_caller=True) + self.end_write(frame) + + def visit_FilterBlock(self, node: nodes.FilterBlock, frame: Frame) -> None: + filter_frame = frame.inner() + filter_frame.symbols.analyze_node(node) + self.enter_frame(filter_frame) + self.buffer(filter_frame) + self.blockvisit(node.body, filter_frame) + self.start_write(frame, node) + self.visit_Filter(node.filter, filter_frame) + self.end_write(frame) + self.leave_frame(filter_frame) + + def visit_With(self, node: nodes.With, frame: Frame) -> None: + with_frame = frame.inner() + with_frame.symbols.analyze_node(node) + self.enter_frame(with_frame) + for target, expr in zip(node.targets, node.values): + self.newline() + self.visit(target, with_frame) + self.write(" = ") + self.visit(expr, frame) + self.blockvisit(node.body, with_frame) + self.leave_frame(with_frame) + + def visit_ExprStmt(self, node: nodes.ExprStmt, frame: Frame) -> None: + self.newline(node) + self.visit(node.node, frame) + + class _FinalizeInfo(t.NamedTuple): + const: t.Optional[t.Callable[..., str]] + src: t.Optional[str] + + @staticmethod + def _default_finalize(value: t.Any) -> t.Any: + """The default finalize function if the environment isn't + configured with one. Or, if the environment has one, this is + called on that function's output for constants. + """ + return str(value) + + _finalize: t.Optional[_FinalizeInfo] = None + + def _make_finalize(self) -> _FinalizeInfo: + """Build the finalize function to be used on constants and at + runtime. Cached so it's only created once for all output nodes. + + Returns a ``namedtuple`` with the following attributes: + + ``const`` + A function to finalize constant data at compile time. + + ``src`` + Source code to output around nodes to be evaluated at + runtime. + """ + if self._finalize is not None: + return self._finalize + + finalize: t.Optional[t.Callable[..., t.Any]] + finalize = default = self._default_finalize + src = None + + if self.environment.finalize: + src = "environment.finalize(" + env_finalize = self.environment.finalize + pass_arg = { + _PassArg.context: "context", + _PassArg.eval_context: "context.eval_ctx", + _PassArg.environment: "environment", + }.get( + _PassArg.from_obj(env_finalize) # type: ignore + ) + finalize = None + + if pass_arg is None: + + def finalize(value: t.Any) -> t.Any: # noqa: F811 + return default(env_finalize(value)) + + else: + src = f"{src}{pass_arg}, " + + if pass_arg == "environment": + + def finalize(value: t.Any) -> t.Any: # noqa: F811 + return default(env_finalize(self.environment, value)) + + self._finalize = self._FinalizeInfo(finalize, src) + return self._finalize + + def _output_const_repr(self, group: t.Iterable[t.Any]) -> str: + """Given a group of constant values converted from ``Output`` + child nodes, produce a string to write to the template module + source. + """ + return repr(concat(group)) + + def _output_child_to_const( + self, node: nodes.Expr, frame: Frame, finalize: _FinalizeInfo + ) -> str: + """Try to optimize a child of an ``Output`` node by trying to + convert it to constant, finalized data at compile time. + + If :exc:`Impossible` is raised, the node is not constant and + will be evaluated at runtime. Any other exception will also be + evaluated at runtime for easier debugging. + """ + const = node.as_const(frame.eval_ctx) + + if frame.eval_ctx.autoescape: + const = escape(const) + + # Template data doesn't go through finalize. + if isinstance(node, nodes.TemplateData): + return str(const) + + return finalize.const(const) # type: ignore + + def _output_child_pre( + self, node: nodes.Expr, frame: Frame, finalize: _FinalizeInfo + ) -> None: + """Output extra source code before visiting a child of an + ``Output`` node. + """ + if frame.eval_ctx.volatile: + self.write("(escape if context.eval_ctx.autoescape else str)(") + elif frame.eval_ctx.autoescape: + self.write("escape(") + else: + self.write("str(") + + if finalize.src is not None: + self.write(finalize.src) + + def _output_child_post( + self, node: nodes.Expr, frame: Frame, finalize: _FinalizeInfo + ) -> None: + """Output extra source code after visiting a child of an + ``Output`` node. + """ + self.write(")") + + if finalize.src is not None: + self.write(")") + + def visit_Output(self, node: nodes.Output, frame: Frame) -> None: + # If an extends is active, don't render outside a block. + if frame.require_output_check: + # A top-level extends is known to exist at compile time. + if self.has_known_extends: + return + + self.writeline("if parent_template is None:") + self.indent() + + finalize = self._make_finalize() + body: t.List[t.Union[t.List[t.Any], nodes.Expr]] = [] + + # Evaluate constants at compile time if possible. Each item in + # body will be either a list of static data or a node to be + # evaluated at runtime. + for child in node.nodes: + try: + if not ( + # If the finalize function requires runtime context, + # constants can't be evaluated at compile time. + finalize.const + # Unless it's basic template data that won't be + # finalized anyway. + or isinstance(child, nodes.TemplateData) + ): + raise nodes.Impossible() + + const = self._output_child_to_const(child, frame, finalize) + except (nodes.Impossible, Exception): + # The node was not constant and needs to be evaluated at + # runtime. Or another error was raised, which is easier + # to debug at runtime. + body.append(child) + continue + + if body and isinstance(body[-1], list): + body[-1].append(const) + else: + body.append([const]) + + if frame.buffer is not None: + if len(body) == 1: + self.writeline(f"{frame.buffer}.append(") + else: + self.writeline(f"{frame.buffer}.extend((") + + self.indent() + + for item in body: + if isinstance(item, list): + # A group of constant data to join and output. + val = self._output_const_repr(item) + + if frame.buffer is None: + self.writeline("yield " + val) + else: + self.writeline(val + ",") + else: + if frame.buffer is None: + self.writeline("yield ", item) + else: + self.newline(item) + + # A node to be evaluated at runtime. + self._output_child_pre(item, frame, finalize) + self.visit(item, frame) + self._output_child_post(item, frame, finalize) + + if frame.buffer is not None: + self.write(",") + + if frame.buffer is not None: + self.outdent() + self.writeline(")" if len(body) == 1 else "))") + + if frame.require_output_check: + self.outdent() + + def visit_Assign(self, node: nodes.Assign, frame: Frame) -> None: + self.push_assign_tracking() + + # ``a.b`` is allowed for assignment, and is parsed as an NSRef. However, + # it is only valid if it references a Namespace object. Emit a check for + # that for each ref here, before assignment code is emitted. This can't + # be done in visit_NSRef as the ref could be in the middle of a tuple. + seen_refs: t.Set[str] = set() + + for nsref in node.find_all(nodes.NSRef): + if nsref.name in seen_refs: + # Only emit the check for each reference once, in case the same + # ref is used multiple times in a tuple, `ns.a, ns.b = c, d`. + continue + + seen_refs.add(nsref.name) + ref = frame.symbols.ref(nsref.name) + self.writeline(f"if not isinstance({ref}, Namespace):") + self.indent() + self.writeline( + "raise TemplateRuntimeError" + '("cannot assign attribute on non-namespace object")' + ) + self.outdent() + + self.newline(node) + self.visit(node.target, frame) + self.write(" = ") + self.visit(node.node, frame) + self.pop_assign_tracking(frame) + + def visit_AssignBlock(self, node: nodes.AssignBlock, frame: Frame) -> None: + self.push_assign_tracking() + block_frame = frame.inner() + # This is a special case. Since a set block always captures we + # will disable output checks. This way one can use set blocks + # toplevel even in extended templates. + block_frame.require_output_check = False + block_frame.symbols.analyze_node(node) + self.enter_frame(block_frame) + self.buffer(block_frame) + self.blockvisit(node.body, block_frame) + self.newline(node) + self.visit(node.target, frame) + self.write(" = (Markup if context.eval_ctx.autoescape else identity)(") + if node.filter is not None: + self.visit_Filter(node.filter, block_frame) + else: + self.write(f"concat({block_frame.buffer})") + self.write(")") + self.pop_assign_tracking(frame) + self.leave_frame(block_frame) + + # -- Expression Visitors + + def visit_Name(self, node: nodes.Name, frame: Frame) -> None: + if node.ctx == "store" and ( + frame.toplevel or frame.loop_frame or frame.block_frame + ): + if self._assign_stack: + self._assign_stack[-1].add(node.name) + ref = frame.symbols.ref(node.name) + + # If we are looking up a variable we might have to deal with the + # case where it's undefined. We can skip that case if the load + # instruction indicates a parameter which are always defined. + if node.ctx == "load": + load = frame.symbols.find_load(ref) + if not ( + load is not None + and load[0] == VAR_LOAD_PARAMETER + and not self.parameter_is_undeclared(ref) + ): + self.write( + f"(undefined(name={node.name!r}) if {ref} is missing else {ref})" + ) + return + + self.write(ref) + + def visit_NSRef(self, node: nodes.NSRef, frame: Frame) -> None: + # NSRef is a dotted assignment target a.b=c, but uses a[b]=c internally. + # visit_Assign emits code to validate that each ref is to a Namespace + # object only. That can't be emitted here as the ref could be in the + # middle of a tuple assignment. + ref = frame.symbols.ref(node.name) + self.writeline(f"{ref}[{node.attr!r}]") + + def visit_Const(self, node: nodes.Const, frame: Frame) -> None: + val = node.as_const(frame.eval_ctx) + if isinstance(val, float): + self.write(str(val)) + else: + self.write(repr(val)) + + def visit_TemplateData(self, node: nodes.TemplateData, frame: Frame) -> None: + try: + self.write(repr(node.as_const(frame.eval_ctx))) + except nodes.Impossible: + self.write( + f"(Markup if context.eval_ctx.autoescape else identity)({node.data!r})" + ) + + def visit_Tuple(self, node: nodes.Tuple, frame: Frame) -> None: + self.write("(") + idx = -1 + for idx, item in enumerate(node.items): + if idx: + self.write(", ") + self.visit(item, frame) + self.write(",)" if idx == 0 else ")") + + def visit_List(self, node: nodes.List, frame: Frame) -> None: + self.write("[") + for idx, item in enumerate(node.items): + if idx: + self.write(", ") + self.visit(item, frame) + self.write("]") + + def visit_Dict(self, node: nodes.Dict, frame: Frame) -> None: + self.write("{") + for idx, item in enumerate(node.items): + if idx: + self.write(", ") + self.visit(item.key, frame) + self.write(": ") + self.visit(item.value, frame) + self.write("}") + + visit_Add = _make_binop("+") + visit_Sub = _make_binop("-") + visit_Mul = _make_binop("*") + visit_Div = _make_binop("/") + visit_FloorDiv = _make_binop("//") + visit_Pow = _make_binop("**") + visit_Mod = _make_binop("%") + visit_And = _make_binop("and") + visit_Or = _make_binop("or") + visit_Pos = _make_unop("+") + visit_Neg = _make_unop("-") + visit_Not = _make_unop("not ") + + @optimizeconst + def visit_Concat(self, node: nodes.Concat, frame: Frame) -> None: + if frame.eval_ctx.volatile: + func_name = "(markup_join if context.eval_ctx.volatile else str_join)" + elif frame.eval_ctx.autoescape: + func_name = "markup_join" + else: + func_name = "str_join" + self.write(f"{func_name}((") + for arg in node.nodes: + self.visit(arg, frame) + self.write(", ") + self.write("))") + + @optimizeconst + def visit_Compare(self, node: nodes.Compare, frame: Frame) -> None: + self.write("(") + self.visit(node.expr, frame) + for op in node.ops: + self.visit(op, frame) + self.write(")") + + def visit_Operand(self, node: nodes.Operand, frame: Frame) -> None: + self.write(f" {operators[node.op]} ") + self.visit(node.expr, frame) + + @optimizeconst + def visit_Getattr(self, node: nodes.Getattr, frame: Frame) -> None: + if self.environment.is_async: + self.write("(await auto_await(") + + self.write("environment.getattr(") + self.visit(node.node, frame) + self.write(f", {node.attr!r})") + + if self.environment.is_async: + self.write("))") + + @optimizeconst + def visit_Getitem(self, node: nodes.Getitem, frame: Frame) -> None: + # slices bypass the environment getitem method. + if isinstance(node.arg, nodes.Slice): + self.visit(node.node, frame) + self.write("[") + self.visit(node.arg, frame) + self.write("]") + else: + if self.environment.is_async: + self.write("(await auto_await(") + + self.write("environment.getitem(") + self.visit(node.node, frame) + self.write(", ") + self.visit(node.arg, frame) + self.write(")") + + if self.environment.is_async: + self.write("))") + + def visit_Slice(self, node: nodes.Slice, frame: Frame) -> None: + if node.start is not None: + self.visit(node.start, frame) + self.write(":") + if node.stop is not None: + self.visit(node.stop, frame) + if node.step is not None: + self.write(":") + self.visit(node.step, frame) + + @contextmanager + def _filter_test_common( + self, node: t.Union[nodes.Filter, nodes.Test], frame: Frame, is_filter: bool + ) -> t.Iterator[None]: + if self.environment.is_async: + self.write("(await auto_await(") + + if is_filter: + self.write(f"{self.filters[node.name]}(") + func = self.environment.filters.get(node.name) + else: + self.write(f"{self.tests[node.name]}(") + func = self.environment.tests.get(node.name) + + # When inside an If or CondExpr frame, allow the filter to be + # undefined at compile time and only raise an error if it's + # actually called at runtime. See pull_dependencies. + if func is None and not frame.soft_frame: + type_name = "filter" if is_filter else "test" + self.fail(f"No {type_name} named {node.name!r}.", node.lineno) + + pass_arg = { + _PassArg.context: "context", + _PassArg.eval_context: "context.eval_ctx", + _PassArg.environment: "environment", + }.get( + _PassArg.from_obj(func) # type: ignore + ) + + if pass_arg is not None: + self.write(f"{pass_arg}, ") + + # Back to the visitor function to handle visiting the target of + # the filter or test. + yield + + self.signature(node, frame) + self.write(")") + + if self.environment.is_async: + self.write("))") + + @optimizeconst + def visit_Filter(self, node: nodes.Filter, frame: Frame) -> None: + with self._filter_test_common(node, frame, True): + # if the filter node is None we are inside a filter block + # and want to write to the current buffer + if node.node is not None: + self.visit(node.node, frame) + elif frame.eval_ctx.volatile: + self.write( + f"(Markup(concat({frame.buffer}))" + f" if context.eval_ctx.autoescape else concat({frame.buffer}))" + ) + elif frame.eval_ctx.autoescape: + self.write(f"Markup(concat({frame.buffer}))") + else: + self.write(f"concat({frame.buffer})") + + @optimizeconst + def visit_Test(self, node: nodes.Test, frame: Frame) -> None: + with self._filter_test_common(node, frame, False): + self.visit(node.node, frame) + + @optimizeconst + def visit_CondExpr(self, node: nodes.CondExpr, frame: Frame) -> None: + frame = frame.soft() + + def write_expr2() -> None: + if node.expr2 is not None: + self.visit(node.expr2, frame) + return + + self.write( + f'cond_expr_undefined("the inline if-expression on' + f" {self.position(node)} evaluated to false and no else" + f' section was defined.")' + ) + + self.write("(") + self.visit(node.expr1, frame) + self.write(" if ") + self.visit(node.test, frame) + self.write(" else ") + write_expr2() + self.write(")") + + @optimizeconst + def visit_Call( + self, node: nodes.Call, frame: Frame, forward_caller: bool = False + ) -> None: + if self.environment.is_async: + self.write("(await auto_await(") + if self.environment.sandboxed: + self.write("environment.call(context, ") + else: + self.write("context.call(") + self.visit(node.node, frame) + extra_kwargs = {"caller": "caller"} if forward_caller else None + loop_kwargs = {"_loop_vars": "_loop_vars"} if frame.loop_frame else {} + block_kwargs = {"_block_vars": "_block_vars"} if frame.block_frame else {} + if extra_kwargs: + extra_kwargs.update(loop_kwargs, **block_kwargs) + elif loop_kwargs or block_kwargs: + extra_kwargs = dict(loop_kwargs, **block_kwargs) + self.signature(node, frame, extra_kwargs) + self.write(")") + if self.environment.is_async: + self.write("))") + + def visit_Keyword(self, node: nodes.Keyword, frame: Frame) -> None: + self.write(node.key + "=") + self.visit(node.value, frame) + + # -- Unused nodes for extensions + + def visit_MarkSafe(self, node: nodes.MarkSafe, frame: Frame) -> None: + self.write("Markup(") + self.visit(node.expr, frame) + self.write(")") + + def visit_MarkSafeIfAutoescape( + self, node: nodes.MarkSafeIfAutoescape, frame: Frame + ) -> None: + self.write("(Markup if context.eval_ctx.autoescape else identity)(") + self.visit(node.expr, frame) + self.write(")") + + def visit_EnvironmentAttribute( + self, node: nodes.EnvironmentAttribute, frame: Frame + ) -> None: + self.write("environment." + node.name) + + def visit_ExtensionAttribute( + self, node: nodes.ExtensionAttribute, frame: Frame + ) -> None: + self.write(f"environment.extensions[{node.identifier!r}].{node.name}") + + def visit_ImportedName(self, node: nodes.ImportedName, frame: Frame) -> None: + self.write(self.import_aliases[node.importname]) + + def visit_InternalName(self, node: nodes.InternalName, frame: Frame) -> None: + self.write(node.name) + + def visit_ContextReference( + self, node: nodes.ContextReference, frame: Frame + ) -> None: + self.write("context") + + def visit_DerivedContextReference( + self, node: nodes.DerivedContextReference, frame: Frame + ) -> None: + self.write(self.derive_context(frame)) + + def visit_Continue(self, node: nodes.Continue, frame: Frame) -> None: + self.writeline("continue", node) + + def visit_Break(self, node: nodes.Break, frame: Frame) -> None: + self.writeline("break", node) + + def visit_Scope(self, node: nodes.Scope, frame: Frame) -> None: + scope_frame = frame.inner() + scope_frame.symbols.analyze_node(node) + self.enter_frame(scope_frame) + self.blockvisit(node.body, scope_frame) + self.leave_frame(scope_frame) + + def visit_OverlayScope(self, node: nodes.OverlayScope, frame: Frame) -> None: + ctx = self.temporary_identifier() + self.writeline(f"{ctx} = {self.derive_context(frame)}") + self.writeline(f"{ctx}.vars = ") + self.visit(node.context, frame) + self.push_context_reference(ctx) + + scope_frame = frame.inner(isolated=True) + scope_frame.symbols.analyze_node(node) + self.enter_frame(scope_frame) + self.blockvisit(node.body, scope_frame) + self.leave_frame(scope_frame) + self.pop_context_reference() + + def visit_EvalContextModifier( + self, node: nodes.EvalContextModifier, frame: Frame + ) -> None: + for keyword in node.options: + self.writeline(f"context.eval_ctx.{keyword.key} = ") + self.visit(keyword.value, frame) + try: + val = keyword.value.as_const(frame.eval_ctx) + except nodes.Impossible: + frame.eval_ctx.volatile = True + else: + setattr(frame.eval_ctx, keyword.key, val) + + def visit_ScopedEvalContextModifier( + self, node: nodes.ScopedEvalContextModifier, frame: Frame + ) -> None: + old_ctx_name = self.temporary_identifier() + saved_ctx = frame.eval_ctx.save() + self.writeline(f"{old_ctx_name} = context.eval_ctx.save()") + self.visit_EvalContextModifier(node, frame) + for child in node.body: + self.visit(child, frame) + frame.eval_ctx.revert(saved_ctx) + self.writeline(f"context.eval_ctx.revert({old_ctx_name})") diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/jinja2/constants.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/jinja2/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..41a1c23b0a7fe134b1f662545876eb65b31b071e --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/jinja2/constants.py @@ -0,0 +1,20 @@ +#: list of lorem ipsum words used by the lipsum() helper function +LOREM_IPSUM_WORDS = """\ +a ac accumsan ad adipiscing aenean aliquam aliquet amet ante aptent arcu at +auctor augue bibendum blandit class commodo condimentum congue consectetuer +consequat conubia convallis cras cubilia cum curabitur curae cursus dapibus +diam dictum dictumst dignissim dis dolor donec dui duis egestas eget eleifend +elementum elit enim erat eros est et etiam eu euismod facilisi facilisis fames +faucibus felis fermentum feugiat fringilla fusce gravida habitant habitasse hac +hendrerit hymenaeos iaculis id imperdiet in inceptos integer interdum ipsum +justo lacinia lacus laoreet lectus leo libero ligula litora lobortis lorem +luctus maecenas magna magnis malesuada massa mattis mauris metus mi molestie +mollis montes morbi mus nam nascetur natoque nec neque netus nibh nisi nisl non +nonummy nostra nulla nullam nunc odio orci ornare parturient pede pellentesque +penatibus per pharetra phasellus placerat platea porta porttitor posuere +potenti praesent pretium primis proin pulvinar purus quam quis quisque rhoncus +ridiculus risus rutrum sagittis sapien scelerisque sed sem semper senectus sit +sociis sociosqu sodales sollicitudin suscipit suspendisse taciti tellus tempor +tempus tincidunt torquent tortor tristique turpis ullamcorper ultrices +ultricies urna ut varius vehicula vel velit venenatis vestibulum vitae vivamus +viverra volutpat vulputate""" diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/jinja2/debug.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/jinja2/debug.py new file mode 100644 index 0000000000000000000000000000000000000000..eeeeee78b620f5d0745133b4629647973cd7af87 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/jinja2/debug.py @@ -0,0 +1,191 @@ +import sys +import typing as t +from types import CodeType +from types import TracebackType + +from .exceptions import TemplateSyntaxError +from .utils import internal_code +from .utils import missing + +if t.TYPE_CHECKING: + from .runtime import Context + + +def rewrite_traceback_stack(source: t.Optional[str] = None) -> BaseException: + """Rewrite the current exception to replace any tracebacks from + within compiled template code with tracebacks that look like they + came from the template source. + + This must be called within an ``except`` block. + + :param source: For ``TemplateSyntaxError``, the original source if + known. + :return: The original exception with the rewritten traceback. + """ + _, exc_value, tb = sys.exc_info() + exc_value = t.cast(BaseException, exc_value) + tb = t.cast(TracebackType, tb) + + if isinstance(exc_value, TemplateSyntaxError) and not exc_value.translated: + exc_value.translated = True + exc_value.source = source + # Remove the old traceback, otherwise the frames from the + # compiler still show up. + exc_value.with_traceback(None) + # Outside of runtime, so the frame isn't executing template + # code, but it still needs to point at the template. + tb = fake_traceback( + exc_value, None, exc_value.filename or "", exc_value.lineno + ) + else: + # Skip the frame for the render function. + tb = tb.tb_next + + stack = [] + + # Build the stack of traceback object, replacing any in template + # code with the source file and line information. + while tb is not None: + # Skip frames decorated with @internalcode. These are internal + # calls that aren't useful in template debugging output. + if tb.tb_frame.f_code in internal_code: + tb = tb.tb_next + continue + + template = tb.tb_frame.f_globals.get("__jinja_template__") + + if template is not None: + lineno = template.get_corresponding_lineno(tb.tb_lineno) + fake_tb = fake_traceback(exc_value, tb, template.filename, lineno) + stack.append(fake_tb) + else: + stack.append(tb) + + tb = tb.tb_next + + tb_next = None + + # Assign tb_next in reverse to avoid circular references. + for tb in reversed(stack): + tb.tb_next = tb_next + tb_next = tb + + return exc_value.with_traceback(tb_next) + + +def fake_traceback( # type: ignore + exc_value: BaseException, tb: t.Optional[TracebackType], filename: str, lineno: int +) -> TracebackType: + """Produce a new traceback object that looks like it came from the + template source instead of the compiled code. The filename, line + number, and location name will point to the template, and the local + variables will be the current template context. + + :param exc_value: The original exception to be re-raised to create + the new traceback. + :param tb: The original traceback to get the local variables and + code info from. + :param filename: The template filename. + :param lineno: The line number in the template source. + """ + if tb is not None: + # Replace the real locals with the context that would be + # available at that point in the template. + locals = get_template_locals(tb.tb_frame.f_locals) + locals.pop("__jinja_exception__", None) + else: + locals = {} + + globals = { + "__name__": filename, + "__file__": filename, + "__jinja_exception__": exc_value, + } + # Raise an exception at the correct line number. + code: CodeType = compile( + "\n" * (lineno - 1) + "raise __jinja_exception__", filename, "exec" + ) + + # Build a new code object that points to the template file and + # replaces the location with a block name. + location = "template" + + if tb is not None: + function = tb.tb_frame.f_code.co_name + + if function == "root": + location = "top-level template code" + elif function.startswith("block_"): + location = f"block {function[6:]!r}" + + if sys.version_info >= (3, 8): + code = code.replace(co_name=location) + else: + code = CodeType( + code.co_argcount, + code.co_kwonlyargcount, + code.co_nlocals, + code.co_stacksize, + code.co_flags, + code.co_code, + code.co_consts, + code.co_names, + code.co_varnames, + code.co_filename, + location, + code.co_firstlineno, + code.co_lnotab, + code.co_freevars, + code.co_cellvars, + ) + + # Execute the new code, which is guaranteed to raise, and return + # the new traceback without this frame. + try: + exec(code, globals, locals) + except BaseException: + return sys.exc_info()[2].tb_next # type: ignore + + +def get_template_locals(real_locals: t.Mapping[str, t.Any]) -> t.Dict[str, t.Any]: + """Based on the runtime locals, get the context that would be + available at that point in the template. + """ + # Start with the current template context. + ctx: t.Optional[Context] = real_locals.get("context") + + if ctx is not None: + data: t.Dict[str, t.Any] = ctx.get_all().copy() + else: + data = {} + + # Might be in a derived context that only sets local variables + # rather than pushing a context. Local variables follow the scheme + # l_depth_name. Find the highest-depth local that has a value for + # each name. + local_overrides: t.Dict[str, t.Tuple[int, t.Any]] = {} + + for name, value in real_locals.items(): + if not name.startswith("l_") or value is missing: + # Not a template variable, or no longer relevant. + continue + + try: + _, depth_str, name = name.split("_", 2) + depth = int(depth_str) + except ValueError: + continue + + cur_depth = local_overrides.get(name, (-1,))[0] + + if cur_depth < depth: + local_overrides[name] = (depth, value) + + # Modify the context with any derived context. + for name, (_, value) in local_overrides.items(): + if value is missing: + data.pop(name, None) + else: + data[name] = value + + return data diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/jinja2/defaults.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/jinja2/defaults.py new file mode 100644 index 0000000000000000000000000000000000000000..638cad3d2d8907330bde56e2b76c9b185c523b45 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/jinja2/defaults.py @@ -0,0 +1,48 @@ +import typing as t + +from .filters import FILTERS as DEFAULT_FILTERS # noqa: F401 +from .tests import TESTS as DEFAULT_TESTS # noqa: F401 +from .utils import Cycler +from .utils import generate_lorem_ipsum +from .utils import Joiner +from .utils import Namespace + +if t.TYPE_CHECKING: + import typing_extensions as te + +# defaults for the parser / lexer +BLOCK_START_STRING = "{%" +BLOCK_END_STRING = "%}" +VARIABLE_START_STRING = "{{" +VARIABLE_END_STRING = "}}" +COMMENT_START_STRING = "{#" +COMMENT_END_STRING = "#}" +LINE_STATEMENT_PREFIX: t.Optional[str] = None +LINE_COMMENT_PREFIX: t.Optional[str] = None +TRIM_BLOCKS = False +LSTRIP_BLOCKS = False +NEWLINE_SEQUENCE: "te.Literal['\\n', '\\r\\n', '\\r']" = "\n" +KEEP_TRAILING_NEWLINE = False + +# default filters, tests and namespace + +DEFAULT_NAMESPACE = { + "range": range, + "dict": dict, + "lipsum": generate_lorem_ipsum, + "cycler": Cycler, + "joiner": Joiner, + "namespace": Namespace, +} + +# default policies +DEFAULT_POLICIES: t.Dict[str, t.Any] = { + "compiler.ascii_str": True, + "urlize.rel": "noopener", + "urlize.target": None, + "urlize.extra_schemes": None, + "truncate.leeway": 5, + "json.dumps_function": None, + "json.dumps_kwargs": {"sort_keys": True}, + "ext.i18n.trimmed": False, +} diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/jinja2/environment.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/jinja2/environment.py new file mode 100644 index 0000000000000000000000000000000000000000..0fc6e5be87ab8273f6056ddfede07e1be28f1495 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/jinja2/environment.py @@ -0,0 +1,1672 @@ +"""Classes for managing templates and their runtime and compile time +options. +""" + +import os +import typing +import typing as t +import weakref +from collections import ChainMap +from functools import lru_cache +from functools import partial +from functools import reduce +from types import CodeType + +from markupsafe import Markup + +from . import nodes +from .compiler import CodeGenerator +from .compiler import generate +from .defaults import BLOCK_END_STRING +from .defaults import BLOCK_START_STRING +from .defaults import COMMENT_END_STRING +from .defaults import COMMENT_START_STRING +from .defaults import DEFAULT_FILTERS # type: ignore[attr-defined] +from .defaults import DEFAULT_NAMESPACE +from .defaults import DEFAULT_POLICIES +from .defaults import DEFAULT_TESTS # type: ignore[attr-defined] +from .defaults import KEEP_TRAILING_NEWLINE +from .defaults import LINE_COMMENT_PREFIX +from .defaults import LINE_STATEMENT_PREFIX +from .defaults import LSTRIP_BLOCKS +from .defaults import NEWLINE_SEQUENCE +from .defaults import TRIM_BLOCKS +from .defaults import VARIABLE_END_STRING +from .defaults import VARIABLE_START_STRING +from .exceptions import TemplateNotFound +from .exceptions import TemplateRuntimeError +from .exceptions import TemplatesNotFound +from .exceptions import TemplateSyntaxError +from .exceptions import UndefinedError +from .lexer import get_lexer +from .lexer import Lexer +from .lexer import TokenStream +from .nodes import EvalContext +from .parser import Parser +from .runtime import Context +from .runtime import new_context +from .runtime import Undefined +from .utils import _PassArg +from .utils import concat +from .utils import consume +from .utils import import_string +from .utils import internalcode +from .utils import LRUCache +from .utils import missing + +if t.TYPE_CHECKING: + import typing_extensions as te + + from .bccache import BytecodeCache + from .ext import Extension + from .loaders import BaseLoader + +_env_bound = t.TypeVar("_env_bound", bound="Environment") + + +# for direct template usage we have up to ten living environments +@lru_cache(maxsize=10) +def get_spontaneous_environment(cls: t.Type[_env_bound], *args: t.Any) -> _env_bound: + """Return a new spontaneous environment. A spontaneous environment + is used for templates created directly rather than through an + existing environment. + + :param cls: Environment class to create. + :param args: Positional arguments passed to environment. + """ + env = cls(*args) + env.shared = True + return env + + +def create_cache( + size: int, +) -> t.Optional[t.MutableMapping[t.Tuple["weakref.ref[t.Any]", str], "Template"]]: + """Return the cache class for the given size.""" + if size == 0: + return None + + if size < 0: + return {} + + return LRUCache(size) # type: ignore + + +def copy_cache( + cache: t.Optional[t.MutableMapping[t.Any, t.Any]], +) -> t.Optional[t.MutableMapping[t.Tuple["weakref.ref[t.Any]", str], "Template"]]: + """Create an empty copy of the given cache.""" + if cache is None: + return None + + if type(cache) is dict: # noqa E721 + return {} + + return LRUCache(cache.capacity) # type: ignore + + +def load_extensions( + environment: "Environment", + extensions: t.Sequence[t.Union[str, t.Type["Extension"]]], +) -> t.Dict[str, "Extension"]: + """Load the extensions from the list and bind it to the environment. + Returns a dict of instantiated extensions. + """ + result = {} + + for extension in extensions: + if isinstance(extension, str): + extension = t.cast(t.Type["Extension"], import_string(extension)) + + result[extension.identifier] = extension(environment) + + return result + + +def _environment_config_check(environment: _env_bound) -> _env_bound: + """Perform a sanity check on the environment.""" + assert issubclass( + environment.undefined, Undefined + ), "'undefined' must be a subclass of 'jinja2.Undefined'." + assert ( + environment.block_start_string + != environment.variable_start_string + != environment.comment_start_string + ), "block, variable and comment start strings must be different." + assert environment.newline_sequence in { + "\r", + "\r\n", + "\n", + }, "'newline_sequence' must be one of '\\n', '\\r\\n', or '\\r'." + return environment + + +class Environment: + r"""The core component of Jinja is the `Environment`. It contains + important shared variables like configuration, filters, tests, + globals and others. Instances of this class may be modified if + they are not shared and if no template was loaded so far. + Modifications on environments after the first template was loaded + will lead to surprising effects and undefined behavior. + + Here are the possible initialization parameters: + + `block_start_string` + The string marking the beginning of a block. Defaults to ``'{%'``. + + `block_end_string` + The string marking the end of a block. Defaults to ``'%}'``. + + `variable_start_string` + The string marking the beginning of a print statement. + Defaults to ``'{{'``. + + `variable_end_string` + The string marking the end of a print statement. Defaults to + ``'}}'``. + + `comment_start_string` + The string marking the beginning of a comment. Defaults to ``'{#'``. + + `comment_end_string` + The string marking the end of a comment. Defaults to ``'#}'``. + + `line_statement_prefix` + If given and a string, this will be used as prefix for line based + statements. See also :ref:`line-statements`. + + `line_comment_prefix` + If given and a string, this will be used as prefix for line based + comments. See also :ref:`line-statements`. + + .. versionadded:: 2.2 + + `trim_blocks` + If this is set to ``True`` the first newline after a block is + removed (block, not variable tag!). Defaults to `False`. + + `lstrip_blocks` + If this is set to ``True`` leading spaces and tabs are stripped + from the start of a line to a block. Defaults to `False`. + + `newline_sequence` + The sequence that starts a newline. Must be one of ``'\r'``, + ``'\n'`` or ``'\r\n'``. The default is ``'\n'`` which is a + useful default for Linux and OS X systems as well as web + applications. + + `keep_trailing_newline` + Preserve the trailing newline when rendering templates. + The default is ``False``, which causes a single newline, + if present, to be stripped from the end of the template. + + .. versionadded:: 2.7 + + `extensions` + List of Jinja extensions to use. This can either be import paths + as strings or extension classes. For more information have a + look at :ref:`the extensions documentation `. + + `optimized` + should the optimizer be enabled? Default is ``True``. + + `undefined` + :class:`Undefined` or a subclass of it that is used to represent + undefined values in the template. + + `finalize` + A callable that can be used to process the result of a variable + expression before it is output. For example one can convert + ``None`` implicitly into an empty string here. + + `autoescape` + If set to ``True`` the XML/HTML autoescaping feature is enabled by + default. For more details about autoescaping see + :class:`~markupsafe.Markup`. As of Jinja 2.4 this can also + be a callable that is passed the template name and has to + return ``True`` or ``False`` depending on autoescape should be + enabled by default. + + .. versionchanged:: 2.4 + `autoescape` can now be a function + + `loader` + The template loader for this environment. + + `cache_size` + The size of the cache. Per default this is ``400`` which means + that if more than 400 templates are loaded the loader will clean + out the least recently used template. If the cache size is set to + ``0`` templates are recompiled all the time, if the cache size is + ``-1`` the cache will not be cleaned. + + .. versionchanged:: 2.8 + The cache size was increased to 400 from a low 50. + + `auto_reload` + Some loaders load templates from locations where the template + sources may change (ie: file system or database). If + ``auto_reload`` is set to ``True`` (default) every time a template is + requested the loader checks if the source changed and if yes, it + will reload the template. For higher performance it's possible to + disable that. + + `bytecode_cache` + If set to a bytecode cache object, this object will provide a + cache for the internal Jinja bytecode so that templates don't + have to be parsed if they were not changed. + + See :ref:`bytecode-cache` for more information. + + `enable_async` + If set to true this enables async template execution which + allows using async functions and generators. + """ + + #: if this environment is sandboxed. Modifying this variable won't make + #: the environment sandboxed though. For a real sandboxed environment + #: have a look at jinja2.sandbox. This flag alone controls the code + #: generation by the compiler. + sandboxed = False + + #: True if the environment is just an overlay + overlayed = False + + #: the environment this environment is linked to if it is an overlay + linked_to: t.Optional["Environment"] = None + + #: shared environments have this set to `True`. A shared environment + #: must not be modified + shared = False + + #: the class that is used for code generation. See + #: :class:`~jinja2.compiler.CodeGenerator` for more information. + code_generator_class: t.Type["CodeGenerator"] = CodeGenerator + + concat = "".join + + #: the context class that is used for templates. See + #: :class:`~jinja2.runtime.Context` for more information. + context_class: t.Type[Context] = Context + + template_class: t.Type["Template"] + + def __init__( + self, + block_start_string: str = BLOCK_START_STRING, + block_end_string: str = BLOCK_END_STRING, + variable_start_string: str = VARIABLE_START_STRING, + variable_end_string: str = VARIABLE_END_STRING, + comment_start_string: str = COMMENT_START_STRING, + comment_end_string: str = COMMENT_END_STRING, + line_statement_prefix: t.Optional[str] = LINE_STATEMENT_PREFIX, + line_comment_prefix: t.Optional[str] = LINE_COMMENT_PREFIX, + trim_blocks: bool = TRIM_BLOCKS, + lstrip_blocks: bool = LSTRIP_BLOCKS, + newline_sequence: "te.Literal['\\n', '\\r\\n', '\\r']" = NEWLINE_SEQUENCE, + keep_trailing_newline: bool = KEEP_TRAILING_NEWLINE, + extensions: t.Sequence[t.Union[str, t.Type["Extension"]]] = (), + optimized: bool = True, + undefined: t.Type[Undefined] = Undefined, + finalize: t.Optional[t.Callable[..., t.Any]] = None, + autoescape: t.Union[bool, t.Callable[[t.Optional[str]], bool]] = False, + loader: t.Optional["BaseLoader"] = None, + cache_size: int = 400, + auto_reload: bool = True, + bytecode_cache: t.Optional["BytecodeCache"] = None, + enable_async: bool = False, + ): + # !!Important notice!! + # The constructor accepts quite a few arguments that should be + # passed by keyword rather than position. However it's important to + # not change the order of arguments because it's used at least + # internally in those cases: + # - spontaneous environments (i18n extension and Template) + # - unittests + # If parameter changes are required only add parameters at the end + # and don't change the arguments (or the defaults!) of the arguments + # existing already. + + # lexer / parser information + self.block_start_string = block_start_string + self.block_end_string = block_end_string + self.variable_start_string = variable_start_string + self.variable_end_string = variable_end_string + self.comment_start_string = comment_start_string + self.comment_end_string = comment_end_string + self.line_statement_prefix = line_statement_prefix + self.line_comment_prefix = line_comment_prefix + self.trim_blocks = trim_blocks + self.lstrip_blocks = lstrip_blocks + self.newline_sequence = newline_sequence + self.keep_trailing_newline = keep_trailing_newline + + # runtime information + self.undefined: t.Type[Undefined] = undefined + self.optimized = optimized + self.finalize = finalize + self.autoescape = autoescape + + # defaults + self.filters = DEFAULT_FILTERS.copy() + self.tests = DEFAULT_TESTS.copy() + self.globals = DEFAULT_NAMESPACE.copy() + + # set the loader provided + self.loader = loader + self.cache = create_cache(cache_size) + self.bytecode_cache = bytecode_cache + self.auto_reload = auto_reload + + # configurable policies + self.policies = DEFAULT_POLICIES.copy() + + # load extensions + self.extensions = load_extensions(self, extensions) + + self.is_async = enable_async + _environment_config_check(self) + + def add_extension(self, extension: t.Union[str, t.Type["Extension"]]) -> None: + """Adds an extension after the environment was created. + + .. versionadded:: 2.5 + """ + self.extensions.update(load_extensions(self, [extension])) + + def extend(self, **attributes: t.Any) -> None: + """Add the items to the instance of the environment if they do not exist + yet. This is used by :ref:`extensions ` to register + callbacks and configuration values without breaking inheritance. + """ + for key, value in attributes.items(): + if not hasattr(self, key): + setattr(self, key, value) + + def overlay( + self, + block_start_string: str = missing, + block_end_string: str = missing, + variable_start_string: str = missing, + variable_end_string: str = missing, + comment_start_string: str = missing, + comment_end_string: str = missing, + line_statement_prefix: t.Optional[str] = missing, + line_comment_prefix: t.Optional[str] = missing, + trim_blocks: bool = missing, + lstrip_blocks: bool = missing, + newline_sequence: "te.Literal['\\n', '\\r\\n', '\\r']" = missing, + keep_trailing_newline: bool = missing, + extensions: t.Sequence[t.Union[str, t.Type["Extension"]]] = missing, + optimized: bool = missing, + undefined: t.Type[Undefined] = missing, + finalize: t.Optional[t.Callable[..., t.Any]] = missing, + autoescape: t.Union[bool, t.Callable[[t.Optional[str]], bool]] = missing, + loader: t.Optional["BaseLoader"] = missing, + cache_size: int = missing, + auto_reload: bool = missing, + bytecode_cache: t.Optional["BytecodeCache"] = missing, + enable_async: bool = missing, + ) -> "te.Self": + """Create a new overlay environment that shares all the data with the + current environment except for cache and the overridden attributes. + Extensions cannot be removed for an overlayed environment. An overlayed + environment automatically gets all the extensions of the environment it + is linked to plus optional extra extensions. + + Creating overlays should happen after the initial environment was set + up completely. Not all attributes are truly linked, some are just + copied over so modifications on the original environment may not shine + through. + + .. versionchanged:: 3.1.5 + ``enable_async`` is applied correctly. + + .. versionchanged:: 3.1.2 + Added the ``newline_sequence``, ``keep_trailing_newline``, + and ``enable_async`` parameters to match ``__init__``. + """ + args = dict(locals()) + del args["self"], args["cache_size"], args["extensions"], args["enable_async"] + + rv = object.__new__(self.__class__) + rv.__dict__.update(self.__dict__) + rv.overlayed = True + rv.linked_to = self + + for key, value in args.items(): + if value is not missing: + setattr(rv, key, value) + + if cache_size is not missing: + rv.cache = create_cache(cache_size) + else: + rv.cache = copy_cache(self.cache) + + rv.extensions = {} + for key, value in self.extensions.items(): + rv.extensions[key] = value.bind(rv) + if extensions is not missing: + rv.extensions.update(load_extensions(rv, extensions)) + + if enable_async is not missing: + rv.is_async = enable_async + + return _environment_config_check(rv) + + @property + def lexer(self) -> Lexer: + """The lexer for this environment.""" + return get_lexer(self) + + def iter_extensions(self) -> t.Iterator["Extension"]: + """Iterates over the extensions by priority.""" + return iter(sorted(self.extensions.values(), key=lambda x: x.priority)) + + def getitem( + self, obj: t.Any, argument: t.Union[str, t.Any] + ) -> t.Union[t.Any, Undefined]: + """Get an item or attribute of an object but prefer the item.""" + try: + return obj[argument] + except (AttributeError, TypeError, LookupError): + if isinstance(argument, str): + try: + attr = str(argument) + except Exception: + pass + else: + try: + return getattr(obj, attr) + except AttributeError: + pass + return self.undefined(obj=obj, name=argument) + + def getattr(self, obj: t.Any, attribute: str) -> t.Any: + """Get an item or attribute of an object but prefer the attribute. + Unlike :meth:`getitem` the attribute *must* be a string. + """ + try: + return getattr(obj, attribute) + except AttributeError: + pass + try: + return obj[attribute] + except (TypeError, LookupError, AttributeError): + return self.undefined(obj=obj, name=attribute) + + def _filter_test_common( + self, + name: t.Union[str, Undefined], + value: t.Any, + args: t.Optional[t.Sequence[t.Any]], + kwargs: t.Optional[t.Mapping[str, t.Any]], + context: t.Optional[Context], + eval_ctx: t.Optional[EvalContext], + is_filter: bool, + ) -> t.Any: + if is_filter: + env_map = self.filters + type_name = "filter" + else: + env_map = self.tests + type_name = "test" + + func = env_map.get(name) # type: ignore + + if func is None: + msg = f"No {type_name} named {name!r}." + + if isinstance(name, Undefined): + try: + name._fail_with_undefined_error() + except Exception as e: + msg = f"{msg} ({e}; did you forget to quote the callable name?)" + + raise TemplateRuntimeError(msg) + + args = [value, *(args if args is not None else ())] + kwargs = kwargs if kwargs is not None else {} + pass_arg = _PassArg.from_obj(func) + + if pass_arg is _PassArg.context: + if context is None: + raise TemplateRuntimeError( + f"Attempted to invoke a context {type_name} without context." + ) + + args.insert(0, context) + elif pass_arg is _PassArg.eval_context: + if eval_ctx is None: + if context is not None: + eval_ctx = context.eval_ctx + else: + eval_ctx = EvalContext(self) + + args.insert(0, eval_ctx) + elif pass_arg is _PassArg.environment: + args.insert(0, self) + + return func(*args, **kwargs) + + def call_filter( + self, + name: str, + value: t.Any, + args: t.Optional[t.Sequence[t.Any]] = None, + kwargs: t.Optional[t.Mapping[str, t.Any]] = None, + context: t.Optional[Context] = None, + eval_ctx: t.Optional[EvalContext] = None, + ) -> t.Any: + """Invoke a filter on a value the same way the compiler does. + + This might return a coroutine if the filter is running from an + environment in async mode and the filter supports async + execution. It's your responsibility to await this if needed. + + .. versionadded:: 2.7 + """ + return self._filter_test_common( + name, value, args, kwargs, context, eval_ctx, True + ) + + def call_test( + self, + name: str, + value: t.Any, + args: t.Optional[t.Sequence[t.Any]] = None, + kwargs: t.Optional[t.Mapping[str, t.Any]] = None, + context: t.Optional[Context] = None, + eval_ctx: t.Optional[EvalContext] = None, + ) -> t.Any: + """Invoke a test on a value the same way the compiler does. + + This might return a coroutine if the test is running from an + environment in async mode and the test supports async execution. + It's your responsibility to await this if needed. + + .. versionchanged:: 3.0 + Tests support ``@pass_context``, etc. decorators. Added + the ``context`` and ``eval_ctx`` parameters. + + .. versionadded:: 2.7 + """ + return self._filter_test_common( + name, value, args, kwargs, context, eval_ctx, False + ) + + @internalcode + def parse( + self, + source: str, + name: t.Optional[str] = None, + filename: t.Optional[str] = None, + ) -> nodes.Template: + """Parse the sourcecode and return the abstract syntax tree. This + tree of nodes is used by the compiler to convert the template into + executable source- or bytecode. This is useful for debugging or to + extract information from templates. + + If you are :ref:`developing Jinja extensions ` + this gives you a good overview of the node tree generated. + """ + try: + return self._parse(source, name, filename) + except TemplateSyntaxError: + self.handle_exception(source=source) + + def _parse( + self, source: str, name: t.Optional[str], filename: t.Optional[str] + ) -> nodes.Template: + """Internal parsing function used by `parse` and `compile`.""" + return Parser(self, source, name, filename).parse() + + def lex( + self, + source: str, + name: t.Optional[str] = None, + filename: t.Optional[str] = None, + ) -> t.Iterator[t.Tuple[int, str, str]]: + """Lex the given sourcecode and return a generator that yields + tokens as tuples in the form ``(lineno, token_type, value)``. + This can be useful for :ref:`extension development ` + and debugging templates. + + This does not perform preprocessing. If you want the preprocessing + of the extensions to be applied you have to filter source through + the :meth:`preprocess` method. + """ + source = str(source) + try: + return self.lexer.tokeniter(source, name, filename) + except TemplateSyntaxError: + self.handle_exception(source=source) + + def preprocess( + self, + source: str, + name: t.Optional[str] = None, + filename: t.Optional[str] = None, + ) -> str: + """Preprocesses the source with all extensions. This is automatically + called for all parsing and compiling methods but *not* for :meth:`lex` + because there you usually only want the actual source tokenized. + """ + return reduce( + lambda s, e: e.preprocess(s, name, filename), + self.iter_extensions(), + str(source), + ) + + def _tokenize( + self, + source: str, + name: t.Optional[str], + filename: t.Optional[str] = None, + state: t.Optional[str] = None, + ) -> TokenStream: + """Called by the parser to do the preprocessing and filtering + for all the extensions. Returns a :class:`~jinja2.lexer.TokenStream`. + """ + source = self.preprocess(source, name, filename) + stream = self.lexer.tokenize(source, name, filename, state) + + for ext in self.iter_extensions(): + stream = ext.filter_stream(stream) # type: ignore + + if not isinstance(stream, TokenStream): + stream = TokenStream(stream, name, filename) + + return stream + + def _generate( + self, + source: nodes.Template, + name: t.Optional[str], + filename: t.Optional[str], + defer_init: bool = False, + ) -> str: + """Internal hook that can be overridden to hook a different generate + method in. + + .. versionadded:: 2.5 + """ + return generate( # type: ignore + source, + self, + name, + filename, + defer_init=defer_init, + optimized=self.optimized, + ) + + def _compile(self, source: str, filename: str) -> CodeType: + """Internal hook that can be overridden to hook a different compile + method in. + + .. versionadded:: 2.5 + """ + return compile(source, filename, "exec") + + @typing.overload + def compile( + self, + source: t.Union[str, nodes.Template], + name: t.Optional[str] = None, + filename: t.Optional[str] = None, + raw: "te.Literal[False]" = False, + defer_init: bool = False, + ) -> CodeType: ... + + @typing.overload + def compile( + self, + source: t.Union[str, nodes.Template], + name: t.Optional[str] = None, + filename: t.Optional[str] = None, + raw: "te.Literal[True]" = ..., + defer_init: bool = False, + ) -> str: ... + + @internalcode + def compile( + self, + source: t.Union[str, nodes.Template], + name: t.Optional[str] = None, + filename: t.Optional[str] = None, + raw: bool = False, + defer_init: bool = False, + ) -> t.Union[str, CodeType]: + """Compile a node or template source code. The `name` parameter is + the load name of the template after it was joined using + :meth:`join_path` if necessary, not the filename on the file system. + the `filename` parameter is the estimated filename of the template on + the file system. If the template came from a database or memory this + can be omitted. + + The return value of this method is a python code object. If the `raw` + parameter is `True` the return value will be a string with python + code equivalent to the bytecode returned otherwise. This method is + mainly used internally. + + `defer_init` is use internally to aid the module code generator. This + causes the generated code to be able to import without the global + environment variable to be set. + + .. versionadded:: 2.4 + `defer_init` parameter added. + """ + source_hint = None + try: + if isinstance(source, str): + source_hint = source + source = self._parse(source, name, filename) + source = self._generate(source, name, filename, defer_init=defer_init) + if raw: + return source + if filename is None: + filename = "