Add files using upload-large-folder tool

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/fsspec/__init__.py

@@ -0,0 +1,71 @@
+from . import caching
+from ._version import __version__  # noqa: F401
+from .callbacks import Callback
+from .compression import available_compressions
+from .core import get_fs_token_paths, open, open_files, open_local, url_to_fs
+from .exceptions import FSTimeoutError
+from .mapping import FSMap, get_mapper
+from .registry import (
+    available_protocols,
+    filesystem,
+    get_filesystem_class,
+    register_implementation,
+    registry,
+)
+from .spec import AbstractFileSystem
+
+__all__ = [
+    "AbstractFileSystem",
+    "FSTimeoutError",
+    "FSMap",
+    "filesystem",
+    "register_implementation",
+    "get_filesystem_class",
+    "get_fs_token_paths",
+    "get_mapper",
+    "open",
+    "open_files",
+    "open_local",
+    "registry",
+    "caching",
+    "Callback",
+    "available_protocols",
+    "available_compressions",
+    "url_to_fs",
+]
+
+
+def process_entries():
+    try:
+        from importlib.metadata import entry_points
+    except ImportError:
+        return
+    if entry_points is not None:
+        try:
+            eps = entry_points()
+        except TypeError:
+            pass  # importlib-metadata < 0.8
+        else:
+            if hasattr(eps, "select"):  # Python 3.10+ / importlib_metadata >= 3.9.0
+                specs = eps.select(group="fsspec.specs")
+            else:
+                specs = eps.get("fsspec.specs", [])
+            registered_names = {}
+            for spec in specs:
+                err_msg = f"Unable to load filesystem from {spec}"
+                name = spec.name
+                if name in registered_names:
+                    continue
+                registered_names[name] = True
+                register_implementation(
+                    name,
+                    spec.value.replace(":", "."),
+                    errtxt=err_msg,
+                    # We take our implementations as the ones to overload with if
+                    # for some reason we encounter some, may be the same, already
+                    # registered
+                    clobber=True,
+                )
+
+
+process_entries()

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/fsspec/asyn.py

@@ -0,0 +1,1127 @@
+import asyncio
+import asyncio.events
+import functools
+import inspect
+import io
+import numbers
+import os
+import re
+import threading
+from collections.abc import Iterable
+from glob import has_magic
+from typing import TYPE_CHECKING
+
+from .callbacks import DEFAULT_CALLBACK
+from .exceptions import FSTimeoutError
+from .implementations.local import LocalFileSystem, make_path_posix, trailing_sep
+from .spec import AbstractBufferedFile, AbstractFileSystem
+from .utils import glob_translate, is_exception, other_paths
+
+private = re.compile("_[^_]")
+iothread = [None]  # dedicated fsspec IO thread
+loop = [None]  # global event loop for any non-async instance
+_lock = None  # global lock placeholder
+get_running_loop = asyncio.get_running_loop
+
+
+def get_lock():
+    """Allocate or return a threading lock.
+
+    The lock is allocated on first use to allow setting one lock per forked process.
+    """
+    global _lock
+    if not _lock:
+        _lock = threading.Lock()
+    return _lock
+
+
+def reset_lock():
+    """Reset the global lock.
+
+    This should be called only on the init of a forked process to reset the lock to
+    None, enabling the new forked process to get a new lock.
+    """
+    global _lock
+
+    iothread[0] = None
+    loop[0] = None
+    _lock = None
+
+
+async def _runner(event, coro, result, timeout=None):
+    timeout = timeout if timeout else None  # convert 0 or 0.0 to None
+    if timeout is not None:
+        coro = asyncio.wait_for(coro, timeout=timeout)
+    try:
+        result[0] = await coro
+    except Exception as ex:
+        result[0] = ex
+    finally:
+        event.set()
+
+
+def sync(loop, func, *args, timeout=None, **kwargs):
+    """
+    Make loop run coroutine until it returns. Runs in other thread
+
+    Examples
+    --------
+    >>> fsspec.asyn.sync(fsspec.asyn.get_loop(), func, *args,
+                         timeout=timeout, **kwargs)
+    """
+    timeout = timeout if timeout else None  # convert 0 or 0.0 to None
+    # NB: if the loop is not running *yet*, it is OK to submit work
+    # and we will wait for it
+    if loop is None or loop.is_closed():
+        raise RuntimeError("Loop is not running")
+    try:
+        loop0 = asyncio.events.get_running_loop()
+        if loop0 is loop:
+            raise NotImplementedError("Calling sync() from within a running loop")
+    except NotImplementedError:
+        raise
+    except RuntimeError:
+        pass
+    coro = func(*args, **kwargs)
+    result = [None]
+    event = threading.Event()
+    asyncio.run_coroutine_threadsafe(_runner(event, coro, result, timeout), loop)
+    while True:
+        # this loops allows thread to get interrupted
+        if event.wait(1):
+            break
+        if timeout is not None:
+            timeout -= 1
+            if timeout < 0:
+                raise FSTimeoutError
+
+    return_result = result[0]
+    if isinstance(return_result, asyncio.TimeoutError):
+        # suppress asyncio.TimeoutError, raise FSTimeoutError
+        raise FSTimeoutError from return_result
+    elif isinstance(return_result, BaseException):
+        raise return_result
+    else:
+        return return_result
+
+
+def sync_wrapper(func, obj=None):
+    """Given a function, make so can be called in blocking contexts
+
+    Leave obj=None if defining within a class. Pass the instance if attaching
+    as an attribute of the instance.
+    """
+
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        self = obj or args[0]
+        return sync(self.loop, func, *args, **kwargs)
+
+    return wrapper
+
+
+def get_loop():
+    """Create or return the default fsspec IO loop
+
+    The loop will be running on a separate thread.
+    """
+    if loop[0] is None:
+        with get_lock():
+            # repeat the check just in case the loop got filled between the
+            # previous two calls from another thread
+            if loop[0] is None:
+                loop[0] = asyncio.new_event_loop()
+                th = threading.Thread(target=loop[0].run_forever, name="fsspecIO")
+                th.daemon = True
+                th.start()
+                iothread[0] = th
+    return loop[0]
+
+
+def reset_after_fork():
+    global lock
+    loop[0] = None
+    iothread[0] = None
+    lock = None
+
+
+if hasattr(os, "register_at_fork"):
+    # should be posix; this will do nothing for spawn or forkserver subprocesses
+    os.register_at_fork(after_in_child=reset_after_fork)
+
+
+if TYPE_CHECKING:
+    import resource
+
+    ResourceError = resource.error
+else:
+    try:
+        import resource
+    except ImportError:
+        resource = None
+        ResourceError = OSError
+    else:
+        ResourceError = getattr(resource, "error", OSError)
+
+_DEFAULT_BATCH_SIZE = 128
+_NOFILES_DEFAULT_BATCH_SIZE = 1280
+
+
+def _get_batch_size(nofiles=False):
+    from fsspec.config import conf
+
+    if nofiles:
+        if "nofiles_gather_batch_size" in conf:
+            return conf["nofiles_gather_batch_size"]
+    else:
+        if "gather_batch_size" in conf:
+            return conf["gather_batch_size"]
+    if nofiles:
+        return _NOFILES_DEFAULT_BATCH_SIZE
+    if resource is None:
+        return _DEFAULT_BATCH_SIZE
+
+    try:
+        soft_limit, _ = resource.getrlimit(resource.RLIMIT_NOFILE)
+    except (ImportError, ValueError, ResourceError):
+        return _DEFAULT_BATCH_SIZE
+
+    if soft_limit == resource.RLIM_INFINITY:
+        return -1
+    else:
+        return soft_limit // 8
+
+
+def running_async() -> bool:
+    """Being executed by an event loop?"""
+    try:
+        asyncio.get_running_loop()
+        return True
+    except RuntimeError:
+        return False
+
+
+async def _run_coros_in_chunks(
+    coros,
+    batch_size=None,
+    callback=DEFAULT_CALLBACK,
+    timeout=None,
+    return_exceptions=False,
+    nofiles=False,
+):
+    """Run the given coroutines in  chunks.
+
+    Parameters
+    ----------
+    coros: list of coroutines to run
+    batch_size: int or None
+        Number of coroutines to submit/wait on simultaneously.
+        If -1, then it will not be any throttling. If
+        None, it will be inferred from _get_batch_size()
+    callback: fsspec.callbacks.Callback instance
+        Gets a relative_update when each coroutine completes
+    timeout: number or None
+        If given, each coroutine times out after this time. Note that, since
+        there are multiple batches, the total run time of this function will in
+        general be longer
+    return_exceptions: bool
+        Same meaning as in asyncio.gather
+    nofiles: bool
+        If inferring the batch_size, does this operation involve local files?
+        If yes, you normally expect smaller batches.
+    """
+
+    if batch_size is None:
+        batch_size = _get_batch_size(nofiles=nofiles)
+
+    if batch_size == -1:
+        batch_size = len(coros)
+
+    assert batch_size > 0
+
+    async def _run_coro(coro, i):
+        try:
+            return await asyncio.wait_for(coro, timeout=timeout), i
+        except Exception as e:
+            if not return_exceptions:
+                raise
+            return e, i
+        finally:
+            callback.relative_update(1)
+
+    i = 0
+    n = len(coros)
+    results = [None] * n
+    pending = set()
+
+    while pending or i < n:
+        while len(pending) < batch_size and i < n:
+            pending.add(asyncio.ensure_future(_run_coro(coros[i], i)))
+            i += 1
+
+        if not pending:
+            break
+
+        done, pending = await asyncio.wait(pending, return_when=asyncio.FIRST_COMPLETED)
+        first_exc = None
+        while done:
+            task = done.pop()
+            try:
+                result, k = await task
+                results[k] = result
+            except Exception as exc:
+                if first_exc is None:
+                    first_exc = exc
+
+        if first_exc is not None:
+            for task in pending:
+                task.cancel()
+            if pending:
+                await asyncio.gather(*pending, return_exceptions=True)
+            raise first_exc
+
+    return results
+
+
+# these methods should be implemented as async by any async-able backend
+async_methods = [
+    "_ls",
+    "_cat_file",
+    "_get_file",
+    "_put_file",
+    "_rm_file",
+    "_cp_file",
+    "_pipe_file",
+    "_expand_path",
+    "_info",
+    "_isfile",
+    "_isdir",
+    "_exists",
+    "_walk",
+    "_glob",
+    "_find",
+    "_du",
+    "_size",
+    "_mkdir",
+    "_makedirs",
+]
+
+
+class AsyncFileSystem(AbstractFileSystem):
+    """Async file operations, default implementations
+
+    Passes bulk operations to asyncio.gather for concurrent operation.
+
+    Implementations that have concurrent batch operations and/or async methods
+    should inherit from this class instead of AbstractFileSystem. Docstrings are
+    copied from the un-underscored method in AbstractFileSystem, if not given.
+    """
+
+    # note that methods do not have docstring here; they will be copied
+    # for _* methods and inferred for overridden methods.
+
+    async_impl = True
+    mirror_sync_methods = True
+    disable_throttling = False
+
+    def __init__(self, *args, asynchronous=False, loop=None, batch_size=None, **kwargs):
+        self.asynchronous = asynchronous
+        self._pid = os.getpid()
+        if not asynchronous:
+            self._loop = loop or get_loop()
+        else:
+            self._loop = None
+        self.batch_size = batch_size
+        super().__init__(*args, **kwargs)
+
+    @property
+    def loop(self):
+        if self._pid != os.getpid():
+            raise RuntimeError("This class is not fork-safe")
+        return self._loop
+
+    async def _rm_file(self, path, **kwargs):
+        if (
+            inspect.iscoroutinefunction(self._rm)
+            and type(self)._rm is not AsyncFileSystem._rm
+        ):
+            return await self._rm(path, recursive=False, batch_size=1, **kwargs)
+        raise NotImplementedError
+
+    async def _rm(self, path, recursive=False, batch_size=None, **kwargs):
+        # TODO: implement on_error
+        batch_size = batch_size or self.batch_size
+        path = await self._expand_path(path, recursive=recursive)
+        return await _run_coros_in_chunks(
+            [self._rm_file(p, **kwargs) for p in reversed(path)],
+            batch_size=batch_size,
+            nofiles=True,
+        )
+
+    async def _cp_file(self, path1, path2, **kwargs):
+        raise NotImplementedError
+
+    async def _mv_file(self, path1, path2):
+        await self._cp_file(path1, path2)
+        await self._rm_file(path1)
+
+    async def _copy(
+        self,
+        path1,
+        path2,
+        recursive=False,
+        on_error=None,
+        maxdepth=None,
+        batch_size=None,
+        **kwargs,
+    ):
+        if on_error is None and recursive:
+            on_error = "ignore"
+        elif on_error is None:
+            on_error = "raise"
+
+        if isinstance(path1, list) and isinstance(path2, list):
+            # No need to expand paths when both source and destination
+            # are provided as lists
+            paths1 = path1
+            paths2 = path2
+        else:
+            source_is_str = isinstance(path1, str)
+            paths1 = await self._expand_path(
+                path1, maxdepth=maxdepth, recursive=recursive
+            )
+            if source_is_str and (not recursive or maxdepth is not None):
+                # Non-recursive glob does not copy directories
+                paths1 = [
+                    p for p in paths1 if not (trailing_sep(p) or await self._isdir(p))
+                ]
+                if not paths1:
+                    return
+
+            source_is_file = len(paths1) == 1
+            dest_is_dir = isinstance(path2, str) and (
+                trailing_sep(path2) or await self._isdir(path2)
+            )
+
+            exists = source_is_str and (
+                (has_magic(path1) and source_is_file)
+                or (not has_magic(path1) and dest_is_dir and not trailing_sep(path1))
+            )
+            paths2 = other_paths(
+                paths1,
+                path2,
+                exists=exists,
+                flatten=not source_is_str,
+            )
+
+        batch_size = batch_size or self.batch_size
+        coros = [self._cp_file(p1, p2, **kwargs) for p1, p2 in zip(paths1, paths2)]
+        result = await _run_coros_in_chunks(
+            coros, batch_size=batch_size, return_exceptions=True, nofiles=True
+        )
+
+        for ex in filter(is_exception, result):
+            if on_error == "ignore" and isinstance(ex, FileNotFoundError):
+                continue
+            raise ex
+
+    async def _pipe_file(self, path, value, mode="overwrite", **kwargs):
+        raise NotImplementedError
+
+    async def _pipe(self, path, value=None, batch_size=None, **kwargs):
+        if isinstance(path, str):
+            path = {path: value}
+        batch_size = batch_size or self.batch_size
+        return await _run_coros_in_chunks(
+            [self._pipe_file(k, v, **kwargs) for k, v in path.items()],
+            batch_size=batch_size,
+            nofiles=True,
+        )
+
+    async def _process_limits(self, url, start, end):
+        """Helper for "Range"-based _cat_file"""
+        size = None
+        suff = False
+        if start is not None and start < 0:
+            # if start is negative and end None, end is the "suffix length"
+            if end is None:
+                end = -start
+                start = ""
+                suff = True
+            else:
+                size = size or (await self._info(url))["size"]
+                start = size + start
+        elif start is None:
+            start = 0
+        if not suff:
+            if end is not None and end < 0:
+                if start is not None:
+                    size = size or (await self._info(url))["size"]
+                    end = size + end
+            elif end is None:
+                end = ""
+            if isinstance(end, numbers.Integral):
+                end -= 1  # bytes range is inclusive
+        return f"bytes={start}-{end}"
+
+    async def _cat_file(self, path, start=None, end=None, **kwargs):
+        raise NotImplementedError
+
+    async def _cat(
+        self, path, recursive=False, on_error="raise", batch_size=None, **kwargs
+    ):
+        paths = await self._expand_path(path, recursive=recursive)
+        coros = [self._cat_file(path, **kwargs) for path in paths]
+        batch_size = batch_size or self.batch_size
+        out = await _run_coros_in_chunks(
+            coros, batch_size=batch_size, nofiles=True, return_exceptions=True
+        )
+        if on_error == "raise":
+            ex = next(filter(is_exception, out), False)
+            if ex:
+                raise ex
+        if (
+            len(paths) > 1
+            or isinstance(path, list)
+            or paths[0] != self._strip_protocol(path)
+        ):
+            return {
+                k: v
+                for k, v in zip(paths, out)
+                if on_error != "omit" or not is_exception(v)
+            }
+        else:
+            return out[0]
+
+    async def _cat_ranges(
+        self,
+        paths,
+        starts,
+        ends,
+        max_gap=None,
+        batch_size=None,
+        on_error="return",
+        **kwargs,
+    ):
+        """Get the contents of byte ranges from one or more files
+
+        Parameters
+        ----------
+        paths: list
+            A list of of filepaths on this filesystems
+        starts, ends: int or list
+            Bytes limits of the read. If using a single int, the same value will be
+            used to read all the specified files.
+        """
+        # TODO: on_error
+        if max_gap is not None:
+            # use utils.merge_offset_ranges
+            raise NotImplementedError
+        if not isinstance(paths, list):
+            raise TypeError
+        if not isinstance(starts, Iterable):
+            starts = [starts] * len(paths)
+        if not isinstance(ends, Iterable):
+            ends = [ends] * len(paths)
+        if len(starts) != len(paths) or len(ends) != len(paths):
+            raise ValueError
+        coros = [
+            self._cat_file(p, start=s, end=e, **kwargs)
+            for p, s, e in zip(paths, starts, ends)
+        ]
+        batch_size = batch_size or self.batch_size
+        return await _run_coros_in_chunks(
+            coros, batch_size=batch_size, nofiles=True, return_exceptions=True
+        )
+
+    async def _put_file(self, lpath, rpath, mode="overwrite", **kwargs):
+        raise NotImplementedError
+
+    async def _put(
+        self,
+        lpath,
+        rpath,
+        recursive=False,
+        callback=DEFAULT_CALLBACK,
+        batch_size=None,
+        maxdepth=None,
+        **kwargs,
+    ):
+        """Copy file(s) from local.
+
+        Copies a specific file or tree of files (if recursive=True). If rpath
+        ends with a "/", it will be assumed to be a directory, and target files
+        will go within.
+
+        The put_file method will be called concurrently on a batch of files. The
+        batch_size option can configure the amount of futures that can be executed
+        at the same time. If it is -1, then all the files will be uploaded concurrently.
+        The default can be set for this instance by passing "batch_size" in the
+        constructor, or for all instances by setting the "gather_batch_size" key
+        in ``fsspec.config.conf``, falling back to 1/8th of the system limit .
+        """
+        if isinstance(lpath, list) and isinstance(rpath, list):
+            # No need to expand paths when both source and destination
+            # are provided as lists
+            rpaths = rpath
+            lpaths = lpath
+        else:
+            source_is_str = isinstance(lpath, str)
+            if source_is_str:
+                lpath = make_path_posix(lpath)
+            fs = LocalFileSystem()
+            lpaths = fs.expand_path(lpath, recursive=recursive, maxdepth=maxdepth)
+            if source_is_str and (not recursive or maxdepth is not None):
+                # Non-recursive glob does not copy directories
+                lpaths = [p for p in lpaths if not (trailing_sep(p) or fs.isdir(p))]
+                if not lpaths:
+                    return
+
+            source_is_file = len(lpaths) == 1
+            dest_is_dir = isinstance(rpath, str) and (
+                trailing_sep(rpath) or await self._isdir(rpath)
+            )
+
+            rpath = self._strip_protocol(rpath)
+            exists = source_is_str and (
+                (has_magic(lpath) and source_is_file)
+                or (not has_magic(lpath) and dest_is_dir and not trailing_sep(lpath))
+            )
+            rpaths = other_paths(
+                lpaths,
+                rpath,
+                exists=exists,
+                flatten=not source_is_str,
+            )
+
+        is_dir = {l: os.path.isdir(l) for l in lpaths}
+        rdirs = [r for l, r in zip(lpaths, rpaths) if is_dir[l]]
+        file_pairs = [(l, r) for l, r in zip(lpaths, rpaths) if not is_dir[l]]
+
+        await asyncio.gather(*[self._makedirs(d, exist_ok=True) for d in rdirs])
+        batch_size = batch_size or self.batch_size
+
+        coros = []
+        callback.set_size(len(file_pairs))
+        for lfile, rfile in file_pairs:
+            put_file = callback.branch_coro(self._put_file)
+            coros.append(put_file(lfile, rfile, **kwargs))
+
+        return await _run_coros_in_chunks(
+            coros, batch_size=batch_size, callback=callback
+        )
+
+    async def _get_file(self, rpath, lpath, **kwargs):
+        raise NotImplementedError
+
+    async def _get(
+        self,
+        rpath,
+        lpath,
+        recursive=False,
+        callback=DEFAULT_CALLBACK,
+        maxdepth=None,
+        **kwargs,
+    ):
+        """Copy file(s) to local.
+
+        Copies a specific file or tree of files (if recursive=True). If lpath
+        ends with a "/", it will be assumed to be a directory, and target files
+        will go within. Can submit a list of paths, which may be glob-patterns
+        and will be expanded.
+
+        The get_file method will be called concurrently on a batch of files. The
+        batch_size option can configure the amount of futures that can be executed
+        at the same time. If it is -1, then all the files will be uploaded concurrently.
+        The default can be set for this instance by passing "batch_size" in the
+        constructor, or for all instances by setting the "gather_batch_size" key
+        in ``fsspec.config.conf``, falling back to 1/8th of the system limit .
+        """
+        if isinstance(lpath, list) and isinstance(rpath, list):
+            # No need to expand paths when both source and destination
+            # are provided as lists
+            rpaths = rpath
+            lpaths = lpath
+        else:
+            source_is_str = isinstance(rpath, str)
+            # First check for rpath trailing slash as _strip_protocol removes it.
+            source_not_trailing_sep = source_is_str and not trailing_sep(rpath)
+            rpath = self._strip_protocol(rpath)
+            rpaths = await self._expand_path(
+                rpath, recursive=recursive, maxdepth=maxdepth
+            )
+            if source_is_str and (not recursive or maxdepth is not None):
+                # Non-recursive glob does not copy directories
+                rpaths = [
+                    p for p in rpaths if not (trailing_sep(p) or await self._isdir(p))
+                ]
+                if not rpaths:
+                    return
+
+            lpath = make_path_posix(lpath)
+            source_is_file = len(rpaths) == 1
+            dest_is_dir = isinstance(lpath, str) and (
+                trailing_sep(lpath) or LocalFileSystem().isdir(lpath)
+            )
+
+            exists = source_is_str and (
+                (has_magic(rpath) and source_is_file)
+                or (not has_magic(rpath) and dest_is_dir and source_not_trailing_sep)
+            )
+            lpaths = other_paths(
+                rpaths,
+                lpath,
+                exists=exists,
+                flatten=not source_is_str,
+            )
+
+        [os.makedirs(os.path.dirname(lp), exist_ok=True) for lp in lpaths]
+        batch_size = kwargs.pop("batch_size", self.batch_size)
+
+        coros = []
+        callback.set_size(len(lpaths))
+        for lpath, rpath in zip(lpaths, rpaths):
+            get_file = callback.branch_coro(self._get_file)
+            coros.append(get_file(rpath, lpath, **kwargs))
+        return await _run_coros_in_chunks(
+            coros, batch_size=batch_size, callback=callback
+        )
+
+    async def _isfile(self, path):
+        try:
+            return (await self._info(path))["type"] == "file"
+        except:  # noqa: E722
+            return False
+
+    async def _isdir(self, path):
+        try:
+            return (await self._info(path))["type"] == "directory"
+        except OSError:
+            return False
+
+    async def _size(self, path):
+        return (await self._info(path)).get("size", None)
+
+    async def _sizes(self, paths, batch_size=None):
+        batch_size = batch_size or self.batch_size
+        return await _run_coros_in_chunks(
+            [self._size(p) for p in paths], batch_size=batch_size
+        )
+
+    async def _exists(self, path, **kwargs):
+        try:
+            await self._info(path, **kwargs)
+            return True
+        except FileNotFoundError:
+            return False
+
+    async def _info(self, path, **kwargs):
+        raise NotImplementedError
+
+    async def _ls(self, path, detail=True, **kwargs):
+        raise NotImplementedError
+
+    async def _walk(self, path, maxdepth=None, on_error="omit", **kwargs):
+        if maxdepth is not None and maxdepth < 1:
+            raise ValueError("maxdepth must be at least 1")
+
+        path = self._strip_protocol(path)
+        full_dirs = {}
+        dirs = {}
+        files = {}
+
+        detail = kwargs.pop("detail", False)
+        try:
+            listing = await self._ls(path, detail=True, **kwargs)
+        except (FileNotFoundError, OSError) as e:
+            if on_error == "raise":
+                raise
+            elif callable(on_error):
+                on_error(e)
+            if detail:
+                yield path, {}, {}
+            else:
+                yield path, [], []
+            return
+
+        for info in listing:
+            # each info name must be at least [path]/part , but here
+            # we check also for names like [path]/part/
+            pathname = info["name"].rstrip("/")
+            name = pathname.rsplit("/", 1)[-1]
+            if info["type"] == "directory" and pathname != path:
+                # do not include "self" path
+                full_dirs[name] = pathname
+                dirs[name] = info
+            elif pathname == path:
+                # file-like with same name as give path
+                files[""] = info
+            else:
+                files[name] = info
+
+        if detail:
+            yield path, dirs, files
+        else:
+            yield path, list(dirs), list(files)
+
+        if maxdepth is not None:
+            maxdepth -= 1
+            if maxdepth < 1:
+                return
+
+        for d in dirs:
+            async for _ in self._walk(
+                full_dirs[d], maxdepth=maxdepth, detail=detail, **kwargs
+            ):
+                yield _
+
+    async def _glob(self, path, maxdepth=None, **kwargs):
+        if maxdepth is not None and maxdepth < 1:
+            raise ValueError("maxdepth must be at least 1")
+
+        import re
+
+        seps = (os.path.sep, os.path.altsep) if os.path.altsep else (os.path.sep,)
+        ends_with_sep = path.endswith(seps)  # _strip_protocol strips trailing slash
+        path = self._strip_protocol(path)
+        append_slash_to_dirname = ends_with_sep or path.endswith(
+            tuple(sep + "**" for sep in seps)
+        )
+        idx_star = path.find("*") if path.find("*") >= 0 else len(path)
+        idx_qmark = path.find("?") if path.find("?") >= 0 else len(path)
+        idx_brace = path.find("[") if path.find("[") >= 0 else len(path)
+
+        min_idx = min(idx_star, idx_qmark, idx_brace)
+
+        detail = kwargs.pop("detail", False)
+        withdirs = kwargs.pop("withdirs", True)
+
+        if not has_magic(path):
+            if await self._exists(path, **kwargs):
+                if not detail:
+                    return [path]
+                else:
+                    return {path: await self._info(path, **kwargs)}
+            else:
+                if not detail:
+                    return []  # glob of non-existent returns empty
+                else:
+                    return {}
+        elif "/" in path[:min_idx]:
+            first_wildcard_idx = min_idx
+            min_idx = path[:min_idx].rindex("/")
+            root = path[
+                : min_idx + 1
+            ]  # everything up to the last / before the first wildcard
+            prefix = path[
+                min_idx + 1 : first_wildcard_idx
+            ]  # stem between last "/" and first wildcard
+            depth = path[min_idx + 1 :].count("/") + 1
+        else:
+            root = ""
+            prefix = path[:min_idx]  # stem up to the first wildcard
+            depth = path[min_idx + 1 :].count("/") + 1
+
+        if "**" in path:
+            if maxdepth is not None:
+                idx_double_stars = path.find("**")
+                depth_double_stars = path[idx_double_stars:].count("/") + 1
+                depth = depth - depth_double_stars + maxdepth
+            else:
+                depth = None
+
+        # Pass the filename stem as prefix= so backends that support it such as
+        # gcsfs, s3fs and adlfs can filter server-side up to the first wildcard.
+        if prefix:
+            kwargs["prefix"] = prefix
+        allpaths = await self._find(
+            root, maxdepth=depth, withdirs=withdirs, detail=True, **kwargs
+        )
+
+        pattern = glob_translate(path + ("/" if ends_with_sep else ""))
+        pattern = re.compile(pattern)
+
+        out = {
+            p: info
+            for p, info in sorted(allpaths.items())
+            if pattern.match(
+                p + "/"
+                if append_slash_to_dirname and info["type"] == "directory"
+                else p
+            )
+        }
+
+        if detail:
+            return out
+        else:
+            return list(out)
+
+    async def _du(self, path, total=True, maxdepth=None, **kwargs):
+        sizes = {}
+        # async for?
+        for f in await self._find(path, maxdepth=maxdepth, **kwargs):
+            info = await self._info(f)
+            sizes[info["name"]] = info["size"]
+        if total:
+            return sum(sizes.values())
+        else:
+            return sizes
+
+    async def _find(self, path, maxdepth=None, withdirs=False, **kwargs):
+        path = self._strip_protocol(path)
+        out = {}
+        detail = kwargs.pop("detail", False)
+
+        # Add the root directory if withdirs is requested
+        # This is needed for posix glob compliance
+        if withdirs and path != "" and await self._isdir(path):
+            out[path] = await self._info(path)
+
+        # async for?
+        async for _, dirs, files in self._walk(path, maxdepth, detail=True, **kwargs):
+            if withdirs:
+                files.update(dirs)
+            out.update({info["name"]: info for name, info in files.items()})
+        if not out and (await self._isfile(path)):
+            # walk works on directories, but find should also return [path]
+            # when path happens to be a file
+            out[path] = {}
+        names = sorted(out)
+        if not detail:
+            return names
+        else:
+            return {name: out[name] for name in names}
+
+    async def _expand_path(self, path, recursive=False, maxdepth=None):
+        if maxdepth is not None and maxdepth < 1:
+            raise ValueError("maxdepth must be at least 1")
+
+        if isinstance(path, str):
+            out = await self._expand_path([path], recursive, maxdepth)
+        else:
+            out = set()
+            path = [self._strip_protocol(p) for p in path]
+            for p in path:  # can gather here
+                if has_magic(p):
+                    bit = set(await self._glob(p, maxdepth=maxdepth))
+                    out |= bit
+                    if recursive:
+                        # glob call above expanded one depth so if maxdepth is defined
+                        # then decrement it in expand_path call below. If it is zero
+                        # after decrementing then avoid expand_path call.
+                        if maxdepth is not None and maxdepth <= 1:
+                            continue
+                        out |= set(
+                            await self._expand_path(
+                                list(bit),
+                                recursive=recursive,
+                                maxdepth=maxdepth - 1 if maxdepth is not None else None,
+                            )
+                        )
+                    continue
+                elif recursive:
+                    rec = set(await self._find(p, maxdepth=maxdepth, withdirs=True))
+                    out |= rec
+                if p not in out and (recursive is False or (await self._exists(p))):
+                    # should only check once, for the root
+                    out.add(p)
+        if not out:
+            raise FileNotFoundError(path)
+        return sorted(out)
+
+    async def _mkdir(self, path, create_parents=True, **kwargs):
+        pass  # not necessary to implement, may not have directories
+
+    async def _makedirs(self, path, exist_ok=False):
+        pass  # not necessary to implement, may not have directories
+
+    async def open_async(self, path, mode="rb", **kwargs):
+        if "b" not in mode or kwargs.get("compression"):
+            raise ValueError
+        raise NotImplementedError
+
+
+def mirror_sync_methods(obj):
+    """Populate sync and async methods for obj
+
+    For each method will create a sync version if the name refers to an async method
+    (coroutine) and there is no override in the child class; will create an async
+    method for the corresponding sync method if there is no implementation.
+
+    Uses the methods specified in
+    - async_methods: the set that an implementation is expected to provide
+    - default_async_methods: that can be derived from their sync version in
+      AbstractFileSystem
+    - AsyncFileSystem: async-specific default coroutines
+    """
+    from fsspec import AbstractFileSystem
+
+    for method in async_methods + dir(AsyncFileSystem):
+        if not method.startswith("_"):
+            continue
+        smethod = method[1:]
+        if private.match(method):
+            isco = inspect.iscoroutinefunction(getattr(obj, method, None))
+            unsync = getattr(getattr(obj, smethod, False), "__func__", None)
+            is_default = unsync is getattr(AbstractFileSystem, smethod, "")
+            if isco and is_default:
+                mth = sync_wrapper(getattr(obj, method), obj=obj)
+                setattr(obj, smethod, mth)
+                if not mth.__doc__:
+                    mth.__doc__ = getattr(
+                        getattr(AbstractFileSystem, smethod, None), "__doc__", ""
+                    )
+
+
+class FSSpecCoroutineCancel(Exception):
+    pass
+
+
+def _dump_running_tasks(
+    printout=True, cancel=True, exc=FSSpecCoroutineCancel, with_task=False
+):
+    import traceback
+
+    tasks = [t for t in asyncio.tasks.all_tasks(loop[0]) if not t.done()]
+    if printout:
+        [task.print_stack() for task in tasks]
+    out = [
+        {
+            "locals": task._coro.cr_frame.f_locals,
+            "file": task._coro.cr_frame.f_code.co_filename,
+            "firstline": task._coro.cr_frame.f_code.co_firstlineno,
+            "linelo": task._coro.cr_frame.f_lineno,
+            "stack": traceback.format_stack(task._coro.cr_frame),
+            "task": task if with_task else None,
+        }
+        for task in tasks
+    ]
+    if cancel:
+        for t in tasks:
+            cbs = t._callbacks
+            t.cancel()
+            asyncio.futures.Future.set_exception(t, exc)
+            asyncio.futures.Future.cancel(t)
+            [cb[0](t) for cb in cbs]  # cancels any dependent concurrent.futures
+            try:
+                t._coro.throw(exc)  # exits coro, unless explicitly handled
+            except exc:
+                pass
+    return out
+
+
+class AbstractAsyncStreamedFile(AbstractBufferedFile):
+    # no read buffering, and always auto-commit
+    # TODO: readahead might still be useful here, but needs async version
+
+    async def read(self, length=-1):
+        """
+        Return data from cache, or fetch pieces as necessary
+
+        Parameters
+        ----------
+        length: int (-1)
+            Number of bytes to read; if <0, all remaining bytes.
+        """
+        length = -1 if length is None else int(length)
+        if self.mode != "rb":
+            raise ValueError("File not in read mode")
+        if length < 0:
+            length = self.size - self.loc
+        if self.closed:
+            raise ValueError("I/O operation on closed file.")
+        if length == 0:
+            # don't even bother calling fetch
+            return b""
+        out = await self._fetch_range(self.loc, self.loc + length)
+        self.loc += len(out)
+        return out
+
+    async def write(self, data):
+        """
+        Write data to buffer.
+
+        Buffer only sent on flush() or if buffer is greater than
+        or equal to blocksize.
+
+        Parameters
+        ----------
+        data: bytes
+            Set of bytes to be written.
+        """
+        if self.mode not in {"wb", "ab"}:
+            raise ValueError("File not in write mode")
+        if self.closed:
+            raise ValueError("I/O operation on closed file.")
+        if self.forced:
+            raise ValueError("This file has been force-flushed, can only close")
+        out = self.buffer.write(data)
+        self.loc += out
+        if self.buffer.tell() >= self.blocksize:
+            await self.flush()
+        return out
+
+    async def close(self):
+        """Close file
+
+        Finalizes writes, discards cache
+        """
+        if getattr(self, "_unclosable", False):
+            return
+        if self.closed:
+            return
+        if self.mode == "rb":
+            self.cache = None
+        else:
+            if not self.forced:
+                await self.flush(force=True)
+
+            if self.fs is not None:
+                self.fs.invalidate_cache(self.path)
+                self.fs.invalidate_cache(self.fs._parent(self.path))
+
+        self.closed = True
+
+    async def flush(self, force=False):
+        if self.closed:
+            raise ValueError("Flush on closed file")
+        if force and self.forced:
+            raise ValueError("Force flush cannot be called more than once")
+        if force:
+            self.forced = True
+
+        if self.mode not in {"wb", "ab"}:
+            # no-op to flush on read-mode
+            return
+
+        if not force and self.buffer.tell() < self.blocksize:
+            # Defer write on small block
+            return
+
+        if self.offset is None:
+            # Initialize a multipart upload
+            self.offset = 0
+            try:
+                await self._initiate_upload()
+            except:
+                self.closed = True
+                raise
+
+        if await self._upload_chunk(final=force) is not False:
+            self.offset += self.buffer.seek(0, 2)
+            self.buffer = io.BytesIO()
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        await self.close()
+
+    async def _fetch_range(self, start, end):
+        raise NotImplementedError
+
+    async def _initiate_upload(self):
+        pass
+
+    async def _upload_chunk(self, final=False):
+        raise NotImplementedError

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/fsspec/fuse.py

@@ -0,0 +1,324 @@
+import argparse
+import logging
+import os
+import stat
+import threading
+import time
+from errno import EIO, ENOENT
+
+from fuse import FUSE, FuseOSError, LoggingMixIn, Operations
+
+from fsspec import __version__
+from fsspec.core import url_to_fs
+
+logger = logging.getLogger("fsspec.fuse")
+
+
+class FUSEr(Operations):
+    def __init__(self, fs, path, ready_file=False):
+        self.fs = fs
+        self.cache = {}
+        self.root = path.rstrip("/") + "/"
+        self.counter = 0
+        logger.info("Starting FUSE at %s", path)
+        self._ready_file = ready_file
+
+    def getattr(self, path, fh=None):
+        logger.debug("getattr %s", path)
+        if self._ready_file and path in ["/.fuse_ready", ".fuse_ready"]:
+            return {"type": "file", "st_size": 5}
+
+        path = "".join([self.root, path.lstrip("/")]).rstrip("/")
+        try:
+            info = self.fs.info(path)
+        except FileNotFoundError as exc:
+            raise FuseOSError(ENOENT) from exc
+
+        data = {"st_uid": info.get("uid", 1000), "st_gid": info.get("gid", 1000)}
+        perm = info.get("mode", 0o777)
+
+        if info["type"] != "file":
+            data["st_mode"] = stat.S_IFDIR | perm
+            data["st_size"] = 0
+            data["st_blksize"] = 0
+        else:
+            data["st_mode"] = stat.S_IFREG | perm
+            data["st_size"] = info["size"]
+            data["st_blksize"] = 5 * 2**20
+            data["st_nlink"] = 1
+        data["st_atime"] = info["atime"] if "atime" in info else time.time()
+        data["st_ctime"] = info["ctime"] if "ctime" in info else time.time()
+        data["st_mtime"] = info["mtime"] if "mtime" in info else time.time()
+        return data
+
+    def readdir(self, path, fh):
+        logger.debug("readdir %s", path)
+        path = "".join([self.root, path.lstrip("/")])
+        files = self.fs.ls(path, False)
+        files = [os.path.basename(f.rstrip("/")) for f in files]
+        return [".", ".."] + files
+
+    def mkdir(self, path, mode):
+        path = "".join([self.root, path.lstrip("/")])
+        self.fs.mkdir(path)
+        return 0
+
+    def rmdir(self, path):
+        path = "".join([self.root, path.lstrip("/")])
+        self.fs.rmdir(path)
+        return 0
+
+    def read(self, path, size, offset, fh):
+        logger.debug("read %s", (path, size, offset))
+        if self._ready_file and path in ["/.fuse_ready", ".fuse_ready"]:
+            # status indicator
+            return b"ready"
+
+        f = self.cache[fh]
+        f.seek(offset)
+        out = f.read(size)
+        return out
+
+    def write(self, path, data, offset, fh):
+        logger.debug("write %s", (path, offset))
+        f = self.cache[fh]
+        f.seek(offset)
+        f.write(data)
+        return len(data)
+
+    def create(self, path, flags, fi=None):
+        logger.debug("create %s", (path, flags))
+        fn = "".join([self.root, path.lstrip("/")])
+        self.fs.touch(fn)  # OS will want to get attributes immediately
+        f = self.fs.open(fn, "wb")
+        self.cache[self.counter] = f
+        self.counter += 1
+        return self.counter - 1
+
+    def open(self, path, flags):
+        logger.debug("open %s", (path, flags))
+        fn = "".join([self.root, path.lstrip("/")])
+        if flags % 2 == 0:
+            # read
+            mode = "rb"
+        else:
+            # write/create
+            mode = "wb"
+        self.cache[self.counter] = self.fs.open(fn, mode)
+        self.counter += 1
+        return self.counter - 1
+
+    def truncate(self, path, length, fh=None):
+        fn = "".join([self.root, path.lstrip("/")])
+        if length != 0:
+            raise NotImplementedError
+        # maybe should be no-op since open with write sets size to zero anyway
+        self.fs.touch(fn)
+
+    def unlink(self, path):
+        fn = "".join([self.root, path.lstrip("/")])
+        try:
+            self.fs.rm(fn, False)
+        except (OSError, FileNotFoundError) as exc:
+            raise FuseOSError(EIO) from exc
+
+    def release(self, path, fh):
+        try:
+            if fh in self.cache:
+                f = self.cache[fh]
+                f.close()
+                self.cache.pop(fh)
+        except Exception as e:
+            print(e)
+        return 0
+
+    def chmod(self, path, mode):
+        if hasattr(self.fs, "chmod"):
+            path = "".join([self.root, path.lstrip("/")])
+            return self.fs.chmod(path, mode)
+        raise NotImplementedError
+
+
+def run(
+    fs,
+    path,
+    mount_point,
+    foreground=True,
+    threads=False,
+    ready_file=False,
+    ops_class=FUSEr,
+):
+    """Mount stuff in a local directory
+
+    This uses fusepy to make it appear as if a given path on an fsspec
+    instance is in fact resident within the local file-system.
+
+    This requires that fusepy by installed, and that FUSE be available on
+    the system (typically requiring a package to be installed with
+    apt, yum, brew, etc.).
+
+    Parameters
+    ----------
+    fs: file-system instance
+        From one of the compatible implementations
+    path: str
+        Location on that file-system to regard as the root directory to
+        mount. Note that you typically should include the terminating "/"
+        character.
+    mount_point: str
+        An empty directory on the local file-system where the contents of
+        the remote path will appear.
+    foreground: bool
+        Whether or not calling this function will block. Operation will
+        typically be more stable if True.
+    threads: bool
+        Whether or not to create threads when responding to file operations
+        within the mounter directory. Operation will typically be more
+        stable if False.
+    ready_file: bool
+        Whether the FUSE process is ready. The ``.fuse_ready`` file will
+        exist in the ``mount_point`` directory if True. Debugging purpose.
+    ops_class: FUSEr or Subclass of FUSEr
+        To override the default behavior of FUSEr. For Example, logging
+        to file.
+
+    """
+    func = lambda: FUSE(
+        ops_class(fs, path, ready_file=ready_file),
+        mount_point,
+        nothreads=not threads,
+        foreground=foreground,
+    )
+    if not foreground:
+        th = threading.Thread(target=func)
+        th.daemon = True
+        th.start()
+        return th
+    else:  # pragma: no cover
+        try:
+            func()
+        except KeyboardInterrupt:
+            pass
+
+
+def main(args):
+    """Mount filesystem from chained URL to MOUNT_POINT.
+
+    Examples:
+
+    python3 -m fsspec.fuse memory /usr/share /tmp/mem
+
+    python3 -m fsspec.fuse local /tmp/source /tmp/local \\
+            -l /tmp/fsspecfuse.log
+
+    You can also mount chained-URLs and use special settings:
+
+    python3 -m fsspec.fuse 'filecache::zip::file://data.zip' \\
+            / /tmp/zip \\
+            -o 'filecache-cache_storage=/tmp/simplecache'
+
+    You can specify the type of the setting by using `[int]` or `[bool]`,
+    (`true`, `yes`, `1` represents the Boolean value `True`):
+
+    python3 -m fsspec.fuse 'simplecache::ftp://ftp1.at.proftpd.org' \\
+            /historic/packages/RPMS /tmp/ftp \\
+            -o 'simplecache-cache_storage=/tmp/simplecache' \\
+            -o 'simplecache-check_files=false[bool]' \\
+            -o 'ftp-listings_expiry_time=60[int]' \\
+            -o 'ftp-username=anonymous' \\
+            -o 'ftp-password=xieyanbo'
+    """
+
+    class RawDescriptionArgumentParser(argparse.ArgumentParser):
+        def format_help(self):
+            usage = super().format_help()
+            parts = usage.split("\n\n")
+            parts[1] = self.description.rstrip()
+            return "\n\n".join(parts)
+
+    parser = RawDescriptionArgumentParser(prog="fsspec.fuse", description=main.__doc__)
+    parser.add_argument("--version", action="version", version=__version__)
+    parser.add_argument("url", type=str, help="fs url")
+    parser.add_argument("source_path", type=str, help="source directory in fs")
+    parser.add_argument("mount_point", type=str, help="local directory")
+    parser.add_argument(
+        "-o",
+        "--option",
+        action="append",
+        help="Any options of protocol included in the chained URL",
+    )
+    parser.add_argument(
+        "-l", "--log-file", type=str, help="Logging FUSE debug info (Default: '')"
+    )
+    parser.add_argument(
+        "-f",
+        "--foreground",
+        action="store_false",
+        help="Running in foreground or not (Default: False)",
+    )
+    parser.add_argument(
+        "-t",
+        "--threads",
+        action="store_false",
+        help="Running with threads support (Default: False)",
+    )
+    parser.add_argument(
+        "-r",
+        "--ready-file",
+        action="store_false",
+        help="The `.fuse_ready` file will exist after FUSE is ready. "
+        "(Debugging purpose, Default: False)",
+    )
+    args = parser.parse_args(args)
+
+    kwargs = {}
+    for item in args.option or []:
+        key, sep, value = item.partition("=")
+        if not sep:
+            parser.error(message=f"Wrong option: {item!r}")
+        val = value.lower()
+        if val.endswith("[int]"):
+            value = int(value[: -len("[int]")])
+        elif val.endswith("[bool]"):
+            value = val[: -len("[bool]")] in ["1", "yes", "true"]
+
+        if "-" in key:
+            fs_name, setting_name = key.split("-", 1)
+            if fs_name in kwargs:
+                kwargs[fs_name][setting_name] = value
+            else:
+                kwargs[fs_name] = {setting_name: value}
+        else:
+            kwargs[key] = value
+
+    if args.log_file:
+        logging.basicConfig(
+            level=logging.DEBUG,
+            filename=args.log_file,
+            format="%(asctime)s %(message)s",
+        )
+
+        class LoggingFUSEr(FUSEr, LoggingMixIn):
+            pass
+
+        fuser = LoggingFUSEr
+    else:
+        fuser = FUSEr
+
+    fs, url_path = url_to_fs(args.url, **kwargs)
+    logger.debug("Mounting %s to %s", url_path, str(args.mount_point))
+    run(
+        fs,
+        args.source_path,
+        args.mount_point,
+        foreground=args.foreground,
+        threads=args.threads,
+        ready_file=args.ready_file,
+        ops_class=fuser,
+    )
+
+
+if __name__ == "__main__":
+    import sys
+
+    main(sys.argv[1:])

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/fsspec/generic.py

@@ -0,0 +1,396 @@
+from __future__ import annotations
+
+import inspect
+import logging
+import os
+import shutil
+import uuid
+
+from .asyn import AsyncFileSystem, _run_coros_in_chunks, sync_wrapper
+from .callbacks import DEFAULT_CALLBACK
+from .core import filesystem, get_filesystem_class, split_protocol, url_to_fs
+
+_generic_fs = {}
+logger = logging.getLogger("fsspec.generic")
+
+
+def set_generic_fs(protocol, **storage_options):
+    """Populate the dict used for method=="generic" lookups"""
+    _generic_fs[protocol] = filesystem(protocol, **storage_options)
+
+
+def _resolve_fs(url, method, protocol=None, storage_options=None):
+    """Pick instance of backend FS"""
+    url = url[0] if isinstance(url, (list, tuple)) else url
+    protocol = protocol or split_protocol(url)[0]
+    storage_options = storage_options or {}
+    if method == "default":
+        return filesystem(protocol)
+    if method == "generic":
+        return _generic_fs[protocol]
+    if method == "current":
+        cls = get_filesystem_class(protocol)
+        return cls.current()
+    if method == "options":
+        fs, _ = url_to_fs(url, **storage_options.get(protocol, {}))
+        return fs
+    raise ValueError(f"Unknown FS resolution method: {method}")
+
+
+def rsync(
+    source,
+    destination,
+    delete_missing=False,
+    source_field="size",
+    dest_field="size",
+    update_cond="different",
+    inst_kwargs=None,
+    fs=None,
+    **kwargs,
+):
+    """Sync files between two directory trees
+
+    (experimental)
+
+    Parameters
+    ----------
+    source: str
+        Root of the directory tree to take files from. This must be a directory, but
+        do not include any terminating "/" character
+    destination: str
+        Root path to copy into. The contents of this location should be
+        identical to the contents of ``source`` when done. This will be made a
+        directory, and the terminal "/" should not be included.
+    delete_missing: bool
+        If there are paths in the destination that don't exist in the
+        source and this is True, delete them. Otherwise, leave them alone.
+    source_field: str | callable
+        If ``update_field`` is "different", this is the key in the info
+        of source files to consider for difference. Maybe a function of the
+        info dict.
+    dest_field: str | callable
+        If ``update_field`` is "different", this is the key in the info
+        of destination files to consider for difference. May be a function of
+        the info dict.
+    update_cond: "different"|"always"|"never"
+        If "always", every file is copied, regardless of whether it exists in
+        the destination. If "never", files that exist in the destination are
+        not copied again. If "different" (default), only copy if the info
+        fields given by ``source_field`` and ``dest_field`` (usually "size")
+        are different. Other comparisons may be added in the future.
+    inst_kwargs: dict|None
+        If ``fs`` is None, use this set of keyword arguments to make a
+        GenericFileSystem instance
+    fs: GenericFileSystem|None
+        Instance to use if explicitly given. The instance defines how to
+        to make downstream file system instances from paths.
+
+    Returns
+    -------
+    dict of the copy operations that were performed, {source: destination}
+    """
+    fs = fs or GenericFileSystem(**(inst_kwargs or {}))
+    source = fs._strip_protocol(source)
+    destination = fs._strip_protocol(destination)
+    allfiles = fs.find(source, withdirs=True, detail=True)
+    if not fs.isdir(source):
+        raise ValueError("Can only rsync on a directory")
+    otherfiles = fs.find(destination, withdirs=True, detail=True)
+    dirs = [
+        a
+        for a, v in allfiles.items()
+        if v["type"] == "directory" and a.replace(source, destination) not in otherfiles
+    ]
+    logger.debug(f"{len(dirs)} directories to create")
+    if dirs:
+        fs.make_many_dirs(
+            [dirn.replace(source, destination) for dirn in dirs], exist_ok=True
+        )
+    allfiles = {a: v for a, v in allfiles.items() if v["type"] == "file"}
+    logger.debug(f"{len(allfiles)} files to consider for copy")
+    to_delete = [
+        o
+        for o, v in otherfiles.items()
+        if o.replace(destination, source) not in allfiles and v["type"] == "file"
+    ]
+    for k, v in allfiles.copy().items():
+        otherfile = k.replace(source, destination)
+        if otherfile in otherfiles:
+            if update_cond == "always":
+                allfiles[k] = otherfile
+            elif update_cond == "never":
+                allfiles.pop(k)
+            elif update_cond == "different":
+                inf1 = source_field(v) if callable(source_field) else v[source_field]
+                v2 = otherfiles[otherfile]
+                inf2 = dest_field(v2) if callable(dest_field) else v2[dest_field]
+                if inf1 != inf2:
+                    # details mismatch, make copy
+                    allfiles[k] = otherfile
+                else:
+                    # details match, don't copy
+                    allfiles.pop(k)
+        else:
+            # file not in target yet
+            allfiles[k] = otherfile
+    logger.debug(f"{len(allfiles)} files to copy")
+    if allfiles:
+        source_files, target_files = zip(*allfiles.items())
+        fs.cp(source_files, target_files, **kwargs)
+    logger.debug(f"{len(to_delete)} files to delete")
+    if delete_missing and to_delete:
+        fs.rm(to_delete)
+    return allfiles
+
+
+class GenericFileSystem(AsyncFileSystem):
+    """Wrapper over all other FS types
+
+    <experimental!>
+
+    This implementation is a single unified interface to be able to run FS operations
+    over generic URLs, and dispatch to the specific implementations using the URL
+    protocol prefix.
+
+    Note: instances of this FS are always async, even if you never use it with any async
+    backend.
+    """
+
+    protocol = "generic"  # there is no real reason to ever use a protocol with this FS
+
+    def __init__(self, default_method="default", storage_options=None, **kwargs):
+        """
+
+        Parameters
+        ----------
+        default_method: str (optional)
+            Defines how to configure backend FS instances. Options are:
+            - "default": instantiate like FSClass(), with no
+              extra arguments; this is the default instance of that FS, and can be
+              configured via the config system
+            - "generic": takes instances from the `_generic_fs` dict in this module,
+              which you must populate before use. Keys are by protocol
+            - "options": expects storage_options, a dict mapping protocol to
+              kwargs to use when constructing the filesystem
+            - "current": takes the most recently instantiated version of each FS
+        """
+        self.method = default_method
+        self.st_opts = storage_options
+        super().__init__(**kwargs)
+
+    def _parent(self, path):
+        fs = _resolve_fs(path, self.method, storage_options=self.st_opts)
+        return fs.unstrip_protocol(fs._parent(path))
+
+    def _strip_protocol(self, path):
+        # normalization only
+        fs = _resolve_fs(path, self.method, storage_options=self.st_opts)
+        return fs.unstrip_protocol(fs._strip_protocol(path))
+
+    async def _find(self, path, maxdepth=None, withdirs=False, detail=False, **kwargs):
+        fs = _resolve_fs(path, self.method, storage_options=self.st_opts)
+        if fs.async_impl:
+            out = await fs._find(
+                path, maxdepth=maxdepth, withdirs=withdirs, detail=True, **kwargs
+            )
+        else:
+            out = fs.find(
+                path, maxdepth=maxdepth, withdirs=withdirs, detail=True, **kwargs
+            )
+        result = {}
+        for k, v in out.items():
+            v = v.copy()  # don't corrupt target FS dircache
+            name = fs.unstrip_protocol(k)
+            v["name"] = name
+            result[name] = v
+        if detail:
+            return result
+        return list(result)
+
+    async def _info(self, url, **kwargs):
+        fs = _resolve_fs(url, self.method)
+        if fs.async_impl:
+            out = await fs._info(url, **kwargs)
+        else:
+            out = fs.info(url, **kwargs)
+        out = out.copy()  # don't edit originals
+        out["name"] = fs.unstrip_protocol(out["name"])
+        return out
+
+    async def _ls(
+        self,
+        url,
+        detail=True,
+        **kwargs,
+    ):
+        fs = _resolve_fs(url, self.method)
+        if fs.async_impl:
+            out = await fs._ls(url, detail=True, **kwargs)
+        else:
+            out = fs.ls(url, detail=True, **kwargs)
+        out = [o.copy() for o in out]  # don't edit originals
+        for o in out:
+            o["name"] = fs.unstrip_protocol(o["name"])
+        if detail:
+            return out
+        else:
+            return [o["name"] for o in out]
+
+    async def _cat_file(
+        self,
+        url,
+        **kwargs,
+    ):
+        fs = _resolve_fs(url, self.method)
+        if fs.async_impl:
+            return await fs._cat_file(url, **kwargs)
+        else:
+            return fs.cat_file(url, **kwargs)
+
+    async def _pipe_file(
+        self,
+        path,
+        value,
+        **kwargs,
+    ):
+        fs = _resolve_fs(path, self.method, storage_options=self.st_opts)
+        if fs.async_impl:
+            return await fs._pipe_file(path, value, **kwargs)
+        else:
+            return fs.pipe_file(path, value, **kwargs)
+
+    async def _rm(self, url, **kwargs):
+        urls = url
+        if isinstance(urls, str):
+            urls = [urls]
+        fs = _resolve_fs(urls[0], self.method)
+        if fs.async_impl:
+            await fs._rm(urls, **kwargs)
+        else:
+            fs.rm(url, **kwargs)
+
+    async def _makedirs(self, path, exist_ok=False):
+        logger.debug("Make dir %s", path)
+        fs = _resolve_fs(path, self.method, storage_options=self.st_opts)
+        if fs.async_impl:
+            await fs._makedirs(path, exist_ok=exist_ok)
+        else:
+            fs.makedirs(path, exist_ok=exist_ok)
+
+    def rsync(self, source, destination, **kwargs):
+        """Sync files between two directory trees
+
+        See `func:rsync` for more details.
+        """
+        rsync(source, destination, fs=self, **kwargs)
+
+    async def _cp_file(
+        self,
+        url,
+        url2,
+        blocksize=2**20,
+        callback=DEFAULT_CALLBACK,
+        tempdir: str | None = None,
+        **kwargs,
+    ):
+        fs = _resolve_fs(url, self.method)
+        fs2 = _resolve_fs(url2, self.method)
+        if fs is fs2:
+            # pure remote
+            if fs.async_impl:
+                return await fs._copy(url, url2, **kwargs)
+            else:
+                return fs.copy(url, url2, **kwargs)
+        await copy_file_op(fs, [url], fs2, [url2], tempdir, 1, on_error="raise")
+
+    async def _make_many_dirs(self, urls, exist_ok=True):
+        fs = _resolve_fs(urls[0], self.method)
+        if fs.async_impl:
+            coros = [fs._makedirs(u, exist_ok=exist_ok) for u in urls]
+            await _run_coros_in_chunks(coros)
+        else:
+            for u in urls:
+                fs.makedirs(u, exist_ok=exist_ok)
+
+    make_many_dirs = sync_wrapper(_make_many_dirs)
+
+    async def _copy(
+        self,
+        path1: list[str],
+        path2: list[str],
+        recursive: bool = False,
+        on_error: str = "ignore",
+        maxdepth: int | None = None,
+        batch_size: int | None = None,
+        tempdir: str | None = None,
+        **kwargs,
+    ):
+        # TODO: special case for one FS being local, which can use get/put
+        # TODO: special case for one being memFS, which can use cat/pipe
+        if recursive:
+            raise NotImplementedError("Please use fsspec.generic.rsync")
+        path1 = [path1] if isinstance(path1, str) else path1
+        path2 = [path2] if isinstance(path2, str) else path2
+
+        fs = _resolve_fs(path1, self.method)
+        fs2 = _resolve_fs(path2, self.method)
+
+        if fs is fs2:
+            if fs.async_impl:
+                return await fs._copy(path1, path2, **kwargs)
+            else:
+                return fs.copy(path1, path2, **kwargs)
+
+        await copy_file_op(
+            fs, path1, fs2, path2, tempdir, batch_size, on_error=on_error
+        )
+
+
+async def copy_file_op(
+    fs1, url1, fs2, url2, tempdir=None, batch_size=20, on_error="ignore"
+):
+    import tempfile
+
+    tempdir = tempdir or tempfile.mkdtemp()
+    try:
+        coros = [
+            _copy_file_op(
+                fs1,
+                u1,
+                fs2,
+                u2,
+                os.path.join(tempdir, uuid.uuid4().hex),
+            )
+            for u1, u2 in zip(url1, url2)
+        ]
+        out = await _run_coros_in_chunks(
+            coros, batch_size=batch_size, return_exceptions=True
+        )
+    finally:
+        shutil.rmtree(tempdir)
+    if on_error == "return":
+        return out
+    elif on_error == "raise":
+        for o in out:
+            if isinstance(o, Exception):
+                raise o
+
+
+async def _copy_file_op(fs1, url1, fs2, url2, local, on_error="ignore"):
+    if fs1.async_impl:
+        await fs1._get_file(url1, local)
+    else:
+        fs1.get_file(url1, local)
+    if fs2.async_impl:
+        await fs2._put_file(local, url2)
+    else:
+        fs2.put_file(local, url2)
+    os.unlink(local)
+    logger.debug("Copy %s -> %s; done", url1, url2)
+
+
+async def maybe_await(cor):
+    if inspect.iscoroutine(cor):
+        return await cor
+    else:
+        return cor

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/fsspec/mapping.py

@@ -0,0 +1,251 @@
+import array
+import logging
+import posixpath
+import warnings
+from collections.abc import MutableMapping
+from functools import cached_property
+
+from fsspec.core import url_to_fs
+
+logger = logging.getLogger("fsspec.mapping")
+
+
+class FSMap(MutableMapping):
+    """Wrap a FileSystem instance as a mutable wrapping.
+
+    The keys of the mapping become files under the given root, and the
+    values (which must be bytes) the contents of those files.
+
+    Parameters
+    ----------
+    root: string
+        prefix for all the files
+    fs: FileSystem instance
+    check: bool (=True)
+        performs a touch at the location, to check for write access.
+
+    Examples
+    --------
+    >>> fs = FileSystem(**parameters)  # doctest: +SKIP
+    >>> d = FSMap('my-data/path/', fs)  # doctest: +SKIP
+    or, more likely
+    >>> d = fs.get_mapper('my-data/path/')
+
+    >>> d['loc1'] = b'Hello World'  # doctest: +SKIP
+    >>> list(d.keys())  # doctest: +SKIP
+    ['loc1']
+    >>> d['loc1']  # doctest: +SKIP
+    b'Hello World'
+    """
+
+    def __init__(self, root, fs, check=False, create=False, missing_exceptions=None):
+        self.fs = fs
+        self.root = fs._strip_protocol(root)
+        self._root_key_to_str = fs._strip_protocol(posixpath.join(root, "x"))[:-1]
+        if missing_exceptions is None:
+            missing_exceptions = (
+                FileNotFoundError,
+                IsADirectoryError,
+                NotADirectoryError,
+            )
+        self.missing_exceptions = missing_exceptions
+        self.check = check
+        self.create = create
+        if create:
+            if not self.fs.exists(root):
+                self.fs.mkdir(root)
+        if check:
+            if not self.fs.exists(root):
+                raise ValueError(
+                    f"Path {root} does not exist. Create "
+                    f" with the ``create=True`` keyword"
+                )
+            self.fs.touch(root + "/a")
+            self.fs.rm(root + "/a")
+
+    @cached_property
+    def dirfs(self):
+        """dirfs instance that can be used with the same keys as the mapper"""
+        from .implementations.dirfs import DirFileSystem
+
+        return DirFileSystem(path=self._root_key_to_str, fs=self.fs)
+
+    def clear(self):
+        """Remove all keys below root - empties out mapping"""
+        logger.info("Clear mapping at %s", self.root)
+        try:
+            self.fs.rm(self.root, True)
+            self.fs.mkdir(self.root)
+        except:  # noqa: E722
+            pass
+
+    def getitems(self, keys, on_error="raise"):
+        """Fetch multiple items from the store
+
+        If the backend is async-able, this might proceed concurrently
+
+        Parameters
+        ----------
+        keys: list(str)
+            They keys to be fetched
+        on_error : "raise", "omit", "return"
+            If raise, an underlying exception will be raised (converted to KeyError
+            if the type is in self.missing_exceptions); if omit, keys with exception
+            will simply not be included in the output; if "return", all keys are
+            included in the output, but the value will be bytes or an exception
+            instance.
+
+        Returns
+        -------
+        dict(key, bytes|exception)
+        """
+        keys2 = [self._key_to_str(k) for k in keys]
+        oe = on_error if on_error == "raise" else "return"
+        try:
+            out = self.fs.cat(keys2, on_error=oe)
+            if isinstance(out, bytes):
+                out = {keys2[0]: out}
+        except self.missing_exceptions as e:
+            raise KeyError from e
+        out = {
+            k: (KeyError() if isinstance(v, self.missing_exceptions) else v)
+            for k, v in out.items()
+        }
+        return {
+            key: out[k2] if on_error == "raise" else out.get(k2, KeyError(k2))
+            for key, k2 in zip(keys, keys2)
+            if on_error == "return" or not isinstance(out[k2], BaseException)
+        }
+
+    def setitems(self, values_dict):
+        """Set the values of multiple items in the store
+
+        Parameters
+        ----------
+        values_dict: dict(str, bytes)
+        """
+        values = {self._key_to_str(k): maybe_convert(v) for k, v in values_dict.items()}
+        self.fs.pipe(values)
+
+    def delitems(self, keys):
+        """Remove multiple keys from the store"""
+        self.fs.rm([self._key_to_str(k) for k in keys])
+
+    def _key_to_str(self, key):
+        """Generate full path for the key"""
+        if not isinstance(key, str):
+            # raise TypeError("key must be of type `str`, got `{type(key).__name__}`"
+            warnings.warn(
+                "from fsspec 2023.5 onward FSMap non-str keys will raise TypeError",
+                DeprecationWarning,
+            )
+            if isinstance(key, list):
+                key = tuple(key)
+            key = str(key)
+        return f"{self._root_key_to_str}{key}".rstrip("/")
+
+    def _str_to_key(self, s):
+        """Strip path of to leave key name"""
+        return s[len(self.root) :].lstrip("/")
+
+    def __getitem__(self, key, default=None):
+        """Retrieve data"""
+        k = self._key_to_str(key)
+        try:
+            result = self.fs.cat(k)
+        except self.missing_exceptions as exc:
+            if default is not None:
+                return default
+            raise KeyError(key) from exc
+        return result
+
+    def pop(self, key, default=None):
+        """Pop data"""
+        result = self.__getitem__(key, default)
+        try:
+            del self[key]
+        except KeyError:
+            pass
+        return result
+
+    def __setitem__(self, key, value):
+        """Store value in key"""
+        key = self._key_to_str(key)
+        self.fs.mkdirs(self.fs._parent(key), exist_ok=True)
+        self.fs.pipe_file(key, maybe_convert(value))
+
+    def __iter__(self):
+        return (self._str_to_key(x) for x in self.fs.find(self.root))
+
+    def __len__(self):
+        return len(self.fs.find(self.root))
+
+    def __delitem__(self, key):
+        """Remove key"""
+        try:
+            self.fs.rm(self._key_to_str(key))
+        except Exception as exc:
+            raise KeyError from exc
+
+    def __contains__(self, key):
+        """Does key exist in mapping?"""
+        path = self._key_to_str(key)
+        return self.fs.isfile(path)
+
+    def __reduce__(self):
+        return FSMap, (self.root, self.fs, False, False, self.missing_exceptions)
+
+
+def maybe_convert(value):
+    if isinstance(value, array.array) or hasattr(value, "__array__"):
+        # bytes-like things
+        if hasattr(value, "dtype") and value.dtype.kind in "Mm":
+            # The buffer interface doesn't support datetime64/timdelta64 numpy
+            # arrays
+            value = value.view("int64")
+        value = bytes(memoryview(value))
+    return value
+
+
+def get_mapper(
+    url="",
+    check=False,
+    create=False,
+    missing_exceptions=None,
+    alternate_root=None,
+    **kwargs,
+):
+    """Create key-value interface for given URL and options
+
+    The URL will be of the form "protocol://location" and point to the root
+    of the mapper required. All keys will be file-names below this location,
+    and their values the contents of each key.
+
+    Also accepts compound URLs like zip::s3://bucket/file.zip , see ``fsspec.open``.
+
+    Parameters
+    ----------
+    url: str
+        Root URL of mapping
+    check: bool
+        Whether to attempt to read from the location before instantiation, to
+        check that the mapping does exist
+    create: bool
+        Whether to make the directory corresponding to the root before
+        instantiating
+    missing_exceptions: None or tuple
+        If given, these exception types will be regarded as missing keys and
+        return KeyError when trying to read data. By default, you get
+        (FileNotFoundError, IsADirectoryError, NotADirectoryError)
+    alternate_root: None or str
+        In cases of complex URLs, the parser may fail to pick the correct part
+        for the mapper root, so this arg can override
+
+    Returns
+    -------
+    ``FSMap`` instance, the dict-like key-value store.
+    """
+    # Removing protocol here - could defer to each open() on the backend
+    fs, urlpath = url_to_fs(url, **kwargs)
+    root = alternate_root if alternate_root is not None else urlpath
+    return FSMap(root, fs, check, create, missing_exceptions=missing_exceptions)

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/fsspec/utils.py

@@ -0,0 +1,748 @@
+from __future__ import annotations
+
+import contextlib
+import logging
+import math
+import os
+import re
+import sys
+import tempfile
+from collections.abc import Callable, Iterable, Iterator, Sequence
+from functools import partial
+from hashlib import md5
+from importlib.metadata import version
+from typing import IO, TYPE_CHECKING, Any, TypeVar
+from urllib.parse import urlsplit
+
+if TYPE_CHECKING:
+    import pathlib
+    from typing import TypeGuard
+
+    from fsspec.spec import AbstractFileSystem
+
+
+DEFAULT_BLOCK_SIZE = 5 * 2**20
+
+T = TypeVar("T")
+
+
+def infer_storage_options(
+    urlpath: str, inherit_storage_options: dict[str, Any] | None = None
+) -> dict[str, Any]:
+    """Infer storage options from URL path and merge it with existing storage
+    options.
+
+    Parameters
+    ----------
+    urlpath: str or unicode
+        Either local absolute file path or URL (hdfs://namenode:8020/file.csv)
+    inherit_storage_options: dict (optional)
+        Its contents will get merged with the inferred information from the
+        given path
+
+    Returns
+    -------
+    Storage options dict.
+
+    Examples
+    --------
+    >>> infer_storage_options('/mnt/datasets/test.csv')  # doctest: +SKIP
+    {"protocol": "file", "path", "/mnt/datasets/test.csv"}
+    >>> infer_storage_options(
+    ...     'hdfs://username:pwd@node:123/mnt/datasets/test.csv?q=1',
+    ...     inherit_storage_options={'extra': 'value'},
+    ... )  # doctest: +SKIP
+    {"protocol": "hdfs", "username": "username", "password": "pwd",
+    "host": "node", "port": 123, "path": "/mnt/datasets/test.csv",
+    "url_query": "q=1", "extra": "value"}
+    """
+    # Handle Windows paths including disk name in this special case
+    if (
+        re.match(r"^[a-zA-Z]:[\\/]", urlpath)
+        or re.match(r"^[a-zA-Z0-9]+://", urlpath) is None
+    ):
+        return {"protocol": "file", "path": urlpath}
+
+    parsed_path = urlsplit(urlpath)
+    protocol = parsed_path.scheme or "file"
+    if parsed_path.fragment:
+        path = "#".join([parsed_path.path, parsed_path.fragment])
+    else:
+        path = parsed_path.path
+    if protocol == "file":
+        # Special case parsing file protocol URL on Windows according to:
+        # https://msdn.microsoft.com/en-us/library/jj710207.aspx
+        windows_path = re.match(r"^/([a-zA-Z])[:|]([\\/].*)$", path)
+        if windows_path:
+            drive, path = windows_path.groups()
+            path = f"{drive}:{path}"
+
+    if protocol in ["http", "https"]:
+        # for HTTP, we don't want to parse, as requests will anyway
+        return {"protocol": protocol, "path": urlpath}
+
+    options: dict[str, Any] = {"protocol": protocol, "path": path}
+
+    if parsed_path.netloc:
+        # Parse `hostname` from netloc manually because `parsed_path.hostname`
+        # lowercases the hostname which is not always desirable (e.g. in S3):
+        # https://github.com/dask/dask/issues/1417
+        options["host"] = parsed_path.netloc.rsplit("@", 1)[-1].rsplit(":", 1)[0]
+
+        if protocol in ("s3", "s3a", "gcs", "gs"):
+            options["path"] = options["host"] + options["path"]
+        else:
+            options["host"] = options["host"]
+        if parsed_path.port:
+            options["port"] = parsed_path.port
+        if parsed_path.username:
+            options["username"] = parsed_path.username
+        if parsed_path.password:
+            options["password"] = parsed_path.password
+
+    if parsed_path.query:
+        options["url_query"] = parsed_path.query
+    if parsed_path.fragment:
+        options["url_fragment"] = parsed_path.fragment
+
+    if inherit_storage_options:
+        update_storage_options(options, inherit_storage_options)
+
+    return options
+
+
+def update_storage_options(
+    options: dict[str, Any], inherited: dict[str, Any] | None = None
+) -> None:
+    if not inherited:
+        inherited = {}
+    collisions = set(options) & set(inherited)
+    if collisions:
+        for collision in collisions:
+            if options.get(collision) != inherited.get(collision):
+                raise KeyError(
+                    f"Collision between inferred and specified storage "
+                    f"option:\n{collision}"
+                )
+    options.update(inherited)
+
+
+# Compression extensions registered via fsspec.compression.register_compression
+compressions: dict[str, str] = {}
+
+
+def infer_compression(filename: str) -> str | None:
+    """Infer compression, if available, from filename.
+
+    Infer a named compression type, if registered and available, from filename
+    extension. This includes builtin (gz, bz2, zip) compressions, as well as
+    optional compressions. See fsspec.compression.register_compression.
+    """
+    extension = os.path.splitext(filename)[-1].strip(".").lower()
+    if extension in compressions:
+        return compressions[extension]
+    return None
+
+
+def build_name_function(max_int: float) -> Callable[[int], str]:
+    """Returns a function that receives a single integer
+    and returns it as a string padded by enough zero characters
+    to align with maximum possible integer
+
+    >>> name_f = build_name_function(57)
+
+    >>> name_f(7)
+    '07'
+    >>> name_f(31)
+    '31'
+    >>> build_name_function(1000)(42)
+    '0042'
+    >>> build_name_function(999)(42)
+    '042'
+    >>> build_name_function(0)(0)
+    '0'
+    """
+    # handle corner cases max_int is 0 or exact power of 10
+    max_int += 1e-8
+
+    pad_length = int(math.ceil(math.log10(max_int)))
+
+    def name_function(i: int) -> str:
+        return str(i).zfill(pad_length)
+
+    return name_function
+
+
+def seek_delimiter(file: IO[bytes], delimiter: bytes, blocksize: int) -> bool:
+    r"""Seek current file to file start, file end, or byte after delimiter seq.
+
+    Seeks file to next chunk delimiter, where chunks are defined on file start,
+    a delimiting sequence, and file end. Use file.tell() to see location afterwards.
+    Note that file start is a valid split, so must be at offset > 0 to seek for
+    delimiter.
+
+    Parameters
+    ----------
+    file: a file
+    delimiter: bytes
+        a delimiter like ``b'\n'`` or message sentinel, matching file .read() type
+    blocksize: int
+        Number of bytes to read from the file at once.
+
+
+    Returns
+    -------
+    Returns True if a delimiter was found, False if at file start or end.
+
+    """
+
+    if file.tell() == 0:
+        # beginning-of-file, return without seek
+        return False
+
+    # Interface is for binary IO, with delimiter as bytes, but initialize last
+    # with result of file.read to preserve compatibility with text IO.
+    last: bytes | None = None
+    while True:
+        current = file.read(blocksize)
+        if not current:
+            # end-of-file without delimiter
+            return False
+        full = last + current if last else current
+        try:
+            if delimiter in full:
+                i = full.index(delimiter)
+                file.seek(file.tell() - (len(full) - i) + len(delimiter))
+                return True
+            elif len(current) < blocksize:
+                # end-of-file without delimiter
+                return False
+        except (OSError, ValueError):
+            pass
+        last = full[-len(delimiter) :]
+
+
+def read_block(
+    f: IO[bytes],
+    offset: int,
+    length: int | None,
+    delimiter: bytes | None = None,
+    split_before: bool = False,
+) -> bytes:
+    """Read a block of bytes from a file
+
+    Parameters
+    ----------
+    f: File
+        Open file
+    offset: int
+        Byte offset to start read
+    length: int
+        Number of bytes to read, read through end of file if None
+    delimiter: bytes (optional)
+        Ensure reading starts and stops at delimiter bytestring
+    split_before: bool (optional)
+        Start/stop read *before* delimiter bytestring.
+
+
+    If using the ``delimiter=`` keyword argument we ensure that the read
+    starts and stops at delimiter boundaries that follow the locations
+    ``offset`` and ``offset + length``.  If ``offset`` is zero then we
+    start at zero, regardless of delimiter.  The bytestring returned WILL
+    include the terminating delimiter string.
+
+    Examples
+    --------
+
+    >>> from io import BytesIO  # doctest: +SKIP
+    >>> f = BytesIO(b'Alice, 100\\nBob, 200\\nCharlie, 300')  # doctest: +SKIP
+    >>> read_block(f, 0, 13)  # doctest: +SKIP
+    b'Alice, 100\\nBo'
+
+    >>> read_block(f, 0, 13, delimiter=b'\\n')  # doctest: +SKIP
+    b'Alice, 100\\nBob, 200\\n'
+
+    >>> read_block(f, 10, 10, delimiter=b'\\n')  # doctest: +SKIP
+    b'Bob, 200\\nCharlie, 300'
+    """
+    if delimiter:
+        f.seek(offset)
+        found_start_delim = seek_delimiter(f, delimiter, 2**16)
+        if length is None:
+            return f.read()
+        start = f.tell()
+        length -= start - offset
+
+        f.seek(start + length)
+        found_end_delim = seek_delimiter(f, delimiter, 2**16)
+        end = f.tell()
+
+        # Adjust split location to before delimiter if seek found the
+        # delimiter sequence, not start or end of file.
+        if found_start_delim and split_before:
+            start -= len(delimiter)
+
+        if found_end_delim and split_before:
+            end -= len(delimiter)
+
+        offset = start
+        length = end - start
+
+    f.seek(offset)
+
+    # TODO: allow length to be None and read to the end of the file?
+    assert length is not None
+    b = f.read(length)
+    return b
+
+
+def tokenize(*args: Any, **kwargs: Any) -> str:
+    """Deterministic token
+
+    (modified from dask.base)
+
+    >>> tokenize([1, 2, '3'])
+    '9d71491b50023b06fc76928e6eddb952'
+
+    >>> tokenize('Hello') == tokenize('Hello')
+    True
+    """
+    if kwargs:
+        args += (kwargs,)
+    try:
+        h = md5(str(args).encode())
+    except ValueError:
+        # FIPS systems: https://github.com/fsspec/filesystem_spec/issues/380
+        h = md5(str(args).encode(), usedforsecurity=False)
+    return h.hexdigest()
+
+
+def stringify_path(filepath: str | os.PathLike[str] | pathlib.Path) -> str:
+    """Attempt to convert a path-like object to a string.
+
+    Parameters
+    ----------
+    filepath: object to be converted
+
+    Returns
+    -------
+    filepath_str: maybe a string version of the object
+
+    Notes
+    -----
+    Objects supporting the fspath protocol are coerced according to its
+    __fspath__ method.
+
+    For backwards compatibility with older Python version, pathlib.Path
+    objects are specially coerced.
+
+    Any other object is passed through unchanged, which includes bytes,
+    strings, buffers, or anything else that's not even path-like.
+    """
+    if isinstance(filepath, str):
+        return filepath
+    elif hasattr(filepath, "__fspath__"):
+        return filepath.__fspath__()
+    elif hasattr(filepath, "path"):
+        return filepath.path
+    else:
+        return filepath  # type: ignore[return-value]
+
+
+def make_instance(
+    cls: Callable[..., T], args: Sequence[Any], kwargs: dict[str, Any]
+) -> T:
+    inst = cls(*args, **kwargs)
+    inst._determine_worker()  # type: ignore[attr-defined]
+    return inst
+
+
+def common_prefix(paths: Iterable[str]) -> str:
+    """For a list of paths, find the shortest prefix common to all"""
+    parts = [p.split("/") for p in paths]
+    lmax = min(len(p) for p in parts)
+    end = 0
+    for i in range(lmax):
+        end = all(p[i] == parts[0][i] for p in parts)
+        if not end:
+            break
+    i += end
+    return "/".join(parts[0][:i])
+
+
+def other_paths(
+    paths: list[str],
+    path2: str | list[str],
+    exists: bool = False,
+    flatten: bool = False,
+) -> list[str]:
+    """In bulk file operations, construct a new file tree from a list of files
+
+    Parameters
+    ----------
+    paths: list of str
+        The input file tree
+    path2: str or list of str
+        Root to construct the new list in. If this is already a list of str, we just
+        assert it has the right number of elements.
+    exists: bool (optional)
+        For a str destination, it is already exists (and is a dir), files should
+        end up inside.
+    flatten: bool (optional)
+        Whether to flatten the input directory tree structure so that the output files
+        are in the same directory.
+
+    Returns
+    -------
+    list of str
+    """
+
+    if isinstance(path2, str):
+        path2 = path2.rstrip("/")
+
+        if flatten:
+            path2 = ["/".join((path2, p.split("/")[-1])) for p in paths]
+        else:
+            cp = common_prefix(paths)
+            if exists:
+                cp = cp.rsplit("/", 1)[0]
+            if not cp and all(not s.startswith("/") for s in paths):
+                path2 = ["/".join([path2, p]) for p in paths]
+            else:
+                path2 = [p.replace(cp, path2, 1) for p in paths]
+    else:
+        assert len(paths) == len(path2)
+    return path2
+
+
+def is_exception(obj: Any) -> bool:
+    return isinstance(obj, BaseException)
+
+
+def isfilelike(f: Any) -> TypeGuard[IO[bytes]]:
+    return all(hasattr(f, attr) for attr in ["read", "close", "tell"])
+
+
+def get_protocol(url: str) -> str:
+    url = stringify_path(url)
+    parts = re.split(r"(\:\:|\://)", url, maxsplit=1)
+    if len(parts) > 1:
+        return parts[0]
+    return "file"
+
+
+def get_file_extension(url: str) -> str:
+    url = stringify_path(url)
+    ext_parts = url.rsplit(".", 1)
+    if len(ext_parts) > 1:
+        return ext_parts[-1]
+    return ""
+
+
+def can_be_local(path: str) -> bool:
+    """Can the given URL be used with open_local?"""
+    from fsspec import get_filesystem_class
+
+    try:
+        return getattr(get_filesystem_class(get_protocol(path)), "local_file", False)
+    except (ValueError, ImportError):
+        # not in registry or import failed
+        return False
+
+
+def get_package_version_without_import(name: str) -> str | None:
+    """For given package name, try to find the version without importing it
+
+    Import and package.__version__ is still the backup here, so an import
+    *might* happen.
+
+    Returns either the version string, or None if the package
+    or the version was not readily  found.
+    """
+    if name in sys.modules:
+        mod = sys.modules[name]
+        if hasattr(mod, "__version__"):
+            return mod.__version__
+    try:
+        return version(name)
+    except:  # noqa: E722
+        pass
+    try:
+        import importlib
+
+        mod = importlib.import_module(name)
+        return mod.__version__
+    except (ImportError, AttributeError):
+        return None
+
+
+def setup_logging(
+    logger: logging.Logger | None = None,
+    logger_name: str | None = None,
+    level: str = "DEBUG",
+    clear: bool = True,
+) -> logging.Logger:
+    if logger is None and logger_name is None:
+        raise ValueError("Provide either logger object or logger name")
+    logger = logger or logging.getLogger(logger_name)
+    handle = logging.StreamHandler()
+    formatter = logging.Formatter(
+        "%(asctime)s - %(name)s - %(levelname)s - %(funcName)s -- %(message)s"
+    )
+    handle.setFormatter(formatter)
+    if clear:
+        logger.handlers.clear()
+    logger.addHandler(handle)
+    logger.setLevel(level)
+    return logger
+
+
+def _unstrip_protocol(name: str, fs: AbstractFileSystem) -> str:
+    return fs.unstrip_protocol(name)
+
+
+def mirror_from(
+    origin_name: str, methods: Iterable[str]
+) -> Callable[[type[T]], type[T]]:
+    """Mirror attributes and methods from the given
+    origin_name attribute of the instance to the
+    decorated class"""
+
+    def origin_getter(method: str, self: Any) -> Any:
+        origin = getattr(self, origin_name)
+        return getattr(origin, method)
+
+    def wrapper(cls: type[T]) -> type[T]:
+        for method in methods:
+            wrapped_method = partial(origin_getter, method)
+            setattr(cls, method, property(wrapped_method))
+        return cls
+
+    return wrapper
+
+
+@contextlib.contextmanager
+def nullcontext(obj: T) -> Iterator[T]:
+    yield obj
+
+
+def merge_offset_ranges(
+    paths: list[str],
+    starts: list[int] | int,
+    ends: list[int] | int,
+    max_gap: int = 0,
+    max_block: int | None = None,
+    sort: bool = True,
+) -> tuple[list[str], list[int], list[int]]:
+    """Merge adjacent byte-offset ranges when the inter-range
+    gap is <= `max_gap`, and when the merged byte range does not
+    exceed `max_block` (if specified). By default, this function
+    will re-order the input paths and byte ranges to ensure sorted
+    order. If the user can guarantee that the inputs are already
+    sorted, passing `sort=False` will skip the re-ordering.
+    """
+    # Check input
+    if not isinstance(paths, list):
+        raise TypeError
+    if not isinstance(starts, list):
+        starts = [starts] * len(paths)
+    if not isinstance(ends, list):
+        ends = [ends] * len(paths)
+    if len(starts) != len(paths) or len(ends) != len(paths):
+        raise ValueError
+
+    # Early Return
+    if len(starts) <= 1:
+        return paths, starts, ends
+
+    starts = [s or 0 for s in starts]
+    # Sort by paths and then ranges if `sort=True`
+    if sort:
+        paths, starts, ends = (
+            list(v)
+            for v in zip(
+                *sorted(
+                    zip(paths, starts, ends),
+                )
+            )
+        )
+    remove = []
+    for i, (path, start, end) in enumerate(zip(paths, starts, ends)):
+        if any(
+            e is not None and p == path and start >= s and end <= e and i != i2
+            for i2, (p, s, e) in enumerate(zip(paths, starts, ends))
+        ):
+            remove.append(i)
+    paths = [p for i, p in enumerate(paths) if i not in remove]
+    starts = [s for i, s in enumerate(starts) if i not in remove]
+    ends = [e for i, e in enumerate(ends) if i not in remove]
+
+    if paths:
+        # Loop through the coupled `paths`, `starts`, and
+        # `ends`, and merge adjacent blocks when appropriate
+        new_paths = paths[:1]
+        new_starts = starts[:1]
+        new_ends = ends[:1]
+        for i in range(1, len(paths)):
+            if paths[i] == paths[i - 1] and new_ends[-1] is None:
+                continue
+            elif (
+                paths[i] != paths[i - 1]
+                or ((starts[i] - new_ends[-1]) > max_gap)
+                or (max_block is not None and (ends[i] - new_starts[-1]) > max_block)
+            ):
+                # Cannot merge with previous block.
+                # Add new `paths`, `starts`, and `ends` elements
+                new_paths.append(paths[i])
+                new_starts.append(starts[i])
+                new_ends.append(ends[i])
+            else:
+                # Merge with the previous block by updating the
+                # last element of `ends`
+                new_ends[-1] = ends[i]
+        return new_paths, new_starts, new_ends
+
+    # `paths` is empty. Just return input lists
+    return paths, starts, ends
+
+
+def file_size(filelike: IO[bytes]) -> int:
+    """Find length of any open read-mode file-like"""
+    pos = filelike.tell()
+    try:
+        return filelike.seek(0, 2)
+    finally:
+        filelike.seek(pos)
+
+
+@contextlib.contextmanager
+def atomic_write(path: str, mode: str = "wb"):
+    """
+    A context manager that opens a temporary file next to `path` and, on exit,
+    replaces `path` with the temporary file, thereby updating `path`
+    atomically.
+    """
+    fd, fn = tempfile.mkstemp(
+        dir=os.path.dirname(path), prefix=os.path.basename(path) + "-"
+    )
+    try:
+        with open(fd, mode) as fp:
+            yield fp
+    except BaseException:
+        with contextlib.suppress(FileNotFoundError):
+            os.unlink(fn)
+        raise
+    else:
+        os.replace(fn, path)
+
+
+def _translate(pat, STAR, QUESTION_MARK):
+    # Copied from: https://github.com/python/cpython/pull/106703.
+    res: list[str] = []
+    add = res.append
+    i, n = 0, len(pat)
+    while i < n:
+        c = pat[i]
+        i = i + 1
+        if c == "*":
+            # compress consecutive `*` into one
+            if (not res) or res[-1] is not STAR:
+                add(STAR)
+        elif c == "?":
+            add(QUESTION_MARK)
+        elif c == "[":
+            j = i
+            if j < n and pat[j] == "!":
+                j = j + 1
+            if j < n and pat[j] == "]":
+                j = j + 1
+            while j < n and pat[j] != "]":
+                j = j + 1
+            if j >= n:
+                add("\\[")
+            else:
+                stuff = pat[i:j]
+                if "-" not in stuff:
+                    stuff = stuff.replace("\\", r"\\")
+                else:
+                    chunks = []
+                    k = i + 2 if pat[i] == "!" else i + 1
+                    while True:
+                        k = pat.find("-", k, j)
+                        if k < 0:
+                            break
+                        chunks.append(pat[i:k])
+                        i = k + 1
+                        k = k + 3
+                    chunk = pat[i:j]
+                    if chunk:
+                        chunks.append(chunk)
+                    else:
+                        chunks[-1] += "-"
+                    # Remove empty ranges -- invalid in RE.
+                    for k in range(len(chunks) - 1, 0, -1):
+                        if chunks[k - 1][-1] > chunks[k][0]:
+                            chunks[k - 1] = chunks[k - 1][:-1] + chunks[k][1:]
+                            del chunks[k]
+                    # Escape backslashes and hyphens for set difference (--).
+                    # Hyphens that create ranges shouldn't be escaped.
+                    stuff = "-".join(
+                        s.replace("\\", r"\\").replace("-", r"\-") for s in chunks
+                    )
+                # Escape set operations (&&, ~~ and ||).
+                stuff = re.sub(r"([&~|])", r"\\\1", stuff)
+                i = j + 1
+                if not stuff:
+                    # Empty range: never match.
+                    add("(?!)")
+                elif stuff == "!":
+                    # Negated empty range: match any character.
+                    add(".")
+                else:
+                    if stuff[0] == "!":
+                        stuff = "^" + stuff[1:]
+                    elif stuff[0] in ("^", "["):
+                        stuff = "\\" + stuff
+                    add(f"[{stuff}]")
+        else:
+            add(re.escape(c))
+    assert i == n
+    return res
+
+
+def glob_translate(pat):
+    # Copied from: https://github.com/python/cpython/pull/106703.
+    # The keyword parameters' values are fixed to:
+    # recursive=True, include_hidden=True, seps=None
+    """Translate a pathname with shell wildcards to a regular expression."""
+    if os.path.altsep:
+        seps = os.path.sep + os.path.altsep
+    else:
+        seps = os.path.sep
+    escaped_seps = "".join(map(re.escape, seps))
+    any_sep = f"[{escaped_seps}]" if len(seps) > 1 else escaped_seps
+    not_sep = f"[^{escaped_seps}]"
+    one_last_segment = f"{not_sep}+"
+    one_segment = f"{one_last_segment}{any_sep}"
+    any_segments = f"(?:.+{any_sep})?"
+    any_last_segments = ".*"
+    results = []
+    parts = re.split(any_sep, pat)
+    last_part_idx = len(parts) - 1
+    for idx, part in enumerate(parts):
+        if part == "*":
+            results.append(one_segment if idx < last_part_idx else one_last_segment)
+            continue
+        if part == "**":
+            results.append(any_segments if idx < last_part_idx else any_last_segments)
+            continue
+        elif "**" in part:
+            raise ValueError(
+                "Invalid pattern: '**' can only be an entire path component"
+            )
+        if part:
+            results.extend(_translate(part, f"{not_sep}*", not_sep))
+        if idx < last_part_idx:
+            results.append(any_sep)
+    res = "".join(results)
+    return rf"(?s:{res})\Z"

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/ma/API_CHANGES.txt

@@ -0,0 +1,135 @@
+.. -*- rest -*-
+
+==================================================
+API changes in the new masked array implementation
+==================================================
+
+Masked arrays are subclasses of ndarray
+---------------------------------------
+
+Contrary to the original implementation, masked arrays are now regular
+ndarrays::
+
+  >>> x = masked_array([1,2,3],mask=[0,0,1])
+  >>> print isinstance(x, numpy.ndarray)
+  True
+
+
+``_data`` returns a view of the masked array
+--------------------------------------------
+
+Masked arrays are composed of a ``_data`` part and a ``_mask``. Accessing the
+``_data`` part will return a regular ndarray or any of its subclass, depending
+on the initial data::
+
+  >>> x = masked_array(numpy.matrix([[1,2],[3,4]]),mask=[[0,0],[0,1]])
+  >>> print x._data
+  [[1 2]
+   [3 4]]
+  >>> print type(x._data)
+  <class 'numpy.matrixlib.defmatrix.matrix'>
+
+
+In practice, ``_data`` is implemented as a property, not as an attribute.
+Therefore, you cannot access it directly, and some simple tests such as the
+following one will fail::
+
+  >>>x._data is x._data
+  False
+
+
+``filled(x)`` can return a subclass of ndarray
+----------------------------------------------
+The function ``filled(a)`` returns an array of the same type as ``a._data``::
+
+  >>> x = masked_array(numpy.matrix([[1,2],[3,4]]),mask=[[0,0],[0,1]])
+  >>> y = filled(x)
+  >>> print type(y)
+  <class 'numpy.matrixlib.defmatrix.matrix'>
+  >>> print y
+  matrix([[     1,      2],
+          [     3, 999999]])
+
+
+``put``, ``putmask`` behave like their ndarray counterparts
+-----------------------------------------------------------
+
+Previously, ``putmask`` was used like this::
+
+  mask = [False,True,True]
+  x = array([1,4,7],mask=mask)
+  putmask(x,mask,[3])
+
+which translated to::
+
+  x[~mask] = [3]
+
+(Note that a ``True``-value in a mask suppresses a value.)
+
+In other words, the mask had the same length as ``x``, whereas
+``values`` had ``sum(~mask)`` elements.
+
+Now, the behaviour is similar to that of ``ndarray.putmask``, where
+the mask and the values are both the same length as ``x``, i.e.
+
+::
+
+  putmask(x,mask,[3,0,0])
+
+
+``fill_value`` is a property
+----------------------------
+
+``fill_value`` is no longer a method, but a property::
+
+  >>> print x.fill_value
+  999999
+
+``cumsum`` and ``cumprod`` ignore missing values
+------------------------------------------------
+
+Missing values are assumed to be the identity element, i.e. 0 for
+``cumsum`` and 1 for ``cumprod``::
+
+  >>> x = N.ma.array([1,2,3,4],mask=[False,True,False,False])
+  >>> print x
+  [1 -- 3 4]
+  >>> print x.cumsum()
+  [1 -- 4 8]
+  >> print x.cumprod()
+  [1 -- 3 12]
+
+``bool(x)`` raises a ValueError
+-------------------------------
+
+Masked arrays now behave like regular ``ndarrays``, in that they cannot be
+converted to booleans:
+
+::
+
+  >>> x = N.ma.array([1,2,3])
+  >>> bool(x)
+  Traceback (most recent call last):
+    File "<stdin>", line 1, in <module>
+  ValueError: The truth value of an array with more than one element is ambiguous. Use a.any() or a.all()
+
+
+==================================
+New features (non exhaustive list)
+==================================
+
+``mr_``
+-------
+
+``mr_`` mimics the behavior of ``r_`` for masked arrays::
+
+  >>> np.ma.mr_[3,4,5]
+  masked_array(data = [3 4 5],
+        mask = False,
+        fill_value=999999)
+
+
+``anom``
+--------
+
+The ``anom`` method returns the deviations from the average (anomalies).

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/ma/LICENSE

@@ -0,0 +1,24 @@
+* Copyright (c) 2006, University of Georgia and Pierre G.F. Gerard-Marchant
+* All rights reserved.
+* Redistribution and use in source and binary forms, with or without
+* modification, are permitted provided that the following conditions are met:
+*
+*     * Redistributions of source code must retain the above copyright
+*       notice, this list of conditions and the following disclaimer.
+*     * Redistributions in binary form must reproduce the above copyright
+*       notice, this list of conditions and the following disclaimer in the
+*       documentation and/or other materials provided with the distribution.
+*     * Neither the name of the University of Georgia nor the
+*       names of its contributors may be used to endorse or promote products
+*       derived from this software without specific prior written permission.
+*
+* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND ANY
+* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
+* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/ma/README.rst

@@ -0,0 +1,236 @@
+==================================
+A Guide to Masked Arrays in NumPy
+==================================
+
+.. Contents::
+
+See http://www.scipy.org/scipy/numpy/wiki/MaskedArray (dead link)
+for updates of this document.
+
+
+History
+-------
+
+As a regular user of MaskedArray, I (Pierre G.F. Gerard-Marchant) became
+increasingly frustrated with the subclassing of masked arrays (even if
+I can only blame my inexperience). I needed to develop a class of arrays
+that could store some additional information along with numerical values,
+while keeping the possibility for missing data (picture storing a series
+of dates along with measurements, what would later become the `TimeSeries
+Scikit <http://projects.scipy.org/scipy/scikits/wiki/TimeSeries>`__
+(dead link).
+
+I started to implement such a class, but then quickly realized that
+any additional information disappeared when processing these subarrays
+(for example, adding a constant value to a subarray would erase its
+dates). I ended up writing the equivalent of *numpy.core.ma* for my
+particular class, ufuncs included. Everything went fine until I needed to
+subclass my new class, when more problems showed up: some attributes of
+the new subclass were lost during processing. I identified the culprit as
+MaskedArray, which returns masked ndarrays when I expected masked
+arrays of my class. I was preparing myself to rewrite *numpy.core.ma*
+when I forced myself to learn how to subclass ndarrays. As I became more
+familiar with the *__new__* and *__array_finalize__* methods,
+I started to wonder why masked arrays were objects, and not ndarrays,
+and whether it wouldn't be more convenient for subclassing if they did
+behave like regular ndarrays.
+
+The new *maskedarray* is what I eventually come up with. The
+main differences with the initial *numpy.core.ma* package are
+that MaskedArray is now a subclass of *ndarray* and that the
+*_data* section can now be any subclass of *ndarray*. Apart from a
+couple of issues listed below, the behavior of the new MaskedArray
+class reproduces the old one. Initially the *maskedarray*
+implementation was marginally slower than *numpy.ma* in some areas,
+but work is underway to speed it up; the expectation is that it can be
+made substantially faster than the present *numpy.ma*.
+
+
+Note that if the subclass has some special methods and
+attributes, they are not propagated to the masked version:
+this would require a modification of the *__getattribute__*
+method (first trying *ndarray.__getattribute__*, then trying
+*self._data.__getattribute__* if an exception is raised in the first
+place), which really slows things down.
+
+Main differences
+----------------
+
+ * The *_data* part of the masked array can be any subclass of ndarray (but not recarray, cf below).
+ * *fill_value* is now a property, not a function.
+ * in the majority of cases, the mask is forced to *nomask* when no value is actually masked. A notable exception is when a masked array (with no masked values) has just been unpickled.
+ * I got rid of the *share_mask* flag, I never understood its purpose.
+ * *put*, *putmask* and *take* now mimic the ndarray methods, to avoid unpleasant surprises. Moreover, *put* and *putmask* both update the mask when needed.  * if *a* is a masked array, *bool(a)* raises a *ValueError*, as it does with ndarrays.
+ * in the same way, the comparison of two masked arrays is a masked array, not a boolean
+ * *filled(a)* returns an array of the same subclass as *a._data*, and no test is performed on whether it is contiguous or not.
+ * the mask is always printed, even if it's *nomask*, which makes things easy (for me at least) to remember that a masked array is used.
+ * *cumsum* works as if the *_data* array was filled with 0. The mask is preserved, but not updated.
+ * *cumprod* works as if the *_data* array was filled with 1. The mask is preserved, but not updated.
+
+New features
+------------
+
+This list is non-exhaustive...
+
+ * the *mr_* function mimics *r_* for masked arrays.
+ * the *anom* method returns the anomalies (deviations from the average)
+
+Using the new package with numpy.core.ma
+----------------------------------------
+
+I tried to make sure that the new package can understand old masked
+arrays. Unfortunately, there's no upward compatibility.
+
+For example:
+
+>>> import numpy.core.ma as old_ma
+>>> import maskedarray as new_ma
+>>> x = old_ma.array([1,2,3,4,5], mask=[0,0,1,0,0])
+>>> x
+array(data =
+ [     1      2 999999      4      5],
+      mask =
+ [False False True False False],
+      fill_value=999999)
+>>> y = new_ma.array([1,2,3,4,5], mask=[0,0,1,0,0])
+>>> y
+array(data = [1 2 -- 4 5],
+      mask = [False False True False False],
+      fill_value=999999)
+>>> x==y
+array(data =
+ [True True True True True],
+      mask =
+ [False False True False False],
+      fill_value=?)
+>>> old_ma.getmask(x) == new_ma.getmask(x)
+array([True, True, True, True, True])
+>>> old_ma.getmask(y) == new_ma.getmask(y)
+array([True, True, False, True, True])
+>>> old_ma.getmask(y)
+False
+
+
+Using maskedarray with matplotlib
+---------------------------------
+
+Starting with matplotlib 0.91.2, the masked array importing will work with
+the maskedarray branch) as well as with earlier versions.
+
+By default matplotlib still uses numpy.ma, but there is an rcParams setting
+that you can use to select maskedarray instead.  In the matplotlibrc file
+you will find::
+
+  #maskedarray : False       # True to use external maskedarray module
+                             # instead of numpy.ma; this is a temporary #
+                             setting for testing maskedarray.
+
+
+Uncomment and set to True to select maskedarray everywhere.
+Alternatively, you can test a script with maskedarray by using a
+command-line option, e.g.::
+
+  python simple_plot.py --maskedarray
+
+
+Masked records
+--------------
+
+Like *numpy.core.ma*, the *ndarray*-based implementation
+of MaskedArray is limited when working with records: you can
+mask any record of the array, but not a field in a record. If you
+need this feature, you may want to give the *mrecords* package
+a try (available in the *maskedarray* directory in the scipy
+sandbox). This module defines a new class, *MaskedRecord*. An
+instance of this class accepts a *recarray* as data, and uses two
+masks: the *fieldmask* has as many entries as records in the array,
+each entry with the same fields as a record, but of boolean types:
+they indicate whether the field is masked or not; a record entry
+is flagged as masked in the *mask* array if all the fields are
+masked. A few examples in the file should give you an idea of what
+can be done. Note that *mrecords* is still experimental...
+
+Optimizing maskedarray
+----------------------
+
+Should masked arrays be filled before processing or not?
+--------------------------------------------------------
+
+In the current implementation, most operations on masked arrays involve
+the following steps:
+
+ * the input arrays are filled
+ * the operation is performed on the filled arrays
+ * the mask is set for the results, from the combination of the input masks and the mask corresponding to the domain of the operation.
+
+For example, consider the division of two masked arrays::
+
+  import numpy
+  import maskedarray as ma
+  x = ma.array([1,2,3,4],mask=[1,0,0,0], dtype=numpy.float_)
+  y = ma.array([-1,0,1,2], mask=[0,0,0,1], dtype=numpy.float_)
+
+The division of x by y is then computed as::
+
+  d1 = x.filled(0) # d1 = array([0., 2., 3., 4.])
+  d2 = y.filled(1) # array([-1.,  0.,  1.,  1.])
+  m = ma.mask_or(ma.getmask(x), ma.getmask(y)) # m =
+  array([True,False,False,True])
+  dm = ma.divide.domain(d1,d2) # array([False,  True, False, False])
+  result = (d1/d2).view(MaskedArray) # masked_array([-0. inf, 3., 4.])
+  result._mask = logical_or(m, dm)
+
+Note that a division by zero takes place. To avoid it, we can consider
+to fill the input arrays, taking the domain mask into account, so that::
+
+  d1 = x._data.copy() # d1 = array([1., 2., 3., 4.])
+  d2 = y._data.copy() # array([-1.,  0.,  1.,  2.])
+  dm = ma.divide.domain(d1,d2) # array([False,  True, False, False])
+  numpy.putmask(d2, dm, 1) # d2 = array([-1.,  1.,  1.,  2.])
+  m = ma.mask_or(ma.getmask(x), ma.getmask(y)) # m =
+  array([True,False,False,True])
+  result = (d1/d2).view(MaskedArray) # masked_array([-1. 0., 3., 2.])
+  result._mask = logical_or(m, dm)
+
+Note that the *.copy()* is required to avoid updating the inputs with
+*putmask*.  The *.filled()* method also involves a *.copy()*.
+
+A third possibility consists in avoid filling the arrays::
+
+  d1 = x._data # d1 = array([1., 2., 3., 4.])
+  d2 = y._data # array([-1.,  0.,  1.,  2.])
+  dm = ma.divide.domain(d1,d2) # array([False,  True, False, False])
+  m = ma.mask_or(ma.getmask(x), ma.getmask(y)) # m =
+  array([True,False,False,True])
+  result = (d1/d2).view(MaskedArray) # masked_array([-1. inf, 3., 2.])
+  result._mask = logical_or(m, dm)
+
+Note that here again the division by zero takes place.
+
+A quick benchmark gives the following results:
+
+ * *numpy.ma.divide*  : 2.69 ms per loop
+ * classical division     : 2.21 ms per loop
+ * division w/ prefilling : 2.34 ms per loop
+ * division w/o filling   : 1.55 ms per loop
+
+So, is it worth filling the arrays beforehand ? Yes, if we are interested
+in avoiding floating-point exceptions that may fill the result with infs
+and nans. No, if we are only interested into speed...
+
+
+Thanks
+------
+
+I'd like to thank Paul Dubois, Travis Oliphant and Sasha for the
+original masked array package: without you, I would never have started
+that (it might be argued that I shouldn't have anyway, but that's
+another story...).  I also wish to extend these thanks to Reggie Dugard
+and Eric Firing for their suggestions and numerous improvements.
+
+
+Revision notes
+--------------
+
+  * 08/25/2007 : Creation of this page
+  * 01/23/2007 : The package has been moved to the SciPy sandbox, and is regularly updated: please check out your SVN version!

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/ma/__init__.py

@@ -0,0 +1,54 @@
+"""
+=============
+Masked Arrays
+=============
+
+Arrays sometimes contain invalid or missing data.  When doing operations
+on such arrays, we wish to suppress invalid values, which is the purpose masked
+arrays fulfill (an example of typical use is given below).
+
+For example, examine the following array:
+
+>>> x = np.array([2, 1, 3, np.nan, 5, 2, 3, np.nan])
+
+When we try to calculate the mean of the data, the result is undetermined:
+
+>>> np.mean(x)
+nan
+
+The mean is calculated using roughly ``np.sum(x)/len(x)``, but since
+any number added to ``NaN`` [1]_ produces ``NaN``, this doesn't work.  Enter
+masked arrays:
+
+>>> m = np.ma.masked_array(x, np.isnan(x))
+>>> m
+masked_array(data = [2.0 1.0 3.0 -- 5.0 2.0 3.0 --],
+      mask = [False False False  True False False False  True],
+      fill_value=1e+20)
+
+Here, we construct a masked array that suppress all ``NaN`` values.  We
+may now proceed to calculate the mean of the other values:
+
+>>> np.mean(m)
+2.6666666666666665
+
+.. [1] Not-a-Number, a floating point value that is the result of an
+       invalid operation.
+
+.. moduleauthor:: Pierre Gerard-Marchant
+.. moduleauthor:: Jarrod Millman
+
+"""
+from . import core
+from .core import *
+
+from . import extras
+from .extras import *
+
+__all__ = ['core', 'extras']
+__all__ += core.__all__
+__all__ += extras.__all__
+
+from numpy._pytesttester import PytestTester
+test = PytestTester(__name__)
+del PytestTester

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/ma/__init__.pyi

@@ -0,0 +1,234 @@
+from numpy._pytesttester import PytestTester
+
+from numpy.ma import extras as extras
+
+from numpy.ma.core import (
+    MAError as MAError,
+    MaskError as MaskError,
+    MaskType as MaskType,
+    MaskedArray as MaskedArray,
+    abs as abs,
+    absolute as absolute,
+    add as add,
+    all as all,
+    allclose as allclose,
+    allequal as allequal,
+    alltrue as alltrue,
+    amax as amax,
+    amin as amin,
+    angle as angle,
+    anom as anom,
+    anomalies as anomalies,
+    any as any,
+    append as append,
+    arange as arange,
+    arccos as arccos,
+    arccosh as arccosh,
+    arcsin as arcsin,
+    arcsinh as arcsinh,
+    arctan as arctan,
+    arctan2 as arctan2,
+    arctanh as arctanh,
+    argmax as argmax,
+    argmin as argmin,
+    argsort as argsort,
+    around as around,
+    array as array,
+    asanyarray as asanyarray,
+    asarray as asarray,
+    bitwise_and as bitwise_and,
+    bitwise_or as bitwise_or,
+    bitwise_xor as bitwise_xor,
+    bool_ as bool_,
+    ceil as ceil,
+    choose as choose,
+    clip as clip,
+    common_fill_value as common_fill_value,
+    compress as compress,
+    compressed as compressed,
+    concatenate as concatenate,
+    conjugate as conjugate,
+    convolve as convolve,
+    copy as copy,
+    correlate as correlate,
+    cos as cos,
+    cosh as cosh,
+    count as count,
+    cumprod as cumprod,
+    cumsum as cumsum,
+    default_fill_value as default_fill_value,
+    diag as diag,
+    diagonal as diagonal,
+    diff as diff,
+    divide as divide,
+    empty as empty,
+    empty_like as empty_like,
+    equal as equal,
+    exp as exp,
+    expand_dims as expand_dims,
+    fabs as fabs,
+    filled as filled,
+    fix_invalid as fix_invalid,
+    flatten_mask as flatten_mask,
+    flatten_structured_array as flatten_structured_array,
+    floor as floor,
+    floor_divide as floor_divide,
+    fmod as fmod,
+    frombuffer as frombuffer,
+    fromflex as fromflex,
+    fromfunction as fromfunction,
+    getdata as getdata,
+    getmask as getmask,
+    getmaskarray as getmaskarray,
+    greater as greater,
+    greater_equal as greater_equal,
+    harden_mask as harden_mask,
+    hypot as hypot,
+    identity as identity,
+    ids as ids,
+    indices as indices,
+    inner as inner,
+    innerproduct as innerproduct,
+    isMA as isMA,
+    isMaskedArray as isMaskedArray,
+    is_mask as is_mask,
+    is_masked as is_masked,
+    isarray as isarray,
+    left_shift as left_shift,
+    less as less,
+    less_equal as less_equal,
+    log as log,
+    log10 as log10,
+    log2 as log2,
+    logical_and as logical_and,
+    logical_not as logical_not,
+    logical_or as logical_or,
+    logical_xor as logical_xor,
+    make_mask as make_mask,
+    make_mask_descr as make_mask_descr,
+    make_mask_none as make_mask_none,
+    mask_or as mask_or,
+    masked as masked,
+    masked_array as masked_array,
+    masked_equal as masked_equal,
+    masked_greater as masked_greater,
+    masked_greater_equal as masked_greater_equal,
+    masked_inside as masked_inside,
+    masked_invalid as masked_invalid,
+    masked_less as masked_less,
+    masked_less_equal as masked_less_equal,
+    masked_not_equal as masked_not_equal,
+    masked_object as masked_object,
+    masked_outside as masked_outside,
+    masked_print_option as masked_print_option,
+    masked_singleton as masked_singleton,
+    masked_values as masked_values,
+    masked_where as masked_where,
+    max as max,
+    maximum as maximum,
+    maximum_fill_value as maximum_fill_value,
+    mean as mean,
+    min as min,
+    minimum as minimum,
+    minimum_fill_value as minimum_fill_value,
+    mod as mod,
+    multiply as multiply,
+    mvoid as mvoid,
+    ndim as ndim,
+    negative as negative,
+    nomask as nomask,
+    nonzero as nonzero,
+    not_equal as not_equal,
+    ones as ones,
+    outer as outer,
+    outerproduct as outerproduct,
+    power as power,
+    prod as prod,
+    product as product,
+    ptp as ptp,
+    put as put,
+    putmask as putmask,
+    ravel as ravel,
+    remainder as remainder,
+    repeat as repeat,
+    reshape as reshape,
+    resize as resize,
+    right_shift as right_shift,
+    round as round,
+    set_fill_value as set_fill_value,
+    shape as shape,
+    sin as sin,
+    sinh as sinh,
+    size as size,
+    soften_mask as soften_mask,
+    sometrue as sometrue,
+    sort as sort,
+    sqrt as sqrt,
+    squeeze as squeeze,
+    std as std,
+    subtract as subtract,
+    sum as sum,
+    swapaxes as swapaxes,
+    take as take,
+    tan as tan,
+    tanh as tanh,
+    trace as trace,
+    transpose as transpose,
+    true_divide as true_divide,
+    var as var,
+    where as where,
+    zeros as zeros,
+)
+
+from numpy.ma.extras import (
+    apply_along_axis as apply_along_axis,
+    apply_over_axes as apply_over_axes,
+    atleast_1d as atleast_1d,
+    atleast_2d as atleast_2d,
+    atleast_3d as atleast_3d,
+    average as average,
+    clump_masked as clump_masked,
+    clump_unmasked as clump_unmasked,
+    column_stack as column_stack,
+    compress_cols as compress_cols,
+    compress_nd as compress_nd,
+    compress_rowcols as compress_rowcols,
+    compress_rows as compress_rows,
+    count_masked as count_masked,
+    corrcoef as corrcoef,
+    cov as cov,
+    diagflat as diagflat,
+    dot as dot,
+    dstack as dstack,
+    ediff1d as ediff1d,
+    flatnotmasked_contiguous as flatnotmasked_contiguous,
+    flatnotmasked_edges as flatnotmasked_edges,
+    hsplit as hsplit,
+    hstack as hstack,
+    isin as isin,
+    in1d as in1d,
+    intersect1d as intersect1d,
+    mask_cols as mask_cols,
+    mask_rowcols as mask_rowcols,
+    mask_rows as mask_rows,
+    masked_all as masked_all,
+    masked_all_like as masked_all_like,
+    median as median,
+    mr_ as mr_,
+    ndenumerate as ndenumerate,
+    notmasked_contiguous as notmasked_contiguous,
+    notmasked_edges as notmasked_edges,
+    polyfit as polyfit,
+    row_stack as row_stack,
+    setdiff1d as setdiff1d,
+    setxor1d as setxor1d,
+    stack as stack,
+    unique as unique,
+    union1d as union1d,
+    vander as vander,
+    vstack as vstack,
+)
+
+__all__: list[str]
+__path__: list[str]
+test: PytestTester

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/ma/core.pyi

@@ -0,0 +1,471 @@
+from collections.abc import Callable
+from typing import Any, TypeVar
+from numpy import ndarray, dtype, float64
+
+from numpy import (
+    amax as amax,
+    amin as amin,
+    bool_ as bool_,
+    expand_dims as expand_dims,
+    clip as clip,
+    indices as indices,
+    ones_like as ones_like,
+    squeeze as squeeze,
+    zeros_like as zeros_like,
+)
+
+from numpy.lib.function_base import (
+    angle as angle,
+)
+
+# TODO: Set the `bound` to something more suitable once we
+# have proper shape support
+_ShapeType = TypeVar("_ShapeType", bound=Any)
+_DType_co = TypeVar("_DType_co", bound=dtype[Any], covariant=True)
+
+__all__: list[str]
+
+MaskType = bool_
+nomask: bool_
+
+class MaskedArrayFutureWarning(FutureWarning): ...
+class MAError(Exception): ...
+class MaskError(MAError): ...
+
+def default_fill_value(obj): ...
+def minimum_fill_value(obj): ...
+def maximum_fill_value(obj): ...
+def set_fill_value(a, fill_value): ...
+def common_fill_value(a, b): ...
+def filled(a, fill_value=...): ...
+def getdata(a, subok=...): ...
+get_data = getdata
+
+def fix_invalid(a, mask=..., copy=..., fill_value=...): ...
+
+class _MaskedUFunc:
+    f: Any
+    __doc__: Any
+    __name__: Any
+    def __init__(self, ufunc): ...
+
+class _MaskedUnaryOperation(_MaskedUFunc):
+    fill: Any
+    domain: Any
+    def __init__(self, mufunc, fill=..., domain=...): ...
+    def __call__(self, a, *args, **kwargs): ...
+
+class _MaskedBinaryOperation(_MaskedUFunc):
+    fillx: Any
+    filly: Any
+    def __init__(self, mbfunc, fillx=..., filly=...): ...
+    def __call__(self, a, b, *args, **kwargs): ...
+    def reduce(self, target, axis=..., dtype=...): ...
+    def outer(self, a, b): ...
+    def accumulate(self, target, axis=...): ...
+
+class _DomainedBinaryOperation(_MaskedUFunc):
+    domain: Any
+    fillx: Any
+    filly: Any
+    def __init__(self, dbfunc, domain, fillx=..., filly=...): ...
+    def __call__(self, a, b, *args, **kwargs): ...
+
+exp: _MaskedUnaryOperation
+conjugate: _MaskedUnaryOperation
+sin: _MaskedUnaryOperation
+cos: _MaskedUnaryOperation
+arctan: _MaskedUnaryOperation
+arcsinh: _MaskedUnaryOperation
+sinh: _MaskedUnaryOperation
+cosh: _MaskedUnaryOperation
+tanh: _MaskedUnaryOperation
+abs: _MaskedUnaryOperation
+absolute: _MaskedUnaryOperation
+fabs: _MaskedUnaryOperation
+negative: _MaskedUnaryOperation
+floor: _MaskedUnaryOperation
+ceil: _MaskedUnaryOperation
+around: _MaskedUnaryOperation
+logical_not: _MaskedUnaryOperation
+sqrt: _MaskedUnaryOperation
+log: _MaskedUnaryOperation
+log2: _MaskedUnaryOperation
+log10: _MaskedUnaryOperation
+tan: _MaskedUnaryOperation
+arcsin: _MaskedUnaryOperation
+arccos: _MaskedUnaryOperation
+arccosh: _MaskedUnaryOperation
+arctanh: _MaskedUnaryOperation
+
+add: _MaskedBinaryOperation
+subtract: _MaskedBinaryOperation
+multiply: _MaskedBinaryOperation
+arctan2: _MaskedBinaryOperation
+equal: _MaskedBinaryOperation
+not_equal: _MaskedBinaryOperation
+less_equal: _MaskedBinaryOperation
+greater_equal: _MaskedBinaryOperation
+less: _MaskedBinaryOperation
+greater: _MaskedBinaryOperation
+logical_and: _MaskedBinaryOperation
+alltrue: _MaskedBinaryOperation
+logical_or: _MaskedBinaryOperation
+sometrue: Callable[..., Any]
+logical_xor: _MaskedBinaryOperation
+bitwise_and: _MaskedBinaryOperation
+bitwise_or: _MaskedBinaryOperation
+bitwise_xor: _MaskedBinaryOperation
+hypot: _MaskedBinaryOperation
+divide: _MaskedBinaryOperation
+true_divide: _MaskedBinaryOperation
+floor_divide: _MaskedBinaryOperation
+remainder: _MaskedBinaryOperation
+fmod: _MaskedBinaryOperation
+mod: _MaskedBinaryOperation
+
+def make_mask_descr(ndtype): ...
+def getmask(a): ...
+get_mask = getmask
+
+def getmaskarray(arr): ...
+def is_mask(m): ...
+def make_mask(m, copy=..., shrink=..., dtype=...): ...
+def make_mask_none(newshape, dtype=...): ...
+def mask_or(m1, m2, copy=..., shrink=...): ...
+def flatten_mask(mask): ...
+def masked_where(condition, a, copy=...): ...
+def masked_greater(x, value, copy=...): ...
+def masked_greater_equal(x, value, copy=...): ...
+def masked_less(x, value, copy=...): ...
+def masked_less_equal(x, value, copy=...): ...
+def masked_not_equal(x, value, copy=...): ...
+def masked_equal(x, value, copy=...): ...
+def masked_inside(x, v1, v2, copy=...): ...
+def masked_outside(x, v1, v2, copy=...): ...
+def masked_object(x, value, copy=..., shrink=...): ...
+def masked_values(x, value, rtol=..., atol=..., copy=..., shrink=...): ...
+def masked_invalid(a, copy=...): ...
+
+class _MaskedPrintOption:
+    def __init__(self, display): ...
+    def display(self): ...
+    def set_display(self, s): ...
+    def enabled(self): ...
+    def enable(self, shrink=...): ...
+
+masked_print_option: _MaskedPrintOption
+
+def flatten_structured_array(a): ...
+
+class MaskedIterator:
+    ma: Any
+    dataiter: Any
+    maskiter: Any
+    def __init__(self, ma): ...
+    def __iter__(self): ...
+    def __getitem__(self, indx): ...
+    def __setitem__(self, index, value): ...
+    def __next__(self): ...
+
+class MaskedArray(ndarray[_ShapeType, _DType_co]):
+    __array_priority__: Any
+    def __new__(cls, data=..., mask=..., dtype=..., copy=..., subok=..., ndmin=..., fill_value=..., keep_mask=..., hard_mask=..., shrink=..., order=...): ...
+    def __array_finalize__(self, obj): ...
+    def __array_wrap__(self, obj, context=...): ...
+    def view(self, dtype=..., type=..., fill_value=...): ...
+    def __getitem__(self, indx): ...
+    def __setitem__(self, indx, value): ...
+    @property
+    def dtype(self): ...
+    @dtype.setter
+    def dtype(self, dtype): ...
+    @property
+    def shape(self): ...
+    @shape.setter
+    def shape(self, shape): ...
+    def __setmask__(self, mask, copy=...): ...
+    @property
+    def mask(self): ...
+    @mask.setter
+    def mask(self, value): ...
+    @property
+    def recordmask(self): ...
+    @recordmask.setter
+    def recordmask(self, mask): ...
+    def harden_mask(self): ...
+    def soften_mask(self): ...
+    @property
+    def hardmask(self): ...
+    def unshare_mask(self): ...
+    @property
+    def sharedmask(self): ...
+    def shrink_mask(self): ...
+    @property
+    def baseclass(self): ...
+    data: Any
+    @property
+    def flat(self): ...
+    @flat.setter
+    def flat(self, value): ...
+    @property
+    def fill_value(self): ...
+    @fill_value.setter
+    def fill_value(self, value=...): ...
+    get_fill_value: Any
+    set_fill_value: Any
+    def filled(self, fill_value=...): ...
+    def compressed(self): ...
+    def compress(self, condition, axis=..., out=...): ...
+    def __eq__(self, other): ...
+    def __ne__(self, other): ...
+    def __ge__(self, other): ...
+    def __gt__(self, other): ...
+    def __le__(self, other): ...
+    def __lt__(self, other): ...
+    def __add__(self, other): ...
+    def __radd__(self, other): ...
+    def __sub__(self, other): ...
+    def __rsub__(self, other): ...
+    def __mul__(self, other): ...
+    def __rmul__(self, other): ...
+    def __div__(self, other): ...
+    def __truediv__(self, other): ...
+    def __rtruediv__(self, other): ...
+    def __floordiv__(self, other): ...
+    def __rfloordiv__(self, other): ...
+    def __pow__(self, other): ...
+    def __rpow__(self, other): ...
+    def __iadd__(self, other): ...
+    def __isub__(self, other): ...
+    def __imul__(self, other): ...
+    def __idiv__(self, other): ...
+    def __ifloordiv__(self, other): ...
+    def __itruediv__(self, other): ...
+    def __ipow__(self, other): ...
+    def __float__(self): ...
+    def __int__(self): ...
+    @property  # type: ignore[misc]
+    def imag(self): ...
+    get_imag: Any
+    @property  # type: ignore[misc]
+    def real(self): ...
+    get_real: Any
+    def count(self, axis=..., keepdims=...): ...
+    def ravel(self, order=...): ...
+    def reshape(self, *s, **kwargs): ...
+    def resize(self, newshape, refcheck=..., order=...): ...
+    def put(self, indices, values, mode=...): ...
+    def ids(self): ...
+    def iscontiguous(self): ...
+    def all(self, axis=..., out=..., keepdims=...): ...
+    def any(self, axis=..., out=..., keepdims=...): ...
+    def nonzero(self): ...
+    def trace(self, offset=..., axis1=..., axis2=..., dtype=..., out=...): ...
+    def dot(self, b, out=..., strict=...): ...
+    def sum(self, axis=..., dtype=..., out=..., keepdims=...): ...
+    def cumsum(self, axis=..., dtype=..., out=...): ...
+    def prod(self, axis=..., dtype=..., out=..., keepdims=...): ...
+    product: Any
+    def cumprod(self, axis=..., dtype=..., out=...): ...
+    def mean(self, axis=..., dtype=..., out=..., keepdims=...): ...
+    def anom(self, axis=..., dtype=...): ...
+    def var(self, axis=..., dtype=..., out=..., ddof=..., keepdims=...): ...
+    def std(self, axis=..., dtype=..., out=..., ddof=..., keepdims=...): ...
+    def round(self, decimals=..., out=...): ...
+    def argsort(self, axis=..., kind=..., order=..., endwith=..., fill_value=...): ...
+    def argmin(self, axis=..., fill_value=..., out=..., *, keepdims=...): ...
+    def argmax(self, axis=..., fill_value=..., out=..., *, keepdims=...): ...
+    def sort(self, axis=..., kind=..., order=..., endwith=..., fill_value=...): ...
+    def min(self, axis=..., out=..., fill_value=..., keepdims=...): ...
+    # NOTE: deprecated
+    # def tostring(self, fill_value=..., order=...): ...
+    def max(self, axis=..., out=..., fill_value=..., keepdims=...): ...
+    def ptp(self, axis=..., out=..., fill_value=..., keepdims=...): ...
+    def partition(self, *args, **kwargs): ...
+    def argpartition(self, *args, **kwargs): ...
+    def take(self, indices, axis=..., out=..., mode=...): ...
+    copy: Any
+    diagonal: Any
+    flatten: Any
+    repeat: Any
+    squeeze: Any
+    swapaxes: Any
+    T: Any
+    transpose: Any
+    def tolist(self, fill_value=...): ...
+    def tobytes(self, fill_value=..., order=...): ...
+    def tofile(self, fid, sep=..., format=...): ...
+    def toflex(self): ...
+    torecords: Any
+    def __reduce__(self): ...
+    def __deepcopy__(self, memo=...): ...
+
+class mvoid(MaskedArray[_ShapeType, _DType_co]):
+    def __new__(
+        self,
+        data,
+        mask=...,
+        dtype=...,
+        fill_value=...,
+        hardmask=...,
+        copy=...,
+        subok=...,
+    ): ...
+    def __getitem__(self, indx): ...
+    def __setitem__(self, indx, value): ...
+    def __iter__(self): ...
+    def __len__(self): ...
+    def filled(self, fill_value=...): ...
+    def tolist(self): ...
+
+def isMaskedArray(x): ...
+isarray = isMaskedArray
+isMA = isMaskedArray
+
+# 0D float64 array
+class MaskedConstant(MaskedArray[Any, dtype[float64]]):
+    def __new__(cls): ...
+    __class__: Any
+    def __array_finalize__(self, obj): ...
+    def __array_prepare__(self, obj, context=...): ...
+    def __array_wrap__(self, obj, context=...): ...
+    def __format__(self, format_spec): ...
+    def __reduce__(self): ...
+    def __iop__(self, other): ...
+    __iadd__: Any
+    __isub__: Any
+    __imul__: Any
+    __ifloordiv__: Any
+    __itruediv__: Any
+    __ipow__: Any
+    def copy(self, *args, **kwargs): ...
+    def __copy__(self): ...
+    def __deepcopy__(self, memo): ...
+    def __setattr__(self, attr, value): ...
+
+masked: MaskedConstant
+masked_singleton: MaskedConstant
+masked_array = MaskedArray
+
+def array(
+    data,
+    dtype=...,
+    copy=...,
+    order=...,
+    mask=...,
+    fill_value=...,
+    keep_mask=...,
+    hard_mask=...,
+    shrink=...,
+    subok=...,
+    ndmin=...,
+): ...
+def is_masked(x): ...
+
+class _extrema_operation(_MaskedUFunc):
+    compare: Any
+    fill_value_func: Any
+    def __init__(self, ufunc, compare, fill_value): ...
+    # NOTE: in practice `b` has a default value, but users should
+    # explicitly provide a value here as the default is deprecated
+    def __call__(self, a, b): ...
+    def reduce(self, target, axis=...): ...
+    def outer(self, a, b): ...
+
+def min(obj, axis=..., out=..., fill_value=..., keepdims=...): ...
+def max(obj, axis=..., out=..., fill_value=..., keepdims=...): ...
+def ptp(obj, axis=..., out=..., fill_value=..., keepdims=...): ...
+
+class _frommethod:
+    __name__: Any
+    __doc__: Any
+    reversed: Any
+    def __init__(self, methodname, reversed=...): ...
+    def getdoc(self): ...
+    def __call__(self, a, *args, **params): ...
+
+all: _frommethod
+anomalies: _frommethod
+anom: _frommethod
+any: _frommethod
+compress: _frommethod
+cumprod: _frommethod
+cumsum: _frommethod
+copy: _frommethod
+diagonal: _frommethod
+harden_mask: _frommethod
+ids: _frommethod
+mean: _frommethod
+nonzero: _frommethod
+prod: _frommethod
+product: _frommethod
+ravel: _frommethod
+repeat: _frommethod
+soften_mask: _frommethod
+std: _frommethod
+sum: _frommethod
+swapaxes: _frommethod
+trace: _frommethod
+var: _frommethod
+count: _frommethod
+argmin: _frommethod
+argmax: _frommethod
+
+minimum: _extrema_operation
+maximum: _extrema_operation
+
+def take(a, indices, axis=..., out=..., mode=...): ...
+def power(a, b, third=...): ...
+def argsort(a, axis=..., kind=..., order=..., endwith=..., fill_value=...): ...
+def sort(a, axis=..., kind=..., order=..., endwith=..., fill_value=...): ...
+def compressed(x): ...
+def concatenate(arrays, axis=...): ...
+def diag(v, k=...): ...
+def left_shift(a, n): ...
+def right_shift(a, n): ...
+def put(a, indices, values, mode=...): ...
+def putmask(a, mask, values): ...
+def transpose(a, axes=...): ...
+def reshape(a, new_shape, order=...): ...
+def resize(x, new_shape): ...
+def ndim(obj): ...
+def shape(obj): ...
+def size(obj, axis=...): ...
+def diff(a, /, n=..., axis=..., prepend=..., append=...): ...
+def where(condition, x=..., y=...): ...
+def choose(indices, choices, out=..., mode=...): ...
+def round(a, decimals=..., out=...): ...
+
+def inner(a, b): ...
+innerproduct = inner
+
+def outer(a, b): ...
+outerproduct = outer
+
+def correlate(a, v, mode=..., propagate_mask=...): ...
+def convolve(a, v, mode=..., propagate_mask=...): ...
+def allequal(a, b, fill_value=...): ...
+def allclose(a, b, masked_equal=..., rtol=..., atol=...): ...
+def asarray(a, dtype=..., order=...): ...
+def asanyarray(a, dtype=...): ...
+def fromflex(fxarray): ...
+
+class _convert2ma:
+    __doc__: Any
+    def __init__(self, funcname, params=...): ...
+    def getdoc(self): ...
+    def __call__(self, *args, **params): ...
+
+arange: _convert2ma
+empty: _convert2ma
+empty_like: _convert2ma
+frombuffer: _convert2ma
+fromfunction: _convert2ma
+identity: _convert2ma
+ones: _convert2ma
+zeros: _convert2ma
+
+def append(a, b, axis=...): ...
+def dot(a, b, strict=..., out=...): ...
+def mask_rowcols(a, axis=...): ...

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/ma/extras.py

@@ -0,0 +1,2133 @@
+"""
+Masked arrays add-ons.
+
+A collection of utilities for `numpy.ma`.
+
+:author: Pierre Gerard-Marchant
+:contact: pierregm_at_uga_dot_edu
+:version: $Id: extras.py 3473 2007-10-29 15:18:13Z jarrod.millman $
+
+"""
+__all__ = [
+    'apply_along_axis', 'apply_over_axes', 'atleast_1d', 'atleast_2d',
+    'atleast_3d', 'average', 'clump_masked', 'clump_unmasked', 'column_stack',
+    'compress_cols', 'compress_nd', 'compress_rowcols', 'compress_rows',
+    'count_masked', 'corrcoef', 'cov', 'diagflat', 'dot', 'dstack', 'ediff1d',
+    'flatnotmasked_contiguous', 'flatnotmasked_edges', 'hsplit', 'hstack',
+    'isin', 'in1d', 'intersect1d', 'mask_cols', 'mask_rowcols', 'mask_rows',
+    'masked_all', 'masked_all_like', 'median', 'mr_', 'ndenumerate',
+    'notmasked_contiguous', 'notmasked_edges', 'polyfit', 'row_stack',
+    'setdiff1d', 'setxor1d', 'stack', 'unique', 'union1d', 'vander', 'vstack',
+    ]
+
+import itertools
+import warnings
+
+from . import core as ma
+from .core import (
+    MaskedArray, MAError, add, array, asarray, concatenate, filled, count,
+    getmask, getmaskarray, make_mask_descr, masked, masked_array, mask_or,
+    nomask, ones, sort, zeros, getdata, get_masked_subclass, dot
+    )
+
+import numpy as np
+from numpy import ndarray, array as nxarray
+from numpy.core.multiarray import normalize_axis_index
+from numpy.core.numeric import normalize_axis_tuple
+from numpy.lib.function_base import _ureduce
+from numpy.lib.index_tricks import AxisConcatenator
+
+
+def issequence(seq):
+    """
+    Is seq a sequence (ndarray, list or tuple)?
+
+    """
+    return isinstance(seq, (ndarray, tuple, list))
+
+
+def count_masked(arr, axis=None):
+    """
+    Count the number of masked elements along the given axis.
+
+    Parameters
+    ----------
+    arr : array_like
+        An array with (possibly) masked elements.
+    axis : int, optional
+        Axis along which to count. If None (default), a flattened
+        version of the array is used.
+
+    Returns
+    -------
+    count : int, ndarray
+        The total number of masked elements (axis=None) or the number
+        of masked elements along each slice of the given axis.
+
+    See Also
+    --------
+    MaskedArray.count : Count non-masked elements.
+
+    Examples
+    --------
+    >>> import numpy.ma as ma
+    >>> a = np.arange(9).reshape((3,3))
+    >>> a = ma.array(a)
+    >>> a[1, 0] = ma.masked
+    >>> a[1, 2] = ma.masked
+    >>> a[2, 1] = ma.masked
+    >>> a
+    masked_array(
+      data=[[0, 1, 2],
+            [--, 4, --],
+            [6, --, 8]],
+      mask=[[False, False, False],
+            [ True, False,  True],
+            [False,  True, False]],
+      fill_value=999999)
+    >>> ma.count_masked(a)
+    3
+
+    When the `axis` keyword is used an array is returned.
+
+    >>> ma.count_masked(a, axis=0)
+    array([1, 1, 1])
+    >>> ma.count_masked(a, axis=1)
+    array([0, 2, 1])
+
+    """
+    m = getmaskarray(arr)
+    return m.sum(axis)
+
+
+def masked_all(shape, dtype=float):
+    """
+    Empty masked array with all elements masked.
+
+    Return an empty masked array of the given shape and dtype, where all the
+    data are masked.
+
+    Parameters
+    ----------
+    shape : int or tuple of ints
+        Shape of the required MaskedArray, e.g., ``(2, 3)`` or ``2``.
+    dtype : dtype, optional
+        Data type of the output.
+
+    Returns
+    -------
+    a : MaskedArray
+        A masked array with all data masked.
+
+    See Also
+    --------
+    masked_all_like : Empty masked array modelled on an existing array.
+
+    Examples
+    --------
+    >>> import numpy.ma as ma
+    >>> ma.masked_all((3, 3))
+    masked_array(
+      data=[[--, --, --],
+            [--, --, --],
+            [--, --, --]],
+      mask=[[ True,  True,  True],
+            [ True,  True,  True],
+            [ True,  True,  True]],
+      fill_value=1e+20,
+      dtype=float64)
+
+    The `dtype` parameter defines the underlying data type.
+
+    >>> a = ma.masked_all((3, 3))
+    >>> a.dtype
+    dtype('float64')
+    >>> a = ma.masked_all((3, 3), dtype=np.int32)
+    >>> a.dtype
+    dtype('int32')
+
+    """
+    a = masked_array(np.empty(shape, dtype),
+                     mask=np.ones(shape, make_mask_descr(dtype)))
+    return a
+
+
+def masked_all_like(arr):
+    """
+    Empty masked array with the properties of an existing array.
+
+    Return an empty masked array of the same shape and dtype as
+    the array `arr`, where all the data are masked.
+
+    Parameters
+    ----------
+    arr : ndarray
+        An array describing the shape and dtype of the required MaskedArray.
+
+    Returns
+    -------
+    a : MaskedArray
+        A masked array with all data masked.
+
+    Raises
+    ------
+    AttributeError
+        If `arr` doesn't have a shape attribute (i.e. not an ndarray)
+
+    See Also
+    --------
+    masked_all : Empty masked array with all elements masked.
+
+    Examples
+    --------
+    >>> import numpy.ma as ma
+    >>> arr = np.zeros((2, 3), dtype=np.float32)
+    >>> arr
+    array([[0., 0., 0.],
+           [0., 0., 0.]], dtype=float32)
+    >>> ma.masked_all_like(arr)
+    masked_array(
+      data=[[--, --, --],
+            [--, --, --]],
+      mask=[[ True,  True,  True],
+            [ True,  True,  True]],
+      fill_value=1e+20,
+      dtype=float32)
+
+    The dtype of the masked array matches the dtype of `arr`.
+
+    >>> arr.dtype
+    dtype('float32')
+    >>> ma.masked_all_like(arr).dtype
+    dtype('float32')
+
+    """
+    a = np.empty_like(arr).view(MaskedArray)
+    a._mask = np.ones(a.shape, dtype=make_mask_descr(a.dtype))
+    return a
+
+
+#####--------------------------------------------------------------------------
+#---- --- Standard functions ---
+#####--------------------------------------------------------------------------
+class _fromnxfunction:
+    """
+    Defines a wrapper to adapt NumPy functions to masked arrays.
+
+
+    An instance of `_fromnxfunction` can be called with the same parameters
+    as the wrapped NumPy function. The docstring of `newfunc` is adapted from
+    the wrapped function as well, see `getdoc`.
+
+    This class should not be used directly. Instead, one of its extensions that
+    provides support for a specific type of input should be used.
+
+    Parameters
+    ----------
+    funcname : str
+        The name of the function to be adapted. The function should be
+        in the NumPy namespace (i.e. ``np.funcname``).
+
+    """
+
+    def __init__(self, funcname):
+        self.__name__ = funcname
+        self.__doc__ = self.getdoc()
+
+    def getdoc(self):
+        """
+        Retrieve the docstring and signature from the function.
+
+        The ``__doc__`` attribute of the function is used as the docstring for
+        the new masked array version of the function. A note on application
+        of the function to the mask is appended.
+
+        Parameters
+        ----------
+        None
+
+        """
+        npfunc = getattr(np, self.__name__, None)
+        doc = getattr(npfunc, '__doc__', None)
+        if doc:
+            sig = self.__name__ + ma.get_object_signature(npfunc)
+            doc = ma.doc_note(doc, "The function is applied to both the _data "
+                                   "and the _mask, if any.")
+            return '\n\n'.join((sig, doc))
+        return
+
+    def __call__(self, *args, **params):
+        pass
+
+
+class _fromnxfunction_single(_fromnxfunction):
+    """
+    A version of `_fromnxfunction` that is called with a single array
+    argument followed by auxiliary args that are passed verbatim for
+    both the data and mask calls.
+    """
+    def __call__(self, x, *args, **params):
+        func = getattr(np, self.__name__)
+        if isinstance(x, ndarray):
+            _d = func(x.__array__(), *args, **params)
+            _m = func(getmaskarray(x), *args, **params)
+            return masked_array(_d, mask=_m)
+        else:
+            _d = func(np.asarray(x), *args, **params)
+            _m = func(getmaskarray(x), *args, **params)
+            return masked_array(_d, mask=_m)
+
+
+class _fromnxfunction_seq(_fromnxfunction):
+    """
+    A version of `_fromnxfunction` that is called with a single sequence
+    of arrays followed by auxiliary args that are passed verbatim for
+    both the data and mask calls.
+    """
+    def __call__(self, x, *args, **params):
+        func = getattr(np, self.__name__)
+        _d = func(tuple([np.asarray(a) for a in x]), *args, **params)
+        _m = func(tuple([getmaskarray(a) for a in x]), *args, **params)
+        return masked_array(_d, mask=_m)
+
+
+class _fromnxfunction_args(_fromnxfunction):
+    """
+    A version of `_fromnxfunction` that is called with multiple array
+    arguments. The first non-array-like input marks the beginning of the
+    arguments that are passed verbatim for both the data and mask calls.
+    Array arguments are processed independently and the results are
+    returned in a list. If only one array is found, the return value is
+    just the processed array instead of a list.
+    """
+    def __call__(self, *args, **params):
+        func = getattr(np, self.__name__)
+        arrays = []
+        args = list(args)
+        while len(args) > 0 and issequence(args[0]):
+            arrays.append(args.pop(0))
+        res = []
+        for x in arrays:
+            _d = func(np.asarray(x), *args, **params)
+            _m = func(getmaskarray(x), *args, **params)
+            res.append(masked_array(_d, mask=_m))
+        if len(arrays) == 1:
+            return res[0]
+        return res
+
+
+class _fromnxfunction_allargs(_fromnxfunction):
+    """
+    A version of `_fromnxfunction` that is called with multiple array
+    arguments. Similar to `_fromnxfunction_args` except that all args
+    are converted to arrays even if they are not so already. This makes
+    it possible to process scalars as 1-D arrays. Only keyword arguments
+    are passed through verbatim for the data and mask calls. Arrays
+    arguments are processed independently and the results are returned
+    in a list. If only one arg is present, the return value is just the
+    processed array instead of a list.
+    """
+    def __call__(self, *args, **params):
+        func = getattr(np, self.__name__)
+        res = []
+        for x in args:
+            _d = func(np.asarray(x), **params)
+            _m = func(getmaskarray(x), **params)
+            res.append(masked_array(_d, mask=_m))
+        if len(args) == 1:
+            return res[0]
+        return res
+
+
+atleast_1d = _fromnxfunction_allargs('atleast_1d')
+atleast_2d = _fromnxfunction_allargs('atleast_2d')
+atleast_3d = _fromnxfunction_allargs('atleast_3d')
+
+vstack = row_stack = _fromnxfunction_seq('vstack')
+hstack = _fromnxfunction_seq('hstack')
+column_stack = _fromnxfunction_seq('column_stack')
+dstack = _fromnxfunction_seq('dstack')
+stack = _fromnxfunction_seq('stack')
+
+hsplit = _fromnxfunction_single('hsplit')
+
+diagflat = _fromnxfunction_single('diagflat')
+
+
+#####--------------------------------------------------------------------------
+#----
+#####--------------------------------------------------------------------------
+def flatten_inplace(seq):
+    """Flatten a sequence in place."""
+    k = 0
+    while (k != len(seq)):
+        while hasattr(seq[k], '__iter__'):
+            seq[k:(k + 1)] = seq[k]
+        k += 1
+    return seq
+
+
+def apply_along_axis(func1d, axis, arr, *args, **kwargs):
+    """
+    (This docstring should be overwritten)
+    """
+    arr = array(arr, copy=False, subok=True)
+    nd = arr.ndim
+    axis = normalize_axis_index(axis, nd)
+    ind = [0] * (nd - 1)
+    i = np.zeros(nd, 'O')
+    indlist = list(range(nd))
+    indlist.remove(axis)
+    i[axis] = slice(None, None)
+    outshape = np.asarray(arr.shape).take(indlist)
+    i.put(indlist, ind)
+    res = func1d(arr[tuple(i.tolist())], *args, **kwargs)
+    #  if res is a number, then we have a smaller output array
+    asscalar = np.isscalar(res)
+    if not asscalar:
+        try:
+            len(res)
+        except TypeError:
+            asscalar = True
+    # Note: we shouldn't set the dtype of the output from the first result
+    # so we force the type to object, and build a list of dtypes.  We'll
+    # just take the largest, to avoid some downcasting
+    dtypes = []
+    if asscalar:
+        dtypes.append(np.asarray(res).dtype)
+        outarr = zeros(outshape, object)
+        outarr[tuple(ind)] = res
+        Ntot = np.prod(outshape)
+        k = 1
+        while k < Ntot:
+            # increment the index
+            ind[-1] += 1
+            n = -1
+            while (ind[n] >= outshape[n]) and (n > (1 - nd)):
+                ind[n - 1] += 1
+                ind[n] = 0
+                n -= 1
+            i.put(indlist, ind)
+            res = func1d(arr[tuple(i.tolist())], *args, **kwargs)
+            outarr[tuple(ind)] = res
+            dtypes.append(asarray(res).dtype)
+            k += 1
+    else:
+        res = array(res, copy=False, subok=True)
+        j = i.copy()
+        j[axis] = ([slice(None, None)] * res.ndim)
+        j.put(indlist, ind)
+        Ntot = np.prod(outshape)
+        holdshape = outshape
+        outshape = list(arr.shape)
+        outshape[axis] = res.shape
+        dtypes.append(asarray(res).dtype)
+        outshape = flatten_inplace(outshape)
+        outarr = zeros(outshape, object)
+        outarr[tuple(flatten_inplace(j.tolist()))] = res
+        k = 1
+        while k < Ntot:
+            # increment the index
+            ind[-1] += 1
+            n = -1
+            while (ind[n] >= holdshape[n]) and (n > (1 - nd)):
+                ind[n - 1] += 1
+                ind[n] = 0
+                n -= 1
+            i.put(indlist, ind)
+            j.put(indlist, ind)
+            res = func1d(arr[tuple(i.tolist())], *args, **kwargs)
+            outarr[tuple(flatten_inplace(j.tolist()))] = res
+            dtypes.append(asarray(res).dtype)
+            k += 1
+    max_dtypes = np.dtype(np.asarray(dtypes).max())
+    if not hasattr(arr, '_mask'):
+        result = np.asarray(outarr, dtype=max_dtypes)
+    else:
+        result = asarray(outarr, dtype=max_dtypes)
+        result.fill_value = ma.default_fill_value(result)
+    return result
+apply_along_axis.__doc__ = np.apply_along_axis.__doc__
+
+
+def apply_over_axes(func, a, axes):
+    """
+    (This docstring will be overwritten)
+    """
+    val = asarray(a)
+    N = a.ndim
+    if array(axes).ndim == 0:
+        axes = (axes,)
+    for axis in axes:
+        if axis < 0:
+            axis = N + axis
+        args = (val, axis)
+        res = func(*args)
+        if res.ndim == val.ndim:
+            val = res
+        else:
+            res = ma.expand_dims(res, axis)
+            if res.ndim == val.ndim:
+                val = res
+            else:
+                raise ValueError("function is not returning "
+                        "an array of the correct shape")
+    return val
+
+
+if apply_over_axes.__doc__ is not None:
+    apply_over_axes.__doc__ = np.apply_over_axes.__doc__[
+        :np.apply_over_axes.__doc__.find('Notes')].rstrip() + \
+    """
+
+    Examples
+    --------
+    >>> a = np.ma.arange(24).reshape(2,3,4)
+    >>> a[:,0,1] = np.ma.masked
+    >>> a[:,1,:] = np.ma.masked
+    >>> a
+    masked_array(
+      data=[[[0, --, 2, 3],
+             [--, --, --, --],
+             [8, 9, 10, 11]],
+            [[12, --, 14, 15],
+             [--, --, --, --],
+             [20, 21, 22, 23]]],
+      mask=[[[False,  True, False, False],
+             [ True,  True,  True,  True],
+             [False, False, False, False]],
+            [[False,  True, False, False],
+             [ True,  True,  True,  True],
+             [False, False, False, False]]],
+      fill_value=999999)
+    >>> np.ma.apply_over_axes(np.ma.sum, a, [0,2])
+    masked_array(
+      data=[[[46],
+             [--],
+             [124]]],
+      mask=[[[False],
+             [ True],
+             [False]]],
+      fill_value=999999)
+
+    Tuple axis arguments to ufuncs are equivalent:
+
+    >>> np.ma.sum(a, axis=(0,2)).reshape((1,-1,1))
+    masked_array(
+      data=[[[46],
+             [--],
+             [124]]],
+      mask=[[[False],
+             [ True],
+             [False]]],
+      fill_value=999999)
+    """
+
+
+def average(a, axis=None, weights=None, returned=False, *,
+            keepdims=np._NoValue):
+    """
+    Return the weighted average of array over the given axis.
+
+    Parameters
+    ----------
+    a : array_like
+        Data to be averaged.
+        Masked entries are not taken into account in the computation.
+    axis : int, optional
+        Axis along which to average `a`. If None, averaging is done over
+        the flattened array.
+    weights : array_like, optional
+        The importance that each element has in the computation of the average.
+        The weights array can either be 1-D (in which case its length must be
+        the size of `a` along the given axis) or of the same shape as `a`.
+        If ``weights=None``, then all data in `a` are assumed to have a
+        weight equal to one.  The 1-D calculation is::
+
+            avg = sum(a * weights) / sum(weights)
+
+        The only constraint on `weights` is that `sum(weights)` must not be 0.
+    returned : bool, optional
+        Flag indicating whether a tuple ``(result, sum of weights)``
+        should be returned as output (True), or just the result (False).
+        Default is False.
+    keepdims : bool, optional
+        If this is set to True, the axes which are reduced are left
+        in the result as dimensions with size one. With this option,
+        the result will broadcast correctly against the original `a`.
+        *Note:* `keepdims` will not work with instances of `numpy.matrix`
+        or other classes whose methods do not support `keepdims`.
+
+        .. versionadded:: 1.23.0
+
+    Returns
+    -------
+    average, [sum_of_weights] : (tuple of) scalar or MaskedArray
+        The average along the specified axis. When returned is `True`,
+        return a tuple with the average as the first element and the sum
+        of the weights as the second element. The return type is `np.float64`
+        if `a` is of integer type and floats smaller than `float64`, or the
+        input data-type, otherwise. If returned, `sum_of_weights` is always
+        `float64`.
+
+    Examples
+    --------
+    >>> a = np.ma.array([1., 2., 3., 4.], mask=[False, False, True, True])
+    >>> np.ma.average(a, weights=[3, 1, 0, 0])
+    1.25
+
+    >>> x = np.ma.arange(6.).reshape(3, 2)
+    >>> x
+    masked_array(
+      data=[[0., 1.],
+            [2., 3.],
+            [4., 5.]],
+      mask=False,
+      fill_value=1e+20)
+    >>> avg, sumweights = np.ma.average(x, axis=0, weights=[1, 2, 3],
+    ...                                 returned=True)
+    >>> avg
+    masked_array(data=[2.6666666666666665, 3.6666666666666665],
+                 mask=[False, False],
+           fill_value=1e+20)
+
+    With ``keepdims=True``, the following result has shape (3, 1).
+
+    >>> np.ma.average(x, axis=1, keepdims=True)
+    masked_array(
+      data=[[0.5],
+            [2.5],
+            [4.5]],
+      mask=False,
+      fill_value=1e+20)
+    """
+    a = asarray(a)
+    m = getmask(a)
+
+    # inspired by 'average' in numpy/lib/function_base.py
+
+    if keepdims is np._NoValue:
+        # Don't pass on the keepdims argument if one wasn't given.
+        keepdims_kw = {}
+    else:
+        keepdims_kw = {'keepdims': keepdims}
+
+    if weights is None:
+        avg = a.mean(axis, **keepdims_kw)
+        scl = avg.dtype.type(a.count(axis))
+    else:
+        wgt = asarray(weights)
+
+        if issubclass(a.dtype.type, (np.integer, np.bool_)):
+            result_dtype = np.result_type(a.dtype, wgt.dtype, 'f8')
+        else:
+            result_dtype = np.result_type(a.dtype, wgt.dtype)
+
+        # Sanity checks
+        if a.shape != wgt.shape:
+            if axis is None:
+                raise TypeError(
+                    "Axis must be specified when shapes of a and weights "
+                    "differ.")
+            if wgt.ndim != 1:
+                raise TypeError(
+                    "1D weights expected when shapes of a and weights differ.")
+            if wgt.shape[0] != a.shape[axis]:
+                raise ValueError(
+                    "Length of weights not compatible with specified axis.")
+
+            # setup wgt to broadcast along axis
+            wgt = np.broadcast_to(wgt, (a.ndim-1)*(1,) + wgt.shape, subok=True)
+            wgt = wgt.swapaxes(-1, axis)
+
+        if m is not nomask:
+            wgt = wgt*(~a.mask)
+            wgt.mask |= a.mask
+
+        scl = wgt.sum(axis=axis, dtype=result_dtype, **keepdims_kw)
+        avg = np.multiply(a, wgt,
+                          dtype=result_dtype).sum(axis, **keepdims_kw) / scl
+
+    if returned:
+        if scl.shape != avg.shape:
+            scl = np.broadcast_to(scl, avg.shape).copy()
+        return avg, scl
+    else:
+        return avg
+
+
+def median(a, axis=None, out=None, overwrite_input=False, keepdims=False):
+    """
+    Compute the median along the specified axis.
+
+    Returns the median of the array elements.
+
+    Parameters
+    ----------
+    a : array_like
+        Input array or object that can be converted to an array.
+    axis : int, optional
+        Axis along which the medians are computed. The default (None) is
+        to compute the median along a flattened version of the array.
+    out : ndarray, optional
+        Alternative output array in which to place the result. It must
+        have the same shape and buffer length as the expected output
+        but the type will be cast if necessary.
+    overwrite_input : bool, optional
+        If True, then allow use of memory of input array (a) for
+        calculations. The input array will be modified by the call to
+        median. This will save memory when you do not need to preserve
+        the contents of the input array. Treat the input as undefined,
+        but it will probably be fully or partially sorted. Default is
+        False. Note that, if `overwrite_input` is True, and the input
+        is not already an `ndarray`, an error will be raised.
+    keepdims : bool, optional
+        If this is set to True, the axes which are reduced are left
+        in the result as dimensions with size one. With this option,
+        the result will broadcast correctly against the input array.
+
+        .. versionadded:: 1.10.0
+
+    Returns
+    -------
+    median : ndarray
+        A new array holding the result is returned unless out is
+        specified, in which case a reference to out is returned.
+        Return data-type is `float64` for integers and floats smaller than
+        `float64`, or the input data-type, otherwise.
+
+    See Also
+    --------
+    mean
+
+    Notes
+    -----
+    Given a vector ``V`` with ``N`` non masked values, the median of ``V``
+    is the middle value of a sorted copy of ``V`` (``Vs``) - i.e.
+    ``Vs[(N-1)/2]``, when ``N`` is odd, or ``{Vs[N/2 - 1] + Vs[N/2]}/2``
+    when ``N`` is even.
+
+    Examples
+    --------
+    >>> x = np.ma.array(np.arange(8), mask=[0]*4 + [1]*4)
+    >>> np.ma.median(x)
+    1.5
+
+    >>> x = np.ma.array(np.arange(10).reshape(2, 5), mask=[0]*6 + [1]*4)
+    >>> np.ma.median(x)
+    2.5
+    >>> np.ma.median(x, axis=-1, overwrite_input=True)
+    masked_array(data=[2.0, 5.0],
+                 mask=[False, False],
+           fill_value=1e+20)
+
+    """
+    if not hasattr(a, 'mask'):
+        m = np.median(getdata(a, subok=True), axis=axis,
+                      out=out, overwrite_input=overwrite_input,
+                      keepdims=keepdims)
+        if isinstance(m, np.ndarray) and 1 <= m.ndim:
+            return masked_array(m, copy=False)
+        else:
+            return m
+
+    return _ureduce(a, func=_median, keepdims=keepdims, axis=axis, out=out,
+                    overwrite_input=overwrite_input)
+
+
+def _median(a, axis=None, out=None, overwrite_input=False):
+    # when an unmasked NaN is present return it, so we need to sort the NaN
+    # values behind the mask
+    if np.issubdtype(a.dtype, np.inexact):
+        fill_value = np.inf
+    else:
+        fill_value = None
+    if overwrite_input:
+        if axis is None:
+            asorted = a.ravel()
+            asorted.sort(fill_value=fill_value)
+        else:
+            a.sort(axis=axis, fill_value=fill_value)
+            asorted = a
+    else:
+        asorted = sort(a, axis=axis, fill_value=fill_value)
+
+    if axis is None:
+        axis = 0
+    else:
+        axis = normalize_axis_index(axis, asorted.ndim)
+
+    if asorted.shape[axis] == 0:
+        # for empty axis integer indices fail so use slicing to get same result
+        # as median (which is mean of empty slice = nan)
+        indexer = [slice(None)] * asorted.ndim
+        indexer[axis] = slice(0, 0)
+        indexer = tuple(indexer)
+        return np.ma.mean(asorted[indexer], axis=axis, out=out)
+
+    if asorted.ndim == 1:
+        idx, odd = divmod(count(asorted), 2)
+        mid = asorted[idx + odd - 1:idx + 1]
+        if np.issubdtype(asorted.dtype, np.inexact) and asorted.size > 0:
+            # avoid inf / x = masked
+            s = mid.sum(out=out)
+            if not odd:
+                s = np.true_divide(s, 2., casting='safe', out=out)
+            s = np.lib.utils._median_nancheck(asorted, s, axis)
+        else:
+            s = mid.mean(out=out)
+
+        # if result is masked either the input contained enough
+        # minimum_fill_value so that it would be the median or all values
+        # masked
+        if np.ma.is_masked(s) and not np.all(asorted.mask):
+            return np.ma.minimum_fill_value(asorted)
+        return s
+
+    counts = count(asorted, axis=axis, keepdims=True)
+    h = counts // 2
+
+    # duplicate high if odd number of elements so mean does nothing
+    odd = counts % 2 == 1
+    l = np.where(odd, h, h-1)
+
+    lh = np.concatenate([l,h], axis=axis)
+
+    # get low and high median
+    low_high = np.take_along_axis(asorted, lh, axis=axis)
+
+    def replace_masked(s):
+        # Replace masked entries with minimum_full_value unless it all values
+        # are masked. This is required as the sort order of values equal or
+        # larger than the fill value is undefined and a valid value placed
+        # elsewhere, e.g. [4, --, inf].
+        if np.ma.is_masked(s):
+            rep = (~np.all(asorted.mask, axis=axis, keepdims=True)) & s.mask
+            s.data[rep] = np.ma.minimum_fill_value(asorted)
+            s.mask[rep] = False
+
+    replace_masked(low_high)
+
+    if np.issubdtype(asorted.dtype, np.inexact):
+        # avoid inf / x = masked
+        s = np.ma.sum(low_high, axis=axis, out=out)
+        np.true_divide(s.data, 2., casting='unsafe', out=s.data)
+
+        s = np.lib.utils._median_nancheck(asorted, s, axis)
+    else:
+        s = np.ma.mean(low_high, axis=axis, out=out)
+
+    return s
+
+
+def compress_nd(x, axis=None):
+    """Suppress slices from multiple dimensions which contain masked values.
+
+    Parameters
+    ----------
+    x : array_like, MaskedArray
+        The array to operate on. If not a MaskedArray instance (or if no array
+        elements are masked), `x` is interpreted as a MaskedArray with `mask`
+        set to `nomask`.
+    axis : tuple of ints or int, optional
+        Which dimensions to suppress slices from can be configured with this
+        parameter.
+        - If axis is a tuple of ints, those are the axes to suppress slices from.
+        - If axis is an int, then that is the only axis to suppress slices from.
+        - If axis is None, all axis are selected.
+
+    Returns
+    -------
+    compress_array : ndarray
+        The compressed array.
+    """
+    x = asarray(x)
+    m = getmask(x)
+    # Set axis to tuple of ints
+    if axis is None:
+        axis = tuple(range(x.ndim))
+    else:
+        axis = normalize_axis_tuple(axis, x.ndim)
+
+    # Nothing is masked: return x
+    if m is nomask or not m.any():
+        return x._data
+    # All is masked: return empty
+    if m.all():
+        return nxarray([])
+    # Filter elements through boolean indexing
+    data = x._data
+    for ax in axis:
+        axes = tuple(list(range(ax)) + list(range(ax + 1, x.ndim)))
+        data = data[(slice(None),)*ax + (~m.any(axis=axes),)]
+    return data
+
+
+def compress_rowcols(x, axis=None):
+    """
+    Suppress the rows and/or columns of a 2-D array that contain
+    masked values.
+
+    The suppression behavior is selected with the `axis` parameter.
+
+    - If axis is None, both rows and columns are suppressed.
+    - If axis is 0, only rows are suppressed.
+    - If axis is 1 or -1, only columns are suppressed.
+
+    Parameters
+    ----------
+    x : array_like, MaskedArray
+        The array to operate on.  If not a MaskedArray instance (or if no array
+        elements are masked), `x` is interpreted as a MaskedArray with
+        `mask` set to `nomask`. Must be a 2D array.
+    axis : int, optional
+        Axis along which to perform the operation. Default is None.
+
+    Returns
+    -------
+    compressed_array : ndarray
+        The compressed array.
+
+    Examples
+    --------
+    >>> x = np.ma.array(np.arange(9).reshape(3, 3), mask=[[1, 0, 0],
+    ...                                                   [1, 0, 0],
+    ...                                                   [0, 0, 0]])
+    >>> x
+    masked_array(
+      data=[[--, 1, 2],
+            [--, 4, 5],
+            [6, 7, 8]],
+      mask=[[ True, False, False],
+            [ True, False, False],
+            [False, False, False]],
+      fill_value=999999)
+
+    >>> np.ma.compress_rowcols(x)
+    array([[7, 8]])
+    >>> np.ma.compress_rowcols(x, 0)
+    array([[6, 7, 8]])
+    >>> np.ma.compress_rowcols(x, 1)
+    array([[1, 2],
+           [4, 5],
+           [7, 8]])
+
+    """
+    if asarray(x).ndim != 2:
+        raise NotImplementedError("compress_rowcols works for 2D arrays only.")
+    return compress_nd(x, axis=axis)
+
+
+def compress_rows(a):
+    """
+    Suppress whole rows of a 2-D array that contain masked values.
+
+    This is equivalent to ``np.ma.compress_rowcols(a, 0)``, see
+    `compress_rowcols` for details.
+
+    See Also
+    --------
+    compress_rowcols
+
+    """
+    a = asarray(a)
+    if a.ndim != 2:
+        raise NotImplementedError("compress_rows works for 2D arrays only.")
+    return compress_rowcols(a, 0)
+
+
+def compress_cols(a):
+    """
+    Suppress whole columns of a 2-D array that contain masked values.
+
+    This is equivalent to ``np.ma.compress_rowcols(a, 1)``, see
+    `compress_rowcols` for details.
+
+    See Also
+    --------
+    compress_rowcols
+
+    """
+    a = asarray(a)
+    if a.ndim != 2:
+        raise NotImplementedError("compress_cols works for 2D arrays only.")
+    return compress_rowcols(a, 1)
+
+
+def mask_rowcols(a, axis=None):
+    """
+    Mask rows and/or columns of a 2D array that contain masked values.
+
+    Mask whole rows and/or columns of a 2D array that contain
+    masked values.  The masking behavior is selected using the
+    `axis` parameter.
+
+      - If `axis` is None, rows *and* columns are masked.
+      - If `axis` is 0, only rows are masked.
+      - If `axis` is 1 or -1, only columns are masked.
+
+    Parameters
+    ----------
+    a : array_like, MaskedArray
+        The array to mask.  If not a MaskedArray instance (or if no array
+        elements are masked), the result is a MaskedArray with `mask` set
+        to `nomask` (False). Must be a 2D array.
+    axis : int, optional
+        Axis along which to perform the operation. If None, applies to a
+        flattened version of the array.
+
+    Returns
+    -------
+    a : MaskedArray
+        A modified version of the input array, masked depending on the value
+        of the `axis` parameter.
+
+    Raises
+    ------
+    NotImplementedError
+        If input array `a` is not 2D.
+
+    See Also
+    --------
+    mask_rows : Mask rows of a 2D array that contain masked values.
+    mask_cols : Mask cols of a 2D array that contain masked values.
+    masked_where : Mask where a condition is met.
+
+    Notes
+    -----
+    The input array's mask is modified by this function.
+
+    Examples
+    --------
+    >>> import numpy.ma as ma
+    >>> a = np.zeros((3, 3), dtype=int)
+    >>> a[1, 1] = 1
+    >>> a
+    array([[0, 0, 0],
+           [0, 1, 0],
+           [0, 0, 0]])
+    >>> a = ma.masked_equal(a, 1)
+    >>> a
+    masked_array(
+      data=[[0, 0, 0],
+            [0, --, 0],
+            [0, 0, 0]],
+      mask=[[False, False, False],
+            [False,  True, False],
+            [False, False, False]],
+      fill_value=1)
+    >>> ma.mask_rowcols(a)
+    masked_array(
+      data=[[0, --, 0],
+            [--, --, --],
+            [0, --, 0]],
+      mask=[[False,  True, False],
+            [ True,  True,  True],
+            [False,  True, False]],
+      fill_value=1)
+
+    """
+    a = array(a, subok=False)
+    if a.ndim != 2:
+        raise NotImplementedError("mask_rowcols works for 2D arrays only.")
+    m = getmask(a)
+    # Nothing is masked: return a
+    if m is nomask or not m.any():
+        return a
+    maskedval = m.nonzero()
+    a._mask = a._mask.copy()
+    if not axis:
+        a[np.unique(maskedval[0])] = masked
+    if axis in [None, 1, -1]:
+        a[:, np.unique(maskedval[1])] = masked
+    return a
+
+
+def mask_rows(a, axis=np._NoValue):
+    """
+    Mask rows of a 2D array that contain masked values.
+
+    This function is a shortcut to ``mask_rowcols`` with `axis` equal to 0.
+
+    See Also
+    --------
+    mask_rowcols : Mask rows and/or columns of a 2D array.
+    masked_where : Mask where a condition is met.
+
+    Examples
+    --------
+    >>> import numpy.ma as ma
+    >>> a = np.zeros((3, 3), dtype=int)
+    >>> a[1, 1] = 1
+    >>> a
+    array([[0, 0, 0],
+           [0, 1, 0],
+           [0, 0, 0]])
+    >>> a = ma.masked_equal(a, 1)
+    >>> a
+    masked_array(
+      data=[[0, 0, 0],
+            [0, --, 0],
+            [0, 0, 0]],
+      mask=[[False, False, False],
+            [False,  True, False],
+            [False, False, False]],
+      fill_value=1)
+
+    >>> ma.mask_rows(a)
+    masked_array(
+      data=[[0, 0, 0],
+            [--, --, --],
+            [0, 0, 0]],
+      mask=[[False, False, False],
+            [ True,  True,  True],
+            [False, False, False]],
+      fill_value=1)
+
+    """
+    if axis is not np._NoValue:
+        # remove the axis argument when this deprecation expires
+        # NumPy 1.18.0, 2019-11-28
+        warnings.warn(
+            "The axis argument has always been ignored, in future passing it "
+            "will raise TypeError", DeprecationWarning, stacklevel=2)
+    return mask_rowcols(a, 0)
+
+
+def mask_cols(a, axis=np._NoValue):
+    """
+    Mask columns of a 2D array that contain masked values.
+
+    This function is a shortcut to ``mask_rowcols`` with `axis` equal to 1.
+
+    See Also
+    --------
+    mask_rowcols : Mask rows and/or columns of a 2D array.
+    masked_where : Mask where a condition is met.
+
+    Examples
+    --------
+    >>> import numpy.ma as ma
+    >>> a = np.zeros((3, 3), dtype=int)
+    >>> a[1, 1] = 1
+    >>> a
+    array([[0, 0, 0],
+           [0, 1, 0],
+           [0, 0, 0]])
+    >>> a = ma.masked_equal(a, 1)
+    >>> a
+    masked_array(
+      data=[[0, 0, 0],
+            [0, --, 0],
+            [0, 0, 0]],
+      mask=[[False, False, False],
+            [False,  True, False],
+            [False, False, False]],
+      fill_value=1)
+    >>> ma.mask_cols(a)
+    masked_array(
+      data=[[0, --, 0],
+            [0, --, 0],
+            [0, --, 0]],
+      mask=[[False,  True, False],
+            [False,  True, False],
+            [False,  True, False]],
+      fill_value=1)
+
+    """
+    if axis is not np._NoValue:
+        # remove the axis argument when this deprecation expires
+        # NumPy 1.18.0, 2019-11-28
+        warnings.warn(
+            "The axis argument has always been ignored, in future passing it "
+            "will raise TypeError", DeprecationWarning, stacklevel=2)
+    return mask_rowcols(a, 1)
+
+
+#####--------------------------------------------------------------------------
+#---- --- arraysetops ---
+#####--------------------------------------------------------------------------
+
+def ediff1d(arr, to_end=None, to_begin=None):
+    """
+    Compute the differences between consecutive elements of an array.
+
+    This function is the equivalent of `numpy.ediff1d` that takes masked
+    values into account, see `numpy.ediff1d` for details.
+
+    See Also
+    --------
+    numpy.ediff1d : Equivalent function for ndarrays.
+
+    """
+    arr = ma.asanyarray(arr).flat
+    ed = arr[1:] - arr[:-1]
+    arrays = [ed]
+    #
+    if to_begin is not None:
+        arrays.insert(0, to_begin)
+    if to_end is not None:
+        arrays.append(to_end)
+    #
+    if len(arrays) != 1:
+        # We'll save ourselves a copy of a potentially large array in the common
+        # case where neither to_begin or to_end was given.
+        ed = hstack(arrays)
+    #
+    return ed
+
+
+def unique(ar1, return_index=False, return_inverse=False):
+    """
+    Finds the unique elements of an array.
+
+    Masked values are considered the same element (masked). The output array
+    is always a masked array. See `numpy.unique` for more details.
+
+    See Also
+    --------
+    numpy.unique : Equivalent function for ndarrays.
+
+    Examples
+    --------
+    >>> import numpy.ma as ma
+    >>> a = [1, 2, 1000, 2, 3]
+    >>> mask = [0, 0, 1, 0, 0]
+    >>> masked_a = ma.masked_array(a, mask)
+    >>> masked_a
+    masked_array(data=[1, 2, --, 2, 3],
+                mask=[False, False,  True, False, False],
+        fill_value=999999)
+    >>> ma.unique(masked_a)
+    masked_array(data=[1, 2, 3, --],
+                mask=[False, False, False,  True],
+        fill_value=999999)
+    >>> ma.unique(masked_a, return_index=True)
+    (masked_array(data=[1, 2, 3, --],
+                mask=[False, False, False,  True],
+        fill_value=999999), array([0, 1, 4, 2]))
+    >>> ma.unique(masked_a, return_inverse=True)
+    (masked_array(data=[1, 2, 3, --],
+                mask=[False, False, False,  True],
+        fill_value=999999), array([0, 1, 3, 1, 2]))
+    >>> ma.unique(masked_a, return_index=True, return_inverse=True)
+    (masked_array(data=[1, 2, 3, --],
+                mask=[False, False, False,  True],
+        fill_value=999999), array([0, 1, 4, 2]), array([0, 1, 3, 1, 2]))
+    """
+    output = np.unique(ar1,
+                       return_index=return_index,
+                       return_inverse=return_inverse)
+    if isinstance(output, tuple):
+        output = list(output)
+        output[0] = output[0].view(MaskedArray)
+        output = tuple(output)
+    else:
+        output = output.view(MaskedArray)
+    return output
+
+
+def intersect1d(ar1, ar2, assume_unique=False):
+    """
+    Returns the unique elements common to both arrays.
+
+    Masked values are considered equal one to the other.
+    The output is always a masked array.
+
+    See `numpy.intersect1d` for more details.
+
+    See Also
+    --------
+    numpy.intersect1d : Equivalent function for ndarrays.
+
+    Examples
+    --------
+    >>> x = np.ma.array([1, 3, 3, 3], mask=[0, 0, 0, 1])
+    >>> y = np.ma.array([3, 1, 1, 1], mask=[0, 0, 0, 1])
+    >>> np.ma.intersect1d(x, y)
+    masked_array(data=[1, 3, --],
+                 mask=[False, False,  True],
+           fill_value=999999)
+
+    """
+    if assume_unique:
+        aux = ma.concatenate((ar1, ar2))
+    else:
+        # Might be faster than unique( intersect1d( ar1, ar2 ) )?
+        aux = ma.concatenate((unique(ar1), unique(ar2)))
+    aux.sort()
+    return aux[:-1][aux[1:] == aux[:-1]]
+
+
+def setxor1d(ar1, ar2, assume_unique=False):
+    """
+    Set exclusive-or of 1-D arrays with unique elements.
+
+    The output is always a masked array. See `numpy.setxor1d` for more details.
+
+    See Also
+    --------
+    numpy.setxor1d : Equivalent function for ndarrays.
+
+    """
+    if not assume_unique:
+        ar1 = unique(ar1)
+        ar2 = unique(ar2)
+
+    aux = ma.concatenate((ar1, ar2))
+    if aux.size == 0:
+        return aux
+    aux.sort()
+    auxf = aux.filled()
+#    flag = ediff1d( aux, to_end = 1, to_begin = 1 ) == 0
+    flag = ma.concatenate(([True], (auxf[1:] != auxf[:-1]), [True]))
+#    flag2 = ediff1d( flag ) == 0
+    flag2 = (flag[1:] == flag[:-1])
+    return aux[flag2]
+
+
+def in1d(ar1, ar2, assume_unique=False, invert=False):
+    """
+    Test whether each element of an array is also present in a second
+    array.
+
+    The output is always a masked array. See `numpy.in1d` for more details.
+
+    We recommend using :func:`isin` instead of `in1d` for new code.
+
+    See Also
+    --------
+    isin       : Version of this function that preserves the shape of ar1.
+    numpy.in1d : Equivalent function for ndarrays.
+
+    Notes
+    -----
+    .. versionadded:: 1.4.0
+
+    """
+    if not assume_unique:
+        ar1, rev_idx = unique(ar1, return_inverse=True)
+        ar2 = unique(ar2)
+
+    ar = ma.concatenate((ar1, ar2))
+    # We need this to be a stable sort, so always use 'mergesort'
+    # here. The values from the first array should always come before
+    # the values from the second array.
+    order = ar.argsort(kind='mergesort')
+    sar = ar[order]
+    if invert:
+        bool_ar = (sar[1:] != sar[:-1])
+    else:
+        bool_ar = (sar[1:] == sar[:-1])
+    flag = ma.concatenate((bool_ar, [invert]))
+    indx = order.argsort(kind='mergesort')[:len(ar1)]
+
+    if assume_unique:
+        return flag[indx]
+    else:
+        return flag[indx][rev_idx]
+
+
+def isin(element, test_elements, assume_unique=False, invert=False):
+    """
+    Calculates `element in test_elements`, broadcasting over
+    `element` only.
+
+    The output is always a masked array of the same shape as `element`.
+    See `numpy.isin` for more details.
+
+    See Also
+    --------
+    in1d       : Flattened version of this function.
+    numpy.isin : Equivalent function for ndarrays.
+
+    Notes
+    -----
+    .. versionadded:: 1.13.0
+
+    """
+    element = ma.asarray(element)
+    return in1d(element, test_elements, assume_unique=assume_unique,
+                invert=invert).reshape(element.shape)
+
+
+def union1d(ar1, ar2):
+    """
+    Union of two arrays.
+
+    The output is always a masked array. See `numpy.union1d` for more details.
+
+    See Also
+    --------
+    numpy.union1d : Equivalent function for ndarrays.
+
+    """
+    return unique(ma.concatenate((ar1, ar2), axis=None))
+
+
+def setdiff1d(ar1, ar2, assume_unique=False):
+    """
+    Set difference of 1D arrays with unique elements.
+
+    The output is always a masked array. See `numpy.setdiff1d` for more
+    details.
+
+    See Also
+    --------
+    numpy.setdiff1d : Equivalent function for ndarrays.
+
+    Examples
+    --------
+    >>> x = np.ma.array([1, 2, 3, 4], mask=[0, 1, 0, 1])
+    >>> np.ma.setdiff1d(x, [1, 2])
+    masked_array(data=[3, --],
+                 mask=[False,  True],
+           fill_value=999999)
+
+    """
+    if assume_unique:
+        ar1 = ma.asarray(ar1).ravel()
+    else:
+        ar1 = unique(ar1)
+        ar2 = unique(ar2)
+    return ar1[in1d(ar1, ar2, assume_unique=True, invert=True)]
+
+
+###############################################################################
+#                                Covariance                                   #
+###############################################################################
+
+
+def _covhelper(x, y=None, rowvar=True, allow_masked=True):
+    """
+    Private function for the computation of covariance and correlation
+    coefficients.
+
+    """
+    x = ma.array(x, ndmin=2, copy=True, dtype=float)
+    xmask = ma.getmaskarray(x)
+    # Quick exit if we can't process masked data
+    if not allow_masked and xmask.any():
+        raise ValueError("Cannot process masked data.")
+    #
+    if x.shape[0] == 1:
+        rowvar = True
+    # Make sure that rowvar is either 0 or 1
+    rowvar = int(bool(rowvar))
+    axis = 1 - rowvar
+    if rowvar:
+        tup = (slice(None), None)
+    else:
+        tup = (None, slice(None))
+    #
+    if y is None:
+        xnotmask = np.logical_not(xmask).astype(int)
+    else:
+        y = array(y, copy=False, ndmin=2, dtype=float)
+        ymask = ma.getmaskarray(y)
+        if not allow_masked and ymask.any():
+            raise ValueError("Cannot process masked data.")
+        if xmask.any() or ymask.any():
+            if y.shape == x.shape:
+                # Define some common mask
+                common_mask = np.logical_or(xmask, ymask)
+                if common_mask is not nomask:
+                    xmask = x._mask = y._mask = ymask = common_mask
+                    x._sharedmask = False
+                    y._sharedmask = False
+        x = ma.concatenate((x, y), axis)
+        xnotmask = np.logical_not(np.concatenate((xmask, ymask), axis)).astype(int)
+    x -= x.mean(axis=rowvar)[tup]
+    return (x, xnotmask, rowvar)
+
+
+def cov(x, y=None, rowvar=True, bias=False, allow_masked=True, ddof=None):
+    """
+    Estimate the covariance matrix.
+
+    Except for the handling of missing data this function does the same as
+    `numpy.cov`. For more details and examples, see `numpy.cov`.
+
+    By default, masked values are recognized as such. If `x` and `y` have the
+    same shape, a common mask is allocated: if ``x[i,j]`` is masked, then
+    ``y[i,j]`` will also be masked.
+    Setting `allow_masked` to False will raise an exception if values are
+    missing in either of the input arrays.
+
+    Parameters
+    ----------
+    x : array_like
+        A 1-D or 2-D array containing multiple variables and observations.
+        Each row of `x` represents a variable, and each column a single
+        observation of all those variables. Also see `rowvar` below.
+    y : array_like, optional
+        An additional set of variables and observations. `y` has the same
+        shape as `x`.
+    rowvar : bool, optional
+        If `rowvar` is True (default), then each row represents a
+        variable, with observations in the columns. Otherwise, the relationship
+        is transposed: each column represents a variable, while the rows
+        contain observations.
+    bias : bool, optional
+        Default normalization (False) is by ``(N-1)``, where ``N`` is the
+        number of observations given (unbiased estimate). If `bias` is True,
+        then normalization is by ``N``. This keyword can be overridden by
+        the keyword ``ddof`` in numpy versions >= 1.5.
+    allow_masked : bool, optional
+        If True, masked values are propagated pair-wise: if a value is masked
+        in `x`, the corresponding value is masked in `y`.
+        If False, raises a `ValueError` exception when some values are missing.
+    ddof : {None, int}, optional
+        If not ``None`` normalization is by ``(N - ddof)``, where ``N`` is
+        the number of observations; this overrides the value implied by
+        ``bias``. The default value is ``None``.
+
+        .. versionadded:: 1.5
+
+    Raises
+    ------
+    ValueError
+        Raised if some values are missing and `allow_masked` is False.
+
+    See Also
+    --------
+    numpy.cov
+
+    """
+    # Check inputs
+    if ddof is not None and ddof != int(ddof):
+        raise ValueError("ddof must be an integer")
+    # Set up ddof
+    if ddof is None:
+        if bias:
+            ddof = 0
+        else:
+            ddof = 1
+
+    (x, xnotmask, rowvar) = _covhelper(x, y, rowvar, allow_masked)
+    if not rowvar:
+        fact = np.dot(xnotmask.T, xnotmask) * 1. - ddof
+        result = (dot(x.T, x.conj(), strict=False) / fact).squeeze()
+    else:
+        fact = np.dot(xnotmask, xnotmask.T) * 1. - ddof
+        result = (dot(x, x.T.conj(), strict=False) / fact).squeeze()
+    return result
+
+
+def corrcoef(x, y=None, rowvar=True, bias=np._NoValue, allow_masked=True,
+             ddof=np._NoValue):
+    """
+    Return Pearson product-moment correlation coefficients.
+
+    Except for the handling of missing data this function does the same as
+    `numpy.corrcoef`. For more details and examples, see `numpy.corrcoef`.
+
+    Parameters
+    ----------
+    x : array_like
+        A 1-D or 2-D array containing multiple variables and observations.
+        Each row of `x` represents a variable, and each column a single
+        observation of all those variables. Also see `rowvar` below.
+    y : array_like, optional
+        An additional set of variables and observations. `y` has the same
+        shape as `x`.
+    rowvar : bool, optional
+        If `rowvar` is True (default), then each row represents a
+        variable, with observations in the columns. Otherwise, the relationship
+        is transposed: each column represents a variable, while the rows
+        contain observations.
+    bias : _NoValue, optional
+        Has no effect, do not use.
+
+        .. deprecated:: 1.10.0
+    allow_masked : bool, optional
+        If True, masked values are propagated pair-wise: if a value is masked
+        in `x`, the corresponding value is masked in `y`.
+        If False, raises an exception.  Because `bias` is deprecated, this
+        argument needs to be treated as keyword only to avoid a warning.
+    ddof : _NoValue, optional
+        Has no effect, do not use.
+
+        .. deprecated:: 1.10.0
+
+    See Also
+    --------
+    numpy.corrcoef : Equivalent function in top-level NumPy module.
+    cov : Estimate the covariance matrix.
+
+    Notes
+    -----
+    This function accepts but discards arguments `bias` and `ddof`.  This is
+    for backwards compatibility with previous versions of this function.  These
+    arguments had no effect on the return values of the function and can be
+    safely ignored in this and previous versions of numpy.
+    """
+    msg = 'bias and ddof have no effect and are deprecated'
+    if bias is not np._NoValue or ddof is not np._NoValue:
+        # 2015-03-15, 1.10
+        warnings.warn(msg, DeprecationWarning, stacklevel=2)
+    # Get the data
+    (x, xnotmask, rowvar) = _covhelper(x, y, rowvar, allow_masked)
+    # Compute the covariance matrix
+    if not rowvar:
+        fact = np.dot(xnotmask.T, xnotmask) * 1.
+        c = (dot(x.T, x.conj(), strict=False) / fact).squeeze()
+    else:
+        fact = np.dot(xnotmask, xnotmask.T) * 1.
+        c = (dot(x, x.T.conj(), strict=False) / fact).squeeze()
+    # Check whether we have a scalar
+    try:
+        diag = ma.diagonal(c)
+    except ValueError:
+        return 1
+    #
+    if xnotmask.all():
+        _denom = ma.sqrt(ma.multiply.outer(diag, diag))
+    else:
+        _denom = diagflat(diag)
+        _denom._sharedmask = False  # We know return is always a copy
+        n = x.shape[1 - rowvar]
+        if rowvar:
+            for i in range(n - 1):
+                for j in range(i + 1, n):
+                    _x = mask_cols(vstack((x[i], x[j]))).var(axis=1)
+                    _denom[i, j] = _denom[j, i] = ma.sqrt(ma.multiply.reduce(_x))
+        else:
+            for i in range(n - 1):
+                for j in range(i + 1, n):
+                    _x = mask_cols(
+                            vstack((x[:, i], x[:, j]))).var(axis=1)
+                    _denom[i, j] = _denom[j, i] = ma.sqrt(ma.multiply.reduce(_x))
+    return c / _denom
+
+#####--------------------------------------------------------------------------
+#---- --- Concatenation helpers ---
+#####--------------------------------------------------------------------------
+
+class MAxisConcatenator(AxisConcatenator):
+    """
+    Translate slice objects to concatenation along an axis.
+
+    For documentation on usage, see `mr_class`.
+
+    See Also
+    --------
+    mr_class
+
+    """
+    concatenate = staticmethod(concatenate)
+
+    @classmethod
+    def makemat(cls, arr):
+        # There used to be a view as np.matrix here, but we may eventually
+        # deprecate that class. In preparation, we use the unmasked version
+        # to construct the matrix (with copy=False for backwards compatibility
+        # with the .view)
+        data = super().makemat(arr.data, copy=False)
+        return array(data, mask=arr.mask)
+
+    def __getitem__(self, key):
+        # matrix builder syntax, like 'a, b; c, d'
+        if isinstance(key, str):
+            raise MAError("Unavailable for masked array.")
+
+        return super().__getitem__(key)
+
+
+class mr_class(MAxisConcatenator):
+    """
+    Translate slice objects to concatenation along the first axis.
+
+    This is the masked array version of `lib.index_tricks.RClass`.
+
+    See Also
+    --------
+    lib.index_tricks.RClass
+
+    Examples
+    --------
+    >>> np.ma.mr_[np.ma.array([1,2,3]), 0, 0, np.ma.array([4,5,6])]
+    masked_array(data=[1, 2, 3, ..., 4, 5, 6],
+                 mask=False,
+           fill_value=999999)
+
+    """
+    def __init__(self):
+        MAxisConcatenator.__init__(self, 0)
+
+mr_ = mr_class()
+
+
+#####--------------------------------------------------------------------------
+#---- Find unmasked data ---
+#####--------------------------------------------------------------------------
+
+def ndenumerate(a, compressed=True):
+    """
+    Multidimensional index iterator.
+
+    Return an iterator yielding pairs of array coordinates and values,
+    skipping elements that are masked. With `compressed=False`,
+    `ma.masked` is yielded as the value of masked elements. This
+    behavior differs from that of `numpy.ndenumerate`, which yields the
+    value of the underlying data array.
+
+    Notes
+    -----
+    .. versionadded:: 1.23.0
+
+    Parameters
+    ----------
+    a : array_like
+        An array with (possibly) masked elements.
+    compressed : bool, optional
+        If True (default), masked elements are skipped.
+
+    See Also
+    --------
+    numpy.ndenumerate : Equivalent function ignoring any mask.
+
+    Examples
+    --------
+    >>> a = np.ma.arange(9).reshape((3, 3))
+    >>> a[1, 0] = np.ma.masked
+    >>> a[1, 2] = np.ma.masked
+    >>> a[2, 1] = np.ma.masked
+    >>> a
+    masked_array(
+      data=[[0, 1, 2],
+            [--, 4, --],
+            [6, --, 8]],
+      mask=[[False, False, False],
+            [ True, False,  True],
+            [False,  True, False]],
+      fill_value=999999)
+    >>> for index, x in np.ma.ndenumerate(a):
+    ...     print(index, x)
+    (0, 0) 0
+    (0, 1) 1
+    (0, 2) 2
+    (1, 1) 4
+    (2, 0) 6
+    (2, 2) 8
+
+    >>> for index, x in np.ma.ndenumerate(a, compressed=False):
+    ...     print(index, x)
+    (0, 0) 0
+    (0, 1) 1
+    (0, 2) 2
+    (1, 0) --
+    (1, 1) 4
+    (1, 2) --
+    (2, 0) 6
+    (2, 1) --
+    (2, 2) 8
+    """
+    for it, mask in zip(np.ndenumerate(a), getmaskarray(a).flat):
+        if not mask:
+            yield it
+        elif not compressed:
+            yield it[0], masked
+
+
+def flatnotmasked_edges(a):
+    """
+    Find the indices of the first and last unmasked values.
+
+    Expects a 1-D `MaskedArray`, returns None if all values are masked.
+
+    Parameters
+    ----------
+    a : array_like
+        Input 1-D `MaskedArray`
+
+    Returns
+    -------
+    edges : ndarray or None
+        The indices of first and last non-masked value in the array.
+        Returns None if all values are masked.
+
+    See Also
+    --------
+    flatnotmasked_contiguous, notmasked_contiguous, notmasked_edges
+    clump_masked, clump_unmasked
+
+    Notes
+    -----
+    Only accepts 1-D arrays.
+
+    Examples
+    --------
+    >>> a = np.ma.arange(10)
+    >>> np.ma.flatnotmasked_edges(a)
+    array([0, 9])
+
+    >>> mask = (a < 3) | (a > 8) | (a == 5)
+    >>> a[mask] = np.ma.masked
+    >>> np.array(a[~a.mask])
+    array([3, 4, 6, 7, 8])
+
+    >>> np.ma.flatnotmasked_edges(a)
+    array([3, 8])
+
+    >>> a[:] = np.ma.masked
+    >>> print(np.ma.flatnotmasked_edges(a))
+    None
+
+    """
+    m = getmask(a)
+    if m is nomask or not np.any(m):
+        return np.array([0, a.size - 1])
+    unmasked = np.flatnonzero(~m)
+    if len(unmasked) > 0:
+        return unmasked[[0, -1]]
+    else:
+        return None
+
+
+def notmasked_edges(a, axis=None):
+    """
+    Find the indices of the first and last unmasked values along an axis.
+
+    If all values are masked, return None.  Otherwise, return a list
+    of two tuples, corresponding to the indices of the first and last
+    unmasked values respectively.
+
+    Parameters
+    ----------
+    a : array_like
+        The input array.
+    axis : int, optional
+        Axis along which to perform the operation.
+        If None (default), applies to a flattened version of the array.
+
+    Returns
+    -------
+    edges : ndarray or list
+        An array of start and end indexes if there are any masked data in
+        the array. If there are no masked data in the array, `edges` is a
+        list of the first and last index.
+
+    See Also
+    --------
+    flatnotmasked_contiguous, flatnotmasked_edges, notmasked_contiguous
+    clump_masked, clump_unmasked
+
+    Examples
+    --------
+    >>> a = np.arange(9).reshape((3, 3))
+    >>> m = np.zeros_like(a)
+    >>> m[1:, 1:] = 1
+
+    >>> am = np.ma.array(a, mask=m)
+    >>> np.array(am[~am.mask])
+    array([0, 1, 2, 3, 6])
+
+    >>> np.ma.notmasked_edges(am)
+    array([0, 6])
+
+    """
+    a = asarray(a)
+    if axis is None or a.ndim == 1:
+        return flatnotmasked_edges(a)
+    m = getmaskarray(a)
+    idx = array(np.indices(a.shape), mask=np.asarray([m] * a.ndim))
+    return [tuple([idx[i].min(axis).compressed() for i in range(a.ndim)]),
+            tuple([idx[i].max(axis).compressed() for i in range(a.ndim)]), ]
+
+
+def flatnotmasked_contiguous(a):
+    """
+    Find contiguous unmasked data in a masked array.
+
+    Parameters
+    ----------
+    a : array_like
+        The input array.
+
+    Returns
+    -------
+    slice_list : list
+        A sorted sequence of `slice` objects (start index, end index).
+
+        .. versionchanged:: 1.15.0
+            Now returns an empty list instead of None for a fully masked array
+
+    See Also
+    --------
+    flatnotmasked_edges, notmasked_contiguous, notmasked_edges
+    clump_masked, clump_unmasked
+
+    Notes
+    -----
+    Only accepts 2-D arrays at most.
+
+    Examples
+    --------
+    >>> a = np.ma.arange(10)
+    >>> np.ma.flatnotmasked_contiguous(a)
+    [slice(0, 10, None)]
+
+    >>> mask = (a < 3) | (a > 8) | (a == 5)
+    >>> a[mask] = np.ma.masked
+    >>> np.array(a[~a.mask])
+    array([3, 4, 6, 7, 8])
+
+    >>> np.ma.flatnotmasked_contiguous(a)
+    [slice(3, 5, None), slice(6, 9, None)]
+    >>> a[:] = np.ma.masked
+    >>> np.ma.flatnotmasked_contiguous(a)
+    []
+
+    """
+    m = getmask(a)
+    if m is nomask:
+        return [slice(0, a.size)]
+    i = 0
+    result = []
+    for (k, g) in itertools.groupby(m.ravel()):
+        n = len(list(g))
+        if not k:
+            result.append(slice(i, i + n))
+        i += n
+    return result
+
+
+def notmasked_contiguous(a, axis=None):
+    """
+    Find contiguous unmasked data in a masked array along the given axis.
+
+    Parameters
+    ----------
+    a : array_like
+        The input array.
+    axis : int, optional
+        Axis along which to perform the operation.
+        If None (default), applies to a flattened version of the array, and this
+        is the same as `flatnotmasked_contiguous`.
+
+    Returns
+    -------
+    endpoints : list
+        A list of slices (start and end indexes) of unmasked indexes
+        in the array.
+
+        If the input is 2d and axis is specified, the result is a list of lists.
+
+    See Also
+    --------
+    flatnotmasked_edges, flatnotmasked_contiguous, notmasked_edges
+    clump_masked, clump_unmasked
+
+    Notes
+    -----
+    Only accepts 2-D arrays at most.
+
+    Examples
+    --------
+    >>> a = np.arange(12).reshape((3, 4))
+    >>> mask = np.zeros_like(a)
+    >>> mask[1:, :-1] = 1; mask[0, 1] = 1; mask[-1, 0] = 0
+    >>> ma = np.ma.array(a, mask=mask)
+    >>> ma
+    masked_array(
+      data=[[0, --, 2, 3],
+            [--, --, --, 7],
+            [8, --, --, 11]],
+      mask=[[False,  True, False, False],
+            [ True,  True,  True, False],
+            [False,  True,  True, False]],
+      fill_value=999999)
+    >>> np.array(ma[~ma.mask])
+    array([ 0,  2,  3,  7, 8, 11])
+
+    >>> np.ma.notmasked_contiguous(ma)
+    [slice(0, 1, None), slice(2, 4, None), slice(7, 9, None), slice(11, 12, None)]
+
+    >>> np.ma.notmasked_contiguous(ma, axis=0)
+    [[slice(0, 1, None), slice(2, 3, None)], [], [slice(0, 1, None)], [slice(0, 3, None)]]
+
+    >>> np.ma.notmasked_contiguous(ma, axis=1)
+    [[slice(0, 1, None), slice(2, 4, None)], [slice(3, 4, None)], [slice(0, 1, None), slice(3, 4, None)]]
+
+    """
+    a = asarray(a)
+    nd = a.ndim
+    if nd > 2:
+        raise NotImplementedError("Currently limited to at most 2D array.")
+    if axis is None or nd == 1:
+        return flatnotmasked_contiguous(a)
+    #
+    result = []
+    #
+    other = (axis + 1) % 2
+    idx = [0, 0]
+    idx[axis] = slice(None, None)
+    #
+    for i in range(a.shape[other]):
+        idx[other] = i
+        result.append(flatnotmasked_contiguous(a[tuple(idx)]))
+    return result
+
+
+def _ezclump(mask):
+    """
+    Finds the clumps (groups of data with the same values) for a 1D bool array.
+
+    Returns a series of slices.
+    """
+    if mask.ndim > 1:
+        mask = mask.ravel()
+    idx = (mask[1:] ^ mask[:-1]).nonzero()
+    idx = idx[0] + 1
+
+    if mask[0]:
+        if len(idx) == 0:
+            return [slice(0, mask.size)]
+
+        r = [slice(0, idx[0])]
+        r.extend((slice(left, right)
+                  for left, right in zip(idx[1:-1:2], idx[2::2])))
+    else:
+        if len(idx) == 0:
+            return []
+
+        r = [slice(left, right) for left, right in zip(idx[:-1:2], idx[1::2])]
+
+    if mask[-1]:
+        r.append(slice(idx[-1], mask.size))
+    return r
+
+
+def clump_unmasked(a):
+    """
+    Return list of slices corresponding to the unmasked clumps of a 1-D array.
+    (A "clump" is defined as a contiguous region of the array).
+
+    Parameters
+    ----------
+    a : ndarray
+        A one-dimensional masked array.
+
+    Returns
+    -------
+    slices : list of slice
+        The list of slices, one for each continuous region of unmasked
+        elements in `a`.
+
+    Notes
+    -----
+    .. versionadded:: 1.4.0
+
+    See Also
+    --------
+    flatnotmasked_edges, flatnotmasked_contiguous, notmasked_edges
+    notmasked_contiguous, clump_masked
+
+    Examples
+    --------
+    >>> a = np.ma.masked_array(np.arange(10))
+    >>> a[[0, 1, 2, 6, 8, 9]] = np.ma.masked
+    >>> np.ma.clump_unmasked(a)
+    [slice(3, 6, None), slice(7, 8, None)]
+
+    """
+    mask = getattr(a, '_mask', nomask)
+    if mask is nomask:
+        return [slice(0, a.size)]
+    return _ezclump(~mask)
+
+
+def clump_masked(a):
+    """
+    Returns a list of slices corresponding to the masked clumps of a 1-D array.
+    (A "clump" is defined as a contiguous region of the array).
+
+    Parameters
+    ----------
+    a : ndarray
+        A one-dimensional masked array.
+
+    Returns
+    -------
+    slices : list of slice
+        The list of slices, one for each continuous region of masked elements
+        in `a`.
+
+    Notes
+    -----
+    .. versionadded:: 1.4.0
+
+    See Also
+    --------
+    flatnotmasked_edges, flatnotmasked_contiguous, notmasked_edges
+    notmasked_contiguous, clump_unmasked
+
+    Examples
+    --------
+    >>> a = np.ma.masked_array(np.arange(10))
+    >>> a[[0, 1, 2, 6, 8, 9]] = np.ma.masked
+    >>> np.ma.clump_masked(a)
+    [slice(0, 3, None), slice(6, 7, None), slice(8, 10, None)]
+
+    """
+    mask = ma.getmask(a)
+    if mask is nomask:
+        return []
+    return _ezclump(mask)
+
+
+###############################################################################
+#                              Polynomial fit                                 #
+###############################################################################
+
+
+def vander(x, n=None):
+    """
+    Masked values in the input array result in rows of zeros.
+
+    """
+    _vander = np.vander(x, n)
+    m = getmask(x)
+    if m is not nomask:
+        _vander[m] = 0
+    return _vander
+
+vander.__doc__ = ma.doc_note(np.vander.__doc__, vander.__doc__)
+
+
+def polyfit(x, y, deg, rcond=None, full=False, w=None, cov=False):
+    """
+    Any masked values in x is propagated in y, and vice-versa.
+
+    """
+    x = asarray(x)
+    y = asarray(y)
+
+    m = getmask(x)
+    if y.ndim == 1:
+        m = mask_or(m, getmask(y))
+    elif y.ndim == 2:
+        my = getmask(mask_rows(y))
+        if my is not nomask:
+            m = mask_or(m, my[:, 0])
+    else:
+        raise TypeError("Expected a 1D or 2D array for y!")
+
+    if w is not None:
+        w = asarray(w)
+        if w.ndim != 1:
+            raise TypeError("expected a 1-d array for weights")
+        if w.shape[0] != y.shape[0]:
+            raise TypeError("expected w and y to have the same length")
+        m = mask_or(m, getmask(w))
+
+    if m is not nomask:
+        not_m = ~m
+        if w is not None:
+            w = w[not_m]
+        return np.polyfit(x[not_m], y[not_m], deg, rcond, full, w, cov)
+    else:
+        return np.polyfit(x, y, deg, rcond, full, w, cov)
+
+polyfit.__doc__ = ma.doc_note(np.polyfit.__doc__, polyfit.__doc__)

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/ma/extras.pyi

@@ -0,0 +1,85 @@
+from typing import Any
+from numpy.lib.index_tricks import AxisConcatenator
+
+from numpy.ma.core import (
+    dot as dot,
+    mask_rowcols as mask_rowcols,
+)
+
+__all__: list[str]
+
+def count_masked(arr, axis=...): ...
+def masked_all(shape, dtype = ...): ...
+def masked_all_like(arr): ...
+
+class _fromnxfunction:
+    __name__: Any
+    __doc__: Any
+    def __init__(self, funcname): ...
+    def getdoc(self): ...
+    def __call__(self, *args, **params): ...
+
+class _fromnxfunction_single(_fromnxfunction):
+    def __call__(self, x, *args, **params): ...
+
+class _fromnxfunction_seq(_fromnxfunction):
+    def __call__(self, x, *args, **params): ...
+
+class _fromnxfunction_allargs(_fromnxfunction):
+    def __call__(self, *args, **params): ...
+
+atleast_1d: _fromnxfunction_allargs
+atleast_2d: _fromnxfunction_allargs
+atleast_3d: _fromnxfunction_allargs
+
+vstack: _fromnxfunction_seq
+row_stack: _fromnxfunction_seq
+hstack: _fromnxfunction_seq
+column_stack: _fromnxfunction_seq
+dstack: _fromnxfunction_seq
+stack: _fromnxfunction_seq
+
+hsplit: _fromnxfunction_single
+diagflat: _fromnxfunction_single
+
+def apply_along_axis(func1d, axis, arr, *args, **kwargs): ...
+def apply_over_axes(func, a, axes): ...
+def average(a, axis=..., weights=..., returned=..., keepdims=...): ...
+def median(a, axis=..., out=..., overwrite_input=..., keepdims=...): ...
+def compress_nd(x, axis=...): ...
+def compress_rowcols(x, axis=...): ...
+def compress_rows(a): ...
+def compress_cols(a): ...
+def mask_rows(a, axis = ...): ...
+def mask_cols(a, axis = ...): ...
+def ediff1d(arr, to_end=..., to_begin=...): ...
+def unique(ar1, return_index=..., return_inverse=...): ...
+def intersect1d(ar1, ar2, assume_unique=...): ...
+def setxor1d(ar1, ar2, assume_unique=...): ...
+def in1d(ar1, ar2, assume_unique=..., invert=...): ...
+def isin(element, test_elements, assume_unique=..., invert=...): ...
+def union1d(ar1, ar2): ...
+def setdiff1d(ar1, ar2, assume_unique=...): ...
+def cov(x, y=..., rowvar=..., bias=..., allow_masked=..., ddof=...): ...
+def corrcoef(x, y=..., rowvar=..., bias = ..., allow_masked=..., ddof = ...): ...
+
+class MAxisConcatenator(AxisConcatenator):
+    concatenate: Any
+    @classmethod
+    def makemat(cls, arr): ...
+    def __getitem__(self, key): ...
+
+class mr_class(MAxisConcatenator):
+    def __init__(self): ...
+
+mr_: mr_class
+
+def ndenumerate(a, compressed=...): ...
+def flatnotmasked_edges(a): ...
+def notmasked_edges(a, axis=...): ...
+def flatnotmasked_contiguous(a): ...
+def notmasked_contiguous(a, axis=...): ...
+def clump_unmasked(a): ...
+def clump_masked(a): ...
+def vander(x, n=...): ...
+def polyfit(x, y, deg, rcond=..., full=..., w=..., cov=...): ...

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/ma/mrecords.py

@@ -0,0 +1,783 @@
+""":mod:`numpy.ma..mrecords`
+
+Defines the equivalent of :class:`numpy.recarrays` for masked arrays,
+where fields can be accessed as attributes.
+Note that :class:`numpy.ma.MaskedArray` already supports structured datatypes
+and the masking of individual fields.
+
+.. moduleauthor:: Pierre Gerard-Marchant
+
+"""
+#  We should make sure that no field is called '_mask','mask','_fieldmask',
+#  or whatever restricted keywords.  An idea would be to no bother in the
+#  first place, and then rename the invalid fields with a trailing
+#  underscore. Maybe we could just overload the parser function ?
+
+from numpy.ma import (
+    MAError, MaskedArray, masked, nomask, masked_array, getdata,
+    getmaskarray, filled
+)
+import numpy.ma as ma
+import warnings
+
+import numpy as np
+from numpy import (
+    bool_, dtype, ndarray, recarray, array as narray
+)
+from numpy.core.records import (
+    fromarrays as recfromarrays, fromrecords as recfromrecords
+)
+
+_byteorderconv = np.core.records._byteorderconv
+
+
+_check_fill_value = ma.core._check_fill_value
+
+
+__all__ = [
+    'MaskedRecords', 'mrecarray', 'fromarrays', 'fromrecords',
+    'fromtextfile', 'addfield',
+]
+
+reserved_fields = ['_data', '_mask', '_fieldmask', 'dtype']
+
+
+def _checknames(descr, names=None):
+    """
+    Checks that field names ``descr`` are not reserved keywords.
+
+    If this is the case, a default 'f%i' is substituted.  If the argument
+    `names` is not None, updates the field names to valid names.
+
+    """
+    ndescr = len(descr)
+    default_names = ['f%i' % i for i in range(ndescr)]
+    if names is None:
+        new_names = default_names
+    else:
+        if isinstance(names, (tuple, list)):
+            new_names = names
+        elif isinstance(names, str):
+            new_names = names.split(',')
+        else:
+            raise NameError(f'illegal input names {names!r}')
+        nnames = len(new_names)
+        if nnames < ndescr:
+            new_names += default_names[nnames:]
+    ndescr = []
+    for (n, d, t) in zip(new_names, default_names, descr.descr):
+        if n in reserved_fields:
+            if t[0] in reserved_fields:
+                ndescr.append((d, t[1]))
+            else:
+                ndescr.append(t)
+        else:
+            ndescr.append((n, t[1]))
+    return np.dtype(ndescr)
+
+
+def _get_fieldmask(self):
+    mdescr = [(n, '|b1') for n in self.dtype.names]
+    fdmask = np.empty(self.shape, dtype=mdescr)
+    fdmask.flat = tuple([False] * len(mdescr))
+    return fdmask
+
+
+class MaskedRecords(MaskedArray):
+    """
+
+    Attributes
+    ----------
+    _data : recarray
+        Underlying data, as a record array.
+    _mask : boolean array
+        Mask of the records. A record is masked when all its fields are
+        masked.
+    _fieldmask : boolean recarray
+        Record array of booleans, setting the mask of each individual field
+        of each record.
+    _fill_value : record
+        Filling values for each field.
+
+    """
+
+    def __new__(cls, shape, dtype=None, buf=None, offset=0, strides=None,
+                formats=None, names=None, titles=None,
+                byteorder=None, aligned=False,
+                mask=nomask, hard_mask=False, fill_value=None, keep_mask=True,
+                copy=False,
+                **options):
+
+        self = recarray.__new__(cls, shape, dtype=dtype, buf=buf, offset=offset,
+                                strides=strides, formats=formats, names=names,
+                                titles=titles, byteorder=byteorder,
+                                aligned=aligned,)
+
+        mdtype = ma.make_mask_descr(self.dtype)
+        if mask is nomask or not np.size(mask):
+            if not keep_mask:
+                self._mask = tuple([False] * len(mdtype))
+        else:
+            mask = np.array(mask, copy=copy)
+            if mask.shape != self.shape:
+                (nd, nm) = (self.size, mask.size)
+                if nm == 1:
+                    mask = np.resize(mask, self.shape)
+                elif nm == nd:
+                    mask = np.reshape(mask, self.shape)
+                else:
+                    msg = "Mask and data not compatible: data size is %i, " + \
+                          "mask size is %i."
+                    raise MAError(msg % (nd, nm))
+            if not keep_mask:
+                self.__setmask__(mask)
+                self._sharedmask = True
+            else:
+                if mask.dtype == mdtype:
+                    _mask = mask
+                else:
+                    _mask = np.array([tuple([m] * len(mdtype)) for m in mask],
+                                     dtype=mdtype)
+                self._mask = _mask
+        return self
+
+    def __array_finalize__(self, obj):
+        # Make sure we have a _fieldmask by default
+        _mask = getattr(obj, '_mask', None)
+        if _mask is None:
+            objmask = getattr(obj, '_mask', nomask)
+            _dtype = ndarray.__getattribute__(self, 'dtype')
+            if objmask is nomask:
+                _mask = ma.make_mask_none(self.shape, dtype=_dtype)
+            else:
+                mdescr = ma.make_mask_descr(_dtype)
+                _mask = narray([tuple([m] * len(mdescr)) for m in objmask],
+                               dtype=mdescr).view(recarray)
+        # Update some of the attributes
+        _dict = self.__dict__
+        _dict.update(_mask=_mask)
+        self._update_from(obj)
+        if _dict['_baseclass'] == ndarray:
+            _dict['_baseclass'] = recarray
+        return
+
+    @property
+    def _data(self):
+        """
+        Returns the data as a recarray.
+
+        """
+        return ndarray.view(self, recarray)
+
+    @property
+    def _fieldmask(self):
+        """
+        Alias to mask.
+
+        """
+        return self._mask
+
+    def __len__(self):
+        """
+        Returns the length
+
+        """
+        # We have more than one record
+        if self.ndim:
+            return len(self._data)
+        # We have only one record: return the nb of fields
+        return len(self.dtype)
+
+    def __getattribute__(self, attr):
+        try:
+            return object.__getattribute__(self, attr)
+        except AttributeError:
+            # attr must be a fieldname
+            pass
+        fielddict = ndarray.__getattribute__(self, 'dtype').fields
+        try:
+            res = fielddict[attr][:2]
+        except (TypeError, KeyError) as e:
+            raise AttributeError(
+                f'record array has no attribute {attr}') from e
+        # So far, so good
+        _localdict = ndarray.__getattribute__(self, '__dict__')
+        _data = ndarray.view(self, _localdict['_baseclass'])
+        obj = _data.getfield(*res)
+        if obj.dtype.names is not None:
+            raise NotImplementedError("MaskedRecords is currently limited to"
+                                      "simple records.")
+        # Get some special attributes
+        # Reset the object's mask
+        hasmasked = False
+        _mask = _localdict.get('_mask', None)
+        if _mask is not None:
+            try:
+                _mask = _mask[attr]
+            except IndexError:
+                # Couldn't find a mask: use the default (nomask)
+                pass
+            tp_len = len(_mask.dtype)
+            hasmasked = _mask.view((bool, ((tp_len,) if tp_len else ()))).any()
+        if (obj.shape or hasmasked):
+            obj = obj.view(MaskedArray)
+            obj._baseclass = ndarray
+            obj._isfield = True
+            obj._mask = _mask
+            # Reset the field values
+            _fill_value = _localdict.get('_fill_value', None)
+            if _fill_value is not None:
+                try:
+                    obj._fill_value = _fill_value[attr]
+                except ValueError:
+                    obj._fill_value = None
+        else:
+            obj = obj.item()
+        return obj
+
+    def __setattr__(self, attr, val):
+        """
+        Sets the attribute attr to the value val.
+
+        """
+        # Should we call __setmask__ first ?
+        if attr in ['mask', 'fieldmask']:
+            self.__setmask__(val)
+            return
+        # Create a shortcut (so that we don't have to call getattr all the time)
+        _localdict = object.__getattribute__(self, '__dict__')
+        # Check whether we're creating a new field
+        newattr = attr not in _localdict
+        try:
+            # Is attr a generic attribute ?
+            ret = object.__setattr__(self, attr, val)
+        except Exception:
+            # Not a generic attribute: exit if it's not a valid field
+            fielddict = ndarray.__getattribute__(self, 'dtype').fields or {}
+            optinfo = ndarray.__getattribute__(self, '_optinfo') or {}
+            if not (attr in fielddict or attr in optinfo):
+                raise
+        else:
+            # Get the list of names
+            fielddict = ndarray.__getattribute__(self, 'dtype').fields or {}
+            # Check the attribute
+            if attr not in fielddict:
+                return ret
+            if newattr:
+                # We just added this one or this setattr worked on an
+                # internal attribute.
+                try:
+                    object.__delattr__(self, attr)
+                except Exception:
+                    return ret
+        # Let's try to set the field
+        try:
+            res = fielddict[attr][:2]
+        except (TypeError, KeyError) as e:
+            raise AttributeError(
+                f'record array has no attribute {attr}') from e
+
+        if val is masked:
+            _fill_value = _localdict['_fill_value']
+            if _fill_value is not None:
+                dval = _localdict['_fill_value'][attr]
+            else:
+                dval = val
+            mval = True
+        else:
+            dval = filled(val)
+            mval = getmaskarray(val)
+        obj = ndarray.__getattribute__(self, '_data').setfield(dval, *res)
+        _localdict['_mask'].__setitem__(attr, mval)
+        return obj
+
+    def __getitem__(self, indx):
+        """
+        Returns all the fields sharing the same fieldname base.
+
+        The fieldname base is either `_data` or `_mask`.
+
+        """
+        _localdict = self.__dict__
+        _mask = ndarray.__getattribute__(self, '_mask')
+        _data = ndarray.view(self, _localdict['_baseclass'])
+        # We want a field
+        if isinstance(indx, str):
+            # Make sure _sharedmask is True to propagate back to _fieldmask
+            # Don't use _set_mask, there are some copies being made that
+            # break propagation Don't force the mask to nomask, that wreaks
+            # easy masking
+            obj = _data[indx].view(MaskedArray)
+            obj._mask = _mask[indx]
+            obj._sharedmask = True
+            fval = _localdict['_fill_value']
+            if fval is not None:
+                obj._fill_value = fval[indx]
+            # Force to masked if the mask is True
+            if not obj.ndim and obj._mask:
+                return masked
+            return obj
+        # We want some elements.
+        # First, the data.
+        obj = np.array(_data[indx], copy=False).view(mrecarray)
+        obj._mask = np.array(_mask[indx], copy=False).view(recarray)
+        return obj
+
+    def __setitem__(self, indx, value):
+        """
+        Sets the given record to value.
+
+        """
+        MaskedArray.__setitem__(self, indx, value)
+        if isinstance(indx, str):
+            self._mask[indx] = ma.getmaskarray(value)
+
+    def __str__(self):
+        """
+        Calculates the string representation.
+
+        """
+        if self.size > 1:
+            mstr = [f"({','.join([str(i) for i in s])})"
+                    for s in zip(*[getattr(self, f) for f in self.dtype.names])]
+            return f"[{', '.join(mstr)}]"
+        else:
+            mstr = [f"{','.join([str(i) for i in s])}"
+                    for s in zip([getattr(self, f) for f in self.dtype.names])]
+            return f"({', '.join(mstr)})"
+
+    def __repr__(self):
+        """
+        Calculates the repr representation.
+
+        """
+        _names = self.dtype.names
+        fmt = "%%%is : %%s" % (max([len(n) for n in _names]) + 4,)
+        reprstr = [fmt % (f, getattr(self, f)) for f in self.dtype.names]
+        reprstr.insert(0, 'masked_records(')
+        reprstr.extend([fmt % ('    fill_value', self.fill_value),
+                        '              )'])
+        return str("\n".join(reprstr))
+
+    def view(self, dtype=None, type=None):
+        """
+        Returns a view of the mrecarray.
+
+        """
+        # OK, basic copy-paste from MaskedArray.view.
+        if dtype is None:
+            if type is None:
+                output = ndarray.view(self)
+            else:
+                output = ndarray.view(self, type)
+        # Here again.
+        elif type is None:
+            try:
+                if issubclass(dtype, ndarray):
+                    output = ndarray.view(self, dtype)
+                else:
+                    output = ndarray.view(self, dtype)
+            # OK, there's the change
+            except TypeError:
+                dtype = np.dtype(dtype)
+                # we need to revert to MaskedArray, but keeping the possibility
+                # of subclasses (eg, TimeSeriesRecords), so we'll force a type
+                # set to the first parent
+                if dtype.fields is None:
+                    basetype = self.__class__.__bases__[0]
+                    output = self.__array__().view(dtype, basetype)
+                    output._update_from(self)
+                else:
+                    output = ndarray.view(self, dtype)
+                output._fill_value = None
+        else:
+            output = ndarray.view(self, dtype, type)
+        # Update the mask, just like in MaskedArray.view
+        if (getattr(output, '_mask', nomask) is not nomask):
+            mdtype = ma.make_mask_descr(output.dtype)
+            output._mask = self._mask.view(mdtype, ndarray)
+            output._mask.shape = output.shape
+        return output
+
+    def harden_mask(self):
+        """
+        Forces the mask to hard.
+
+        """
+        self._hardmask = True
+
+    def soften_mask(self):
+        """
+        Forces the mask to soft
+
+        """
+        self._hardmask = False
+
+    def copy(self):
+        """
+        Returns a copy of the masked record.
+
+        """
+        copied = self._data.copy().view(type(self))
+        copied._mask = self._mask.copy()
+        return copied
+
+    def tolist(self, fill_value=None):
+        """
+        Return the data portion of the array as a list.
+
+        Data items are converted to the nearest compatible Python type.
+        Masked values are converted to fill_value. If fill_value is None,
+        the corresponding entries in the output list will be ``None``.
+
+        """
+        if fill_value is not None:
+            return self.filled(fill_value).tolist()
+        result = narray(self.filled().tolist(), dtype=object)
+        mask = narray(self._mask.tolist())
+        result[mask] = None
+        return result.tolist()
+
+    def __getstate__(self):
+        """Return the internal state of the masked array.
+
+        This is for pickling.
+
+        """
+        state = (1,
+                 self.shape,
+                 self.dtype,
+                 self.flags.fnc,
+                 self._data.tobytes(),
+                 self._mask.tobytes(),
+                 self._fill_value,
+                 )
+        return state
+
+    def __setstate__(self, state):
+        """
+        Restore the internal state of the masked array.
+
+        This is for pickling.  ``state`` is typically the output of the
+        ``__getstate__`` output, and is a 5-tuple:
+
+        - class name
+        - a tuple giving the shape of the data
+        - a typecode for the data
+        - a binary string for the data
+        - a binary string for the mask.
+
+        """
+        (ver, shp, typ, isf, raw, msk, flv) = state
+        ndarray.__setstate__(self, (shp, typ, isf, raw))
+        mdtype = dtype([(k, bool_) for (k, _) in self.dtype.descr])
+        self.__dict__['_mask'].__setstate__((shp, mdtype, isf, msk))
+        self.fill_value = flv
+
+    def __reduce__(self):
+        """
+        Return a 3-tuple for pickling a MaskedArray.
+
+        """
+        return (_mrreconstruct,
+                (self.__class__, self._baseclass, (0,), 'b',),
+                self.__getstate__())
+
+
+def _mrreconstruct(subtype, baseclass, baseshape, basetype,):
+    """
+    Build a new MaskedArray from the information stored in a pickle.
+
+    """
+    _data = ndarray.__new__(baseclass, baseshape, basetype).view(subtype)
+    _mask = ndarray.__new__(ndarray, baseshape, 'b1')
+    return subtype.__new__(subtype, _data, mask=_mask, dtype=basetype,)
+
+mrecarray = MaskedRecords
+
+
+###############################################################################
+#                             Constructors                                    #
+###############################################################################
+
+
+def fromarrays(arraylist, dtype=None, shape=None, formats=None,
+               names=None, titles=None, aligned=False, byteorder=None,
+               fill_value=None):
+    """
+    Creates a mrecarray from a (flat) list of masked arrays.
+
+    Parameters
+    ----------
+    arraylist : sequence
+        A list of (masked) arrays. Each element of the sequence is first converted
+        to a masked array if needed. If a 2D array is passed as argument, it is
+        processed line by line
+    dtype : {None, dtype}, optional
+        Data type descriptor.
+    shape : {None, integer}, optional
+        Number of records. If None, shape is defined from the shape of the
+        first array in the list.
+    formats : {None, sequence}, optional
+        Sequence of formats for each individual field. If None, the formats will
+        be autodetected by inspecting the fields and selecting the highest dtype
+        possible.
+    names : {None, sequence}, optional
+        Sequence of the names of each field.
+    fill_value : {None, sequence}, optional
+        Sequence of data to be used as filling values.
+
+    Notes
+    -----
+    Lists of tuples should be preferred over lists of lists for faster processing.
+
+    """
+    datalist = [getdata(x) for x in arraylist]
+    masklist = [np.atleast_1d(getmaskarray(x)) for x in arraylist]
+    _array = recfromarrays(datalist,
+                           dtype=dtype, shape=shape, formats=formats,
+                           names=names, titles=titles, aligned=aligned,
+                           byteorder=byteorder).view(mrecarray)
+    _array._mask.flat = list(zip(*masklist))
+    if fill_value is not None:
+        _array.fill_value = fill_value
+    return _array
+
+
+def fromrecords(reclist, dtype=None, shape=None, formats=None, names=None,
+                titles=None, aligned=False, byteorder=None,
+                fill_value=None, mask=nomask):
+    """
+    Creates a MaskedRecords from a list of records.
+
+    Parameters
+    ----------
+    reclist : sequence
+        A list of records. Each element of the sequence is first converted
+        to a masked array if needed. If a 2D array is passed as argument, it is
+        processed line by line
+    dtype : {None, dtype}, optional
+        Data type descriptor.
+    shape : {None,int}, optional
+        Number of records. If None, ``shape`` is defined from the shape of the
+        first array in the list.
+    formats : {None, sequence}, optional
+        Sequence of formats for each individual field. If None, the formats will
+        be autodetected by inspecting the fields and selecting the highest dtype
+        possible.
+    names : {None, sequence}, optional
+        Sequence of the names of each field.
+    fill_value : {None, sequence}, optional
+        Sequence of data to be used as filling values.
+    mask : {nomask, sequence}, optional.
+        External mask to apply on the data.
+
+    Notes
+    -----
+    Lists of tuples should be preferred over lists of lists for faster processing.
+
+    """
+    # Grab the initial _fieldmask, if needed:
+    _mask = getattr(reclist, '_mask', None)
+    # Get the list of records.
+    if isinstance(reclist, ndarray):
+        # Make sure we don't have some hidden mask
+        if isinstance(reclist, MaskedArray):
+            reclist = reclist.filled().view(ndarray)
+        # Grab the initial dtype, just in case
+        if dtype is None:
+            dtype = reclist.dtype
+        reclist = reclist.tolist()
+    mrec = recfromrecords(reclist, dtype=dtype, shape=shape, formats=formats,
+                          names=names, titles=titles,
+                          aligned=aligned, byteorder=byteorder).view(mrecarray)
+    # Set the fill_value if needed
+    if fill_value is not None:
+        mrec.fill_value = fill_value
+    # Now, let's deal w/ the mask
+    if mask is not nomask:
+        mask = np.array(mask, copy=False)
+        maskrecordlength = len(mask.dtype)
+        if maskrecordlength:
+            mrec._mask.flat = mask
+        elif mask.ndim == 2:
+            mrec._mask.flat = [tuple(m) for m in mask]
+        else:
+            mrec.__setmask__(mask)
+    if _mask is not None:
+        mrec._mask[:] = _mask
+    return mrec
+
+
+def _guessvartypes(arr):
+    """
+    Tries to guess the dtypes of the str_ ndarray `arr`.
+
+    Guesses by testing element-wise conversion. Returns a list of dtypes.
+    The array is first converted to ndarray. If the array is 2D, the test
+    is performed on the first line. An exception is raised if the file is
+    3D or more.
+
+    """
+    vartypes = []
+    arr = np.asarray(arr)
+    if arr.ndim == 2:
+        arr = arr[0]
+    elif arr.ndim > 2:
+        raise ValueError("The array should be 2D at most!")
+    # Start the conversion loop.
+    for f in arr:
+        try:
+            int(f)
+        except (ValueError, TypeError):
+            try:
+                float(f)
+            except (ValueError, TypeError):
+                try:
+                    complex(f)
+                except (ValueError, TypeError):
+                    vartypes.append(arr.dtype)
+                else:
+                    vartypes.append(np.dtype(complex))
+            else:
+                vartypes.append(np.dtype(float))
+        else:
+            vartypes.append(np.dtype(int))
+    return vartypes
+
+
+def openfile(fname):
+    """
+    Opens the file handle of file `fname`.
+
+    """
+    # A file handle
+    if hasattr(fname, 'readline'):
+        return fname
+    # Try to open the file and guess its type
+    try:
+        f = open(fname)
+    except FileNotFoundError as e:
+        raise FileNotFoundError(f"No such file: '{fname}'") from e
+    if f.readline()[:2] != "\\x":
+        f.seek(0, 0)
+        return f
+    f.close()
+    raise NotImplementedError("Wow, binary file")
+
+
+def fromtextfile(fname, delimiter=None, commentchar='#', missingchar='',
+                 varnames=None, vartypes=None,
+                 *, delimitor=np._NoValue):  # backwards compatibility
+    """
+    Creates a mrecarray from data stored in the file `filename`.
+
+    Parameters
+    ----------
+    fname : {file name/handle}
+        Handle of an opened file.
+    delimiter : {None, string}, optional
+        Alphanumeric character used to separate columns in the file.
+        If None, any (group of) white spacestring(s) will be used.
+    commentchar : {'#', string}, optional
+        Alphanumeric character used to mark the start of a comment.
+    missingchar : {'', string}, optional
+        String indicating missing data, and used to create the masks.
+    varnames : {None, sequence}, optional
+        Sequence of the variable names. If None, a list will be created from
+        the first non empty line of the file.
+    vartypes : {None, sequence}, optional
+        Sequence of the variables dtypes. If None, it will be estimated from
+        the first non-commented line.
+
+
+    Ultra simple: the varnames are in the header, one line"""
+    if delimitor is not np._NoValue:
+        if delimiter is not None:
+            raise TypeError("fromtextfile() got multiple values for argument "
+                            "'delimiter'")
+        # NumPy 1.22.0, 2021-09-23
+        warnings.warn("The 'delimitor' keyword argument of "
+                      "numpy.ma.mrecords.fromtextfile() is deprecated "
+                      "since NumPy 1.22.0, use 'delimiter' instead.",
+                      DeprecationWarning, stacklevel=2)
+        delimiter = delimitor
+
+    # Try to open the file.
+    ftext = openfile(fname)
+
+    # Get the first non-empty line as the varnames
+    while True:
+        line = ftext.readline()
+        firstline = line[:line.find(commentchar)].strip()
+        _varnames = firstline.split(delimiter)
+        if len(_varnames) > 1:
+            break
+    if varnames is None:
+        varnames = _varnames
+
+    # Get the data.
+    _variables = masked_array([line.strip().split(delimiter) for line in ftext
+                               if line[0] != commentchar and len(line) > 1])
+    (_, nfields) = _variables.shape
+    ftext.close()
+
+    # Try to guess the dtype.
+    if vartypes is None:
+        vartypes = _guessvartypes(_variables[0])
+    else:
+        vartypes = [np.dtype(v) for v in vartypes]
+        if len(vartypes) != nfields:
+            msg = "Attempting to %i dtypes for %i fields!"
+            msg += " Reverting to default."
+            warnings.warn(msg % (len(vartypes), nfields), stacklevel=2)
+            vartypes = _guessvartypes(_variables[0])
+
+    # Construct the descriptor.
+    mdescr = [(n, f) for (n, f) in zip(varnames, vartypes)]
+    mfillv = [ma.default_fill_value(f) for f in vartypes]
+
+    # Get the data and the mask.
+    # We just need a list of masked_arrays. It's easier to create it like that:
+    _mask = (_variables.T == missingchar)
+    _datalist = [masked_array(a, mask=m, dtype=t, fill_value=f)
+                 for (a, m, t, f) in zip(_variables.T, _mask, vartypes, mfillv)]
+
+    return fromarrays(_datalist, dtype=mdescr)
+
+
+def addfield(mrecord, newfield, newfieldname=None):
+    """Adds a new field to the masked record array
+
+    Uses `newfield` as data and `newfieldname` as name. If `newfieldname`
+    is None, the new field name is set to 'fi', where `i` is the number of
+    existing fields.
+
+    """
+    _data = mrecord._data
+    _mask = mrecord._mask
+    if newfieldname is None or newfieldname in reserved_fields:
+        newfieldname = 'f%i' % len(_data.dtype)
+    newfield = ma.array(newfield)
+    # Get the new data.
+    # Create a new empty recarray
+    newdtype = np.dtype(_data.dtype.descr + [(newfieldname, newfield.dtype)])
+    newdata = recarray(_data.shape, newdtype)
+    # Add the existing field
+    [newdata.setfield(_data.getfield(*f), *f)
+     for f in _data.dtype.fields.values()]
+    # Add the new field
+    newdata.setfield(newfield._data, *newdata.dtype.fields[newfieldname])
+    newdata = newdata.view(MaskedRecords)
+    # Get the new mask
+    # Create a new empty recarray
+    newmdtype = np.dtype([(n, bool_) for n in newdtype.names])
+    newmask = recarray(_data.shape, newmdtype)
+    # Add the old masks
+    [newmask.setfield(_mask.getfield(*f), *f)
+     for f in _mask.dtype.fields.values()]
+    # Add the mask of the new field
+    newmask.setfield(getmaskarray(newfield),
+                     *newmask.dtype.fields[newfieldname])
+    newdata._mask = newmask
+    return newdata

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/ma/mrecords.pyi

@@ -0,0 +1,90 @@
+from typing import Any, TypeVar
+
+from numpy import dtype
+from numpy.ma import MaskedArray
+
+__all__: list[str]
+
+# TODO: Set the `bound` to something more suitable once we
+# have proper shape support
+_ShapeType = TypeVar("_ShapeType", bound=Any)
+_DType_co = TypeVar("_DType_co", bound=dtype[Any], covariant=True)
+
+class MaskedRecords(MaskedArray[_ShapeType, _DType_co]):
+    def __new__(
+        cls,
+        shape,
+        dtype=...,
+        buf=...,
+        offset=...,
+        strides=...,
+        formats=...,
+        names=...,
+        titles=...,
+        byteorder=...,
+        aligned=...,
+        mask=...,
+        hard_mask=...,
+        fill_value=...,
+        keep_mask=...,
+        copy=...,
+        **options,
+    ): ...
+    _mask: Any
+    _fill_value: Any
+    @property
+    def _data(self): ...
+    @property
+    def _fieldmask(self): ...
+    def __array_finalize__(self, obj): ...
+    def __len__(self): ...
+    def __getattribute__(self, attr): ...
+    def __setattr__(self, attr, val): ...
+    def __getitem__(self, indx): ...
+    def __setitem__(self, indx, value): ...
+    def view(self, dtype=..., type=...): ...
+    def harden_mask(self): ...
+    def soften_mask(self): ...
+    def copy(self): ...
+    def tolist(self, fill_value=...): ...
+    def __reduce__(self): ...
+
+mrecarray = MaskedRecords
+
+def fromarrays(
+    arraylist,
+    dtype=...,
+    shape=...,
+    formats=...,
+    names=...,
+    titles=...,
+    aligned=...,
+    byteorder=...,
+    fill_value=...,
+): ...
+
+def fromrecords(
+    reclist,
+    dtype=...,
+    shape=...,
+    formats=...,
+    names=...,
+    titles=...,
+    aligned=...,
+    byteorder=...,
+    fill_value=...,
+    mask=...,
+): ...
+
+def fromtextfile(
+    fname,
+    delimiter=...,
+    commentchar=...,
+    missingchar=...,
+    varnames=...,
+    vartypes=...,
+    # NOTE: deprecated: NumPy 1.22.0, 2021-09-23
+    # delimitor=...,
+): ...
+
+def addfield(mrecord, newfield, newfieldname=...): ...

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/ma/setup.py

@@ -0,0 +1,12 @@
+#!/usr/bin/env python3
+def configuration(parent_package='',top_path=None):
+    from numpy.distutils.misc_util import Configuration
+    config = Configuration('ma', parent_package, top_path)
+    config.add_subpackage('tests')
+    config.add_data_files('*.pyi')
+    return config
+
+if __name__ == "__main__":
+    from numpy.distutils.core import setup
+    config = configuration(top_path='').todict()
+    setup(**config)

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/qwen3_vl_moe/configuration_qwen3_vl_moe.py

@@ -0,0 +1,194 @@
+#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+#           This file was automatically generated from src/transformers/models/qwen3_vl_moe/modular_qwen3_vl_moe.py.
+#               Do NOT edit this file manually as any edits will be overwritten by the generation of
+#             the file from the modular. If any change should be done, please apply the change to the
+#                          modular_qwen3_vl_moe.py file directly. One of our CI enforces this.
+#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+# Copyright 2025 The Qwen Team and The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from huggingface_hub.dataclasses import strict
+
+from ...configuration_utils import PreTrainedConfig
+from ...modeling_rope_utils import RopeParameters
+from ...utils import auto_docstring
+
+
+@auto_docstring(checkpoint="Qwen/Qwen3-VL-30B-A3B-Instruct")
+@strict
+class Qwen3VLMoeTextConfig(PreTrainedConfig):
+    r"""
+    decoder_sparse_step (`int`, *optional*, defaults to 1):
+        The frequency of the MoE layer.
+    mlp_only_layers (`List[int]`, *optional*, defaults to `[]`):
+        Indicate which layers use Qwen3VLMoeMLP rather than Qwen3VLMoeSparseMoeBlock
+        The list contains layer index, from 0 to num_layers-1 if we have num_layers layers
+        If `mlp_only_layers` is empty, `decoder_sparse_step` is used to determine the sparsity.
+
+    ```python
+    >>> from transformers import Qwen3VLMoeForConditionalGeneration, Qwen3VLMoeConfig
+
+    >>> # Initializing a Qwen3VLMoe style configuration
+    >>> configuration = Qwen3VLMoeConfig()
+
+    >>> # Initializing a model from the Qwen3-VL-30B-A3B style configuration
+    >>> model = Qwen3VLMoeForConditionalGeneration(configuration)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```"""
+
+    model_type = "qwen3_vl_moe_text"
+    keys_to_ignore_at_inference = ["past_key_values"]
+
+    attribute_map = {
+        "num_experts": "num_local_experts",
+    }
+    # Default tensor parallel plan for base model `Qwen3VLMoe`
+    base_model_tp_plan = {
+        "layers.*.self_attn.q_proj": "colwise",
+        "layers.*.self_attn.k_proj": "colwise",
+        "layers.*.self_attn.v_proj": "colwise",
+        "layers.*.self_attn.o_proj": "rowwise",
+        "layers.*.mlp.gate_proj": "colwise",
+        "layers.*.mlp.up_proj": "colwise",
+        "layers.*.mlp.down_proj": "rowwise",
+    }
+    base_model_ep_plan = {
+        "layers.*.mlp.gate": "ep_router",
+        "layers.*.mlp.experts.gate_up_proj": "grouped_gemm",
+        "layers.*.mlp.experts.down_proj": "grouped_gemm",
+        "layers.*.mlp.experts": "moe_tp_experts",
+    }
+    base_model_pp_plan = {
+        "embed_tokens": (["input_ids"], ["inputs_embeds"]),
+        "layers": (["hidden_states", "attention_mask"], ["hidden_states"]),
+        "norm": (["hidden_states"], ["hidden_states"]),
+    }
+
+    vocab_size: int = 151936
+    hidden_size: int = 2048
+
+    intermediate_size: int = 5632
+    num_hidden_layers: int = 24
+    num_attention_heads: int = 16
+    num_key_value_heads: int = 16
+    hidden_act: str = "silu"
+    max_position_embeddings: int = 128000
+    initializer_range: float = 0.02
+    rms_norm_eps: float = 1e-6
+    use_cache: bool = True
+    tie_word_embeddings: bool = True
+    rope_parameters: RopeParameters | dict | None = None
+    attention_bias: bool = False
+    attention_dropout: float | int = 0.0
+    decoder_sparse_step: int = 1
+    moe_intermediate_size: int = 1408
+    num_experts_per_tok: int = 4
+    num_experts: int = 60
+    router_aux_loss_coef: float = 0.001
+    mlp_only_layers: list[int] | None = None
+    pad_token_id: int | None = None
+    bos_token_id: int | None = None
+    eos_token_id: int | list[int] | None = None
+    base_config_key = "text_config"
+    default_theta = 500000.0
+    ignore_keys_at_rope_validation = {"mrope_section", "mrope_interleaved"}
+    head_dim: int | None = None
+
+    def __post_init__(self, **kwargs):
+        if self.num_key_value_heads is None:
+            self.num_key_value_heads = self.num_attention_heads
+
+        self.head_dim = self.head_dim or self.hidden_size // self.num_attention_heads
+        self.sliding_window = None
+        self.mlp_only_layers = [] if self.mlp_only_layers is None else self.mlp_only_layers
+        super().__post_init__(**kwargs)
+
+
+@auto_docstring(checkpoint="Qwen/Qwen3-VL-30B-A3B-Instruct")
+@strict
+class Qwen3VLMoeVisionConfig(PreTrainedConfig):
+    r"""
+    out_hidden_size (`int`, *optional*, defaults to 3584):
+        The output hidden size of the vision model.
+    num_position_embeddings (`int`, *optional*, defaults to 2304):
+        The maximum sequence length that this model might ever be used with
+    deepstack_visual_indexes (`list[int]`, *optional*, defaults to `[8, 16, 24]`):
+        Indexed of layers for deepstack embeddings.
+    """
+
+    model_type = "qwen3_vl_moe_vision"
+    base_config_key = "vision_config"
+
+    depth: int = 27
+    hidden_size: int = 1152
+    hidden_act: str = "gelu_pytorch_tanh"
+    intermediate_size: int = 4304
+    num_heads: int = 16
+    in_channels: int = 3
+    patch_size: int | list[int] | tuple[int, int] = 16
+    spatial_merge_size: int = 2
+    temporal_patch_size: int | list[int] | tuple[int, int] = 2
+    out_hidden_size: int = 3584
+    num_position_embeddings: int = 2304
+    deepstack_visual_indexes: list[int] | tuple[int, ...] = (8, 16, 24)
+    initializer_range: float = 0.02
+
+
+@auto_docstring(checkpoint="Qwen/Qwen3-VL-30B-A3B-Instruct")
+@strict
+class Qwen3VLMoeConfig(PreTrainedConfig):
+    r"""
+    Example:
+
+    ```python
+    >>> from transformers import Qwen3VLMoeForConditionalGeneration, Qwen3VLMoeConfig
+
+    >>> # Initializing a Qwen3-VL-MOE style configuration
+    >>> configuration = Qwen3VLMoeConfig()
+
+    >>> # Initializing a model from the Qwen3-VL-30B-A3B style configuration
+    >>> model = Qwen3VLMoeForConditionalGeneration(configuration)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```"""
+
+    model_type = "qwen3_vl_moe"
+    sub_configs = {"vision_config": Qwen3VLMoeVisionConfig, "text_config": Qwen3VLMoeTextConfig}
+    keys_to_ignore_at_inference = ["past_key_values"]
+
+    text_config: dict | PreTrainedConfig | None = None
+    vision_config: dict | PreTrainedConfig | None = None
+    image_token_id: int = 151655
+    video_token_id: int = 151656
+    vision_start_token_id: int = 151652
+    vision_end_token_id: int = 151653
+    tie_word_embeddings: bool = False
+
+    def __post_init__(self, **kwargs):
+        if isinstance(self.vision_config, dict):
+            self.vision_config = self.sub_configs["vision_config"](**self.vision_config)
+        elif self.vision_config is None:
+            self.vision_config = self.sub_configs["vision_config"]()
+
+        if isinstance(self.text_config, dict):
+            self.text_config = self.sub_configs["text_config"](**self.text_config)
+        elif self.text_config is None:
+            self.text_config = self.sub_configs["text_config"]()
+
+        super().__post_init__(**kwargs)
+
+
+__all__ = ["Qwen3VLMoeConfig", "Qwen3VLMoeTextConfig", "Qwen3VLMoeVisionConfig"]

LTA_openwebtext_dualt/mini_owt_logdirichlet/logs/infer_not5_bottleneck128_170k_decode32_ema_20260611/lr2e3.log

@@ -0,0 +1,29 @@
+checkpoint=runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_2x8gpu_50ep_lr2e3_ema0p9999_elfopt_not5_bottleneck128_unfixed_norm_stateprobadd_selfcond_ce_fast_20260609_150828/step_170000.pt
+use_ema=1
+step=170000
+decode_steps=32
+n=64 chunk_n=8 gpu=1
+out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260611
+[2026-06-11T21:37:16+00:00] infer step=170000 decode=32 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260611/owt_t5_not5_bottleneck128_norm_stateprobadd_selfcond_ce_fast_lr2e3_ema0p9999_step170000_ema_sc1p0_decode32_n64
+[2026-06-11T21:37:16+00:00] run decode=32 chunk=0 n=8 seed=123
+[2026-06-11T21:37:23+00:00] done decode=32 chunk=0
+[2026-06-11T21:37:23+00:00] run decode=32 chunk=1 n=8 seed=124
+[2026-06-11T21:37:30+00:00] done decode=32 chunk=1
+[2026-06-11T21:37:30+00:00] run decode=32 chunk=2 n=8 seed=125
+[2026-06-11T21:37:37+00:00] done decode=32 chunk=2
+[2026-06-11T21:37:37+00:00] run decode=32 chunk=3 n=8 seed=126
+[2026-06-11T21:37:43+00:00] done decode=32 chunk=3
+[2026-06-11T21:37:43+00:00] run decode=32 chunk=4 n=8 seed=127
+[2026-06-11T21:37:50+00:00] done decode=32 chunk=4
+[2026-06-11T21:37:50+00:00] run decode=32 chunk=5 n=8 seed=128
+[2026-06-11T21:37:57+00:00] done decode=32 chunk=5
+[2026-06-11T21:37:57+00:00] run decode=32 chunk=6 n=8 seed=129
+[2026-06-11T21:38:04+00:00] done decode=32 chunk=6
+[2026-06-11T21:38:04+00:00] run decode=32 chunk=7 n=8 seed=130
+[2026-06-11T21:38:11+00:00] done decode=32 chunk=7
+merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260611/owt_t5_not5_bottleneck128_norm_stateprobadd_selfcond_ce_fast_lr2e3_ema0p9999_step170000_ema_sc1p0_decode32_n64/sc1p0/samples64.txt
+loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda
+run	kind	ppl	mean_entropy	distinct_1	distinct_2	top_token_mass	eos_rows	eos_total	ppl_tokens	t5_tokens	path
+sc1p0	raw_full	36.813703657387094	5.18392545332874	0.08469329900856587	0.5129332802108554	0.026417812102545242	62	63	60904	65259	/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260611/owt_t5_not5_bottleneck128_norm_stateprobadd_selfcond_ce_fast_lr2e3_ema0p9999_step170000_ema_sc1p0_decode32_n64/sc1p0
+sc1p0	pre_eos	40.30085154584371	5.192043916306752	0.08625290656845457	0.5223477636630357	0.02690429001701025	0	0	58501	64079	/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260611/owt_t5_not5_bottleneck128_norm_stateprobadd_selfcond_ce_fast_lr2e3_ema0p9999_step170000_ema_sc1p0_decode32_n64/sc1p0
+[2026-06-11T21:38:24+00:00] done