diff --git a/.venv/lib/python3.11/site-packages/numpy/f2py/tests/__pycache__/test_array_from_pyobj.cpython-311.pyc b/.venv/lib/python3.11/site-packages/numpy/f2py/tests/__pycache__/test_array_from_pyobj.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2d454ff74198d3852e163a9a1823376164e262e4 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/numpy/f2py/tests/__pycache__/test_array_from_pyobj.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/numpy/f2py/tests/__pycache__/test_callback.cpython-311.pyc b/.venv/lib/python3.11/site-packages/numpy/f2py/tests/__pycache__/test_callback.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e5b6fed191dedbf1533c6b685362c6400f68b57c Binary files /dev/null and b/.venv/lib/python3.11/site-packages/numpy/f2py/tests/__pycache__/test_callback.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/numpy/f2py/tests/__pycache__/test_crackfortran.cpython-311.pyc b/.venv/lib/python3.11/site-packages/numpy/f2py/tests/__pycache__/test_crackfortran.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8f5758df235fc528d4389abb45df0f44c1735f4c Binary files /dev/null and b/.venv/lib/python3.11/site-packages/numpy/f2py/tests/__pycache__/test_crackfortran.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/numpy/f2py/tests/__pycache__/test_f2py2e.cpython-311.pyc b/.venv/lib/python3.11/site-packages/numpy/f2py/tests/__pycache__/test_f2py2e.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..26fd75ad3dbf4622916ca8db57f5ffc43f94f95b Binary files /dev/null and b/.venv/lib/python3.11/site-packages/numpy/f2py/tests/__pycache__/test_f2py2e.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/numpy/f2py/tests/__pycache__/test_module_doc.cpython-311.pyc b/.venv/lib/python3.11/site-packages/numpy/f2py/tests/__pycache__/test_module_doc.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..01841239d098852e50dc81e11acf69d94e35049a Binary files /dev/null and b/.venv/lib/python3.11/site-packages/numpy/f2py/tests/__pycache__/test_module_doc.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/numpy/f2py/tests/__pycache__/test_pyf_src.cpython-311.pyc b/.venv/lib/python3.11/site-packages/numpy/f2py/tests/__pycache__/test_pyf_src.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..397c204c11738962d6869b65145bc4ec12fe1791 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/numpy/f2py/tests/__pycache__/test_pyf_src.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/numpy/f2py/tests/__pycache__/test_quoted_character.cpython-311.pyc b/.venv/lib/python3.11/site-packages/numpy/f2py/tests/__pycache__/test_quoted_character.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fe7c35a0baa4aac88d1364d957ffc8ecaa60eb52 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/numpy/f2py/tests/__pycache__/test_quoted_character.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/numpy/f2py/tests/__pycache__/test_return_character.cpython-311.pyc b/.venv/lib/python3.11/site-packages/numpy/f2py/tests/__pycache__/test_return_character.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a46c45ac51b60af5b859d184af41c6d9b1a7309c Binary files /dev/null and b/.venv/lib/python3.11/site-packages/numpy/f2py/tests/__pycache__/test_return_character.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/numpy/f2py/tests/__pycache__/test_return_complex.cpython-311.pyc b/.venv/lib/python3.11/site-packages/numpy/f2py/tests/__pycache__/test_return_complex.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4a52f4064acdba35e83886f2800b22a4ca7124c7 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/numpy/f2py/tests/__pycache__/test_return_complex.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/numpy/f2py/tests/__pycache__/test_return_logical.cpython-311.pyc b/.venv/lib/python3.11/site-packages/numpy/f2py/tests/__pycache__/test_return_logical.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..42eff64d4702a0bc9c1d50036ee9bc9ecbacf6f5 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/numpy/f2py/tests/__pycache__/test_return_logical.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/ray/dashboard/modules/__pycache__/__init__.cpython-311.pyc b/.venv/lib/python3.11/site-packages/ray/dashboard/modules/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5a5404c642f75bd7f9d2e3aaa4f3cbc9aa263f59 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/ray/dashboard/modules/__pycache__/__init__.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/ray/dashboard/modules/__pycache__/dashboard_sdk.cpython-311.pyc b/.venv/lib/python3.11/site-packages/ray/dashboard/modules/__pycache__/dashboard_sdk.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..56880e7a1468803bbf26d2a578273add4f081f15 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/ray/dashboard/modules/__pycache__/dashboard_sdk.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/ray/dashboard/modules/__pycache__/version.cpython-311.pyc b/.venv/lib/python3.11/site-packages/ray/dashboard/modules/__pycache__/version.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..59d6f189481b3a984248a6a1494309953d2f116d Binary files /dev/null and b/.venv/lib/python3.11/site-packages/ray/dashboard/modules/__pycache__/version.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/ray/dashboard/modules/event/event_utils.py b/.venv/lib/python3.11/site-packages/ray/dashboard/modules/event/event_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..782fa916fe378968fa8744a4f235a560f7c08550 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/ray/dashboard/modules/event/event_utils.py @@ -0,0 +1,208 @@ +import asyncio +import collections +import fnmatch +import itertools +import json +import logging.handlers +import mmap +import os +import time +from concurrent.futures import ThreadPoolExecutor + +from ray._private.utils import get_or_create_event_loop, run_background_task +from ray.dashboard.modules.event import event_consts +from ray.dashboard.utils import async_loop_forever + +logger = logging.getLogger(__name__) + + +def _get_source_files(event_dir, source_types=None, event_file_filter=None): + event_log_names = os.listdir(event_dir) + source_files = {} + all_source_types = set(event_consts.EVENT_SOURCE_ALL) + for source_type in source_types or event_consts.EVENT_SOURCE_ALL: + assert source_type in all_source_types, f"Invalid source type: {source_type}" + files = [] + for n in event_log_names: + if fnmatch.fnmatch(n, f"*{source_type}*"): + f = os.path.join(event_dir, n) + if event_file_filter is not None and not event_file_filter(f): + continue + files.append(f) + if files: + source_files[source_type] = files + return source_files + + +def _restore_newline(event_dict): + try: + event_dict["message"] = ( + event_dict["message"].replace("\\n", "\n").replace("\\r", "\n") + ) + except Exception: + logger.exception("Restore newline for event failed: %s", event_dict) + return event_dict + + +def _parse_line(event_str): + return _restore_newline(json.loads(event_str)) + + +def parse_event_strings(event_string_list): + events = [] + for data in event_string_list: + if not data: + continue + try: + event = _parse_line(data) + events.append(event) + except Exception: + logger.exception("Parse event line failed: %s", repr(data)) + return events + + +ReadFileResult = collections.namedtuple( + "ReadFileResult", ["fid", "size", "mtime", "position", "lines"] +) + + +def _read_file( + file, pos, n_lines=event_consts.EVENT_READ_LINE_COUNT_LIMIT, closefd=True +): + with open(file, "rb", closefd=closefd) as f: + # The ino may be 0 on Windows. + stat = os.stat(f.fileno()) + fid = stat.st_ino or file + lines = [] + with mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as mm: + start = pos + for _ in range(n_lines): + sep = mm.find(b"\n", start) + if sep == -1: + break + if sep - start <= event_consts.EVENT_READ_LINE_LENGTH_LIMIT: + lines.append(mm[start:sep].decode("utf-8")) + else: + truncated_size = min(100, event_consts.EVENT_READ_LINE_LENGTH_LIMIT) + logger.warning( + "Ignored long string: %s...(%s chars)", + mm[start : start + truncated_size].decode("utf-8"), + sep - start, + ) + start = sep + 1 + return ReadFileResult(fid, stat.st_size, stat.st_mtime, start, lines) + + +def monitor_events( + event_dir, + callback, + monitor_thread_pool_executor: ThreadPoolExecutor, + scan_interval_seconds=event_consts.SCAN_EVENT_DIR_INTERVAL_SECONDS, + start_mtime=time.time() + event_consts.SCAN_EVENT_START_OFFSET_SECONDS, + monitor_files=None, + source_types=None, +): + """Monitor events in directory. New events will be read and passed to the + callback. + + Args: + event_dir: The event log directory. + callback (def callback(List[str]): pass): A callback accepts a list of + event strings. + monitor_thread_pool_executor: A thread pool exector to monitor/update + events. None means it will use the default execturo which uses + num_cpus of the machine * 5 threads (before python 3.8) or + min(32, num_cpus + 5) (from Python 3.8). + scan_interval_seconds: An interval seconds between two scans. + start_mtime: Only the event log files whose last modification + time is greater than start_mtime are monitored. + monitor_files (Dict[int, MonitorFile]): The map from event log file id + to MonitorFile object. Monitor all files start from the beginning + if the value is None. + source_types (List[str]): A list of source type name from + event_pb2.Event.SourceType.keys(). Monitor all source types if the + value is None. + """ + loop = get_or_create_event_loop() + if monitor_files is None: + monitor_files = {} + + logger.info( + "Monitor events logs modified after %s on %s, " "the source types are %s.", + start_mtime, + event_dir, + "all" if source_types is None else source_types, + ) + + MonitorFile = collections.namedtuple("MonitorFile", ["size", "mtime", "position"]) + + def _source_file_filter(source_file): + stat = os.stat(source_file) + return stat.st_mtime > start_mtime + + def _read_monitor_file(file, pos): + assert isinstance( + file, str + ), f"File should be a str, but a {type(file)}({file}) found" + fd = os.open(file, os.O_RDONLY) + try: + stat = os.stat(fd) + # Check the file size to avoid raising the exception + # ValueError: cannot mmap an empty file + if stat.st_size <= 0: + return [] + fid = stat.st_ino or file + monitor_file = monitor_files.get(fid) + if monitor_file: + if ( + monitor_file.position == monitor_file.size + and monitor_file.size == stat.st_size + and monitor_file.mtime == stat.st_mtime + ): + logger.debug( + "Skip reading the file because " "there is no change: %s", file + ) + return [] + position = monitor_file.position + else: + logger.info("Found new event log file: %s", file) + position = pos + # Close the fd in finally. + r = _read_file(fd, position, closefd=False) + # It should be fine to update the dict in executor thread. + monitor_files[r.fid] = MonitorFile(r.size, r.mtime, r.position) + loop.call_soon_threadsafe(callback, r.lines) + except Exception as e: + raise Exception(f"Read event file failed: {file}") from e + finally: + os.close(fd) + + @async_loop_forever(scan_interval_seconds, cancellable=True) + async def _scan_event_log_files(): + # Scan event files. + source_files = await loop.run_in_executor( + monitor_thread_pool_executor, + _get_source_files, + event_dir, + source_types, + _source_file_filter, + ) + + # Limit concurrent read to avoid fd exhaustion. + semaphore = asyncio.Semaphore(event_consts.CONCURRENT_READ_LIMIT) + + async def _concurrent_coro(filename): + async with semaphore: + return await loop.run_in_executor( + monitor_thread_pool_executor, _read_monitor_file, filename, 0 + ) + + # Read files. + await asyncio.gather( + *[ + _concurrent_coro(filename) + for filename in list(itertools.chain(*source_files.values())) + ] + ) + + return run_background_task(_scan_event_log_files()) diff --git a/.venv/lib/python3.11/site-packages/ray/dashboard/modules/healthz/__init__.py b/.venv/lib/python3.11/site-packages/ray/dashboard/modules/healthz/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.venv/lib/python3.11/site-packages/ray/dashboard/modules/healthz/__pycache__/__init__.cpython-311.pyc b/.venv/lib/python3.11/site-packages/ray/dashboard/modules/healthz/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ab8287a5ae50fba11795f0b553c3c1bc394fec04 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/ray/dashboard/modules/healthz/__pycache__/__init__.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/ray/dashboard/modules/healthz/__pycache__/healthz_agent.cpython-311.pyc b/.venv/lib/python3.11/site-packages/ray/dashboard/modules/healthz/__pycache__/healthz_agent.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..089d90f1b630af8f280e837ca1aacaace4086361 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/ray/dashboard/modules/healthz/__pycache__/healthz_agent.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/ray/dashboard/modules/healthz/__pycache__/healthz_head.cpython-311.pyc b/.venv/lib/python3.11/site-packages/ray/dashboard/modules/healthz/__pycache__/healthz_head.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6ea41787cd30716be649d4fcbbf42ffc425589d3 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/ray/dashboard/modules/healthz/__pycache__/healthz_head.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/ray/dashboard/modules/healthz/__pycache__/utils.cpython-311.pyc b/.venv/lib/python3.11/site-packages/ray/dashboard/modules/healthz/__pycache__/utils.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f743feec113603f183cebb847c148604f551697a Binary files /dev/null and b/.venv/lib/python3.11/site-packages/ray/dashboard/modules/healthz/__pycache__/utils.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/ray/dashboard/modules/healthz/healthz_agent.py b/.venv/lib/python3.11/site-packages/ray/dashboard/modules/healthz/healthz_agent.py new file mode 100644 index 0000000000000000000000000000000000000000..f9483f14aea64ae998d4bbfa98cfbb879145afd0 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/ray/dashboard/modules/healthz/healthz_agent.py @@ -0,0 +1,54 @@ +from aiohttp.web import Request, Response + +import ray.dashboard.optional_utils as optional_utils +import ray.dashboard.utils as dashboard_utils +import ray.exceptions +from ray.dashboard.modules.healthz.utils import HealthChecker + +routes = optional_utils.DashboardAgentRouteTable + + +class HealthzAgent(dashboard_utils.DashboardAgentModule): + """Health check in the agent. + + This module adds health check related endpoint to the agent to check + local components' health. + """ + + def __init__(self, dashboard_agent): + super().__init__(dashboard_agent) + self._health_checker = HealthChecker( + dashboard_agent.gcs_aio_client, + f"{dashboard_agent.ip}:{dashboard_agent.node_manager_port}", + ) + + @routes.get("/api/local_raylet_healthz") + async def health_check(self, req: Request) -> Response: + try: + alive = await self._health_checker.check_local_raylet_liveness() + if alive is False: + return Response(status=503, text="Local Raylet failed") + except ray.exceptions.RpcError as e: + # We only consider the error other than GCS unreachable as raylet failure + # to avoid false positive. + # In case of GCS failed, Raylet will crash eventually if GCS is not back + # within a given time and the check will fail since agent can't live + # without a local raylet. + if e.rpc_code not in ( + ray._raylet.GRPC_STATUS_CODE_UNAVAILABLE, + ray._raylet.GRPC_STATUS_CODE_UNKNOWN, + ray._raylet.GRPC_STATUS_CODE_DEADLINE_EXCEEDED, + ): + return Response(status=503, text=f"Health check failed due to: {e}") + + return Response( + text="success", + content_type="application/text", + ) + + async def run(self, server): + pass + + @staticmethod + def is_minimal_module(): + return False diff --git a/.venv/lib/python3.11/site-packages/ray/dashboard/modules/healthz/healthz_head.py b/.venv/lib/python3.11/site-packages/ray/dashboard/modules/healthz/healthz_head.py new file mode 100644 index 0000000000000000000000000000000000000000..75272962aa972ec7a0328674b8e63bb452e72762 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/ray/dashboard/modules/healthz/healthz_head.py @@ -0,0 +1,41 @@ +from aiohttp.web import HTTPServiceUnavailable, Request, Response + +import ray.dashboard.optional_utils as optional_utils +import ray.dashboard.utils as dashboard_utils +from ray.dashboard.modules.healthz.utils import HealthChecker + +routes = optional_utils.DashboardHeadRouteTable + + +class HealthzHead(dashboard_utils.DashboardHeadModule): + """Health check in the head. + + This module adds health check related endpoint to the head to check + GCS's heath. + """ + + def __init__(self, config: dashboard_utils.DashboardHeadModuleConfig): + super().__init__(config) + self._health_checker = HealthChecker(self.gcs_aio_client) + + @routes.get("/api/gcs_healthz") + async def health_check(self, req: Request) -> Response: + alive = False + try: + alive = await self._health_checker.check_gcs_liveness() + if alive is True: + return Response( + text="success", + content_type="application/text", + ) + except Exception as e: + return HTTPServiceUnavailable(reason=f"Health check failed: {e}") + + return HTTPServiceUnavailable(reason="Health check failed") + + async def run(self, server): + pass + + @staticmethod + def is_minimal_module(): + return True diff --git a/.venv/lib/python3.11/site-packages/ray/dashboard/modules/healthz/utils.py b/.venv/lib/python3.11/site-packages/ray/dashboard/modules/healthz/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..4941c22e6e714d2086d1238070b17636688e39df --- /dev/null +++ b/.venv/lib/python3.11/site-packages/ray/dashboard/modules/healthz/utils.py @@ -0,0 +1,24 @@ +from typing import Optional + +from ray._private.gcs_utils import GcsAioClient + + +class HealthChecker: + def __init__( + self, gcs_aio_client: GcsAioClient, local_node_address: Optional[str] = None + ): + self._gcs_aio_client = gcs_aio_client + self._local_node_address = local_node_address + + async def check_local_raylet_liveness(self) -> bool: + if self._local_node_address is None: + return False + + liveness = await self._gcs_aio_client.check_alive( + [self._local_node_address.encode()], 0.1 + ) + return liveness[0] + + async def check_gcs_liveness(self) -> bool: + await self._gcs_aio_client.check_alive([], 0.1) + return True diff --git a/.venv/lib/python3.11/site-packages/ray/dashboard/modules/job/__pycache__/__init__.cpython-311.pyc b/.venv/lib/python3.11/site-packages/ray/dashboard/modules/job/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..72830e14b77e41fb8f3224c9ad98aab6dd735a2e Binary files /dev/null and b/.venv/lib/python3.11/site-packages/ray/dashboard/modules/job/__pycache__/__init__.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/ray/dashboard/modules/job/__pycache__/common.cpython-311.pyc b/.venv/lib/python3.11/site-packages/ray/dashboard/modules/job/__pycache__/common.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8ebcaaf52e5421e7d57e5cc6d140148410270104 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/ray/dashboard/modules/job/__pycache__/common.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/ray/dashboard/modules/job/__pycache__/job_log_storage_client.cpython-311.pyc b/.venv/lib/python3.11/site-packages/ray/dashboard/modules/job/__pycache__/job_log_storage_client.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..33b2770775437e6e6847a8509d93744e2c49d320 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/ray/dashboard/modules/job/__pycache__/job_log_storage_client.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/ray/dashboard/modules/job/__pycache__/sdk.cpython-311.pyc b/.venv/lib/python3.11/site-packages/ray/dashboard/modules/job/__pycache__/sdk.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ed08693350c7a5a78566c32fc53a3c15e08f8d57 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/ray/dashboard/modules/job/__pycache__/sdk.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/ray/dashboard/modules/job/__pycache__/utils.cpython-311.pyc b/.venv/lib/python3.11/site-packages/ray/dashboard/modules/job/__pycache__/utils.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1493c2b0b9f07945bd1b9b9c795d4c8956a6c9f4 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/ray/dashboard/modules/job/__pycache__/utils.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/ray/dashboard/modules/snapshot/__init__.py b/.venv/lib/python3.11/site-packages/ray/dashboard/modules/snapshot/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.venv/lib/python3.11/site-packages/ray/dashboard/modules/snapshot/__pycache__/__init__.cpython-311.pyc b/.venv/lib/python3.11/site-packages/ray/dashboard/modules/snapshot/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b748015e2e2d7d2feed141a0f643923bf248d04a Binary files /dev/null and b/.venv/lib/python3.11/site-packages/ray/dashboard/modules/snapshot/__pycache__/__init__.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/ray/dashboard/modules/snapshot/__pycache__/snapshot_head.cpython-311.pyc b/.venv/lib/python3.11/site-packages/ray/dashboard/modules/snapshot/__pycache__/snapshot_head.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b995cb49a6da94a919ac8d588041f075695e1e8b Binary files /dev/null and b/.venv/lib/python3.11/site-packages/ray/dashboard/modules/snapshot/__pycache__/snapshot_head.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/ray/dashboard/modules/snapshot/snapshot_head.py b/.venv/lib/python3.11/site-packages/ray/dashboard/modules/snapshot/snapshot_head.py new file mode 100644 index 0000000000000000000000000000000000000000..fe42d4f4f626cec59d7ea0fa54e94dfd89388524 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/ray/dashboard/modules/snapshot/snapshot_head.py @@ -0,0 +1,235 @@ +import concurrent.futures +import enum +import json +import logging +import os +from datetime import datetime +from typing import Optional + +import aiohttp.web + +import ray.dashboard.optional_utils as dashboard_optional_utils +import ray.dashboard.utils as dashboard_utils +from ray import ActorID +from ray._private.pydantic_compat import BaseModel, Extra, Field, validator +from ray._private.storage import _load_class +from ray.dashboard.consts import RAY_CLUSTER_ACTIVITY_HOOK + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + +routes = dashboard_optional_utils.DashboardHeadRouteTable + +SNAPSHOT_API_TIMEOUT_SECONDS = 30 + + +class RayActivityStatus(str, enum.Enum): + ACTIVE = "ACTIVE" + INACTIVE = "INACTIVE" + ERROR = "ERROR" + + +class RayActivityResponse(BaseModel, extra=Extra.allow): + """ + Pydantic model used to inform if a particular Ray component can be considered + active, and metadata about observation. + """ + + is_active: RayActivityStatus = Field( + ..., + description=( + "Whether the corresponding Ray component is considered active or inactive, " + "or if there was an error while collecting this observation." + ), + ) + reason: Optional[str] = Field( + None, description="Reason if Ray component is considered active or errored." + ) + timestamp: float = Field( + ..., + description=( + "Timestamp of when this observation about the Ray component was made. " + "This is in the format of seconds since unix epoch." + ), + ) + last_activity_at: Optional[float] = Field( + None, + description=( + "Timestamp when last actvity of this Ray component finished in format of " + "seconds since unix epoch. This field does not need to be populated " + "for Ray components where it is not meaningful." + ), + ) + + @validator("reason", always=True) + def reason_required(cls, v, values, **kwargs): + if "is_active" in values and values["is_active"] != RayActivityStatus.INACTIVE: + if v is None: + raise ValueError( + 'Reason is required if is_active is "active" or "error"' + ) + return v + + +class APIHead(dashboard_utils.DashboardHeadModule): + def __init__(self, config: dashboard_utils.DashboardHeadModuleConfig): + super().__init__(config) + # For offloading CPU intensive work. + self._thread_pool = concurrent.futures.ThreadPoolExecutor( + max_workers=2, thread_name_prefix="api_head" + ) + + @routes.get("/api/actors/kill") + async def kill_actor_gcs(self, req) -> aiohttp.web.Response: + actor_id = req.query.get("actor_id") + force_kill = req.query.get("force_kill", False) in ("true", "True") + no_restart = req.query.get("no_restart", False) in ("true", "True") + if not actor_id: + return dashboard_optional_utils.rest_response( + success=False, message="actor_id is required." + ) + + await self.gcs_aio_client.kill_actor( + ActorID.from_hex(actor_id), + force_kill, + no_restart, + timeout=SNAPSHOT_API_TIMEOUT_SECONDS, + ) + + message = ( + f"Force killed actor with id {actor_id}" + if force_kill + else f"Requested actor with id {actor_id} to terminate. " + + "It will exit once running tasks complete" + ) + + return dashboard_optional_utils.rest_response(success=True, message=message) + + @routes.get("/api/component_activities") + async def get_component_activities(self, req) -> aiohttp.web.Response: + timeout = req.query.get("timeout", None) + if timeout and timeout.isdigit(): + timeout = int(timeout) + else: + timeout = SNAPSHOT_API_TIMEOUT_SECONDS + + # Get activity information for driver + driver_activity_info = await self._get_job_activity_info(timeout=timeout) + resp = {"driver": dict(driver_activity_info)} + + if RAY_CLUSTER_ACTIVITY_HOOK in os.environ: + try: + cluster_activity_callable = _load_class( + os.environ[RAY_CLUSTER_ACTIVITY_HOOK] + ) + external_activity_output = cluster_activity_callable() + assert isinstance(external_activity_output, dict), ( + f"Output of hook {os.environ[RAY_CLUSTER_ACTIVITY_HOOK]} " + "should be Dict[str, RayActivityResponse]. Got " + f"output: {external_activity_output}" + ) + for component_type in external_activity_output: + try: + component_activity_output = external_activity_output[ + component_type + ] + # Parse and validate output to type RayActivityResponse + component_activity_output = RayActivityResponse( + **dict(component_activity_output) + ) + resp[component_type] = dict(component_activity_output) + except Exception as e: + logger.exception( + f"Failed to get activity status of {component_type} " + f"from user hook {os.environ[RAY_CLUSTER_ACTIVITY_HOOK]}." + ) + resp[component_type] = { + "is_active": RayActivityStatus.ERROR, + "reason": repr(e), + "timestamp": datetime.now().timestamp(), + } + except Exception as e: + logger.exception( + "Failed to get activity status from user " + f"hook {os.environ[RAY_CLUSTER_ACTIVITY_HOOK]}." + ) + resp["external_component"] = { + "is_active": RayActivityStatus.ERROR, + "reason": repr(e), + "timestamp": datetime.now().timestamp(), + } + + return aiohttp.web.Response( + text=json.dumps(resp), + content_type="application/json", + status=aiohttp.web.HTTPOk.status_code, + ) + + async def _get_job_activity_info(self, timeout: int) -> RayActivityResponse: + # Returns if there is Ray activity from drivers (job). + # Drivers in namespaces that start with _ray_internal_ are not + # considered activity. + # This includes the _ray_internal_dashboard job that gets automatically + # created with every cluster + try: + reply = await self.gcs_aio_client.get_all_job_info( + skip_submission_job_info_field=True, + skip_is_running_tasks_field=True, + timeout=timeout, + ) + + num_active_drivers = 0 + latest_job_end_time = 0 + for job_table_entry in reply.values(): + is_dead = bool(job_table_entry.is_dead) + in_internal_namespace = job_table_entry.config.ray_namespace.startswith( + "_ray_internal_" + ) + latest_job_end_time = ( + max(latest_job_end_time, job_table_entry.end_time) + if job_table_entry.end_time + else latest_job_end_time + ) + if not is_dead and not in_internal_namespace: + num_active_drivers += 1 + + current_timestamp = datetime.now().timestamp() + # Latest job end time must be before or equal to the current timestamp. + # Job end times may be provided in epoch milliseconds. Check if this + # is true, and convert to seconds + if latest_job_end_time > current_timestamp: + latest_job_end_time = latest_job_end_time / 1000 + assert current_timestamp >= latest_job_end_time, ( + f"Most recent job end time {latest_job_end_time} must be " + f"before or equal to the current timestamp {current_timestamp}" + ) + + is_active = ( + RayActivityStatus.ACTIVE + if num_active_drivers > 0 + else RayActivityStatus.INACTIVE + ) + return RayActivityResponse( + is_active=is_active, + reason=f"Number of active drivers: {num_active_drivers}" + if num_active_drivers + else None, + timestamp=current_timestamp, + # If latest_job_end_time == 0, no jobs have finished yet so don't + # populate last_activity_at + last_activity_at=latest_job_end_time if latest_job_end_time else None, + ) + except Exception as e: + logger.exception("Failed to get activity status of Ray drivers.") + return RayActivityResponse( + is_active=RayActivityStatus.ERROR, + reason=repr(e), + timestamp=datetime.now().timestamp(), + ) + + async def run(self, server): + pass + + @staticmethod + def is_minimal_module(): + return False diff --git a/.venv/lib/python3.11/site-packages/ray/data/_internal/arrow_ops/__init__.py b/.venv/lib/python3.11/site-packages/ray/data/_internal/arrow_ops/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.venv/lib/python3.11/site-packages/ray/data/_internal/arrow_ops/__pycache__/transform_polars.cpython-311.pyc b/.venv/lib/python3.11/site-packages/ray/data/_internal/arrow_ops/__pycache__/transform_polars.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..578d927d9b87dcc982b8b52e1180d298759d4478 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/ray/data/_internal/arrow_ops/__pycache__/transform_polars.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/ray/data/_internal/arrow_ops/__pycache__/transform_pyarrow.cpython-311.pyc b/.venv/lib/python3.11/site-packages/ray/data/_internal/arrow_ops/__pycache__/transform_pyarrow.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f7cd047999191571f7ba278d0f6706b959f26ce0 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/ray/data/_internal/arrow_ops/__pycache__/transform_pyarrow.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/ray/data/_internal/arrow_ops/transform_polars.py b/.venv/lib/python3.11/site-packages/ray/data/_internal/arrow_ops/transform_polars.py new file mode 100644 index 0000000000000000000000000000000000000000..93c615f55d21e2bb4f31dfdf419ab12189e8d998 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/ray/data/_internal/arrow_ops/transform_polars.py @@ -0,0 +1,40 @@ +from typing import TYPE_CHECKING, List + +try: + import pyarrow +except ImportError: + pyarrow = None + + +if TYPE_CHECKING: + from ray.data._internal.planner.exchange.sort_task_spec import SortKey + +pl = None + + +def check_polars_installed(): + try: + global pl + import polars as pl + except ImportError: + raise ImportError( + "polars not installed. Install with `pip install polars` or set " + "`DataContext.use_polars = False` to fall back to pyarrow" + ) + + +def sort(table: "pyarrow.Table", sort_key: "SortKey") -> "pyarrow.Table": + check_polars_installed() + df = pl.from_arrow(table) + return df.sort(sort_key.get_columns(), reverse=sort_key.get_descending()).to_arrow() + + +def concat_and_sort( + blocks: List["pyarrow.Table"], sort_key: "SortKey" +) -> "pyarrow.Table": + check_polars_installed() + blocks = [pl.from_arrow(block) for block in blocks] + df = pl.concat(blocks).sort( + sort_key.get_columns(), reverse=sort_key.get_descending() + ) + return df.to_arrow() diff --git a/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__init__.py b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/__init__.cpython-311.pyc b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f1cc677b7ba48b0dee3bc639b5b82aacd7b35217 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/__init__.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/audio_datasource.cpython-311.pyc b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/audio_datasource.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1965d03335a96f1e780b0aaadd9836b85cea067b Binary files /dev/null and b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/audio_datasource.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/avro_datasource.cpython-311.pyc b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/avro_datasource.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..73fdf09f2429602689fe16a59682c83c5e8ec452 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/avro_datasource.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/bigquery_datasink.cpython-311.pyc b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/bigquery_datasink.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1d25bd6781cbcdd57dee9317ab4848824a59aaef Binary files /dev/null and b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/bigquery_datasink.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/bigquery_datasource.cpython-311.pyc b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/bigquery_datasource.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6ca7f891f5589f7cd2d305bcb3cbeda909a232bf Binary files /dev/null and b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/bigquery_datasource.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/binary_datasource.cpython-311.pyc b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/binary_datasource.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..47d2fd47dc85c1aac910d30fa366f03ee25a36c9 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/binary_datasource.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/clickhouse_datasource.cpython-311.pyc b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/clickhouse_datasource.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4e8d881d73e65f9b9f614a9cbddf2516adcb4b98 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/clickhouse_datasource.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/csv_datasink.cpython-311.pyc b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/csv_datasink.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e0ac863610b4488489772b377c37deaebbbef478 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/csv_datasink.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/csv_datasource.cpython-311.pyc b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/csv_datasource.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..78db580b7f73300555155987ff0cc7c394ee5aba Binary files /dev/null and b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/csv_datasource.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/databricks_uc_datasource.cpython-311.pyc b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/databricks_uc_datasource.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ad3d8b2cbdda2d753d062787dfba956ddc3cabf7 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/databricks_uc_datasource.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/delta_sharing_datasource.cpython-311.pyc b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/delta_sharing_datasource.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..946ced5f73a327529d960a448aaeac99d77c01ab Binary files /dev/null and b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/delta_sharing_datasource.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/hudi_datasource.cpython-311.pyc b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/hudi_datasource.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cf5a645e6343aa6953f813f27a9f7b796cb3045e Binary files /dev/null and b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/hudi_datasource.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/huggingface_datasource.cpython-311.pyc b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/huggingface_datasource.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0ccf7b2b8384f8d2cd837e8c99d43f4f7000ac14 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/huggingface_datasource.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/iceberg_datasource.cpython-311.pyc b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/iceberg_datasource.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..98848d6a148a224d98b4a7288539472ce4a2eb44 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/iceberg_datasource.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/image_datasink.cpython-311.pyc b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/image_datasink.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..44867645c2d399ca9729b86a1f799fb337637c99 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/image_datasink.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/image_datasource.cpython-311.pyc b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/image_datasource.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..79510412e900b7ddc394ccb983517ceb959a7292 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/image_datasource.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/json_datasink.cpython-311.pyc b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/json_datasink.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ff65726e695b2335d4827c0c9fd94f94a0559e25 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/json_datasink.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/json_datasource.cpython-311.pyc b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/json_datasource.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0a14b519a9e2d7f6ab501fdb2ad6cb55d6127ef3 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/json_datasource.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/lance_datasource.cpython-311.pyc b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/lance_datasource.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cfb4ee19145a5e0425508d953f379335fe27c25f Binary files /dev/null and b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/lance_datasource.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/mongo_datasink.cpython-311.pyc b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/mongo_datasink.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ce25eac46ebd806fac594f1bf747c516270aebda Binary files /dev/null and b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/mongo_datasink.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/mongo_datasource.cpython-311.pyc b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/mongo_datasource.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7a7aa146a71cf571b8f7c6fe871da477907a5ffa Binary files /dev/null and b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/mongo_datasource.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/numpy_datasink.cpython-311.pyc b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/numpy_datasink.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c8cd587ebf88da738a7a5a12a34c8654bfb2bc7f Binary files /dev/null and b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/numpy_datasink.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/numpy_datasource.cpython-311.pyc b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/numpy_datasource.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ec9ff749e3a18b7ffc43bac0e1176e49d8a0e9cc Binary files /dev/null and b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/numpy_datasource.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/parquet_bulk_datasource.cpython-311.pyc b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/parquet_bulk_datasource.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cc8317933d9515481847772b6ae0a07bb034ae68 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/parquet_bulk_datasource.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/parquet_datasink.cpython-311.pyc b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/parquet_datasink.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2bb0c8ac643c878554282db8d62d3faaf1f72c1e Binary files /dev/null and b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/parquet_datasink.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/parquet_datasource.cpython-311.pyc b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/parquet_datasource.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..471ba4d6694f9e93b12d2971806b888d2f0d2995 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/parquet_datasource.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/range_datasource.cpython-311.pyc b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/range_datasource.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..aa48f47823a05da2b2be9ebd576745ccdfcb5a8e Binary files /dev/null and b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/range_datasource.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/sql_datasink.cpython-311.pyc b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/sql_datasink.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8b3e929927eee7b8f03031ee89e0a2d516396dfe Binary files /dev/null and b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/sql_datasink.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/sql_datasource.cpython-311.pyc b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/sql_datasource.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6da5d8411c81c84fcab93bfe99294a0a0f46e279 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/sql_datasource.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/text_datasource.cpython-311.pyc b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/text_datasource.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ca1d009ba1beb58111c946a91b39e5b635825b4a Binary files /dev/null and b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/text_datasource.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/tfrecords_datasink.cpython-311.pyc b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/tfrecords_datasink.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1d21ce384266a1deba1b4a6e90ef823ac8f3cd9b Binary files /dev/null and b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/tfrecords_datasink.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/tfrecords_datasource.cpython-311.pyc b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/tfrecords_datasource.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2183330f2380e641f17c4512a8b24224aa392f10 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/tfrecords_datasource.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/torch_datasource.cpython-311.pyc b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/torch_datasource.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2c61e971603947b6f1fdf10e7e5c3f367ca2efa8 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/torch_datasource.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/video_datasource.cpython-311.pyc b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/video_datasource.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2a16d134b95ce904d481b0efe6199a389e2a09e6 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/video_datasource.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/webdataset_datasink.cpython-311.pyc b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/webdataset_datasink.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b059c82f765fd578185b9f76ba700b72f840ee5f Binary files /dev/null and b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/webdataset_datasink.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/webdataset_datasource.cpython-311.pyc b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/webdataset_datasource.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e3a06853742b8a55e6182f5b4b2a1e53543e1cce Binary files /dev/null and b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/__pycache__/webdataset_datasource.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/csv_datasource.py b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/csv_datasource.py new file mode 100644 index 0000000000000000000000000000000000000000..f8ddc4bda6bdbe660b0b72f46003578de85b6378 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/csv_datasource.py @@ -0,0 +1,73 @@ +from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Union + +from ray.data.block import Block +from ray.data.datasource.file_based_datasource import FileBasedDatasource + +if TYPE_CHECKING: + import pyarrow + + +class CSVDatasource(FileBasedDatasource): + """CSV datasource, for reading and writing CSV files.""" + + _FILE_EXTENSIONS = [ + "csv", + "csv.gz", # gzip-compressed files + "csv.br", # Brotli-compressed files + "csv.zst", # Zstandard-compressed files + "csv.lz4", # lz4-compressed files + ] + + def __init__( + self, + paths: Union[str, List[str]], + arrow_csv_args: Optional[Dict[str, Any]] = None, + **file_based_datasource_kwargs, + ): + from pyarrow import csv + + super().__init__(paths, **file_based_datasource_kwargs) + + if arrow_csv_args is None: + arrow_csv_args = {} + + self.read_options = arrow_csv_args.pop( + "read_options", csv.ReadOptions(use_threads=False) + ) + self.parse_options = arrow_csv_args.pop("parse_options", csv.ParseOptions()) + self.arrow_csv_args = arrow_csv_args + + def _read_stream(self, f: "pyarrow.NativeFile", path: str) -> Iterator[Block]: + import pyarrow as pa + from pyarrow import csv + + # Re-init invalid row handler: https://issues.apache.org/jira/browse/ARROW-17641 + if hasattr(self.parse_options, "invalid_row_handler"): + self.parse_options.invalid_row_handler = ( + self.parse_options.invalid_row_handler + ) + + try: + reader = csv.open_csv( + f, + read_options=self.read_options, + parse_options=self.parse_options, + **self.arrow_csv_args, + ) + schema = None + while True: + try: + batch = reader.read_next_batch() + table = pa.Table.from_batches([batch], schema=schema) + if schema is None: + schema = table.schema + yield table + except StopIteration: + return + except pa.lib.ArrowInvalid as e: + raise ValueError( + f"Failed to read CSV file: {path}. " + "Please check the CSV file has correct format, or filter out non-CSV " + "file with 'partition_filter' field. See read_csv() documentation for " + "more details." + ) from e diff --git a/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/sql_datasource.py b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/sql_datasource.py new file mode 100644 index 0000000000000000000000000000000000000000..c38f5bdd6a4a66179b5231d9a4c30959510f0631 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/ray/data/_internal/datasource/sql_datasource.py @@ -0,0 +1,89 @@ +from contextlib import contextmanager +from typing import Any, Callable, Iterable, Iterator, List, Optional + +from ray.data.block import Block, BlockMetadata +from ray.data.datasource.datasource import Datasource, ReadTask + +Connection = Any # A Python DB API2-compliant `Connection` object. +Cursor = Any # A Python DB API2-compliant `Cursor` object. + + +def _cursor_to_block(cursor) -> Block: + import pyarrow as pa + + rows = cursor.fetchall() + # Each `column_description` is a 7-element sequence. The first element is the column + # name. To learn more, read https://peps.python.org/pep-0249/#description. + columns = [column_description[0] for column_description in cursor.description] + pydict = {column: [row[i] for row in rows] for i, column in enumerate(columns)} + return pa.Table.from_pydict(pydict) + + +def _check_connection_is_dbapi2_compliant(connection) -> None: + for attr in "close", "commit", "cursor": + if not hasattr(connection, attr): + raise ValueError( + "Your `connection_factory` created a `Connection` object without a " + f"{attr!r} method, but this method is required by the Python DB API2 " + "specification. Check that your database connector is DB API2-" + "compliant. To learn more, read https://peps.python.org/pep-0249/." + ) + + +def _check_cursor_is_dbapi2_compliant(cursor) -> None: + # These aren't all the methods required by the specification, but it's all the ones + # we care about. + for attr in "execute", "executemany", "fetchone", "fetchall", "description": + if not hasattr(cursor, attr): + raise ValueError( + "Your database connector created a `Cursor` object without a " + f"{attr!r} method, but this method is required by the Python DB API2 " + "specification. Check that your database connector is DB API2-" + "compliant. To learn more, read https://peps.python.org/pep-0249/." + ) + + +@contextmanager +def _connect(connection_factory: Callable[[], Connection]) -> Iterator[Cursor]: + connection = connection_factory() + _check_connection_is_dbapi2_compliant(connection) + + try: + cursor = connection.cursor() + _check_cursor_is_dbapi2_compliant(cursor) + yield cursor + connection.commit() + except Exception: + # `rollback` is optional since not all databases provide transaction support. + try: + connection.rollback() + except Exception as e: + # Each connector implements its own `NotSupportError` class, so we check + # the exception's name instead of using `isinstance`. + if ( + isinstance(e, AttributeError) + or e.__class__.__name__ == "NotSupportedError" + ): + pass + raise + finally: + connection.close() + + +class SQLDatasource(Datasource): + def __init__(self, sql: str, connection_factory: Callable[[], Connection]): + self.sql = sql + self.connection_factory = connection_factory + + def estimate_inmemory_data_size(self) -> Optional[int]: + return None + + def get_read_tasks(self, parallelism: int) -> List[ReadTask]: + def read_fn() -> Iterable[Block]: + with _connect(self.connection_factory) as cursor: + cursor.execute(self.sql) + block = _cursor_to_block(cursor) + return [block] + + metadata = BlockMetadata(None, None, None, None, None) + return [ReadTask(read_fn, metadata)] diff --git a/.venv/lib/python3.11/site-packages/ray/job_submission/__init__.py b/.venv/lib/python3.11/site-packages/ray/job_submission/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6a86cf73c329063ce3f730af66073ce3b5da1123 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/ray/job_submission/__init__.py @@ -0,0 +1,12 @@ +from ray.dashboard.modules.job.common import JobInfo, JobStatus +from ray.dashboard.modules.job.pydantic_models import DriverInfo, JobDetails, JobType +from ray.dashboard.modules.job.sdk import JobSubmissionClient + +__all__ = [ + "JobSubmissionClient", + "JobStatus", + "JobInfo", + "JobDetails", + "DriverInfo", + "JobType", +]