Add files using upload-large-folder tool

.venv/lib/python3.11/site-packages/ray/dashboard/modules/actor/actor_consts.py

@@ -0,0 +1,5 @@
+import ray
+
+ACTOR_CHANNEL = "ACTOR"
+NIL_NODE_ID = ray.NodeID.nil().hex()
+RETRY_GET_ALL_ACTOR_INFO_INTERVAL_SECONDS = 1

.venv/lib/python3.11/site-packages/ray/dashboard/modules/actor/actor_head.py

@@ -0,0 +1,290 @@
+import asyncio
+import logging
+from collections import defaultdict, deque
+from concurrent.futures import ThreadPoolExecutor
+from typing import Any, Dict
+
+import aiohttp.web
+
+import ray
+import ray.dashboard.optional_utils as dashboard_optional_utils
+import ray.dashboard.utils as dashboard_utils
+from ray._private.gcs_pubsub import GcsAioActorSubscriber
+from ray._private.utils import get_or_create_event_loop
+from ray.dashboard.consts import GCS_RPC_TIMEOUT_SECONDS
+from ray.dashboard.datacenter import DataOrganizer, DataSource
+from ray.dashboard.modules.actor import actor_consts
+
+logger = logging.getLogger(__name__)
+routes = dashboard_optional_utils.DashboardHeadRouteTable
+
+MAX_DESTROYED_ACTORS_TO_CACHE = max(
+    0, ray._config.maximum_gcs_destroyed_actor_cached_count()
+)
+
+ACTOR_CLEANUP_FREQUENCY = 1  # seconds
+
+
+ACTOR_TABLE_STATE_COLUMNS = (
+    "state",
+    "address",
+    "numRestarts",
+    "timestamp",
+    "pid",
+    "exitDetail",
+    "startTime",
+    "endTime",
+    "reprName",
+)
+
+
+def actor_table_data_to_dict(message):
+    orig_message = dashboard_utils.message_to_dict(
+        message,
+        {
+            "actorId",
+            "parentId",
+            "jobId",
+            "workerId",
+            "rayletId",
+            "callerId",
+            "taskId",
+            "parentTaskId",
+            "sourceActorId",
+            "placementGroupId",
+        },
+        always_print_fields_with_no_presence=True,
+    )
+    # The complete schema for actor table is here:
+    #     src/ray/protobuf/gcs.proto
+    # It is super big and for dashboard, we don't need that much information.
+    # Only preserve the necessary ones here for memory usage.
+    fields = {
+        "actorId",
+        "jobId",
+        "pid",
+        "address",
+        "state",
+        "name",
+        "numRestarts",
+        "timestamp",
+        "className",
+        "startTime",
+        "endTime",
+        "reprName",
+        "placementGroupId",
+        "callSite",
+    }
+    light_message = {k: v for (k, v) in orig_message.items() if k in fields}
+    light_message["actorClass"] = orig_message["className"]
+    exit_detail = "-"
+    if "deathCause" in orig_message:
+        context = orig_message["deathCause"]
+        if "actorDiedErrorContext" in context:
+            exit_detail = context["actorDiedErrorContext"]["errorMessage"]  # noqa
+        elif "runtimeEnvFailedContext" in context:
+            exit_detail = context["runtimeEnvFailedContext"]["errorMessage"]  # noqa
+        elif "actorUnschedulableContext" in context:
+            exit_detail = context["actorUnschedulableContext"]["errorMessage"]  # noqa
+        elif "creationTaskFailureContext" in context:
+            exit_detail = context["creationTaskFailureContext"][
+                "formattedExceptionString"
+            ]  # noqa
+    light_message["exitDetail"] = exit_detail
+    light_message["startTime"] = int(light_message["startTime"])
+    light_message["endTime"] = int(light_message["endTime"])
+    light_message["requiredResources"] = dict(message.required_resources)
+
+    return light_message
+
+
+class ActorHead(dashboard_utils.DashboardHeadModule):
+    def __init__(self, config: dashboard_utils.DashboardHeadModuleConfig):
+        super().__init__(config)
+
+        self._gcs_actor_channel_subscriber = None
+        # A queue of dead actors in order of when they died
+        self.destroyed_actors_queue = deque()
+
+        # -- Internal state --
+        self._loop = get_or_create_event_loop()
+        # NOTE: This executor is intentionally constrained to just 1 thread to
+        #       limit its concurrency, therefore reducing potential for GIL contention
+        self._executor = ThreadPoolExecutor(
+            max_workers=1, thread_name_prefix="actor_head_executor"
+        )
+
+    async def _update_actors(self):
+        """
+        Processes actor info. First gets all actors from GCS, then subscribes to
+        actor updates. For each actor update, updates DataSource.node_actors and
+        DataSource.actors.
+        """
+
+        # To prevent Time-of-check to time-of-use issue [1], the get-all-actor-info
+        # happens after the subscription. That is, an update between get-all-actor-info
+        # and the subscription is not missed.
+        #
+        # [1] https://en.wikipedia.org/wiki/Time-of-check_to_time-of-use
+        gcs_addr = self.gcs_address
+        actor_channel_subscriber = GcsAioActorSubscriber(address=gcs_addr)
+        await actor_channel_subscriber.subscribe()
+
+        # Get all actor info.
+        while True:
+            try:
+                logger.info("Getting all actor info from GCS.")
+
+                actor_dicts = await self._get_all_actors()
+                # Update actors
+                DataSource.actors.reset(actor_dicts)
+
+                # Update node actors and job actors.
+                node_actors = defaultdict(dict)
+                for actor_id_bytes, updated_actor_table in actor_dicts.items():
+                    node_id = updated_actor_table["address"]["rayletId"]
+                    # Update only when node_id is not Nil.
+                    if node_id != actor_consts.NIL_NODE_ID:
+                        node_actors[node_id][actor_id_bytes] = updated_actor_table
+
+                # Update node's actor info
+                DataSource.node_actors.reset(node_actors)
+
+                logger.info("Received %d actor info from GCS.", len(actor_dicts))
+
+                # Break, once all initial actors are successfully fetched
+                break
+            except Exception as e:
+                logger.exception("Error Getting all actor info from GCS", exc_info=e)
+                await asyncio.sleep(
+                    actor_consts.RETRY_GET_ALL_ACTOR_INFO_INTERVAL_SECONDS
+                )
+
+        # Pull incremental updates from the GCS channel
+        while True:
+            try:
+                updated_actor_table_entries = await self._poll_updated_actor_table_data(
+                    actor_channel_subscriber
+                )
+
+                for (
+                    actor_id,
+                    updated_actor_table,
+                ) in updated_actor_table_entries.items():
+                    self._process_updated_actor_table(actor_id, updated_actor_table)
+
+                # TODO emit metrics
+                logger.debug(
+                    f"Total events processed: {len(updated_actor_table_entries)}, "
+                    f"queue size: {actor_channel_subscriber.queue_size}"
+                )
+
+            except Exception as e:
+                logger.exception("Error processing actor info from GCS.", exc_info=e)
+
+    async def _poll_updated_actor_table_data(
+        self, actor_channel_subscriber: GcsAioActorSubscriber
+    ) -> Dict[str, Dict[str, Any]]:
+        # TODO make batch size configurable
+        batch = await actor_channel_subscriber.poll(batch_size=200)
+
+        # NOTE: We're offloading conversion to a TPE to make sure we're not
+        #       blocking the event-loop for prolonged period of time irrespective
+        #       of the batch size
+        def _convert_to_dict():
+            return {
+                actor_id_bytes.hex(): actor_table_data_to_dict(actor_table_data_message)
+                for actor_id_bytes, actor_table_data_message in batch
+                if actor_id_bytes is not None
+            }
+
+        return await self._loop.run_in_executor(self._executor, _convert_to_dict)
+
+    def _process_updated_actor_table(
+        self, actor_id: str, actor_table_data: Dict[str, Any]
+    ):
+        """NOTE: This method has to be executed on the event-loop, provided that it
+        accesses DataSource data structures (to follow its thread-safety model)"""
+
+        # If actor is not new registered but updated, we only update
+        # states related fields.
+        actor = DataSource.actors.get(actor_id)
+
+        if actor and actor_table_data["state"] != "DEPENDENCIES_UNREADY":
+            for k in ACTOR_TABLE_STATE_COLUMNS:
+                if k in actor_table_data:
+                    actor[k] = actor_table_data[k]
+            actor_table_data = actor
+
+        actor_id = actor_table_data["actorId"]
+        node_id = actor_table_data["address"]["rayletId"]
+
+        if actor_table_data["state"] == "DEAD":
+            self.destroyed_actors_queue.append(actor_id)
+
+        # Update actors.
+        DataSource.actors[actor_id] = actor_table_data
+        # Update node actors (only when node_id is not Nil).
+        if node_id != actor_consts.NIL_NODE_ID:
+            node_actors = DataSource.node_actors.get(node_id, {})
+            node_actors[actor_id] = actor_table_data
+            DataSource.node_actors[node_id] = node_actors
+
+    async def _get_all_actors(self) -> Dict[str, dict]:
+        actors = await self.gcs_aio_client.get_all_actor_info(
+            timeout=GCS_RPC_TIMEOUT_SECONDS
+        )
+
+        # NOTE: We're offloading conversion to a TPE to make sure we're not
+        #       blocking the event-loop for prolonged period of time for large clusters
+        def _convert_to_dict():
+            return {
+                actor_id.hex(): actor_table_data_to_dict(actor_table_data)
+                for actor_id, actor_table_data in actors.items()
+            }
+
+        return await self._loop.run_in_executor(self._executor, _convert_to_dict)
+
+    async def _cleanup_actors(self):
+        while True:
+            try:
+                while len(self.destroyed_actors_queue) > MAX_DESTROYED_ACTORS_TO_CACHE:
+                    actor_id = self.destroyed_actors_queue.popleft()
+                    if actor_id in DataSource.actors:
+                        actor = DataSource.actors.pop(actor_id)
+                        node_id = actor["address"].get("rayletId")
+                        if node_id and node_id != actor_consts.NIL_NODE_ID:
+                            del DataSource.node_actors[node_id][actor_id]
+                await asyncio.sleep(ACTOR_CLEANUP_FREQUENCY)
+            except Exception:
+                logger.exception("Error cleaning up actor info from GCS.")
+
+    @routes.get("/logical/actors")
+    @dashboard_optional_utils.aiohttp_cache
+    async def get_all_actors(self, req) -> aiohttp.web.Response:
+        actors = await DataOrganizer.get_actor_infos()
+        return dashboard_optional_utils.rest_response(
+            success=True,
+            message="All actors fetched.",
+            actors=actors,
+            # False to avoid converting Ray resource name to google style.
+            # It's not necessary here because the fields are already
+            # google formatted when protobuf was converted into dict.
+            convert_google_style=False,
+        )
+
+    @routes.get("/logical/actors/{actor_id}")
+    @dashboard_optional_utils.aiohttp_cache
+    async def get_actor(self, req) -> aiohttp.web.Response:
+        actor_id = req.match_info.get("actor_id")
+        actors = await DataOrganizer.get_actor_infos(actor_ids=[actor_id])
+        return dashboard_optional_utils.rest_response(
+            success=True, message="Actor details fetched.", detail=actors[actor_id]
+        )
+
+    async def run(self, server):
+        await asyncio.gather(self._update_actors(), self._cleanup_actors())
+
+    @staticmethod
+    def is_minimal_module():
+        return False

.venv/lib/python3.11/site-packages/ray/dashboard/modules/event/event_agent.py

@@ -0,0 +1,133 @@
+import asyncio
+import logging
+import os
+import time
+from concurrent.futures import ThreadPoolExecutor
+from typing import Union
+
+import ray._private.ray_constants as ray_constants
+import ray._private.utils as utils
+import ray.dashboard.consts as dashboard_consts
+import ray.dashboard.utils as dashboard_utils
+from ray.core.generated import event_pb2, event_pb2_grpc
+from ray.dashboard.modules.event import event_consts
+from ray.dashboard.modules.event.event_utils import monitor_events
+from ray.dashboard.utils import async_loop_forever, create_task
+
+logger = logging.getLogger(__name__)
+
+
+# NOTE: Executor in this head is intentionally constrained to just 1 thread by
+#       default to limit its concurrency, therefore reducing potential for
+#       GIL contention
+RAY_DASHBOARD_EVENT_AGENT_TPE_MAX_WORKERS = ray_constants.env_integer(
+    "RAY_DASHBOARD_EVENT_AGENT_TPE_MAX_WORKERS", 1
+)
+
+
+class EventAgent(dashboard_utils.DashboardAgentModule):
+    def __init__(self, dashboard_agent):
+        super().__init__(dashboard_agent)
+        self._event_dir = os.path.join(self._dashboard_agent.log_dir, "events")
+        os.makedirs(self._event_dir, exist_ok=True)
+        self._monitor: Union[asyncio.Task, None] = None
+        self._stub: Union[event_pb2_grpc.ReportEventServiceStub, None] = None
+        self._cached_events = asyncio.Queue(event_consts.EVENT_AGENT_CACHE_SIZE)
+        self._gcs_aio_client = dashboard_agent.gcs_aio_client
+        # Total number of event created from this agent.
+        self.total_event_reported = 0
+        # Total number of event report request sent.
+        self.total_request_sent = 0
+        self.module_started = time.monotonic()
+
+        self._executor = ThreadPoolExecutor(
+            max_workers=RAY_DASHBOARD_EVENT_AGENT_TPE_MAX_WORKERS,
+            thread_name_prefix="event_agent_executor",
+        )
+
+        logger.info("Event agent cache buffer size: %s", self._cached_events.maxsize)
+
+    async def _connect_to_dashboard(self):
+        """Connect to the dashboard. If the dashboard is not started, then
+        this method will never returns.
+
+        Returns:
+            The ReportEventServiceStub object.
+        """
+        while True:
+            try:
+                dashboard_rpc_address = await self._gcs_aio_client.internal_kv_get(
+                    dashboard_consts.DASHBOARD_RPC_ADDRESS.encode(),
+                    namespace=ray_constants.KV_NAMESPACE_DASHBOARD,
+                    timeout=1,
+                )
+                dashboard_rpc_address = dashboard_rpc_address.decode()
+                if dashboard_rpc_address:
+                    logger.info("Report events to %s", dashboard_rpc_address)
+                    options = ray_constants.GLOBAL_GRPC_OPTIONS
+                    channel = utils.init_grpc_channel(
+                        dashboard_rpc_address, options=options, asynchronous=True
+                    )
+                    return event_pb2_grpc.ReportEventServiceStub(channel)
+            except Exception:
+                logger.exception("Connect to dashboard failed.")
+            await asyncio.sleep(
+                event_consts.RETRY_CONNECT_TO_DASHBOARD_INTERVAL_SECONDS
+            )
+
+    @async_loop_forever(event_consts.EVENT_AGENT_REPORT_INTERVAL_SECONDS)
+    async def report_events(self):
+        """Report events from cached events queue. Reconnect to dashboard if
+        report failed. Log error after retry EVENT_AGENT_RETRY_TIMES.
+
+        This method will never returns.
+        """
+        data = await self._cached_events.get()
+        self.total_event_reported += len(data)
+        for _ in range(event_consts.EVENT_AGENT_RETRY_TIMES):
+            try:
+                logger.debug("Report %s events.", len(data))
+                request = event_pb2.ReportEventsRequest(event_strings=data)
+                await self._stub.ReportEvents(request)
+                self.total_request_sent += 1
+                break
+            except Exception:
+                logger.exception("Report event failed, reconnect to the " "dashboard.")
+                self._stub = await self._connect_to_dashboard()
+        else:
+            data_str = str(data)
+            limit = event_consts.LOG_ERROR_EVENT_STRING_LENGTH_LIMIT
+            logger.error(
+                "Report event failed: %s",
+                data_str[:limit] + (data_str[limit:] and "..."),
+            )
+
+    async def get_internal_states(self):
+        if self.total_event_reported <= 0 or self.total_request_sent <= 0:
+            return
+
+        elapsed = time.monotonic() - self.module_started
+        return {
+            "total_events_reported": self.total_event_reported,
+            "Total_report_request": self.total_request_sent,
+            "queue_size": self._cached_events.qsize(),
+            "total_uptime": elapsed,
+        }
+
+    async def run(self, server):
+        # Connect to dashboard.
+        self._stub = await self._connect_to_dashboard()
+        # Start monitor task.
+        self._monitor = monitor_events(
+            self._event_dir,
+            lambda data: create_task(self._cached_events.put(data)),
+            self._executor,
+        )
+
+        await asyncio.gather(
+            self.report_events(),
+        )
+
+    @staticmethod
+    def is_minimal_module():
+        return False

.venv/lib/python3.11/site-packages/ray/dashboard/modules/event/event_consts.py

@@ -0,0 +1,21 @@
+from ray._private.ray_constants import env_float, env_integer
+from ray.core.generated import event_pb2
+
+LOG_ERROR_EVENT_STRING_LENGTH_LIMIT = 1000
+RETRY_CONNECT_TO_DASHBOARD_INTERVAL_SECONDS = 2
+# Monitor events
+SCAN_EVENT_DIR_INTERVAL_SECONDS = env_integer("SCAN_EVENT_DIR_INTERVAL_SECONDS", 2)
+SCAN_EVENT_START_OFFSET_SECONDS = -30 * 60
+CONCURRENT_READ_LIMIT = 50
+EVENT_READ_LINE_COUNT_LIMIT = 200
+EVENT_READ_LINE_LENGTH_LIMIT = env_integer(
+    "EVENT_READ_LINE_LENGTH_LIMIT", 2 * 1024 * 1024
+)  # 2MB
+# Report events
+EVENT_AGENT_REPORT_INTERVAL_SECONDS = env_float(
+    "EVENT_AGENT_REPORT_INTERVAL_SECONDS", 0.1
+)
+EVENT_AGENT_RETRY_TIMES = 10
+EVENT_AGENT_CACHE_SIZE = 10240
+# Event sources
+EVENT_SOURCE_ALL = event_pb2.Event.SourceType.keys()

.venv/lib/python3.11/site-packages/ray/dashboard/modules/event/event_head.py

@@ -0,0 +1,212 @@
+import asyncio
+import logging
+import os
+import time
+from collections import OrderedDict, defaultdict
+from concurrent.futures import ThreadPoolExecutor
+from datetime import datetime
+from itertools import islice
+from typing import Dict, Union
+
+import aiohttp.web
+
+import ray.dashboard.optional_utils as dashboard_optional_utils
+import ray.dashboard.utils as dashboard_utils
+from ray._private.ray_constants import env_integer
+from ray._private.usage.usage_lib import TagKey, record_extra_usage_tag
+from ray._private.utils import get_or_create_event_loop
+from ray.core.generated import event_pb2, event_pb2_grpc
+from ray.dashboard.consts import (
+    RAY_STATE_SERVER_MAX_HTTP_REQUEST,
+    RAY_STATE_SERVER_MAX_HTTP_REQUEST_ALLOWED,
+    RAY_STATE_SERVER_MAX_HTTP_REQUEST_ENV_NAME,
+)
+from ray.dashboard.modules.event.event_utils import monitor_events, parse_event_strings
+from ray.dashboard.state_api_utils import do_filter, handle_list_api
+from ray.util.state.common import ClusterEventState, ListApiOptions, ListApiResponse
+
+logger = logging.getLogger(__name__)
+routes = dashboard_optional_utils.DashboardHeadRouteTable
+
+JobEvents = OrderedDict
+dashboard_utils._json_compatible_types.add(JobEvents)
+
+MAX_EVENTS_TO_CACHE = int(os.environ.get("RAY_DASHBOARD_MAX_EVENTS_TO_CACHE", 10000))
+
+# NOTE: Executor in this head is intentionally constrained to just 1 thread by
+#       default to limit its concurrency, therefore reducing potential for
+#       GIL contention
+RAY_DASHBOARD_EVENT_HEAD_TPE_MAX_WORKERS = env_integer(
+    "RAY_DASHBOARD_EVENT_HEAD_TPE_MAX_WORKERS", 1
+)
+
+
+async def _list_cluster_events_impl(
+    *, all_events, executor: ThreadPoolExecutor, option: ListApiOptions
+) -> ListApiResponse:
+    """
+    List all cluster events from the cluster. Made a free function to allow unit tests.
+
+    Returns:
+        A list of cluster events in the cluster.
+        The schema of returned "dict" is equivalent to the
+        `ClusterEventState` protobuf message.
+    """
+
+    def transform(all_events) -> ListApiResponse:
+        result = []
+        for _, events in all_events.items():
+            for _, event in events.items():
+                event["time"] = str(datetime.fromtimestamp(int(event["timestamp"])))
+                result.append(event)
+
+        num_after_truncation = len(result)
+        result.sort(key=lambda entry: entry["timestamp"])
+        total = len(result)
+        result = do_filter(result, option.filters, ClusterEventState, option.detail)
+        num_filtered = len(result)
+        # Sort to make the output deterministic.
+        result = list(islice(result, option.limit))
+        return ListApiResponse(
+            result=result,
+            total=total,
+            num_after_truncation=num_after_truncation,
+            num_filtered=num_filtered,
+        )
+
+    return await get_or_create_event_loop().run_in_executor(
+        executor, transform, all_events
+    )
+
+
+class EventHead(
+    dashboard_utils.DashboardHeadModule,
+    dashboard_utils.RateLimitedModule,
+    event_pb2_grpc.ReportEventServiceServicer,
+):
+    def __init__(self, config: dashboard_utils.DashboardHeadModuleConfig):
+        dashboard_utils.DashboardHeadModule.__init__(self, config)
+        dashboard_utils.RateLimitedModule.__init__(
+            self,
+            min(
+                RAY_STATE_SERVER_MAX_HTTP_REQUEST,
+                RAY_STATE_SERVER_MAX_HTTP_REQUEST_ALLOWED,
+            ),
+        )
+        self._event_dir = os.path.join(self.log_dir, "events")
+        os.makedirs(self._event_dir, exist_ok=True)
+        self._monitor: Union[asyncio.Task, None] = None
+        self.total_report_events_count = 0
+        self.total_events_received = 0
+        self.module_started = time.monotonic()
+        # {job_id hex(str): {event_id (str): event (dict)}}
+        self.events: Dict[str, JobEvents] = defaultdict(JobEvents)
+
+        self._executor = ThreadPoolExecutor(
+            max_workers=RAY_DASHBOARD_EVENT_HEAD_TPE_MAX_WORKERS,
+            thread_name_prefix="event_head_executor",
+        )
+
+    async def limit_handler_(self):
+        return dashboard_optional_utils.rest_response(
+            success=False,
+            error_message=(
+                "Max number of in-progress requests="
+                f"{self.max_num_call_} reached. "
+                "To set a higher limit, set environment variable: "
+                f"export {RAY_STATE_SERVER_MAX_HTTP_REQUEST_ENV_NAME}='xxx'. "
+                f"Max allowed = {RAY_STATE_SERVER_MAX_HTTP_REQUEST_ALLOWED}"
+            ),
+            result=None,
+        )
+
+    def _update_events(self, event_list):
+        # {job_id: {event_id: event}}
+        all_job_events = defaultdict(JobEvents)
+        for event in event_list:
+            event_id = event["event_id"]
+            custom_fields = event.get("custom_fields")
+            system_event = False
+            if custom_fields:
+                job_id = custom_fields.get("job_id", "global") or "global"
+            else:
+                job_id = "global"
+            if system_event is False:
+                all_job_events[job_id][event_id] = event
+
+        for job_id, new_job_events in all_job_events.items():
+            job_events = self.events[job_id]
+            job_events.update(new_job_events)
+
+            # Limit the # of events cached if it exceeds the threshold.
+            if len(job_events) > MAX_EVENTS_TO_CACHE * 1.1:
+                while len(job_events) > MAX_EVENTS_TO_CACHE:
+                    job_events.popitem(last=False)
+
+    async def ReportEvents(self, request, context):
+        received_events = []
+        if request.event_strings:
+            received_events.extend(parse_event_strings(request.event_strings))
+        logger.debug("Received %d events", len(received_events))
+        self._update_events(received_events)
+        self.total_report_events_count += 1
+        self.total_events_received += len(received_events)
+        return event_pb2.ReportEventsReply(send_success=True)
+
+    async def _periodic_state_print(self):
+        if self.total_events_received <= 0 or self.total_report_events_count <= 0:
+            return
+
+        elapsed = time.monotonic() - self.module_started
+        return {
+            "total_events_received": self.total_events_received,
+            "Total_requests_received": self.total_report_events_count,
+            "total_uptime": elapsed,
+        }
+
+    @routes.get("/events")
+    @dashboard_optional_utils.aiohttp_cache
+    async def get_event(self, req) -> aiohttp.web.Response:
+        job_id = req.query.get("job_id")
+        if job_id is None:
+            all_events = {
+                job_id: list(job_events.values())
+                for job_id, job_events in self.events.items()
+            }
+            return dashboard_optional_utils.rest_response(
+                success=True, message="All events fetched.", events=all_events
+            )
+
+        job_events = self.events[job_id]
+        return dashboard_optional_utils.rest_response(
+            success=True,
+            message="Job events fetched.",
+            job_id=job_id,
+            events=list(job_events.values()),
+        )
+
+    @routes.get("/api/v0/cluster_events")
+    @dashboard_utils.RateLimitedModule.enforce_max_concurrent_calls
+    async def list_cluster_events(
+        self, req: aiohttp.web.Request
+    ) -> aiohttp.web.Response:
+        record_extra_usage_tag(TagKey.CORE_STATE_API_LIST_CLUSTER_EVENTS, "1")
+
+        async def list_api_fn(option: ListApiOptions):
+            return await _list_cluster_events_impl(
+                all_events=self.events, executor=self._executor, option=option
+            )
+
+        return await handle_list_api(list_api_fn, req)
+
+    async def run(self, server):
+        event_pb2_grpc.add_ReportEventServiceServicer_to_server(self, server)
+        self._monitor = monitor_events(
+            self._event_dir,
+            lambda data: self._update_events(parse_event_strings(data)),
+            self._executor,
+        )
+
+    @staticmethod
+    def is_minimal_module():
+        return False

.venv/lib/python3.11/site-packages/ray/dashboard/modules/metrics/dashboards/common.py

@@ -0,0 +1,70 @@
+from dataclasses import dataclass
+from typing import List, Optional
+
+
+@dataclass
+class GridPos:
+    x: int
+    y: int
+    w: int
+    h: int
+
+
+@dataclass
+class Target:
+    """Defines a Grafana target (time-series query) within a panel.
+
+    A panel will have one or more targets. By default, all targets are rendered as
+    stacked area charts, with the exception of legend="MAX", which is rendered as
+    a blue dotted line. Any legend="FINISHED|FAILED|DEAD|REMOVED" series will also be
+    rendered hidden by default.
+
+    Attributes:
+        expr: The prometheus query to evaluate.
+        legend: The legend string to format for each time-series.
+    """
+
+    expr: str
+    legend: str
+
+
+@dataclass
+class Panel:
+    """Defines a Grafana panel (graph) for the Ray dashboard page.
+
+    A panel contains one or more targets (time-series queries).
+
+    Attributes:
+        title: Short name of the graph. Note: please keep this in sync with the title
+            definitions in Metrics.tsx.
+        description: Long form description of the graph.
+        id: Integer id used to reference the graph from Metrics.tsx.
+        unit: The unit to display on the y-axis of the graph.
+        targets: List of query targets.
+        fill: Whether or not the graph will be filled by a color.
+        stack: Whether or not the lines in the graph will be stacked.
+    """
+
+    title: str
+    description: str
+    id: int
+    unit: str
+    targets: List[Target]
+    fill: int = 10
+    stack: bool = True
+    linewidth: int = 1
+    grid_pos: Optional[GridPos] = None
+
+
+@dataclass
+class DashboardConfig:
+    # This dashboard name is an internal key used to determine which env vars
+    # to check for customization
+    name: str
+    # The uid of the dashboard json if not overridden by a user
+    default_uid: str
+    panels: List[Panel]
+    # The global filters applied to all graphs in this dashboard. Users can
+    # add additional global_filters on top of this.
+    standard_global_filters: List[str]
+    base_json_file_name: str

.venv/lib/python3.11/site-packages/ray/dashboard/modules/metrics/dashboards/data_dashboard_panels.py

@@ -0,0 +1,551 @@
+# ruff: noqa: E501
+
+from ray.dashboard.modules.metrics.dashboards.common import (
+    DashboardConfig,
+    Panel,
+    Target,
+)
+
+# When adding a new panels for an OpRuntimeMetric, follow this format:
+# Panel(
+#     title=title,
+#     description=metric.metadata.get("description"),
+#     id=panel_id,
+#     unit=unit,
+#     targets=[
+#         Target(
+#             expr=f"sum(ray_data_{metric.name}"
+#             + "{{{global_filters}}}) by (dataset, operator)",
+#             legend=legend,
+#         )
+#     ],
+#     fill=fill,
+#     stack=stack,
+# )
+
+
+DATA_GRAFANA_PANELS = [
+    # Ray Data Metrics (Overview)
+    Panel(
+        id=1,
+        title="Bytes Spilled",
+        description="Amount spilled by dataset operators. DataContext.enable_get_object_locations_for_metrics must be set to True to report this metric",
+        unit="bytes",
+        targets=[
+            Target(
+                expr="sum(ray_data_spilled_bytes{{{global_filters}}}) by (dataset, operator)",
+                legend="Bytes Spilled: {{dataset}}, {{operator}}",
+            )
+        ],
+        fill=0,
+        stack=False,
+    ),
+    Panel(
+        id=2,
+        title="Bytes Allocated",
+        description="Amount allocated by dataset operators.",
+        unit="bytes",
+        targets=[
+            Target(
+                expr="sum(ray_data_allocated_bytes{{{global_filters}}}) by (dataset, operator)",
+                legend="Bytes Allocated: {{dataset}}, {{operator}}",
+            )
+        ],
+        fill=0,
+        stack=False,
+    ),
+    Panel(
+        id=3,
+        title="Bytes Freed",
+        description="Amount freed by dataset operators.",
+        unit="bytes",
+        targets=[
+            Target(
+                expr="sum(ray_data_freed_bytes{{{global_filters}}}) by (dataset, operator)",
+                legend="Bytes Freed: {{dataset}}, {{operator}}",
+            )
+        ],
+        fill=0,
+        stack=False,
+    ),
+    Panel(
+        id=4,
+        title="Object Store Memory",
+        description="Amount of memory store used by dataset operators.",
+        unit="bytes",
+        targets=[
+            Target(
+                expr="sum(ray_data_current_bytes{{{global_filters}}}) by (dataset, operator)",
+                legend="Current Usage: {{dataset}}, {{operator}}",
+            )
+        ],
+        fill=0,
+        stack=False,
+    ),
+    Panel(
+        id=5,
+        title="CPUs (logical slots)",
+        description="Logical CPUs allocated to dataset operators.",
+        unit="cores",
+        targets=[
+            Target(
+                expr="sum(ray_data_cpu_usage_cores{{{global_filters}}}) by (dataset, operator)",
+                legend="CPU Usage: {{dataset}}, {{operator}}",
+            )
+        ],
+        fill=0,
+        stack=False,
+    ),
+    Panel(
+        id=6,
+        title="GPUs (logical slots)",
+        description="Logical GPUs allocated to dataset operators.",
+        unit="cores",
+        targets=[
+            Target(
+                expr="sum(ray_data_gpu_usage_cores{{{global_filters}}}) by (dataset, operator)",
+                legend="GPU Usage: {{dataset}}, {{operator}}",
+            )
+        ],
+        fill=0,
+        stack=False,
+    ),
+    Panel(
+        id=7,
+        title="Bytes Output / Second",
+        description="Bytes output per second by dataset operators.",
+        unit="Bps",
+        targets=[
+            Target(
+                expr="sum(rate(ray_data_output_bytes{{{global_filters}}}[1m])) by (dataset, operator)",
+                legend="Bytes Output / Second: {{dataset}}, {{operator}}",
+            )
+        ],
+        fill=0,
+        stack=False,
+    ),
+    Panel(
+        id=11,
+        title="Rows Output / Second",
+        description="Total rows output per second by dataset operators.",
+        unit="rows/sec",
+        targets=[
+            Target(
+                expr="sum(rate(ray_data_output_rows{{{global_filters}}}[1m])) by (dataset, operator)",
+                legend="Rows Output / Second: {{dataset}}, {{operator}}",
+            )
+        ],
+        fill=0,
+        stack=False,
+    ),
+    # Ray Data Metrics (Inputs)
+    Panel(
+        id=17,
+        title="Input Blocks Received by Operator / Second",
+        description="Number of input blocks received by operator per second.",
+        unit="blocks/sec",
+        targets=[
+            Target(
+                expr="sum(rate(ray_data_num_inputs_received{{{global_filters}}}[1m])) by (dataset, operator)",
+                legend="Blocks Received / Second: {{dataset}}, {{operator}}",
+            )
+        ],
+        fill=0,
+        stack=False,
+    ),
+    Panel(
+        id=18,
+        title="Input Bytes Received by Operator / Second",
+        description="Byte size of input blocks received by operator per second.",
+        unit="Bps",
+        targets=[
+            Target(
+                expr="sum(rate(ray_data_bytes_inputs_received{{{global_filters}}}[1m])) by (dataset, operator)",
+                legend="Bytes Received / Second: {{dataset}}, {{operator}}",
+            )
+        ],
+        fill=0,
+        stack=False,
+    ),
+    Panel(
+        id=19,
+        title="Input Blocks Processed by Tasks / Second",
+        description=(
+            "Number of input blocks that operator's tasks have finished processing per second."
+        ),
+        unit="blocks/sec",
+        targets=[
+            Target(
+                expr="sum(rate(ray_data_num_task_inputs_processed{{{global_filters}}}[1m])) by (dataset, operator)",
+                legend="Blocks Processed / Second: {{dataset}}, {{operator}}",
+            )
+        ],
+        fill=0,
+        stack=False,
+    ),
+    Panel(
+        id=20,
+        title="Input Bytes Processed by Tasks / Second",
+        description=(
+            "Byte size of input blocks that operator's tasks have finished processing per second."
+        ),
+        unit="Bps",
+        targets=[
+            Target(
+                expr="sum(rate(ray_data_bytes_task_inputs_processed{{{global_filters}}}[1m])) by (dataset, operator)",
+                legend="Bytes Processed / Second: {{dataset}}, {{operator}}",
+            )
+        ],
+        fill=0,
+        stack=False,
+    ),
+    Panel(
+        id=21,
+        title="Input Bytes Submitted to Tasks / Second",
+        description="Byte size of input blocks passed to submitted tasks per second.",
+        unit="Bps",
+        targets=[
+            Target(
+                expr="sum(rate(ray_data_bytes_inputs_of_submitted_tasks{{{global_filters}}}[1m])) by (dataset, operator)",
+                legend="Bytes Submitted / Second: {{dataset}}, {{operator}}",
+            )
+        ],
+        fill=0,
+        stack=False,
+    ),
+    Panel(
+        id=22,
+        title="Blocks Generated by Tasks / Second",
+        description="Number of output blocks generated by tasks per second.",
+        unit="blocks/sec",
+        targets=[
+            Target(
+                expr="sum(rate(ray_data_num_task_outputs_generated{{{global_filters}}}[1m])) by (dataset, operator)",
+                legend="Blocks Generated / Second: {{dataset}}, {{operator}}",
+            )
+        ],
+        fill=0,
+        stack=False,
+    ),
+    Panel(
+        id=23,
+        title="Bytes Generated by Tasks / Second",
+        description="Byte size of output blocks generated by tasks per second.",
+        unit="Bps",
+        targets=[
+            Target(
+                expr="sum(rate(ray_data_bytes_task_outputs_generated{{{global_filters}}}[1m])) by (dataset, operator)",
+                legend="Bytes Generated / Second: {{dataset}}, {{operator}}",
+            )
+        ],
+        fill=0,
+        stack=False,
+    ),
+    Panel(
+        id=24,
+        title="Rows Generated by Tasks / Second",
+        description="Number of rows in generated output blocks from finished tasks per second.",
+        unit="rows/sec",
+        targets=[
+            Target(
+                expr="sum(rate(ray_data_rows_task_outputs_generated{{{global_filters}}}[1m])) by (dataset, operator)",
+                legend="Rows Generated / Second: {{dataset}}, {{operator}}",
+            )
+        ],
+        fill=0,
+        stack=False,
+    ),
+    Panel(
+        id=25,
+        title="Output Blocks Taken by Downstream Operators / Second",
+        description="Number of output blocks taken by downstream operators per second.",
+        unit="blocks/sec",
+        targets=[
+            Target(
+                expr="sum(rate(ray_data_num_outputs_taken{{{global_filters}}}[1m])) by (dataset, operator)",
+                legend="Blocks Taken / Second: {{dataset}}, {{operator}}",
+            )
+        ],
+        fill=0,
+        stack=False,
+    ),
+    Panel(
+        id=26,
+        title="Output Bytes Taken by Downstream Operators / Second",
+        description=(
+            "Byte size of output blocks taken by downstream operators per second."
+        ),
+        unit="Bps",
+        targets=[
+            Target(
+                expr="sum(rate(ray_data_bytes_outputs_taken{{{global_filters}}}[1m])) by (dataset, operator)",
+                legend="Bytes Taken / Second: {{dataset}}, {{operator}}",
+            )
+        ],
+        fill=0,
+        stack=False,
+    ),
+    # Ray Data Metrics (Tasks)
+    Panel(
+        id=29,
+        title="Submitted Tasks",
+        description="Number of submitted tasks.",
+        unit="tasks",
+        targets=[
+            Target(
+                expr="sum(ray_data_num_tasks_submitted{{{global_filters}}}) by (dataset, operator)",
+                legend="Submitted Tasks: {{dataset}}, {{operator}}",
+            )
+        ],
+        fill=0,
+        stack=False,
+    ),
+    Panel(
+        id=30,
+        title="Running Tasks",
+        description="Number of running tasks.",
+        unit="tasks",
+        targets=[
+            Target(
+                expr="sum(ray_data_num_tasks_running{{{global_filters}}}) by (dataset, operator)",
+                legend="Running Tasks: {{dataset}}, {{operator}}",
+            )
+        ],
+        fill=0,
+        stack=False,
+    ),
+    Panel(
+        id=31,
+        title="Tasks with output blocks",
+        description="Number of tasks that already have output.",
+        unit="tasks",
+        targets=[
+            Target(
+                expr="sum(ray_data_num_tasks_have_outputs{{{global_filters}}}) by (dataset, operator)",
+                legend="Tasks with output blocks: {{dataset}}, {{operator}}",
+            )
+        ],
+        fill=0,
+        stack=False,
+    ),
+    Panel(
+        id=32,
+        title="Finished Tasks",
+        description="Number of finished tasks.",
+        unit="tasks",
+        targets=[
+            Target(
+                expr="sum(ray_data_num_tasks_finished{{{global_filters}}}) by (dataset, operator)",
+                legend="Finished Tasks: {{dataset}}, {{operator}}",
+            )
+        ],
+        fill=0,
+        stack=False,
+    ),
+    Panel(
+        id=33,
+        title="Failed Tasks",
+        description="Number of failed tasks.",
+        unit="tasks",
+        targets=[
+            Target(
+                expr="sum(ray_data_num_tasks_failed{{{global_filters}}}) by (dataset, operator)",
+                legend="Failed Tasks: {{dataset}}, {{operator}}",
+            )
+        ],
+        fill=0,
+        stack=False,
+    ),
+    Panel(
+        id=8,
+        title="Block Generation Time",
+        description="Time spent generating blocks in tasks.",
+        unit="seconds",
+        targets=[
+            Target(
+                expr="sum(ray_data_block_generation_time{{{global_filters}}}) by (dataset, operator)",
+                legend="Block Generation Time: {{dataset}}, {{operator}}",
+            )
+        ],
+        fill=0,
+        stack=False,
+    ),
+    Panel(
+        id=37,
+        title="Task Submission Backpressure Time",
+        description="Time spent in task submission backpressure.",
+        unit="seconds",
+        targets=[
+            Target(
+                expr="sum(ray_data_task_submission_backpressure_time{{{global_filters}}}) by (dataset, operator)",
+                legend="Backpressure Time: {{dataset}}, {{operator}}",
+            )
+        ],
+        fill=0,
+        stack=True,
+    ),
+    # Ray Data Metrics (Object Store Memory)
+    Panel(
+        id=13,
+        title="Operator Internal Inqueue Size (Blocks)",
+        description="Number of blocks in operator's internal input queue",
+        unit="blocks",
+        targets=[
+            Target(
+                expr="sum(ray_data_obj_store_mem_internal_inqueue_blocks{{{global_filters}}}) by (dataset, operator)",
+                legend="Number of Blocks: {{dataset}}, {{operator}}",
+            )
+        ],
+        fill=0,
+        stack=False,
+    ),
+    Panel(
+        id=14,
+        title="Operator Internal Inqueue Size (Bytes)",
+        description="Byte size of input blocks in the operator's internal input queue.",
+        unit="bytes",
+        targets=[
+            Target(
+                expr="sum(ray_data_obj_store_mem_internal_inqueue{{{global_filters}}}) by (dataset, operator)",
+                legend="Bytes Size: {{dataset}}, {{operator}}",
+            )
+        ],
+        fill=0,
+        stack=True,
+    ),
+    Panel(
+        id=15,
+        title="Operator Internal Outqueue Size (Blocks)",
+        description="Number of blocks in operator's internal output queue",
+        unit="blocks",
+        targets=[
+            Target(
+                expr="sum(ray_data_obj_store_mem_internal_outqueue_blocks{{{global_filters}}}) by (dataset, operator)",
+                legend="Number of Blocks: {{dataset}}, {{operator}}",
+            )
+        ],
+        fill=0,
+        stack=False,
+    ),
+    Panel(
+        id=16,
+        title="Operator Internal Outqueue Size (Bytes)",
+        description=(
+            "Byte size of output blocks in the operator's internal output queue."
+        ),
+        unit="bytes",
+        targets=[
+            Target(
+                expr="sum(ray_data_obj_store_mem_internal_outqueue{{{global_filters}}}) by (dataset, operator)",
+                legend="Bytes Size: {{dataset}}, {{operator}}",
+            )
+        ],
+        fill=0,
+        stack=True,
+    ),
+    Panel(
+        id=34,
+        title="Size of Blocks used in Pending Tasks (Bytes)",
+        description="Byte size of input blocks used by pending tasks.",
+        unit="bytes",
+        targets=[
+            Target(
+                expr="sum(ray_data_obj_store_mem_pending_task_inputs{{{global_filters}}}) by (dataset, operator)",
+                legend="Bytes Size: {{dataset}}, {{operator}}",
+            )
+        ],
+        fill=0,
+        stack=True,
+    ),
+    Panel(
+        id=35,
+        title="Freed Memory in Object Store (Bytes)",
+        description="Byte size of freed memory in object store.",
+        unit="bytes",
+        targets=[
+            Target(
+                expr="sum(ray_data_obj_store_mem_freed{{{global_filters}}}) by (dataset, operator)",
+                legend="Bytes Size: {{dataset}}, {{operator}}",
+            )
+        ],
+        fill=0,
+        stack=True,
+    ),
+    Panel(
+        id=36,
+        title="Spilled Memory in Object Store (Bytes)",
+        description="Byte size of spilled memory in object store.",
+        unit="bytes",
+        targets=[
+            Target(
+                expr="sum(ray_data_obj_store_mem_spilled{{{global_filters}}}) by (dataset, operator)",
+                legend="Bytes Size: {{dataset}}, {{operator}}",
+            )
+        ],
+        fill=0,
+        stack=True,
+    ),
+    # Ray Data Metrics (Iteration)
+    Panel(
+        id=12,
+        title="Iteration Initialization Time",
+        description="Seconds spent in iterator initialization code",
+        unit="seconds",
+        targets=[
+            Target(
+                expr="sum(ray_data_iter_initialize_seconds{{{global_filters}}}) by (dataset)",
+                legend="Seconds: {{dataset}}, {{operator}}",
+            )
+        ],
+        fill=0,
+        stack=False,
+    ),
+    Panel(
+        id=9,
+        title="Iteration Blocked Time",
+        description="Seconds user thread is blocked by iter_batches()",
+        unit="seconds",
+        targets=[
+            Target(
+                expr="sum(ray_data_iter_total_blocked_seconds{{{global_filters}}}) by (dataset)",
+                legend="Seconds: {{dataset}}",
+            )
+        ],
+        fill=0,
+        stack=False,
+    ),
+    Panel(
+        id=10,
+        title="Iteration User Time",
+        description="Seconds spent in user code",
+        unit="seconds",
+        targets=[
+            Target(
+                expr="sum(ray_data_iter_user_seconds{{{global_filters}}}) by (dataset)",
+                legend="Seconds: {{dataset}}",
+            )
+        ],
+        fill=0,
+        stack=False,
+    ),
+    # Ray Data Metrics (Miscellaneous)
+]
+
+ids = []
+for panel in DATA_GRAFANA_PANELS:
+    ids.append(panel.id)
+assert len(ids) == len(
+    set(ids)
+), f"Duplicated id found. Use unique id for each panel. {ids}"
+
+data_dashboard_config = DashboardConfig(
+    name="DATA",
+    default_uid="rayDataDashboard",
+    panels=DATA_GRAFANA_PANELS,
+    standard_global_filters=[
+        'dataset=~"$DatasetID"',
+        'SessionName=~"$SessionName"',
+        'ray_io_cluster=~"$Cluster"',
+    ],
+    base_json_file_name="data_grafana_dashboard_base.json",
+)

.venv/lib/python3.11/site-packages/ray/dashboard/modules/metrics/dashboards/default_dashboard_panels.py

@@ -0,0 +1,478 @@
+# ruff: noqa: E501
+
+from ray.dashboard.modules.metrics.dashboards.common import (
+    DashboardConfig,
+    Panel,
+    Target,
+)
+
+"""
+Queries for autoscaler resources.
+"""
+# Note: MAX & USED resources are reported from raylet to provide the most up to date information.
+# But MAX + PENDING data is coming from the autoscaler. That said, MAX + PENDING can be
+# more outdated. it is harmless because the actual MAX will catch up with MAX + PENDING
+# eventually.
+MAX_CPUS = 'sum(autoscaler_cluster_resources{{resource="CPU",{global_filters}}})'
+PENDING_CPUS = 'sum(autoscaler_pending_resources{{resource="CPU",{global_filters}}})'
+MAX_GPUS = 'sum(autoscaler_cluster_resources{{resource="GPU",{global_filters}}})'
+PENDING_GPUS = 'sum(autoscaler_pending_resources{{resource="GPU",{global_filters}}})'
+
+
+def max_plus_pending(max_resource, pending_resource):
+    return f"({max_resource} or vector(0)) + ({pending_resource} or vector(0))"
+
+
+MAX_PLUS_PENDING_CPUS = max_plus_pending(MAX_CPUS, PENDING_CPUS)
+MAX_PLUS_PENDING_GPUS = max_plus_pending(MAX_GPUS, PENDING_GPUS)
+
+
+# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+# IMPORTANT: Please keep this in sync with Metrics.tsx and ray-metrics.rst
+# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+DEFAULT_GRAFANA_PANELS = [
+    Panel(
+        id=26,
+        title="Scheduler Task State",
+        description="Current number of tasks in a particular state.\n\nState: the task state, as described by rpc::TaskState proto in common.proto. Task resubmissions due to failures or object reconstruction are shown with (retry) in the label.",
+        unit="tasks",
+        targets=[
+            Target(
+                expr='sum(max_over_time(ray_tasks{{IsRetry="0",State=~"FINISHED|FAILED",instance=~"$Instance",{global_filters}}}[14d])) by (State) or clamp_min(sum(ray_tasks{{IsRetry="0",State!~"FINISHED|FAILED",instance=~"$Instance",{global_filters}}}) by (State), 0)',
+                legend="{{State}}",
+            ),
+            Target(
+                expr='sum(max_over_time(ray_tasks{{IsRetry!="0",State=~"FINISHED|FAILED",instance=~"$Instance",{global_filters}}}[14d])) by (State) or clamp_min(sum(ray_tasks{{IsRetry!="0",State!~"FINISHED|FAILED",instance=~"$Instance",{global_filters}}}) by (State), 0)',
+                legend="{{State}} (retry)",
+            ),
+        ],
+        fill=0,
+        stack=False,
+    ),
+    Panel(
+        id=35,
+        title="Requested Live Tasks by Name",
+        description="Current number of (live) tasks with a particular name. Task resubmissions due to failures or object reconstruction are shown with (retry) in the label.",
+        unit="tasks",
+        targets=[
+            Target(
+                expr='clamp_min(sum(ray_tasks{{IsRetry="0",State!~"FINISHED|FAILED",instance=~"$Instance",{global_filters}}}) by (Name), 0)',
+                legend="{{Name}}",
+            ),
+            Target(
+                expr='clamp_min(sum(ray_tasks{{IsRetry!="0",State!~"FINISHED|FAILED",instance=~"$Instance",{global_filters}}}) by (Name), 0)',
+                legend="{{Name}} (retry)",
+            ),
+        ],
+        fill=0,
+        stack=False,
+    ),
+    Panel(
+        id=38,
+        title="Running Tasks by Name",
+        description="Current number of (running) tasks with a particular name. Task resubmissions due to failures or object reconstruction are shown with (retry) in the label.",
+        unit="tasks",
+        targets=[
+            Target(
+                expr='clamp_min(sum(ray_tasks{{IsRetry="0",State=~"RUNNING*",instance=~"$Instance",{global_filters}}}) by (Name), 0)',
+                legend="{{Name}}",
+            ),
+            Target(
+                expr='clamp_min(sum(ray_tasks{{IsRetry!="0",State=~"RUNNING*",instance=~"$Instance",{global_filters}}}) by (Name), 0)',
+                legend="{{Name}} (retry)",
+            ),
+        ],
+        fill=0,
+        stack=False,
+    ),
+    Panel(
+        id=33,
+        title="Scheduler Actor State",
+        description='Note: not impacted by "Instance" variable.\n\nCurrent number of actors in a particular state.\n\nState: the actor state, as described by rpc::ActorTableData proto in gcs.proto.',
+        unit="actors",
+        targets=[
+            Target(
+                expr='sum(ray_actors{{Source="gcs",{global_filters}}}) by (State)',
+                legend="{{State}}",
+            )
+        ],
+    ),
+    Panel(
+        id=42,
+        title="Live Actor State",
+        description="Current number of alive actors in a particular state.\n\nState: IDLE, RUNNING_TASK, RUNNING_IN_RAY_GET, RUNNING_IN_RAY_WAIT",
+        unit="actors",
+        targets=[
+            Target(
+                expr='sum(ray_actors{{Source="executor",NodeAddress=~"$Instance",{global_filters}}}) by (State)',
+                legend="{{State}}",
+            )
+        ],
+    ),
+    Panel(
+        id=36,
+        title="Live Actors by Name",
+        description="Current number of alive actors with a particular name.",
+        unit="actors",
+        targets=[
+            Target(
+                expr='sum(ray_actors{{State!="DEAD",Source="executor",NodeAddress=~"$Instance",{global_filters}}}) by (Name)',
+                legend="{{Name}}",
+            )
+        ],
+    ),
+    Panel(
+        id=27,
+        title="Scheduler CPUs (logical slots)",
+        description="Logical CPU usage of Ray. The dotted line indicates the total number of CPUs. The logical CPU is allocated by `num_cpus` arguments from tasks and actors. PENDING means the number of CPUs that will be available when new nodes are up after the autoscaler scales up.\n\nNOTE: Ray's logical CPU is different from physical CPU usage. Ray's logical CPU is allocated by `num_cpus` arguments.",
+        unit="cores",
+        targets=[
+            Target(
+                expr='sum(ray_resources{{Name="CPU",State="USED",instance=~"$Instance",{global_filters}}}) by (instance)',
+                legend="CPU Usage: {{instance}}",
+            ),
+            Target(
+                expr='sum(ray_resources{{Name="CPU",instance=~"$Instance",{global_filters}}})',
+                legend="MAX",
+            ),
+            # If max + pending > max, we display this value.
+            # (A and predicate) means to return A when the predicate satisfies in PromSql.
+            Target(
+                expr=f"({MAX_PLUS_PENDING_CPUS} and {MAX_PLUS_PENDING_CPUS} > ({MAX_CPUS} or vector(0)))",
+                legend="MAX + PENDING",
+            ),
+        ],
+    ),
+    Panel(
+        id=29,
+        title="Object Store Memory",
+        description="Object store memory usage by location. The dotted line indicates the object store memory capacity.\n\nLocation: where the memory was allocated, which is MMAP_SHM or MMAP_DISK to indicate memory-mapped page, SPILLED to indicate spillage to disk, and WORKER_HEAP for objects small enough to be inlined in worker memory. Refer to metric_defs.cc for more information.",
+        unit="bytes",
+        targets=[
+            Target(
+                expr='sum(ray_object_store_memory{{instance=~"$Instance",{global_filters}}}) by (Location)',
+                legend="{{Location}}",
+            ),
+            Target(
+                expr='sum(ray_resources{{Name="object_store_memory",instance=~"$Instance",{global_filters}}})',
+                legend="MAX",
+            ),
+        ],
+    ),
+    Panel(
+        id=28,
+        title="Scheduler GPUs (logical slots)",
+        description="Logical GPU usage of Ray. The dotted line indicates the total number of GPUs. The logical GPU is allocated by `num_gpus` arguments from tasks and actors. PENDING means the number of GPUs that will be available when new nodes are up after the autoscaler scales up.",
+        unit="GPUs",
+        targets=[
+            Target(
+                expr='ray_resources{{Name="GPU",State="USED",instance=~"$Instance",{global_filters}}}',
+                legend="GPU Usage: {{instance}}",
+            ),
+            Target(
+                expr='sum(ray_resources{{Name="GPU",instance=~"$Instance",{global_filters}}})',
+                legend="MAX",
+            ),
+            # If max + pending > max, we display this value.
+            # (A and predicate) means to return A when the predicate satisfies in PromSql.
+            Target(
+                expr=f"({MAX_PLUS_PENDING_GPUS} and {MAX_PLUS_PENDING_GPUS} > ({MAX_GPUS} or vector(0)))",
+                legend="MAX + PENDING",
+            ),
+        ],
+    ),
+    Panel(
+        id=40,
+        title="Scheduler Placement Groups",
+        description='Note: not impacted by "Instance" variable.\n\nCurrent number of placement groups in a particular state.\n\nState: the placement group state, as described by the rpc::PlacementGroupTable proto in gcs.proto.',
+        unit="placement groups",
+        targets=[
+            Target(
+                expr="sum(ray_placement_groups{{{global_filters}}}) by (State)",
+                legend="{{State}}",
+            )
+        ],
+    ),
+    Panel(
+        id=2,
+        title="Node CPU (hardware utilization)",
+        description="",
+        unit="cores",
+        targets=[
+            Target(
+                expr='ray_node_cpu_utilization{{instance=~"$Instance", IsHeadNode="false", {global_filters}}} * ray_node_cpu_count{{instance=~"$Instance",{global_filters}}} / 100',
+                legend="CPU Usage: {{instance}}",
+            ),
+            Target(
+                expr='ray_node_cpu_utilization{{instance=~"$Instance", IsHeadNode="true", {global_filters}}} * ray_node_cpu_count{{instance=~"$Instance",{global_filters}}} / 100',
+                legend="CPU Usage: {{instance}} (head)",
+            ),
+            Target(
+                expr='sum(ray_node_cpu_count{{instance=~"$Instance",{global_filters}}})',
+                legend="MAX",
+            ),
+        ],
+    ),
+    Panel(
+        id=8,
+        title="Node GPU (hardware utilization)",
+        description="Node's physical (hardware) GPU usage. The dotted line means the total number of hardware GPUs from the cluster. ",
+        unit="GPUs",
+        targets=[
+            Target(
+                expr='ray_node_gpus_utilization{{instance=~"$Instance", IsHeadNode="false", {global_filters}}} / 100',
+                legend="GPU Usage: {{instance}}, gpu.{{GpuIndex}}, {{GpuDeviceName}}",
+            ),
+            Target(
+                expr='ray_node_gpus_utilization{{instance=~"$Instance", IsHeadNode="true", {global_filters}}} / 100',
+                legend="GPU Usage: {{instance}} (head), gpu.{{GpuIndex}}, {{GpuDeviceName}}",
+            ),
+            Target(
+                expr='sum(ray_node_gpus_available{{instance=~"$Instance",{global_filters}}})',
+                legend="MAX",
+            ),
+        ],
+    ),
+    Panel(
+        id=6,
+        title="Node Disk",
+        description="Node's physical (hardware) disk usage. The dotted line means the total amount of disk space from the cluster.\n\nNOTE: When Ray is deployed within a container, this shows the disk usage from the host machine. ",
+        unit="bytes",
+        targets=[
+            Target(
+                expr='ray_node_disk_usage{{instance=~"$Instance", IsHeadNode="false", {global_filters}}}',
+                legend="Disk Used: {{instance}}",
+            ),
+            Target(
+                expr='ray_node_disk_usage{{instance=~"$Instance", IsHeadNode="true", {global_filters}}}',
+                legend="Disk Used: {{instance}} (head)",
+            ),
+            Target(
+                expr='sum(ray_node_disk_free{{instance=~"$Instance",{global_filters}}}) + sum(ray_node_disk_usage{{instance=~"$Instance",{global_filters}}})',
+                legend="MAX",
+            ),
+        ],
+    ),
+    Panel(
+        id=32,
+        title="Node Disk IO Speed",
+        description="Disk IO per node.",
+        unit="Bps",
+        targets=[
+            Target(
+                expr='ray_node_disk_io_write_speed{{instance=~"$Instance", IsHeadNode="false", {global_filters}}}',
+                legend="Write: {{instance}}",
+            ),
+            Target(
+                expr='ray_node_disk_io_write_speed{{instance=~"$Instance", IsHeadNode="true", {global_filters}}}',
+                legend="Write: {{instance}} (head)",
+            ),
+            Target(
+                expr='ray_node_disk_io_read_speed{{instance=~"$Instance", IsHeadNode="false", {global_filters}}}',
+                legend="Read: {{instance}}",
+            ),
+            Target(
+                expr='ray_node_disk_io_read_speed{{instance=~"$Instance", IsHeadNode="true", {global_filters}}}',
+                legend="Read: {{instance}} (head)",
+            ),
+        ],
+    ),
+    Panel(
+        id=4,
+        title="Node Memory (heap + object store)",
+        description="The physical (hardware) memory usage for each node. The dotted line means the total amount of memory from the cluster. Node memory is a sum of object store memory (shared memory) and heap memory.\n\nNote: If Ray is deployed within a container, the total memory could be lower than the host machine because Ray may reserve some additional memory space outside the container.",
+        unit="bytes",
+        targets=[
+            Target(
+                expr='ray_node_mem_used{{instance=~"$Instance", IsHeadNode="false", {global_filters}}}',
+                legend="Memory Used: {{instance}}",
+            ),
+            Target(
+                expr='ray_node_mem_used{{instance=~"$Instance", IsHeadNode="true", {global_filters}}}',
+                legend="Memory Used: {{instance}} (head)",
+            ),
+            Target(
+                expr='sum(ray_node_mem_total{{instance=~"$Instance",{global_filters}}})',
+                legend="MAX",
+            ),
+        ],
+    ),
+    Panel(
+        id=48,
+        title="Node Memory Percentage (heap + object store)",
+        description="The percentage of physical (hardware) memory usage for each node.",
+        unit="%",
+        targets=[
+            Target(
+                expr='ray_node_mem_used{{instance=~"$Instance", IsHeadNode="false", {global_filters}}}/ray_node_mem_total{{instance=~"$Instance", IsHeadNode="false", {global_filters}}} * 100',
+                legend="Memory Used: {{instance}}",
+            ),
+            Target(
+                expr='ray_node_mem_used{{instance=~"$Instance", IsHeadNode="true", {global_filters}}}/ray_node_mem_total{{instance=~"$Instance", IsHeadNode="true", {global_filters}}} * 100',
+                legend="Memory Used: {{instance}} (head)",
+            ),
+        ],
+        fill=0,
+        stack=False,
+    ),
+    Panel(
+        id=44,
+        title="Node Out of Memory Failures by Name",
+        description="The number of tasks and actors killed by the Ray Out of Memory killer due to high memory pressure. Metrics are broken down by IP and the name. https://docs.ray.io/en/master/ray-core/scheduling/ray-oom-prevention.html.",
+        unit="failures",
+        targets=[
+            Target(
+                expr='ray_memory_manager_worker_eviction_total{{instance=~"$Instance",{global_filters}}}',
+                legend="OOM Killed: {{Name}}, {{instance}}",
+            ),
+        ],
+    ),
+    Panel(
+        id=34,
+        title="Node Memory by Component",
+        description="The physical (hardware) memory usage across the cluster, broken down by component. This reports the summed RSS-SHM per Ray component, which corresponds to an approximate memory usage per proc. Ray components consist of system components (e.g., raylet, gcs, dashboard, or agent) and the process (that contains method names) names of running tasks/actors.",
+        unit="bytes",
+        targets=[
+            Target(
+                expr='(sum(ray_component_rss_mb{{instance=~"$Instance",{global_filters}}} * 1e6) by (Component)) - (sum(ray_component_mem_shared_bytes{{instance=~"$Instance",{global_filters}}}) by (Component))',
+                legend="{{Component}}",
+            ),
+            Target(
+                expr='sum(ray_node_mem_shared_bytes{{instance=~"$Instance",{global_filters}}})',
+                legend="shared_memory",
+            ),
+            Target(
+                expr='sum(ray_node_mem_total{{instance=~"$Instance",{global_filters}}})',
+                legend="MAX",
+            ),
+        ],
+    ),
+    Panel(
+        id=37,
+        title="Node CPU by Component",
+        description="The physical (hardware) CPU usage across the cluster, broken down by component. This reports the summed CPU usage per Ray component. Ray components consist of system components (e.g., raylet, gcs, dashboard, or agent) and the process (that contains method names) names of running tasks/actors.",
+        unit="cores",
+        targets=[
+            Target(
+                # ray_component_cpu_percentage returns a percentage that can be > 100. It means that it uses more than 1 CPU.
+                expr='sum(ray_component_cpu_percentage{{instance=~"$Instance",{global_filters}}}) by (Component) / 100',
+                legend="{{Component}}",
+            ),
+            Target(
+                expr='sum(ray_node_cpu_count{{instance=~"$Instance",{global_filters}}})',
+                legend="MAX",
+            ),
+        ],
+    ),
+    Panel(
+        id=18,
+        title="Node GPU Memory (GRAM)",
+        description="The physical (hardware) GPU memory usage for each node. The dotted line means the total amount of GPU memory from the cluster.",
+        unit="bytes",
+        targets=[
+            Target(
+                expr='ray_node_gram_used{{instance=~"$Instance",{global_filters}}} * 1024 * 1024',
+                legend="Used GRAM: {{instance}}, gpu.{{GpuIndex}}, {{GpuDeviceName}}",
+            ),
+            Target(
+                expr='(sum(ray_node_gram_available{{instance=~"$Instance",{global_filters}}}) + sum(ray_node_gram_used{{instance=~"$Instance",{global_filters}}})) * 1024 * 1024',
+                legend="MAX",
+            ),
+        ],
+    ),
+    Panel(
+        id=20,
+        title="Node Network",
+        description="Network speed per node",
+        unit="Bps",
+        targets=[
+            Target(
+                expr='ray_node_network_receive_speed{{instance=~"$Instance",{global_filters}}}',
+                legend="Recv: {{instance}}",
+            ),
+            Target(
+                expr='ray_node_network_send_speed{{instance=~"$Instance",{global_filters}}}',
+                legend="Send: {{instance}}",
+            ),
+        ],
+    ),
+    Panel(
+        id=24,
+        title="Node Count",
+        description='Note: not impacted by "Instance" variable.\n\nA total number of active failed, and pending nodes from the cluster. \n\nACTIVE: A node is alive and available.\n\nFAILED: A node is dead and not available. The node is considered dead when the raylet process on the node is terminated. The node will get into the failed state if it cannot be provided (e.g., there\'s no available node from the cloud provider) or failed to setup (e.g., setup_commands have errors). \n\nPending: A node is being started by the Ray cluster launcher. The node is unavailable now because it is being provisioned and initialized.',
+        unit="nodes",
+        targets=[
+            Target(
+                expr="sum(autoscaler_active_nodes{{{global_filters}}}) by (NodeType)",
+                legend="Active Nodes: {{NodeType}}",
+            ),
+            Target(
+                expr="sum(autoscaler_recently_failed_nodes{{{global_filters}}}) by (NodeType)",
+                legend="Failed Nodes: {{NodeType}}",
+            ),
+            Target(
+                expr="sum(autoscaler_pending_nodes{{{global_filters}}}) by (NodeType)",
+                legend="Pending Nodes: {{NodeType}}",
+            ),
+        ],
+    ),
+    Panel(
+        id=41,
+        title="Cluster Utilization",
+        description="Aggregated utilization of all physical resources (CPU, GPU, memory, disk, or etc.) across the cluster.",
+        unit="%",
+        targets=[
+            # CPU
+            Target(
+                expr='avg(ray_node_cpu_utilization{{instance=~"$Instance",{global_filters}}})',
+                legend="CPU (physical)",
+            ),
+            # GPU
+            Target(
+                expr='sum(ray_node_gpus_utilization{{instance=~"$Instance",{global_filters}}}) / on() (sum(autoscaler_cluster_resources{{resource="GPU",instance=~"$Instance",{global_filters}}}) or vector(0))',
+                legend="GPU (physical)",
+            ),
+            # Memory
+            Target(
+                expr='sum(ray_node_mem_used{{instance=~"$Instance",{global_filters}}}) / on() (sum(ray_node_mem_total{{instance=~"$Instance",{global_filters}}})) * 100',
+                legend="Memory (RAM)",
+            ),
+            # GRAM
+            Target(
+                expr='sum(ray_node_gram_used{{instance=~"$Instance",{global_filters}}}) / on() (sum(ray_node_gram_available{{instance=~"$Instance",{global_filters}}}) + sum(ray_node_gram_used{{instance=~"$Instance",{global_filters}}})) * 100',
+                legend="GRAM",
+            ),
+            # Object Store
+            Target(
+                expr='sum(ray_object_store_memory{{instance=~"$Instance",{global_filters}}}) / on() sum(ray_resources{{Name="object_store_memory",instance=~"$Instance",{global_filters}}}) * 100',
+                legend="Object Store Memory",
+            ),
+            # Disk
+            Target(
+                expr='sum(ray_node_disk_usage{{instance=~"$Instance",{global_filters}}}) / on() (sum(ray_node_disk_free{{instance=~"$Instance",{global_filters}}}) + sum(ray_node_disk_usage{{instance=~"$Instance",{global_filters}}})) * 100',
+                legend="Disk",
+            ),
+        ],
+        fill=0,
+        stack=False,
+    ),
+]
+
+
+ids = []
+for panel in DEFAULT_GRAFANA_PANELS:
+    ids.append(panel.id)
+assert len(ids) == len(
+    set(ids)
+), f"Duplicated id found. Use unique id for each panel. {ids}"
+
+default_dashboard_config = DashboardConfig(
+    name="DEFAULT",
+    default_uid="rayDefaultDashboard",
+    panels=DEFAULT_GRAFANA_PANELS,
+    standard_global_filters=[
+        'SessionName=~"$SessionName"',
+        'ray_io_cluster=~"$Cluster"',
+    ],
+    base_json_file_name="default_grafana_dashboard_base.json",
+)

.venv/lib/python3.11/site-packages/ray/dashboard/modules/metrics/dashboards/serve_dashboard_panels.py

@@ -0,0 +1,420 @@
+# ruff: noqa: E501
+
+from ray.dashboard.modules.metrics.dashboards.common import (
+    DashboardConfig,
+    GridPos,
+    Panel,
+    Target,
+)
+
+SERVE_GRAFANA_PANELS = [
+    Panel(
+        id=5,
+        title="Cluster Utilization",
+        description="Aggregated utilization of all physical resources (CPU, GPU, memory, disk, or etc.) across the cluster. Ignores application variable.",
+        unit="%",
+        targets=[
+            # CPU
+            Target(
+                expr="avg(ray_node_cpu_utilization{{{global_filters}}})",
+                legend="CPU (physical)",
+            ),
+            # GPU
+            Target(
+                expr="sum(ray_node_gpus_utilization{{{global_filters}}}) / on() (sum(autoscaler_cluster_resources{{resource='GPU',{global_filters}}}) or vector(0))",
+                legend="GPU (physical)",
+            ),
+            # Memory
+            Target(
+                expr="sum(ray_node_mem_used{{{global_filters}}}) / on() (sum(ray_node_mem_total{{{global_filters}}})) * 100",
+                legend="Memory (RAM)",
+            ),
+            # GRAM
+            Target(
+                expr="sum(ray_node_gram_used{{{global_filters}}}) / on() (sum(ray_node_gram_available{{{global_filters}}}) + sum(ray_node_gram_used{{{global_filters}}})) * 100",
+                legend="GRAM",
+            ),
+            # Object Store
+            Target(
+                expr='sum(ray_object_store_memory{{{global_filters}}}) / on() sum(ray_resources{{Name="object_store_memory",{global_filters}}}) * 100',
+                legend="Object Store Memory",
+            ),
+            # Disk
+            Target(
+                expr="sum(ray_node_disk_usage{{{global_filters}}}) / on() (sum(ray_node_disk_free{{{global_filters}}}) + sum(ray_node_disk_usage{{{global_filters}}})) * 100",
+                legend="Disk",
+            ),
+        ],
+        fill=0,
+        stack=False,
+        grid_pos=GridPos(0, 0, 12, 8),
+    ),
+    Panel(
+        id=7,
+        title="QPS per application",
+        description="QPS for each selected application.",
+        unit="qps",
+        targets=[
+            Target(
+                expr='sum(rate(ray_serve_num_http_requests_total{{application=~"$Application",application!~"",route=~"$HTTP_Route",route!~"/-/.*",{global_filters}}}[5m])) by (application, route)',
+                legend="{{application, route}}",
+            ),
+            Target(
+                expr='sum(rate(ray_serve_num_grpc_requests_total{{application=~"$Application",application!~"",method=~"$gRPC_Method",{global_filters}}}[5m])) by (application, method)',
+                legend="{{application, method}}",
+            ),
+        ],
+        grid_pos=GridPos(12, 0, 12, 8),
+    ),
+    Panel(
+        id=8,
+        title="Error QPS per application",
+        description="Error QPS for each selected application.",
+        unit="qps",
+        targets=[
+            Target(
+                expr='sum(rate(ray_serve_num_http_error_requests_total{{application=~"$Application",application!~"",route=~"$HTTP_Route",route!~"/-/.*",{global_filters}}}[5m])) by (application, route)',
+                legend="{{application, route}}",
+            ),
+            Target(
+                expr='sum(rate(ray_serve_num_grpc_error_requests_total{{application=~"$Application",application!~"",method=~"$gRPC_Method",{global_filters}}}[5m])) by (application, method)',
+                legend="{{application, method}}",
+            ),
+        ],
+        grid_pos=GridPos(0, 1, 12, 8),
+    ),
+    Panel(
+        id=17,
+        title="Error QPS per application per error code",
+        description="Error QPS for each selected application.",
+        unit="qps",
+        targets=[
+            Target(
+                expr='sum(rate(ray_serve_num_http_error_requests_total{{application=~"$Application",application!~"",route=~"$HTTP_Route",route!~"/-/.*",{global_filters}}}[5m])) by (application, route, error_code)',
+                legend="{{application, route, error_code}}",
+            ),
+            Target(
+                expr='sum(rate(ray_serve_num_grpc_error_requests_total{{application=~"$Application",application!~"",method=~"$gRPC_Method",{global_filters}}}[5m])) by (application, method, error_code)',
+                legend="{{application, method, error_code}}",
+            ),
+        ],
+        grid_pos=GridPos(12, 1, 12, 8),
+    ),
+    Panel(
+        id=12,
+        title="P50 latency per application",
+        description="P50 latency for selected applications.",
+        unit="ms",
+        targets=[
+            Target(
+                expr='histogram_quantile(0.5, sum(rate(ray_serve_http_request_latency_ms_bucket{{application=~"$Application",application!~"",route=~"$HTTP_Route",route!~"/-/.*",{global_filters}}}[5m])) by (application, route, le))',
+                legend="{{application, route}}",
+            ),
+            Target(
+                expr='histogram_quantile(0.5, sum(rate(ray_serve_grpc_request_latency_ms_bucket{{application=~"$Application",application!~"",method=~"$gRPC_Method",{global_filters}}}[5m])) by (application, method, le))',
+                legend="{{application, method}}",
+            ),
+            Target(
+                expr='histogram_quantile(0.5, sum(rate({{__name__=~ "ray_serve_(http|grpc)_request_latency_ms_bucket",application=~"$Application",application!~"",{global_filters}}}[5m])) by (le))',
+                legend="Total",
+            ),
+        ],
+        fill=0,
+        stack=False,
+        grid_pos=GridPos(0, 2, 8, 8),
+    ),
+    Panel(
+        id=15,
+        title="P90 latency per application",
+        description="P90 latency for selected applications.",
+        unit="ms",
+        targets=[
+            Target(
+                expr='histogram_quantile(0.9, sum(rate(ray_serve_http_request_latency_ms_bucket{{application=~"$Application",application!~"",route=~"$HTTP_Route",route!~"/-/.*",{global_filters}}}[5m])) by (application, route, le))',
+                legend="{{application, route}}",
+            ),
+            Target(
+                expr='histogram_quantile(0.9, sum(rate(ray_serve_grpc_request_latency_ms_bucket{{application=~"$Application",application!~"",method=~"$gRPC_Method",{global_filters}}}[5m])) by (application, method, le))',
+                legend="{{application, method}}",
+            ),
+            Target(
+                expr='histogram_quantile(0.9, sum(rate({{__name__=~ "ray_serve_(http|grpc)_request_latency_ms_bucket|ray_serve_grpc_request_latency_ms_bucket",application=~"$Application",application!~"",{global_filters}}}[5m])) by (le))',
+                legend="Total",
+            ),
+        ],
+        fill=0,
+        stack=False,
+        grid_pos=GridPos(8, 2, 8, 8),
+    ),
+    Panel(
+        id=16,
+        title="P99 latency per application",
+        description="P99 latency for selected applications.",
+        unit="ms",
+        targets=[
+            Target(
+                expr='histogram_quantile(0.99, sum(rate(ray_serve_http_request_latency_ms_bucket{{application=~"$Application",application!~"",route=~"$HTTP_Route",route!~"/-/.*",{global_filters}}}[5m])) by (application, route, le))',
+                legend="{{application, route}}",
+            ),
+            Target(
+                expr='histogram_quantile(0.99, sum(rate(ray_serve_grpc_request_latency_ms_bucket{{application=~"$Application",application!~"",method=~"$gRPC_Method",{global_filters}}}[5m])) by (application, method, le))',
+                legend="{{application, method}}",
+            ),
+            Target(
+                expr='histogram_quantile(0.99, sum(rate({{__name__=~ "ray_serve_(http|grpc)_request_latency_ms_bucket|ray_serve_grpc_request_latency_ms_bucket",application=~"$Application",application!~"",{global_filters}}}[5m])) by (le))',
+                legend="Total",
+            ),
+        ],
+        fill=0,
+        stack=False,
+        grid_pos=GridPos(16, 2, 8, 8),
+    ),
+    Panel(
+        id=2,
+        title="Replicas per deployment",
+        description='Number of replicas per deployment. Ignores "Application" variable.',
+        unit="replicas",
+        targets=[
+            Target(
+                expr="sum(ray_serve_deployment_replica_healthy{{{global_filters}}}) by (application, deployment)",
+                legend="{{application, deployment}}",
+            ),
+        ],
+        grid_pos=GridPos(0, 3, 8, 8),
+    ),
+    Panel(
+        id=13,
+        title="QPS per deployment",
+        description="QPS for each deployment.",
+        unit="qps",
+        targets=[
+            Target(
+                expr='sum(rate(ray_serve_deployment_request_counter_total{{application=~"$Application",application!~"",{global_filters}}}[5m])) by (application, deployment)',
+                legend="{{application, deployment}}",
+            ),
+        ],
+        grid_pos=GridPos(8, 3, 8, 8),
+    ),
+    Panel(
+        id=14,
+        title="Error QPS per deployment",
+        description="Error QPS for each deplyoment.",
+        unit="qps",
+        targets=[
+            Target(
+                expr='sum(rate(ray_serve_deployment_error_counter_total{{application=~"$Application",application!~"",{global_filters}}}[5m])) by (application, deployment)',
+                legend="{{application, deployment}}",
+            ),
+        ],
+        grid_pos=GridPos(16, 3, 8, 8),
+    ),
+    Panel(
+        id=9,
+        title="P50 latency per deployment",
+        description="P50 latency per deployment.",
+        unit="ms",
+        targets=[
+            Target(
+                expr='histogram_quantile(0.5, sum(rate(ray_serve_deployment_processing_latency_ms_bucket{{application=~"$Application",application!~"",{global_filters}}}[5m])) by (application, deployment, le))',
+                legend="{{application, deployment}}",
+            ),
+            Target(
+                expr='histogram_quantile(0.5, sum(rate(ray_serve_deployment_processing_latency_ms_bucket{{application=~"$Application",application!~"",{global_filters}}}[5m])) by (le))',
+                legend="Total",
+            ),
+        ],
+        fill=0,
+        stack=False,
+        grid_pos=GridPos(0, 4, 8, 8),
+    ),
+    Panel(
+        id=10,
+        title="P90 latency per deployment",
+        description="P90 latency per deployment.",
+        unit="ms",
+        targets=[
+            Target(
+                expr='histogram_quantile(0.9, sum(rate(ray_serve_deployment_processing_latency_ms_bucket{{application=~"$Application",application!~"",{global_filters}}}[5m])) by (application, deployment, le))',
+                legend="{{application, deployment}}",
+            ),
+            Target(
+                expr='histogram_quantile(0.9, sum(rate(ray_serve_deployment_processing_latency_ms_bucket{{application=~"$Application",application!~"",{global_filters}}}[5m])) by (le))',
+                legend="Total",
+            ),
+        ],
+        fill=0,
+        stack=False,
+        grid_pos=GridPos(8, 4, 8, 8),
+    ),
+    Panel(
+        id=11,
+        title="P99 latency per deployment",
+        description="P99 latency per deployment.",
+        unit="ms",
+        targets=[
+            Target(
+                expr='histogram_quantile(0.99, sum(rate(ray_serve_deployment_processing_latency_ms_bucket{{application=~"$Application",application!~"",{global_filters}}}[5m])) by (application, deployment, le))',
+                legend="{{application, deployment}}",
+            ),
+            Target(
+                expr='histogram_quantile(0.99, sum(rate(ray_serve_deployment_processing_latency_ms_bucket{{application=~"$Application",application!~"",{global_filters}}}[5m])) by (le))',
+                legend="Total",
+            ),
+        ],
+        fill=0,
+        stack=False,
+        grid_pos=GridPos(16, 4, 8, 8),
+    ),
+    Panel(
+        id=3,
+        title="Queue size per deployment",
+        description='Number of requests queued per deployment. Ignores "Application" variable.',
+        unit="requests",
+        targets=[
+            Target(
+                expr="sum(ray_serve_deployment_queued_queries{{{global_filters}}}) by (application, deployment)",
+                legend="{{application, deployment}}",
+            ),
+        ],
+        fill=0,
+        stack=False,
+        grid_pos=GridPos(0, 5, 8, 8),
+    ),
+    Panel(
+        id=4,
+        title="Node count",
+        description='Number of nodes in this cluster. Ignores "Application" variable.',
+        unit="nodes",
+        targets=[
+            # TODO(aguo): Update this to use autoscaler metrics instead
+            Target(
+                expr="sum(autoscaler_active_nodes{{{global_filters}}}) by (NodeType)",
+                legend="Active Nodes: {{NodeType}}",
+            ),
+            Target(
+                expr="sum(autoscaler_recently_failed_nodes{{{global_filters}}}) by (NodeType)",
+                legend="Failed Nodes: {{NodeType}}",
+            ),
+            Target(
+                expr="sum(autoscaler_pending_nodes{{{global_filters}}}) by (NodeType)",
+                legend="Pending Nodes: {{NodeType}}",
+            ),
+        ],
+        grid_pos=GridPos(8, 5, 8, 8),
+    ),
+    Panel(
+        id=6,
+        title="Node network",
+        description='Network speed per node. Ignores "Application" variable.',
+        unit="Bps",
+        targets=[
+            Target(
+                expr="sum(ray_node_network_receive_speed{{{global_filters}}}) by (instance)",
+                legend="Recv: {{instance}}",
+            ),
+            Target(
+                expr="sum(ray_node_network_send_speed{{{global_filters}}}) by (instance)",
+                legend="Send: {{instance}}",
+            ),
+        ],
+        fill=1,
+        linewidth=2,
+        stack=False,
+        grid_pos=GridPos(16, 5, 8, 8),
+    ),
+    Panel(
+        id=20,
+        title="Ongoing HTTP Requests",
+        description="The number of ongoing requests in the HTTP Proxy.",
+        unit="requests",
+        targets=[
+            Target(
+                expr="ray_serve_num_ongoing_http_requests{{{global_filters}}}",
+                legend="Ongoing HTTP Requests",
+            ),
+        ],
+        grid_pos=GridPos(0, 6, 8, 8),
+    ),
+    Panel(
+        id=21,
+        title="Ongoing gRPC Requests",
+        description="The number of ongoing requests in the gRPC Proxy.",
+        unit="requests",
+        targets=[
+            Target(
+                expr="ray_serve_num_ongoing_grpc_requests{{{global_filters}}}",
+                legend="Ongoing gRPC Requests",
+            ),
+        ],
+        grid_pos=GridPos(8, 6, 8, 8),
+    ),
+    Panel(
+        id=22,
+        title="Scheduling Tasks",
+        description="The number of request scheduling tasks in the router.",
+        unit="tasks",
+        targets=[
+            Target(
+                expr="ray_serve_num_scheduling_tasks{{{global_filters}}}",
+                legend="Scheduling Tasks",
+            ),
+        ],
+        grid_pos=GridPos(16, 6, 8, 8),
+    ),
+    Panel(
+        id=23,
+        title="Scheduling Tasks in Backoff",
+        description="The number of request scheduling tasks in the router that are undergoing backoff.",
+        unit="tasks",
+        targets=[
+            Target(
+                expr="ray_serve_num_scheduling_tasks_in_backoff{{{global_filters}}}",
+                legend="Scheduling Tasks in Backoff",
+            ),
+        ],
+        grid_pos=GridPos(0, 7, 8, 8),
+    ),
+    Panel(
+        id=24,
+        title="Controller Control Loop Duration",
+        description="The duration of the last control loop.",
+        unit="seconds",
+        targets=[
+            Target(
+                expr="ray_serve_controller_control_loop_duration_s{{{global_filters}}}",
+                legend="Control Loop Duration",
+            ),
+        ],
+        grid_pos=GridPos(8, 7, 8, 8),
+    ),
+    Panel(
+        id=25,
+        title="Number of Control Loops",
+        description="The number of control loops performed by the controller. Increases monotonically over the controller's lifetime.",
+        unit="loops",
+        targets=[
+            Target(
+                expr="ray_serve_controller_num_control_loops{{{global_filters}}}",
+                legend="Control Loops",
+            ),
+        ],
+        grid_pos=GridPos(16, 7, 8, 8),
+    ),
+]
+
+ids = []
+for panel in SERVE_GRAFANA_PANELS:
+    ids.append(panel.id)
+assert len(ids) == len(
+    set(ids)
+), f"Duplicated id found. Use unique id for each panel. {ids}"
+
+serve_dashboard_config = DashboardConfig(
+    name="SERVE",
+    default_uid="rayServeDashboard",
+    panels=SERVE_GRAFANA_PANELS,
+    standard_global_filters=[
+        'ray_io_cluster=~"$Cluster"',
+    ],
+    base_json_file_name="serve_grafana_dashboard_base.json",
+)

.venv/lib/python3.11/site-packages/ray/dashboard/modules/metrics/dashboards/serve_deployment_dashboard_panels.py

@@ -0,0 +1,259 @@
+# ruff: noqa: E501
+
+from ray.dashboard.modules.metrics.dashboards.common import (
+    DashboardConfig,
+    GridPos,
+    Panel,
+    Target,
+)
+
+SERVE_DEPLOYMENT_GRAFANA_PANELS = [
+    Panel(
+        id=1,
+        title="Replicas per deployment",
+        description='Number of replicas per deployment. Ignores "Route" variable.',
+        unit="replicas",
+        targets=[
+            Target(
+                expr="sum(ray_serve_deployment_replica_healthy{{{global_filters}}}) by (application, deployment)",
+                legend="{{application, deployment}}",
+            ),
+        ],
+        grid_pos=GridPos(0, 0, 8, 8),
+    ),
+    Panel(
+        id=2,
+        title="QPS per replica",
+        description="QPS for each replica.",
+        unit="qps",
+        targets=[
+            Target(
+                expr='sum(rate(ray_serve_deployment_request_counter_total{{route=~"$Route",route!~"/-/.*",{global_filters}}}[5m])) by (application, deployment, replica)',
+                legend="{{replica}}",
+            ),
+        ],
+        grid_pos=GridPos(8, 0, 8, 8),
+    ),
+    Panel(
+        id=3,
+        title="Error QPS per replica",
+        description="Error QPS for each replica.",
+        unit="qps",
+        targets=[
+            Target(
+                expr='sum(rate(ray_serve_deployment_error_counter_total{{route=~"$Route",route!~"/-/.*",{global_filters}}}[5m])) by (application, deployment, replica)',
+                legend="{{replica}}",
+            ),
+        ],
+        grid_pos=GridPos(16, 0, 8, 8),
+    ),
+    Panel(
+        id=4,
+        title="P50 latency per replica",
+        description="P50 latency per replica.",
+        unit="ms",
+        targets=[
+            Target(
+                expr='histogram_quantile(0.5, sum(rate(ray_serve_deployment_processing_latency_ms_bucket{{route=~"$Route",route!~"/-/.*",{global_filters}}}[5m])) by (application, deployment, replica, le))',
+                legend="{{replica}}",
+            ),
+            Target(
+                expr='histogram_quantile(0.5, sum(rate(ray_serve_deployment_processing_latency_ms_bucket{{route=~"$Route",route!~"/-/.*",{global_filters}}}[5m])) by (le))',
+                legend="Total",
+            ),
+        ],
+        fill=0,
+        stack=False,
+        grid_pos=GridPos(0, 1, 8, 8),
+    ),
+    Panel(
+        id=5,
+        title="P90 latency per replica",
+        description="P90 latency per replica.",
+        unit="ms",
+        targets=[
+            Target(
+                expr='histogram_quantile(0.9, sum(rate(ray_serve_deployment_processing_latency_ms_bucket{{route=~"$Route",route!~"/-/.*",{global_filters}}}[5m])) by (application, deployment, replica, le))',
+                legend="{{replica}}",
+            ),
+            Target(
+                expr='histogram_quantile(0.9, sum(rate(ray_serve_deployment_processing_latency_ms_bucket{{route=~"$Route",route!~"/-/.*",{global_filters}}}[5m])) by (le))',
+                legend="Total",
+            ),
+        ],
+        fill=0,
+        stack=False,
+        grid_pos=GridPos(8, 1, 8, 8),
+    ),
+    Panel(
+        id=6,
+        title="P99 latency per replica",
+        description="P99 latency per replica.",
+        unit="ms",
+        targets=[
+            Target(
+                expr='histogram_quantile(0.99, sum(rate(ray_serve_deployment_processing_latency_ms_bucket{{route=~"$Route",route!~"/-/.*",{global_filters}}}[5m])) by (application, deployment, replica, le))',
+                legend="{{replica}}",
+            ),
+            Target(
+                expr='histogram_quantile(0.99, sum(rate(ray_serve_deployment_processing_latency_ms_bucket{{route=~"$Route",{global_filters}}}[5m])) by (le))',
+                legend="Total",
+            ),
+        ],
+        fill=0,
+        stack=False,
+        grid_pos=GridPos(16, 1, 8, 8),
+    ),
+    Panel(
+        id=7,
+        title="Queue size per deployment",
+        description='Number of requests queued per deployment. Ignores "Replica" and "Route" variable.',
+        unit="requests",
+        targets=[
+            Target(
+                expr="sum(ray_serve_deployment_queued_queries{{{global_filters}}}) by (application, deployment)",
+                legend="{{application, deployment}}",
+            ),
+        ],
+        fill=0,
+        stack=False,
+        grid_pos=GridPos(0, 2, 12, 8),
+    ),
+    Panel(
+        id=8,
+        title="Running requests per replica",
+        description="Current running requests for each replica.",
+        unit="requests",
+        targets=[
+            Target(
+                expr="sum(ray_serve_replica_processing_queries{{{global_filters}}}) by (application, deployment, replica)",
+                legend="{{replica}}",
+            ),
+        ],
+        fill=0,
+        stack=False,
+        grid_pos=GridPos(12, 2, 12, 8),
+    ),
+    Panel(
+        id=9,
+        title="Multiplexed models per replica",
+        description="The number of multiplexed models for each replica.",
+        unit="models",
+        targets=[
+            Target(
+                expr="sum(ray_serve_num_multiplexed_models{{{global_filters}}}) by (application, deployment, replica)",
+                legend="{{replica}}",
+            ),
+        ],
+        fill=0,
+        stack=False,
+        grid_pos=GridPos(0, 3, 8, 8),
+    ),
+    Panel(
+        id=10,
+        title="Multiplexed model loads per replica",
+        description="The number of times of multiplexed models loaded for each replica.",
+        unit="times",
+        targets=[
+            Target(
+                expr="sum(ray_serve_multiplexed_models_load_counter_total{{{global_filters}}}) by (application, deployment, replica)",
+                legend="{{replica}}",
+            ),
+        ],
+        fill=0,
+        stack=False,
+        grid_pos=GridPos(8, 3, 8, 8),
+    ),
+    Panel(
+        id=11,
+        title="Multiplexed model unloads per replica",
+        description="The number of times of multiplexed models unloaded for each replica.",
+        unit="times",
+        targets=[
+            Target(
+                expr="sum(ray_serve_multiplexed_models_unload_counter_total{{{global_filters}}}) by (application, deployment, replica)",
+                legend="{{replica}}",
+            ),
+        ],
+        fill=0,
+        stack=False,
+        grid_pos=GridPos(16, 3, 8, 8),
+    ),
+    Panel(
+        id=12,
+        title="P99 latency of multiplexed model loads per replica",
+        description="P99 latency of mutliplexed model load per replica.",
+        unit="ms",
+        targets=[
+            Target(
+                expr="histogram_quantile(0.99, sum(rate(ray_serve_multiplexed_model_load_latency_ms_bucket{{{global_filters}}}[5m])) by (application, deployment, replica, le))",
+                legend="{{replica}}",
+            ),
+        ],
+        fill=0,
+        stack=False,
+        grid_pos=GridPos(0, 4, 8, 8),
+    ),
+    Panel(
+        id=13,
+        title="P99 latency of multiplexed model unloads per replica",
+        description="P99 latency of mutliplexed model unload per replica.",
+        unit="ms",
+        targets=[
+            Target(
+                expr="histogram_quantile(0.99, sum(rate(ray_serve_multiplexed_model_unload_latency_ms_bucket{{{global_filters}}}[5m])) by (application, deployment, replica, le))",
+                legend="{{replica}}",
+            ),
+        ],
+        fill=0,
+        stack=False,
+        grid_pos=GridPos(8, 4, 8, 8),
+    ),
+    Panel(
+        id=14,
+        title="Multiplexed model ids per replica",
+        description="The ids of multiplexed models for each replica.",
+        unit="model",
+        targets=[
+            Target(
+                expr="ray_serve_registered_multiplexed_model_id{{{global_filters}}}",
+                legend="{{replica}}:{{model_id}}",
+            ),
+        ],
+        grid_pos=GridPos(16, 4, 8, 8),
+        stack=False,
+    ),
+    Panel(
+        id=15,
+        title="Multiplexed model cache hit rate",
+        description="The cache hit rate of multiplexed models for the deployment.",
+        unit="%",
+        targets=[
+            Target(
+                expr="(1 - sum(rate(ray_serve_multiplexed_models_load_counter_total{{{global_filters}}}[5m]))/sum(rate(ray_serve_multiplexed_get_model_requests_counter_total{{{global_filters}}}[5m])))",
+                legend="{{replica}}",
+            ),
+        ],
+        grid_pos=GridPos(0, 5, 8, 8),
+    ),
+]
+
+ids = []
+for panel in SERVE_DEPLOYMENT_GRAFANA_PANELS:
+    ids.append(panel.id)
+assert len(ids) == len(
+    set(ids)
+), f"Duplicated id found. Use unique id for each panel. {ids}"
+
+serve_deployment_dashboard_config = DashboardConfig(
+    name="SERVE_DEPLOYMENT",
+    default_uid="rayServeDeploymentDashboard",
+    panels=SERVE_DEPLOYMENT_GRAFANA_PANELS,
+    standard_global_filters=[
+        'application=~"$Application"',
+        'deployment=~"$Deployment"',
+        'replica=~"$Replica"',
+        'ray_io_cluster=~"$Cluster"',
+    ],
+    base_json_file_name="serve_deployment_grafana_dashboard_base.json",
+)

.venv/lib/python3.11/site-packages/ray/dashboard/modules/metrics/dashboards/serve_deployment_grafana_dashboard_base.json

@@ -0,0 +1,223 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "gnetId": null,
+  "graphTooltip": 0,
+  "iteration": 1667344411089,
+  "links": [],
+  "panels": [],
+  "refresh": false,
+  "schemaVersion": 27,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "selected": false
+        },
+        "description": "Filter queries to specific prometheus type.",
+        "hide": 2,
+        "includeAll": false,
+        "multi": false,
+        "name": "datasource",
+        "options": [],
+        "query": "prometheus",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "type": "datasource"
+      },
+      {
+        "allValue": ".*",
+        "current": {
+          "selected": true,
+          "text": [
+            "All"
+          ],
+          "value": [
+            "$__all"
+          ]
+        },
+        "datasource": "${datasource}",
+        "definition": "label_values(ray_serve_deployment_replica_healthy{{{global_filters}}}, application)",
+        "description": null,
+        "error": null,
+        "hide": 0,
+        "includeAll": true,
+        "label": null,
+        "multi": true,
+        "name": "Application",
+        "options": [],
+        "query": {
+          "query": "label_values(ray_serve_deployment_replica_healthy{{{global_filters}}}, application)",
+          "refId": "Prometheus-Instance-Variable-Query"
+        },
+        "refresh": 2,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "allValue": ".*",
+        "current": {
+          "selected": true,
+          "text": [
+            "All"
+          ],
+          "value": [
+            "$__all"
+          ]
+        },
+        "datasource": "${datasource}",
+        "definition": "label_values(ray_serve_deployment_replica_healthy{{application=~\"$Application\",{global_filters}}}, deployment)",
+        "description": null,
+        "error": null,
+        "hide": 0,
+        "includeAll": true,
+        "label": null,
+        "multi": true,
+        "name": "Deployment",
+        "options": [],
+        "query": {
+          "query": "label_values(ray_serve_deployment_replica_healthy{{application=~\"$Application\",{global_filters}}}, deployment)",
+          "refId": "Prometheus-Instance-Variable-Query"
+        },
+        "refresh": 2,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "allValue": ".*",
+        "current": {
+          "selected": true,
+          "text": [
+            "All"
+          ],
+          "value": [
+            "$__all"
+          ]
+        },
+        "datasource": "${datasource}",
+        "definition": "label_values(ray_serve_deployment_replica_healthy{{application=~\"$Application\",deployment=~\"$Deployment\",{global_filters}}}, replica)",
+        "description": null,
+        "error": null,
+        "hide": 0,
+        "includeAll": true,
+        "label": null,
+        "multi": true,
+        "name": "Replica",
+        "options": [],
+        "query": {
+          "query": "label_values(ray_serve_deployment_replica_healthy{{application=~\"$Application\",deployment=~\"$Deployment\",{global_filters}}}, replica)",
+          "refId": "Prometheus-Instance-Variable-Query"
+        },
+        "refresh": 2,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "allValue": ".*",
+        "current": {
+          "selected": true,
+          "text": [
+            "All"
+          ],
+          "value": [
+            "$__all"
+          ]
+        },
+        "datasource": "${datasource}",
+        "definition": "label_values(ray_serve_deployment_request_counter{{deployment=~\"$Deployment\",{global_filters}}}, route)",
+        "description": null,
+        "error": null,
+        "hide": 0,
+        "includeAll": true,
+        "label": null,
+        "multi": true,
+        "name": "Route",
+        "options": [],
+        "query": {
+          "query": "label_values(ray_serve_deployment_request_counter{{deployment=~\"$Deployment\",{global_filters}}}, route)",
+          "refId": "Prometheus-Instance-Variable-Query"
+        },
+        "refresh": 2,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "current": {
+          "selected": false
+        },
+        "datasource": "${datasource}",
+        "definition": "label_values(ray_node_network_receive_speed{{{global_filters}}}, ray_io_cluster)",
+        "description": "Filter queries to specific Ray clusters for KubeRay. When ingesting metrics across multiple ray clusters, the ray_io_cluster label should be set per cluster. For KubeRay users, this is done automaticaly with Prometheus PodMonitor.",
+        "error": null,
+        "hide": 0,
+        "includeAll": false,
+        "label": null,
+        "multi": false,
+        "name": "Cluster",
+        "options": [],
+        "query": {
+          "query": "label_values(ray_node_network_receive_speed{{{global_filters}}}, ray_io_cluster)",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 2,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 2,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      }
+    ]
+  },
+  "rayMeta": ["excludesSystemRoutes"],
+  "time": {
+    "from": "now-30m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "Serve Deployment Dashboard",
+  "uid": "rayServeDeploymentDashboard",
+  "version": 1
+}

.venv/lib/python3.11/site-packages/ray/dashboard/modules/metrics/dashboards/serve_grafana_dashboard_base.json

@@ -0,0 +1,188 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "gnetId": null,
+  "graphTooltip": 0,
+  "iteration": 1667344411089,
+  "links": [],
+  "panels": [],
+  "refresh": false,
+  "schemaVersion": 27,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "selected": false
+        },
+        "description": "Filter queries of a specific Prometheus type.",
+        "hide": 2,
+        "includeAll": false,
+        "multi": false,
+        "name": "datasource",
+        "options": [],
+        "query": "prometheus",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "type": "datasource"
+      },
+      {
+        "allValue": ".*",
+        "current": {
+          "selected": true,
+          "text": [
+            "All"
+          ],
+          "value": [
+            "$__all"
+          ]
+        },
+        "datasource": "${datasource}",
+        "definition": "label_values(ray_serve_deployment_replica_healthy{{{global_filters}}}, application)",
+        "description": null,
+        "error": null,
+        "hide": 0,
+        "includeAll": true,
+        "label": null,
+        "multi": true,
+        "name": "Application",
+        "options": [],
+        "query": {
+          "query": "label_values(ray_serve_deployment_replica_healthy{{{global_filters}}}, application)",
+          "refId": "Prometheus-Instance-Variable-Query"
+        },
+        "refresh": 2,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "allValue": ".*",
+        "current": {
+          "selected": true,
+          "text": [
+            "All"
+          ],
+          "value": [
+            "$__all"
+          ]
+        },
+        "datasource": "${datasource}",
+        "definition": "label_values(ray_serve_num_http_requests_total{{{global_filters}}}, route)",
+        "description": null,
+        "error": null,
+        "hide": 0,
+        "includeAll": true,
+        "label": "HTTP Route",
+        "multi": true,
+        "name": "HTTP_Route",
+        "options": [],
+        "query": {
+          "query": "label_values(ray_serve_num_http_requests_total{{{global_filters}}}, route)",
+          "refId": "Prometheus-Instance-Variable-Query"
+        },
+        "refresh": 2,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "allValue": ".*",
+        "current": {
+          "selected": true,
+          "text": [
+            "All"
+          ],
+          "value": [
+            "$__all"
+          ]
+        },
+        "datasource": "${datasource}",
+        "definition": "label_values(ray_serve_num_grpc_requests{{{global_filters}}}, method)",
+        "description": null,
+        "error": null,
+        "hide": 0,
+        "includeAll": true,
+        "label": "gRPC Service Method",
+        "multi": true,
+        "name": "gRPC_Method",
+        "options": [],
+        "query": {
+          "query": "label_values(ray_serve_num_grpc_requests{{{global_filters}}}, method)",
+          "refId": "Prometheus-Instance-Variable-Query"
+        },
+        "refresh": 2,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "current": {
+          "selected": false
+        },
+        "datasource": "${datasource}",
+        "definition": "label_values(ray_node_network_receive_speed{{{global_filters}}}, ray_io_cluster)",
+        "description": "Filter queries to specific Ray clusters for KubeRay. When ingesting metrics across multiple ray clusters, the ray_io_cluster label should be set per cluster. For KubeRay users, this is done automaticaly with Prometheus PodMonitor.",
+        "error": null,
+        "hide": 0,
+        "includeAll": false,
+        "label": null,
+        "multi": false,
+        "name": "Cluster",
+        "options": [],
+        "query": {
+          "query": "label_values(ray_node_network_receive_speed{{{global_filters}}}, ray_io_cluster)",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 2,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 2,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      }
+    ]
+  },
+  "rayMeta": ["excludesSystemRoutes"],
+  "time": {
+    "from": "now-30m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "Serve Dashboard",
+  "uid": "rayServeDashboard",
+  "version": 1
+}

.venv/lib/python3.11/site-packages/ray/dashboard/modules/metrics/export/prometheus/prometheus.yml

@@ -0,0 +1,12 @@
+# my global config
+global:
+  scrape_interval: 10s # Set the scrape interval to every 10 seconds. Default is every 1 minute.
+  evaluation_interval: 10s # Evaluate rules every 10 seconds. The default is every 1 minute.
+  # scrape_timeout is set to the global default (10s).
+
+scrape_configs:
+# Scrape from each Ray node as defined in the service_discovery.json provided by Ray.
+- job_name: 'ray'
+  file_sd_configs:
+  - files:
+    - '/tmp/ray/prom_metrics_service_discovery.json'

.venv/lib/python3.11/site-packages/ray/dashboard/modules/metrics/grafana_dashboard_factory.py

@@ -0,0 +1,301 @@
+import copy
+import json
+import os
+from dataclasses import asdict
+from typing import List, Tuple
+
+import ray
+from ray.dashboard.modules.metrics.dashboards.common import DashboardConfig, Panel
+from ray.dashboard.modules.metrics.dashboards.data_dashboard_panels import (
+    data_dashboard_config,
+)
+from ray.dashboard.modules.metrics.dashboards.default_dashboard_panels import (
+    default_dashboard_config,
+)
+from ray.dashboard.modules.metrics.dashboards.serve_dashboard_panels import (
+    serve_dashboard_config,
+)
+from ray.dashboard.modules.metrics.dashboards.serve_deployment_dashboard_panels import (
+    serve_deployment_dashboard_config,
+)
+
+GRAFANA_DASHBOARD_UID_OVERRIDE_ENV_VAR_TEMPLATE = "RAY_GRAFANA_{name}_DASHBOARD_UID"
+GRAFANA_DASHBOARD_GLOBAL_FILTERS_OVERRIDE_ENV_VAR_TEMPLATE = (
+    "RAY_GRAFANA_{name}_DASHBOARD_GLOBAL_FILTERS"
+)
+
+TARGET_TEMPLATE = {
+    "exemplar": True,
+    "expr": "0",
+    "interval": "",
+    "legendFormat": "",
+    "queryType": "randomWalk",
+    "refId": "A",
+}
+
+
+PANEL_TEMPLATE = {
+    "aliasColors": {},
+    "bars": False,
+    "dashLength": 10,
+    "dashes": False,
+    "datasource": r"${datasource}",
+    "description": "<Description>",
+    "fieldConfig": {"defaults": {}, "overrides": []},
+    "fill": 10,
+    "fillGradient": 0,
+    "gridPos": {"h": 8, "w": 12, "x": 0, "y": 0},
+    "hiddenSeries": False,
+    "id": 26,
+    "legend": {
+        "alignAsTable": True,
+        "avg": False,
+        "current": True,
+        "hideEmpty": False,
+        "hideZero": True,
+        "max": False,
+        "min": False,
+        "rightSide": False,
+        "show": True,
+        "sort": "current",
+        "sortDesc": True,
+        "total": False,
+        "values": True,
+    },
+    "lines": True,
+    "linewidth": 1,
+    "nullPointMode": "null",
+    "options": {"alertThreshold": True},
+    "percentage": False,
+    "pluginVersion": "7.5.17",
+    "pointradius": 2,
+    "points": False,
+    "renderer": "flot",
+    "seriesOverrides": [
+        {
+            "$$hashKey": "object:2987",
+            "alias": "MAX",
+            "dashes": True,
+            "color": "#1F60C4",
+            "fill": 0,
+            "stack": False,
+        },
+        {
+            "$$hashKey": "object:78",
+            "alias": "/FINISHED|FAILED|DEAD|REMOVED|Failed Nodes:/",
+            "hiddenSeries": True,
+        },
+        {
+            "$$hashKey": "object:2987",
+            "alias": "MAX + PENDING",
+            "dashes": True,
+            "color": "#777777",
+            "fill": 0,
+            "stack": False,
+        },
+    ],
+    "spaceLength": 10,
+    "stack": True,
+    "steppedLine": False,
+    "targets": [],
+    "thresholds": [],
+    "timeFrom": None,
+    "timeRegions": [],
+    "timeShift": None,
+    "title": "<Title>",
+    "tooltip": {"shared": True, "sort": 0, "value_type": "individual"},
+    "type": "graph",
+    "xaxis": {
+        "buckets": None,
+        "mode": "time",
+        "name": None,
+        "show": True,
+        "values": [],
+    },
+    "yaxes": [
+        {
+            "$$hashKey": "object:628",
+            "format": "units",
+            "label": "",
+            "logBase": 1,
+            "max": None,
+            "min": "0",
+            "show": True,
+        },
+        {
+            "$$hashKey": "object:629",
+            "format": "short",
+            "label": None,
+            "logBase": 1,
+            "max": None,
+            "min": None,
+            "show": True,
+        },
+    ],
+    "yaxis": {"align": False, "alignLevel": None},
+}
+
+
+def _read_configs_for_dashboard(
+    dashboard_config: DashboardConfig,
+) -> Tuple[str, List[str]]:
+    """
+    Reads environment variable configs for overriding uid or global_filters for a given
+    dashboard.
+
+    Returns:
+      Tuple with format uid, global_filters
+    """
+    uid = (
+        os.environ.get(
+            GRAFANA_DASHBOARD_UID_OVERRIDE_ENV_VAR_TEMPLATE.format(
+                name=dashboard_config.name
+            )
+        )
+        or dashboard_config.default_uid
+    )
+    global_filters_str = (
+        os.environ.get(
+            GRAFANA_DASHBOARD_GLOBAL_FILTERS_OVERRIDE_ENV_VAR_TEMPLATE.format(
+                name=dashboard_config.name
+            )
+        )
+        or ""
+    )
+    global_filters = global_filters_str.split(",")
+
+    return uid, global_filters
+
+
+def generate_default_grafana_dashboard() -> Tuple[str, str]:
+    """
+    Generates the dashboard output for the default dashboard and returns
+    both the content and the uid.
+
+    Returns:
+      Tuple with format content, uid
+    """
+    return _generate_grafana_dashboard(default_dashboard_config)
+
+
+def generate_serve_grafana_dashboard() -> Tuple[str, str]:
+    """
+    Generates the dashboard output for the serve dashboard and returns
+    both the content and the uid.
+
+    Returns:
+      Tuple with format content, uid
+    """
+    return _generate_grafana_dashboard(serve_dashboard_config)
+
+
+def generate_serve_deployment_grafana_dashboard() -> Tuple[str, str]:
+    """
+    Generates the dashboard output for the serve dashboard and returns
+    both the content and the uid.
+
+    Returns:
+      Tuple with format content, uid
+    """
+    return _generate_grafana_dashboard(serve_deployment_dashboard_config)
+
+
+def generate_data_grafana_dashboard() -> Tuple[str, str]:
+    """
+    Generates the dashboard output for the data dashboard and returns
+    both the content and the uid.
+
+    Returns:
+      Tuple with format content, uid
+    """
+    return _generate_grafana_dashboard(data_dashboard_config)
+
+
+def _generate_grafana_dashboard(dashboard_config: DashboardConfig) -> str:
+    """
+    Returns:
+      Tuple with format dashboard_content, uid
+    """
+    uid, global_filters = _read_configs_for_dashboard(dashboard_config)
+    panels = _generate_grafana_panels(dashboard_config, global_filters)
+    base_file_name = dashboard_config.base_json_file_name
+
+    base_json = json.load(
+        open(os.path.join(os.path.dirname(__file__), "dashboards", base_file_name))
+    )
+    base_json["panels"] = panels
+    # Update variables to use global_filters
+    global_filters_str = ",".join(global_filters)
+    variables = base_json.get("templating", {}).get("list", [])
+    for variable in variables:
+        if "definition" not in variable:
+            continue
+        variable["definition"] = variable["definition"].format(
+            global_filters=global_filters_str
+        )
+        variable["query"]["query"] = variable["query"]["query"].format(
+            global_filters=global_filters_str
+        )
+
+    tags = base_json.get("tags", []) or []
+    tags.append(f"rayVersion:{ray.__version__}")
+    base_json["tags"] = tags
+    base_json["uid"] = uid
+    # Ray metadata can be used to put arbitrary metadata
+    ray_meta = base_json.get("rayMeta", []) or []
+    ray_meta.append("supportsGlobalFilterOverride")
+    base_json["rayMeta"] = ray_meta
+    return json.dumps(base_json, indent=4), uid
+
+
+def _generate_grafana_panels(
+    config: DashboardConfig, global_filters: List[str]
+) -> List[dict]:
+    out = []
+    panel_global_filters = [*config.standard_global_filters, *global_filters]
+    for i, panel in enumerate(config.panels):
+        template = copy.deepcopy(PANEL_TEMPLATE)
+        template.update(
+            {
+                "title": panel.title,
+                "description": panel.description,
+                "id": panel.id,
+                "targets": _generate_targets(panel, panel_global_filters),
+            }
+        )
+        if panel.grid_pos:
+            template["gridPos"] = asdict(panel.grid_pos)
+        else:
+            template["gridPos"]["y"] = i // 2
+            template["gridPos"]["x"] = 12 * (i % 2)
+        template["yaxes"][0]["format"] = panel.unit
+        template["fill"] = panel.fill
+        template["stack"] = panel.stack
+        template["linewidth"] = panel.linewidth
+        out.append(template)
+    return out
+
+
+def gen_incrementing_alphabets(length):
+    assert 65 + length < 96, "we only support up to 26 targets at a time."
+    # 65: ascii code of 'A'.
+    return list(map(chr, range(65, 65 + length)))
+
+
+def _generate_targets(panel: Panel, panel_global_filters: List[str]) -> List[dict]:
+    targets = []
+    for target, ref_id in zip(
+        panel.targets, gen_incrementing_alphabets(len(panel.targets))
+    ):
+        template = copy.deepcopy(TARGET_TEMPLATE)
+        template.update(
+            {
+                "expr": target.expr.format(
+                    global_filters=",".join(panel_global_filters)
+                ),
+                "legendFormat": target.legend,
+                "refId": ref_id,
+            }
+        )
+        targets.append(template)
+    return targets