Add files using upload-large-folder tool

lib/python3.12/site-packages/ray/autoscaler/__init__.py

@@ -0,0 +1,8 @@
+import os
+from pathlib import Path
+
+from ray.autoscaler import sdk
+
+__all__ = ["sdk"]
+
+AUTOSCALER_DIR_PATH = Path(os.path.abspath(os.path.dirname(__file__)))

lib/python3.12/site-packages/ray/autoscaler/_private/_azure/azure-config-template.json

@@ -0,0 +1,137 @@
+{
+    "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
+    "contentVersion": "1.0.0.0",
+    "parameters": {
+        "clusterId": {
+            "type": "string",
+            "metadata": {
+                "description": "Unique string appended to resource names to isolate resources from different ray clusters."
+            }
+        },
+        "subnet": {
+            "type": "string",
+            "metadata": {
+                "description": "Subnet parameters."
+            }
+        },
+        "msiName": {
+            "type": "string",
+            "metadata": {
+                "description": "Managed service identity."
+            }
+        },
+        "msiResourceGroup": {
+            "type": "string",
+            "metadata": {
+                "description": "Managed service identity resource group."
+            }
+        },
+        "createMsi": {
+            "type": "bool",
+            "defaultValue": "true"
+        },
+        "roleAssignmentGuid": {
+            "type": "string",
+            "metadata": {
+                "description": "Deterministic resource name for the MSI role assignment."
+            }
+        }
+    },
+    "variables": {
+        "contributor": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b24988ac-6180-42a0-ab88-20f7382dd24c')]",
+        "location": "[resourceGroup().location]",
+        "roleAssignmentName": "[concat('ray-', parameters('clusterId'), '-ra')]",
+        "nsgName": "[concat('ray-', parameters('clusterId'), '-nsg')]",
+        "nsg": "[resourceId('Microsoft.Network/networkSecurityGroups', variables('nsgName'))]",
+        "vnetName": "[concat('ray-', parameters('clusterId'), '-vnet')]",
+        "subnetName": "[concat('ray-', parameters('clusterId'), '-subnet')]"
+    },
+    "resources": [
+       {
+            "condition": "[parameters('createMsi')]",
+            "type": "Microsoft.ManagedIdentity/userAssignedIdentities",
+            "apiVersion": "2024-11-30",
+            "location": "[variables('location')]",
+            "name": "[parameters('msiName')]"
+        },
+        {
+            "type": "Microsoft.Authorization/roleAssignments",
+            "apiVersion": "2022-04-01",
+            "name": "[parameters('roleAssignmentGuid')]",
+            "properties": {
+                "principalId": "[reference(resourceId(parameters('msiResourceGroup'), 'Microsoft.ManagedIdentity/userAssignedIdentities', parameters('msiName')), '2024-11-30').principalId]",
+                "roleDefinitionId": "[variables('contributor')]",
+                "scope": "[resourceGroup().id]",
+                "principalType": "ServicePrincipal",
+                "description": "[concat('Ray autoscaler cluster ', parameters('clusterId'))]"
+            },
+            "dependsOn": [
+                "[parameters('msiName')]"
+            ]
+        },
+        {
+            "type": "Microsoft.Network/networkSecurityGroups",
+            "apiVersion": "2024-10-01",
+            "name": "[variables('nsgName')]",
+            "location": "[variables('location')]",
+            "properties": {
+                "securityRules": [
+                    {
+                        "name": "SSH",
+                        "properties": {
+                            "priority": 1000,
+                            "protocol": "Tcp",
+                            "access": "Allow",
+                            "direction": "Inbound",
+                            "sourceAddressPrefix": "*",
+                            "sourcePortRange": "*",
+                            "destinationAddressPrefix": "*",
+                            "destinationPortRange": "22"
+                        }
+                    }
+                ]
+            }
+        },
+        {
+            "type": "Microsoft.Network/virtualNetworks",
+            "apiVersion": "2024-10-01",
+            "name": "[variables('vnetName')]",
+            "location": "[variables('location')]",
+            "properties": {
+                "addressSpace": {
+                    "addressPrefixes": [
+                        "[parameters('subnet')]"
+                    ]
+                },
+                "subnets": [
+                    {
+                        "name": "[variables('subnetName')]",
+                        "properties": {
+                            "addressPrefix": "[parameters('subnet')]",
+                            "networkSecurityGroup": {
+                                "id": "[variables('nsg')]"
+                              }
+                        }
+                    }
+                ]
+            },
+            "dependsOn": [
+                "[variables('nsg')]"
+            ]
+        }
+    ],
+    "outputs": {
+        "subnet": {
+            "type": "string",
+            "value": "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('vnetName'), variables('subnetName'))]"
+        },
+        "nsg": {
+            "type": "string",
+            "value": "[variables('nsg')]"
+        },
+        "msi": {
+            "type": "string",
+            "value": "[resourceId(parameters('msiResourceGroup'), 'Microsoft.ManagedIdentity/userAssignedIdentities', parameters('msiName'))]"
+        }
+    }
+}

lib/python3.12/site-packages/ray/autoscaler/_private/_azure/azure-vm-template.json

@@ -0,0 +1,318 @@
+{
+    "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
+    "contentVersion": "1.0.0.0",
+    "parameters": {
+        "vmName": {
+            "type": "string",
+            "metadata": {
+                "description": "The name of you Virtual Machine."
+            }
+        },
+        "adminUsername": {
+            "type": "string",
+            "metadata": {
+                "description": "Username for the Virtual Machine."
+            }
+        },
+        "publicKey": {
+            "type": "securestring",
+            "metadata": {
+                "description": "SSH Key for the Virtual Machine"
+            }
+        },
+        "imagePublisher": {
+            "type": "string",
+            "metadata": {
+                "description": "The publisher of the VM image"
+            },
+	    "defaultValue": ""
+        },
+        "imageOffer": {
+            "type": "string",
+            "metadata": {
+                "description": "The offer of the VM image"
+            },
+	    "defaultValue": ""
+        },
+        "imageSku": {
+            "type": "string",
+            "metadata": {
+                "description": "The sku of the VM image"
+            },
+	    "defaultValue": ""
+        },
+        "imageVersion": {
+            "type": "string",
+            "metadata": {
+                "description": "The version of the VM image"
+            },
+	    "defaultValue": ""
+        },
+	"imageId": {
+	    "type": "string",
+	    "metadata": {
+		"description": "The resource id of the VM image. If provided, it will override the imagePublisher, imageOffer, imageSku and imageVersion parameters."
+	    },
+	    "defaultValue": ""
+	},
+        "vmSize": {
+            "type": "string",
+            "metadata": {
+                "description": "The size of the VM"
+            }
+        },
+        "vmTags": {
+            "type": "object",
+            "metadata": {
+                "description": "Tags for the VM"
+            }
+        },
+        "vmCount": {
+            "type": "int",
+            "metadata": {
+                "description": "Number of VMs to deploy"
+            }
+        },
+	"osDiskSize": {
+	    "type": "int",
+	    "defaultValue": 0,
+	    "metadata": {
+		"description": "Size of the OS disk in GB"
+	    }
+	},
+        "provisionPublicIp": {
+            "type": "bool",
+            "defaultValue": true,
+            "metadata": {
+                "description": "If true creates a public ip"
+            }
+        },
+        "priority": {
+            "type": "string",
+            "defaultValue": "Regular",
+            "metadata": {
+                "description": "Specifies the priority for the virtual machine."
+            }
+        },
+        "evictionPolicy": {
+            "type": "string",
+            "defaultValue": "Delete",
+            "metadata": {
+                "description": "Specifies the eviction policy for the virtual machine."
+            }
+        },
+        "billingProfile": {
+            "type": "object",
+            "defaultValue": {},
+            "metadata": {
+                "description": "Specifies the maximum price to pay for Azure Spot VM."
+            }
+        },
+        "msi": {
+            "type": "string",
+            "metadata": {
+                "description": "Managed service identity resource id."
+            }
+        },
+        "nsg": {
+            "type": "string",
+            "metadata": {
+                "description": "Network security group resource id."
+            }
+        },
+        "subnet": {
+            "type": "string",
+            "metadata": {
+                "descriptions": "Subnet resource id."
+            }
+        },
+        "enableAcceleratedNetworking": {
+            "type": "bool",
+            "defaultValue": false,
+            "metadata": {
+                "descriptions": "Whether to enable accelerated networking."
+            }
+        },
+        "zones": {
+            "type": "array",
+            "defaultValue": [],
+            "metadata": {
+                "description": "Availability zones for the virtual machine. If empty, no zones will be specified."
+            }
+        }
+    },
+    "variables": {
+        "location": "[resourceGroup().location]",
+        "useZones": "[greater(length(parameters('zones')), 0)]",
+        "networkInterfaceNamePrivate": "[concat(parameters('vmName'), '-nic')]",
+        "networkInterfaceNamePublic": "[concat(parameters('vmName'), '-nic-public')]",
+        "networkInterfaceName": "[if(parameters('provisionPublicIp'), variables('networkInterfaceNamePublic'), variables('networkInterfaceNamePrivate'))]",
+        "networkIpConfig": "[guid(resourceGroup().id, parameters('vmName'))]",
+        "osDiskType": "Standard_LRS",
+        "publicIpAddressName": "[concat(parameters('vmName'), '-ip')]"
+    },
+    "resources": [
+        {
+            "type": "Microsoft.Network/networkInterfaces",
+            "apiVersion": "2024-07-01",
+            "name": "[concat(variables('networkInterfaceNamePublic'), copyIndex())]",
+            "location": "[variables('location')]",
+            "dependsOn": [
+                "[resourceId('Microsoft.Network/publicIpAddresses/', concat(variables('publicIpAddressName'), copyIndex()))]"
+            ],
+            "copy": {
+                "name": "NICPublicCopy",
+                "count": "[parameters('vmCount')]"
+            },
+            "properties": {
+                "ipConfigurations": [
+                    {
+                        "name": "[variables('networkIpConfig')]",
+                        "properties": {
+                            "subnet": {
+                                "id": "[parameters('subnet')]"
+                            },
+                            "privateIPAllocationMethod": "Dynamic",
+                            "publicIpAddress": {
+                                "id": "[resourceId('Microsoft.Network/publicIPAddresses', concat(variables('publicIPAddressName'), copyIndex()))]"
+                            }
+                        }
+                    }
+                ],
+                "networkSecurityGroup": {
+                    "id": "[parameters('nsg')]"
+                },
+                "enableAcceleratedNetworking": "[parameters('enableAcceleratedNetworking')]"
+            },
+            "condition": "[parameters('provisionPublicIp')]"
+        },
+        {
+            "type": "Microsoft.Network/networkInterfaces",
+            "apiVersion": "2024-07-01",
+            "name": "[concat(variables('networkInterfaceNamePrivate'), copyIndex())]",
+            "location": "[variables('location')]",
+            "copy": {
+                "name": "NICPrivateCopy",
+                "count": "[parameters('vmCount')]"
+            },
+            "properties": {
+                "ipConfigurations": [
+                    {
+                        "name": "[variables('networkIpConfig')]",
+                        "properties": {
+                            "subnet": {
+                                "id": "[parameters('subnet')]"
+                            },
+                            "privateIPAllocationMethod": "Dynamic"
+                        }
+                    }
+                ],
+                "networkSecurityGroup": {
+                    "id": "[parameters('nsg')]"
+                },
+                "enableAcceleratedNetworking": "[parameters('enableAcceleratedNetworking')]"
+            },
+            "condition": "[not(parameters('provisionPublicIp'))]"
+        },
+        {
+            "type": "Microsoft.Network/publicIpAddresses",
+            "apiVersion": "2024-07-01",
+            "name": "[concat(variables('publicIpAddressName'), copyIndex())]",
+            "location": "[variables('location')]",
+            "properties": {
+                "publicIpAllocationMethod": "Static",
+                "publicIPAddressVersion": "IPv4"
+            },
+            "copy": {
+                "name": "PublicIpCopy",
+                "count": "[parameters('vmCount')]"
+            },
+            "zones": "[if(variables('useZones'), createArray(string(parameters('zones')[mod(copyIndex(), length(parameters('zones')))])), json('null'))]",
+            "sku": {
+                "name": "Standard",
+                "tier": "Regional"
+            },
+            "condition": "[parameters('provisionPublicIp')]"
+        },
+        {
+            "type": "Microsoft.Compute/virtualMachines",
+            "apiVersion": "2024-07-01",
+            "name": "[concat(parameters('vmName'), copyIndex())]",
+            "location": "[variables('location')]",
+            "dependsOn": [
+                "[resourceId('Microsoft.Network/networkInterfaces/', concat(variables('networkInterfaceName'), copyIndex()))]"
+            ],
+            "copy": {
+                "name": "VmCopy",
+                "count": "[parameters('vmCount')]"
+            },
+            "tags": "[parameters('vmTags')]",
+            "zones": "[if(variables('useZones'), createArray(string(parameters('zones')[mod(copyIndex(), length(parameters('zones')))])), json('null'))]",
+            "properties": {
+                "hardwareProfile": {
+                    "vmSize": "[parameters('vmSize')]"
+                },
+                "storageProfile": {
+                    "osDisk": {
+                        "createOption": "fromImage",
+			"diskSizeGB": "[if(equals(parameters('osDiskSize'), 0), json('null'), parameters('osDiskSize'))]",
+                        "managedDisk": {
+                            "storageAccountType": "[variables('osDiskType')]"
+                        }
+                    },
+                    "imageReference": "[if(equals(parameters('imageId'), ''), json(concat('{\"publisher\":\"', parameters('imagePublisher'), '\",\"offer\":\"', parameters('imageOffer'), '\",\"sku\":\"', parameters('imageSku'), '\",\"version\":\"', parameters('imageVersion'), '\"}')), json(concat('{\"id\":\"', parameters('imageId'), '\"}')))]"
+                },
+                "networkProfile": {
+                    "networkInterfaces": [
+                        {
+                            "id": "[resourceId('Microsoft.Network/networkInterfaces', concat(variables('networkInterfaceName'), copyIndex()))]"
+                        }
+                    ]
+                },
+                "osProfile": {
+                    "computerName": "[concat(parameters('vmName'), copyIndex())]",
+                    "adminUsername": "[parameters('adminUsername')]",
+                    "adminPassword": "[parameters('publicKey')]",
+                    "linuxConfiguration": {
+                        "disablePasswordAuthentication": true,
+                        "ssh": {
+                            "publicKeys": [
+                                {
+                                    "path": "[concat('/home/', parameters('adminUsername'), '/.ssh/authorized_keys')]",
+                                    "keyData": "[parameters('publicKey')]"
+                                }
+                            ]
+                        }
+                    }
+                },
+                "priority": "[parameters('priority')]",
+                "evictionPolicy": "[if(equals(parameters('priority'), 'Spot'), parameters('evictionPolicy'), '')]",
+                "billingProfile": "[parameters('billingProfile')]"
+            },
+            "identity": {
+                "type": "UserAssigned",
+                "userAssignedIdentities": {
+                    "[parameters('msi')]": {
+                    }
+                }
+            }
+        }
+    ],
+    "outputs": {
+        "publicIp": {
+            "type": "array",
+            "copy": {
+                "count": "[parameters('vmCount')]",
+                "input": "[reference(concat(variables('publicIpAddressName'), copyIndex())).ipAddress]"
+            },
+            "condition": "[parameters('provisionPublicIp')]"
+        },
+        "privateIp": {
+            "type": "array",
+            "copy": {
+                "count": "[parameters('vmCount')]",
+                "input": "[reference(concat(variables('networkInterfaceName'), copyIndex())).ipConfigurations[0].properties.privateIPAddress]"
+            }
+        }
+    }
+}

lib/python3.12/site-packages/ray/autoscaler/_private/_azure/config.py

@@ -0,0 +1,580 @@
+import json
+import logging
+import os
+import random
+import time
+from hashlib import md5, sha256
+from pathlib import Path
+from typing import Any, Callable
+from uuid import UUID
+
+from azure.common.credentials import get_cli_profile
+from azure.core.exceptions import HttpResponseError, ResourceNotFoundError
+from azure.identity import AzureCliCredential
+from azure.mgmt.resource import ResourceManagementClient
+from azure.mgmt.resource.resources.models import DeploymentMode
+
+from ray.autoscaler._private.util import (
+    generate_rsa_key_pair,
+    generate_ssh_key_name,
+    generate_ssh_key_paths,
+)
+
+# Built-in Azure Contributor role definition ID used for role assignments.
+CONTRIBUTOR_ROLE_DEFINITION_ID = "b24988ac-6180-42a0-ab88-20f7382dd24c"
+
+UNIQUE_ID_LEN = 4
+
+logger = logging.getLogger(__name__)
+
+
+def get_azure_sdk_function(client: Any, function_name: str) -> Callable:
+    """Retrieve a callable function from Azure SDK client object.
+
+    Newer versions of the various client SDKs renamed function names to
+    have a begin_ prefix. This function supports both the old and new
+    versions of the SDK by first trying the old name and falling back to
+    the prefixed new name.
+    """
+    func = getattr(
+        client, function_name, getattr(client, f"begin_{function_name}", None)
+    )
+    if func is None:
+        raise AttributeError(
+            "'{obj}' object has no {func} or begin_{func} attribute".format(
+                obj={client.__name__}, func=function_name
+            )
+        )
+    return func
+
+
+def bootstrap_azure(config):
+    config = _configure_key_pair(config)
+    config = _configure_resource_group(config)
+    return config
+
+
+def _configure_resource_group(config):
+    # TODO: look at availability sets
+    # https://docs.microsoft.com/en-us/azure/virtual-machines/windows/tutorial-availability-sets
+    subscription_id = config["provider"].get("subscription_id")
+    if subscription_id is None:
+        subscription_id = get_cli_profile().get_subscription_id()
+    resource_client = ResourceManagementClient(AzureCliCredential(), subscription_id)
+    config["provider"]["subscription_id"] = subscription_id
+    logger.info("Using subscription id: %s", subscription_id)
+
+    assert (
+        "resource_group" in config["provider"]
+    ), "Provider config must include resource_group field"
+    resource_group = config["provider"]["resource_group"]
+
+    assert (
+        "location" in config["provider"]
+    ), "Provider config must include location field"
+    params = {"location": config["provider"]["location"]}
+
+    if "tags" in config["provider"]:
+        params["tags"] = config["provider"]["tags"]
+
+    logger.info("Creating/Updating resource group: %s", resource_group)
+    rg_create_or_update = get_azure_sdk_function(
+        client=resource_client.resource_groups, function_name="create_or_update"
+    )
+    rg_create_or_update(resource_group_name=resource_group, parameters=params)
+
+    # load the template file
+    current_path = Path(__file__).parent
+    template_path = current_path.joinpath("azure-config-template.json")
+    with open(template_path, "r") as template_fp:
+        template = json.load(template_fp)
+
+    logger.info("Using cluster name: %s", config["cluster_name"])
+
+    # set unique id for resources in this cluster
+    unique_id = config["provider"].get("unique_id")
+    if unique_id is None:
+        hasher = sha256()
+        hasher.update(config["provider"]["resource_group"].encode("utf-8"))
+        unique_id = hasher.hexdigest()[:UNIQUE_ID_LEN]
+    else:
+        unique_id = str(unique_id)
+    config["provider"]["unique_id"] = unique_id
+    logger.info("Using unique id: %s", unique_id)
+    cluster_id = "{}-{}".format(config["cluster_name"], unique_id)
+    role_assignment_name = f"ray-{cluster_id}-ra"
+    role_assignment_guid = _generate_arm_guid(role_assignment_name)
+    role_assignment_resource_id = (
+        f"/subscriptions/{subscription_id}/resourceGroups/{resource_group}/providers"
+        f"/Microsoft.Authorization/roleAssignments/{role_assignment_guid}"
+    )
+
+    subnet_mask = config["provider"].get("subnet_mask")
+    if subnet_mask is None:
+        # choose a random subnet, skipping most common value of 0
+        random.seed(unique_id)
+        subnet_mask = "10.{}.0.0/16".format(random.randint(1, 254))
+    logger.info("Using subnet mask: %s", subnet_mask)
+
+    # Copy over properties from existing subnet.
+    # Addresses issue (https://github.com/Azure/azure-quickstart-templates/issues/2786)
+    # where existing subnet properties will get overwritten unless explicitly specified
+    # during multiple deployments even if vnet/subnet do not change.
+    # May eventually be fixed by passing empty subnet list if they already exist:
+    # https://techcommunity.microsoft.com/t5/azure-networking-blog/azure-virtual-network-now-supports-updates-without-subnet/ba-p/4067952
+    list_by_rg = get_azure_sdk_function(
+        client=resource_client.resources, function_name="list_by_resource_group"
+    )
+    existing_vnets = list(
+        list_by_rg(
+            resource_group,
+            f"substringof('{unique_id}', name) and "
+            "resourceType eq 'Microsoft.Network/virtualNetworks'",
+        )
+    )
+    if len(existing_vnets) > 0:
+        vnid = existing_vnets[0].id
+        get_by_id = get_azure_sdk_function(
+            client=resource_client.resources, function_name="get_by_id"
+        )
+
+        # Query for supported API versions for Microsoft.Network/virtualNetworks
+        # because resource_client.DEFAULT_API_VERSION is not always supported.
+        # (Example: "2024-11-01" is the default at the time of this writing)
+        # Use "2024-10-01" as a fallback if we can't determine the latest stable version.
+        vnet_api_version = "2024-10-01"
+        try:
+            # Get supported API versions for Microsoft.Network provider
+            providers = resource_client.providers.get("Microsoft.Network")
+            vnet_resource_type = next(
+                (
+                    rt
+                    for rt in providers.resource_types
+                    if rt.resource_type == "virtualNetworks"
+                ),
+                None,
+            )
+            if vnet_resource_type and vnet_resource_type.api_versions:
+                stable_versions = [
+                    v for v in vnet_resource_type.api_versions if "preview" not in v
+                ]
+                versions_to_consider = (
+                    stable_versions or vnet_resource_type.api_versions
+                )
+                vnet_api_version = sorted(versions_to_consider)[-1]
+                logger.info(
+                    "Using API version: %s for virtualNetworks", vnet_api_version
+                )
+            else:
+                logger.warning(
+                    "Could not determine supported API versions for virtualNetworks, using fallback version %s",
+                    vnet_api_version,
+                )
+        except Exception as e:
+            logger.warning(
+                "Failed to query Microsoft.Network provider: %s. Using fallback API version 2024-10-01",
+                str(e),
+            )
+
+        subnet = get_by_id(vnid, vnet_api_version).properties["subnets"][0]
+        template_vnet = next(
+            (
+                rs
+                for rs in template["resources"]
+                if rs["type"] == "Microsoft.Network/virtualNetworks"
+            ),
+            None,
+        )
+        if template_vnet is not None:
+            template_subnets = template_vnet["properties"].get("subnets")
+            if template_subnets is not None:
+                template_subnets[0]["properties"].update(subnet["properties"])
+
+    # Get or create an MSI name and resource group.
+    # Defaults to current resource group if not provided.
+    use_existing_msi = (
+        "msi_name" in config["provider"] and "msi_resource_group" in config["provider"]
+    )
+    msi_resource_group = config["provider"].get("msi_resource_group", resource_group)
+    msi_name = config["provider"].get("msi_name", f"ray-{cluster_id}-msi")
+    logger.info(
+        "Using msi_name: %s from msi_resource_group: %s", msi_name, msi_resource_group
+    )
+
+    existing_principal_id = None
+    if not use_existing_msi:
+        # When creating a MSI for managing Azure resources, we first need to clean up
+        # any role assignments from the previous MSI's principal ID to avoid
+        # orphaned permissions when the MSI gets recreated with a new principal ID.
+        # Role assignments cannot be updated, only created/removed, so cleanup is required.
+        msi_resource_id = (
+            f"/subscriptions/{subscription_id}/resourceGroups/{msi_resource_group}"
+            f"/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{msi_name}"
+        )
+        try:
+            get_identity = get_azure_sdk_function(
+                client=resource_client.resources, function_name="get_by_id"
+            )
+            existing_msi = get_identity(msi_resource_id, "2023-01-31")
+            existing_principal_id = getattr(existing_msi, "properties", {}).get(
+                "principalId"
+            )
+        except ResourceNotFoundError:
+            existing_principal_id = None
+        except Exception as exc:
+            logger.warning(
+                "Failed to query MSI %s for existing principal: %s",
+                msi_name,
+                exc,
+            )
+
+        if existing_principal_id:
+            logger.info(
+                "Removing existing role assignments for MSI principal %s before recreation",
+                existing_principal_id,
+            )
+            _delete_role_assignments_for_principal(
+                resource_client,
+                resource_group,
+                existing_principal_id,
+            )
+
+        delete_role_assignment = get_azure_sdk_function(
+            client=resource_client.resources, function_name="delete_by_id"
+        )
+        get_role_assignment = get_azure_sdk_function(
+            client=resource_client.resources, function_name="get_by_id"
+        )
+
+        role_assignment_known_missing = False
+        initial_query_failed = False
+        try:
+            get_role_assignment(
+                role_assignment_resource_id,
+                "2022-04-01",
+            )
+        except ResourceNotFoundError:
+            role_assignment_known_missing = True
+            logger.debug(
+                "Role assignment %s not found before MSI creation; skipping deletion",
+                role_assignment_guid,
+            )
+        except Exception as exc:
+            logger.warning(
+                "Failed to query role assignment %s before deletion: %s",
+                role_assignment_guid,
+                exc,
+            )
+            initial_query_failed = True
+
+        if not role_assignment_known_missing:
+            try:
+                delete_lro = delete_role_assignment(
+                    resource_id=role_assignment_resource_id,
+                    api_version="2022-04-01",
+                )
+                if hasattr(delete_lro, "wait"):
+                    delete_lro.wait()
+                logger.info(
+                    "Deleted existing role assignment %s before recreating MSI",
+                    role_assignment_guid,
+                )
+
+                if initial_query_failed:
+                    logger.debug(
+                        "Retrying verification for role assignment %s",
+                        role_assignment_guid,
+                    )
+
+                if not _wait_for_role_assignment_deletion(
+                    get_role_assignment,
+                    role_assignment_resource_id,
+                    role_assignment_guid,
+                ):
+                    logger.warning(
+                        "Role assignment %s not confirmed deleted",
+                        role_assignment_guid,
+                    )
+            except ResourceNotFoundError:
+                logger.debug(
+                    "Role assignment %s disappeared before deletion attempt; continuing",
+                    role_assignment_guid,
+                )
+            except Exception as e:
+                logger.warning(
+                    "Failed to delete role assignment %s before MSI creation: %s",
+                    role_assignment_guid,
+                    e,
+                )
+
+    parameters = {
+        "properties": {
+            "mode": DeploymentMode.incremental,
+            "template": template,
+            "parameters": {
+                "subnet": {"value": subnet_mask},
+                "clusterId": {"value": cluster_id},
+                "msiName": {"value": msi_name},
+                "msiResourceGroup": {"value": msi_resource_group},
+                "createMsi": {"value": not use_existing_msi},
+                "roleAssignmentGuid": {"value": role_assignment_guid},
+            },
+        }
+    }
+
+    create_or_update = get_azure_sdk_function(
+        client=resource_client.deployments, function_name="create_or_update"
+    )
+    outputs = (
+        create_or_update(
+            resource_group_name=resource_group,
+            deployment_name="ray-config",
+            parameters=parameters,
+        )
+        .result()
+        .properties.outputs
+    )
+
+    # append output resource ids to be used with vm creation
+    config["provider"]["msi"] = outputs["msi"]["value"]
+    config["provider"]["nsg"] = outputs["nsg"]["value"]
+    config["provider"]["subnet"] = outputs["subnet"]["value"]
+
+    return config
+
+
+def _configure_key_pair(config):
+    """
+    Configure SSH keypair. Use user specified custom paths, otherwise,
+    generate Ray-specific keypair in this format: "ray-autoscaler_azure_{region}_{resource_group}_{ssh_user}_{index}"
+    """
+    ssh_user = config["auth"]["ssh_user"]
+    public_key = None
+
+    # Check if user specified custom SSH key paths
+    user_specified_private_key = "ssh_private_key" in config["auth"]
+    user_specified_public_key = "ssh_public_key" in config["auth"]
+
+    # Validate that the user either specfied both keys or none, but not just one
+    if user_specified_private_key != user_specified_public_key:
+        if user_specified_private_key:
+            missing_key, specified_key = "ssh_public_key", "ssh_private_key"
+        else:
+            missing_key, specified_key = "ssh_private_key", "ssh_public_key"
+        raise ValueError(
+            f"{specified_key} is specified but {missing_key} is missing. "
+            "Both SSH key paths must be specified together, or omit both from "
+            "your config to use auto-generated keys."
+        )
+
+    if user_specified_private_key and user_specified_public_key:
+        # User specified custom paths
+        private_key_path = Path(config["auth"]["ssh_private_key"]).expanduser()
+        public_key_path = Path(config["auth"]["ssh_public_key"]).expanduser()
+
+        # Validate that user-specified keys exist
+        missing_keys = []
+        if not private_key_path.is_file():
+            missing_keys.append(f"ssh_private_key: {private_key_path}")
+        if not public_key_path.is_file():
+            missing_keys.append(f"ssh_public_key: {public_key_path}")
+
+        if missing_keys:
+            raise ValueError(
+                "SSH key files from config do not exist: {}. "
+                "Please create the keys or remove the custom paths from your config "
+                "to use auto-generated keys.".format(", ".join(missing_keys))
+            )
+        logger.info(
+            "Using specified SSH keys from config: {} and {}".format(
+                private_key_path, public_key_path
+            )
+        )
+
+        with open(public_key_path, "r") as f:
+            public_key = f.read()
+    else:
+        # Generate Ray-specific keys
+        region = config["provider"]["location"]
+        resource_group = config["provider"]["resource_group"]
+
+        # Generate single deterministic key name for this configuration
+        key_name = generate_ssh_key_name(
+            "azure", None, region, resource_group, ssh_user
+        )
+        public_key_path, private_key_path = generate_ssh_key_paths(key_name)
+
+        # Check if this key pair already exists
+        if os.path.exists(private_key_path) and os.path.exists(public_key_path):
+            logger.info(
+                "Using existing Ray-specific SSH keys: {} and {}".format(
+                    private_key_path, public_key_path
+                )
+            )
+            with open(public_key_path, "r") as f:
+                public_key = f.read()
+        else:
+            # Create a key pair since it doesn't exist locally
+            logger.info(
+                "Generating new Ray-specific SSH key pair at {} and {}".format(
+                    private_key_path, public_key_path
+                )
+            )
+            os.makedirs(os.path.dirname(private_key_path), exist_ok=True)
+            public_key, private_key = generate_rsa_key_pair()
+            with open(
+                private_key_path,
+                "w",
+                opener=lambda path, flags: os.open(path, flags, 0o600),
+            ) as f:
+                f.write(private_key)
+            with open(public_key_path, "w") as f:
+                f.write(public_key)
+
+        assert os.path.exists(
+            private_key_path
+        ), "Private key file {} not found for user {}".format(
+            private_key_path, ssh_user
+        )
+
+    config["auth"]["ssh_private_key"] = str(private_key_path)
+    # Remove public key path because bootstrap config must only contain paths that exist on head node
+    config["auth"].pop("ssh_public_key", None)
+
+    for node_type in config["available_node_types"].values():
+        azure_arm_parameters = node_type["node_config"].setdefault(
+            "azure_arm_parameters", {}
+        )
+        azure_arm_parameters["adminUsername"] = ssh_user
+        azure_arm_parameters["publicKey"] = public_key
+
+    return config
+
+
+def _delete_role_assignments_for_principal(
+    resource_client: ResourceManagementClient,
+    resource_group: str,
+    principal_id: str,
+) -> None:
+    """Delete all role assignments in the resource group for the given principal.
+
+    Uses the generic ResourceManagementClient so we avoid depending on
+    azure-mgmt-authorization. All role assignments associated with the
+    provided principal ID are removed.
+    """
+
+    if not principal_id:
+        return
+
+    list_by_rg = get_azure_sdk_function(
+        client=resource_client.resources, function_name="list_by_resource_group"
+    )
+    delete_role_assignment = get_azure_sdk_function(
+        client=resource_client.resources, function_name="delete_by_id"
+    )
+
+    try:
+        assignments = list(
+            list_by_rg(
+                resource_group,
+                "resourceType eq 'Microsoft.Authorization/roleAssignments'",
+            )
+        )
+        logger.debug(
+            "Found %d role assignments in resource group %s while cleaning up principal %s",
+            len(assignments),
+            resource_group,
+            principal_id,
+        )
+    except HttpResponseError as exc:
+        logger.warning(
+            "Failed to enumerate role assignments for resource group %s: %s",
+            resource_group,
+            exc,
+        )
+        return
+
+    for assignment in assignments:
+        properties = getattr(assignment, "properties", {}) or {}
+        logger.debug(
+            "Inspecting role assignment %s with principalId=%s roleDefinitionId=%s",
+            getattr(assignment, "name", "<unknown>"),
+            properties.get("principalId"),
+            properties.get("roleDefinitionId"),
+        )
+        if properties.get("principalId") != principal_id:
+            continue
+
+        try:
+            delete_lro = delete_role_assignment(
+                resource_id=assignment.id,
+                api_version="2022-04-01",
+            )
+            if hasattr(delete_lro, "wait"):
+                delete_lro.wait()
+            logger.info(
+                "Deleted existing role assignment %s for principal %s",
+                assignment.name,
+                principal_id,
+            )
+        except ResourceNotFoundError:
+            logger.debug(
+                "Role assignment %s not found while processing principal %s",
+                assignment.name,
+                principal_id,
+            )
+        except Exception as exc:
+            logger.warning(
+                "Failed to delete role assignment %s for principal %s: %s",
+                assignment.name,
+                principal_id,
+                exc,
+            )
+
+
+def _wait_for_role_assignment_deletion(
+    get_role_assignment: Callable[..., Any],
+    resource_id: str,
+    role_assignment_guid: str,
+    *,
+    max_attempts: int = 10,
+    delay_seconds: int = 2,
+) -> bool:
+    """Poll until a role assignment disappears after deletion.
+
+    Returns True if the assignment is confirmed deleted, False otherwise.
+    Logs detailed progress to aid troubleshooting when transient errors occur.
+    """
+
+    for attempt in range(1, max_attempts + 1):
+        try:
+            get_role_assignment(resource_id, "2022-04-01")
+        except ResourceNotFoundError:
+            return True
+        except Exception as exc:  # noqa: BLE001
+            logger.debug(
+                "Attempt %d/%d to verify removal of role assignment %s failed: %s",
+                attempt,
+                max_attempts,
+                role_assignment_guid,
+                exc,
+            )
+        else:
+            logger.debug(
+                "Role assignment %s still present after deletion (attempt %d/%d)",
+                role_assignment_guid,
+                attempt,
+                max_attempts,
+            )
+
+        if attempt < max_attempts:
+            time.sleep(delay_seconds)
+
+    return False
+
+
+def _generate_arm_guid(*values: Any) -> str:
+    """Replicates ARM template guid() function for creating deterministic IDs."""
+
+    concatenated = "".join(str(v) for v in values)
+    return str(UUID(md5(concatenated.encode("utf-8")).hexdigest()))

lib/python3.12/site-packages/ray/autoscaler/_private/_azure/node_provider.py

@@ -0,0 +1,942 @@
+import json
+import logging
+import os
+import time
+from concurrent.futures import Future, ThreadPoolExecutor
+from pathlib import Path
+from threading import RLock
+from typing import List, Optional
+from uuid import uuid4
+
+from azure.common.credentials import get_cli_profile
+from azure.core.exceptions import ResourceNotFoundError
+from azure.identity import DefaultAzureCredential
+from azure.mgmt.compute import ComputeManagementClient
+from azure.mgmt.network import NetworkManagementClient
+from azure.mgmt.resource import ResourceManagementClient
+from azure.mgmt.resource.resources.models import DeploymentMode
+
+from ray._common.usage.usage_lib import get_cloud_from_metadata_requests
+from ray.autoscaler._private._azure.config import (
+    _delete_role_assignments_for_principal,
+    _generate_arm_guid,
+    bootstrap_azure,
+    get_azure_sdk_function,
+)
+from ray.autoscaler._private.constants import (
+    AUTOSCALER_NODE_START_WAIT_S,
+    AUTOSCALER_NODE_TERMINATE_WAIT_S,
+    MAX_PARALLEL_SHUTDOWN_WORKERS,
+)
+from ray.autoscaler.node_provider import NodeProvider
+from ray.autoscaler.tags import (
+    NODE_KIND_HEAD,
+    TAG_RAY_CLUSTER_NAME,
+    TAG_RAY_LAUNCH_CONFIG,
+    TAG_RAY_NODE_KIND,
+    TAG_RAY_NODE_NAME,
+    TAG_RAY_USER_NODE_TYPE,
+)
+
+VM_NAME_MAX_LEN = 64
+UNIQUE_ID_LEN = 4
+
+logger = logging.getLogger(__name__)
+azure_logger = logging.getLogger("azure.core.pipeline.policies.http_logging_policy")
+azure_logger.setLevel(logging.WARNING)
+
+
+def synchronized(f):
+    def wrapper(self, *args, **kwargs):
+        self.lock.acquire()
+        try:
+            return f(self, *args, **kwargs)
+        finally:
+            self.lock.release()
+
+    return wrapper
+
+
+class AzureNodeProvider(NodeProvider):
+    """Node Provider for Azure
+
+    This provider assumes Azure credentials are set by running ``az login``
+    and the default subscription is configured through ``az account``
+    or set in the ``provider`` field of the autoscaler configuration.
+
+    Nodes may be in one of three states: {pending, running, terminated}. Nodes
+    appear immediately once started by ``create_node``, and transition
+    immediately to terminated when ``terminate_node`` is called.
+    """
+
+    def __init__(self, provider_config, cluster_name):
+        NodeProvider.__init__(self, provider_config, cluster_name)
+        subscription_id = provider_config.get("subscription_id")
+        if subscription_id is None:
+            # Get subscription from logged in azure profile
+            # if it isn't provided in the provider_config
+            # so operations like `get-head-ip` will work
+            subscription_id = get_cli_profile().get_subscription_id()
+            logger.info(
+                "subscription_id not found in provider config, falling back "
+                f"to subscription_id from the logged in azure profile: {subscription_id}"
+            )
+        self.cache_stopped_nodes = provider_config.get("cache_stopped_nodes", True)
+
+        # Detect cloud environment to optimize Azure credential chain.
+        # On non-Azure clouds (AWS, GCP), skip Azure-specific auth methods
+        # (managed identity, workload identity) to avoid IMDS timeout delays / failures.
+        detected_cloud = get_cloud_from_metadata_requests()
+        on_azure = detected_cloud == "azure"
+
+        if on_azure:
+            logger.info(
+                "Initializing Azure node provider for Azure infrastructure "
+                "running on Azure cloud environment"
+            )
+        else:
+            logger.info(
+                f"Initializing Azure node provider for Azure infrastructure "
+                f"but detected this is running on a '{detected_cloud}' environment. "
+                f"Skipping Azure-specific authentication methods to avoid timeouts."
+            )
+
+        credential = DefaultAzureCredential(
+            exclude_shared_token_cache_credential=True,
+            exclude_managed_identity_credential=not on_azure,
+            exclude_workload_identity_credential=not on_azure,
+        )
+        self.compute_client = ComputeManagementClient(credential, subscription_id)
+        self.network_client = NetworkManagementClient(credential, subscription_id)
+        self.resource_client = ResourceManagementClient(credential, subscription_id)
+
+        self.lock = RLock()
+
+        # cache node objects
+        self.cached_nodes = {}
+
+        # Cache terminating node operations
+        self.terminating_nodes: dict[str, Future] = {}
+        self.termination_executor = ThreadPoolExecutor(
+            max_workers=MAX_PARALLEL_SHUTDOWN_WORKERS
+        )
+
+    @synchronized
+    def _get_filtered_nodes(self, tag_filters):
+        # add cluster name filter to only get nodes from this cluster
+        cluster_tag_filters = {**tag_filters, TAG_RAY_CLUSTER_NAME: self.cluster_name}
+
+        def match_tags(tags):
+            for k, v in cluster_tag_filters.items():
+                if tags.get(k) != v:
+                    return False
+            return True
+
+        vms = self.compute_client.virtual_machines.list(
+            resource_group_name=self.provider_config["resource_group"]
+        )
+
+        nodes = [self._extract_metadata(vm) for vm in vms]
+        self.cached_nodes = {node["name"]: node for node in nodes}
+
+        # Update terminating nodes list by removing nodes that
+        # have finished termination.
+        self.terminating_nodes = {
+            k: v for k, v in self.terminating_nodes.items() if not v.done()
+        }
+
+        return {
+            k: v
+            for k, v in self.cached_nodes.items()
+            if v["tags"] is not None and match_tags(v["tags"])
+        }
+
+    def _extract_metadata(self, vm):
+        # get tags
+        metadata = {"name": vm.name, "tags": vm.tags, "status": ""}
+
+        # get status
+        resource_group = self.provider_config["resource_group"]
+        try:
+            instance = self.compute_client.virtual_machines.instance_view(
+                resource_group_name=resource_group, vm_name=vm.name
+            ).as_dict()
+        except ResourceNotFoundError:
+            return metadata
+
+        for status in instance["statuses"]:
+            # If ProvisioningState is "failed" (e.g.,
+            # ProvisioningState/failed/RetryableError), we can get a third
+            # string here, so we need to limit to the first two outputs.
+            code, state = status["code"].split("/")[:2]
+            # skip provisioning status
+            if code == "PowerState":
+                metadata["status"] = state
+                break
+
+        # get ip data
+        nic_id = vm.network_profile.network_interfaces[0].id
+        metadata["nic_name"] = nic_id.split("/")[-1]
+        nic = self.network_client.network_interfaces.get(
+            resource_group_name=resource_group,
+            network_interface_name=metadata["nic_name"],
+        )
+        ip_config = nic.ip_configurations[0]
+
+        # Get public IP if not using internal IPs or if this is the
+        # head node and use_external_head_ip is True
+        if not self.provider_config.get("use_internal_ips", False) or (
+            self.provider_config.get("use_external_head_ip", False)
+            and metadata["tags"][TAG_RAY_NODE_KIND] == NODE_KIND_HEAD
+        ):
+            public_ip_id = ip_config.public_ip_address.id
+            metadata["public_ip_name"] = public_ip_id.split("/")[-1]
+            public_ip = self.network_client.public_ip_addresses.get(
+                resource_group_name=resource_group,
+                public_ip_address_name=metadata["public_ip_name"],
+            )
+            metadata["external_ip"] = public_ip.ip_address
+
+        metadata["internal_ip"] = ip_config.private_ip_address
+
+        return metadata
+
+    def _get_zones_for_vm_size(self, vm_size, location):
+        """Get usable availability zones for a given VM size in a specific location."""
+        try:
+            # Note: Azure ResourceSKUs API filters don't work reliably(?), so we query all SKUs
+            # and filter in code. Each SKU object represents one location for the VM size.
+            skus = self.compute_client.resource_skus.list()
+
+            for sku in skus:
+                if sku.name == vm_size and sku.location_info:
+                    # Each SKU object represents one location, check if it matches our target
+                    for location_info in sku.location_info:
+                        if location_info.location.lower() == location.lower():
+                            zones = location_info.zones if location_info.zones else []
+                            logger.debug(
+                                f"Found {vm_size} in {location} with zones: {zones}"
+                            )
+                            return sorted(zones)
+
+            logger.warning(f"No zones found for {vm_size} in {location}")
+            return []  # No zones available for this VM size
+        except Exception as e:
+            logger.warning(
+                f"Failed to get zones for VM size {vm_size} in location {location}: {str(e)}"
+            )
+            return []
+
+    def _parse_availability_zones(
+        self, availability_zone_config: Optional[str]
+    ) -> Optional[List[str]]:
+        """Parse availability_zone configuration from comma-separated string format.
+
+        Args:
+            availability_zone_config: Can be:
+                - String: comma-separated zones like "1,2,3"
+                - "none": explicitly disable zones
+                - "auto": let Azure automatically pick zones
+                - None: no zones specified (defaults to letting Azure pick)
+
+        Returns:
+            List of zone strings, or None if zones explicitly disabled, or [] if auto/unspecified
+        """
+        if availability_zone_config is None:
+            return []  # Auto - let Azure pick
+
+        # Handle string format (AWS-style comma-separated)
+        if isinstance(availability_zone_config, str):
+            # Strip whitespace and split by comma
+            zones = [zone.strip() for zone in availability_zone_config.split(",")]
+
+            # Handle special cases (case-insensitive)
+            if len(zones) == 1:
+                zone_lower = zones[0].lower()
+                if zone_lower in ["none", "null"]:
+                    return None  # Explicitly disabled
+                elif zone_lower == "auto":
+                    return []  # Auto - let Azure pick
+
+            # Handle empty string or whitespace-only
+            if not zones or all(not zone for zone in zones):
+                return []  # Auto - let Azure pick
+            return zones
+
+        # Unsupported format
+        raise ValueError(
+            f"availability_zone must be a string, got {type(availability_zone_config).__name__}: {availability_zone_config!r}"
+        )
+
+    def _validate_zones_for_node_pool(self, zones, location, vm_size):
+        """Validate that the specified zones are available for the given VM size in the location."""
+        # Special case: zones=None means explicitly disabled availability zones
+        if zones is None:
+            logger.info(
+                "Zones explicitly disabled with 'none' - will create VM without an availability zone"
+            )
+            return None  # Special return value to indicate "no zones by choice"
+
+        vm_zones = self._get_zones_for_vm_size(vm_size, location)
+
+        available_zones = set(vm_zones)
+        if not available_zones:
+            logger.warning("No zones available for this VM size and location")
+            return []
+
+        if zones:
+            requested_zones = {str(z) for z in zones}
+            intersection = sorted(available_zones.intersection(requested_zones))
+            return intersection
+
+        return sorted(available_zones)
+
+    def stopped_nodes(self, tag_filters):
+        """Return a list of stopped node ids filtered by the specified tags dict."""
+        nodes = self._get_filtered_nodes(tag_filters=tag_filters)
+        return [k for k, v in nodes.items() if v["status"].startswith("deallocat")]
+
+    def non_terminated_nodes(self, tag_filters):
+        """Return a list of node ids filtered by the specified tags dict.
+
+        This list must not include terminated nodes. For performance reasons,
+        providers are allowed to cache the result of a call to nodes() to
+        serve single-node queries (e.g. is_running(node_id)). This means that
+        nodes() must be called again to refresh results.
+
+        Examples:
+            >>> from ray.autoscaler.tags import TAG_RAY_NODE_KIND
+            >>> provider = ... # doctest: +SKIP
+            >>> provider.non_terminated_nodes( # doctest: +SKIP
+            ...     {TAG_RAY_NODE_KIND: "worker"})
+            ["node-1", "node-2"]
+        """
+        nodes = self._get_filtered_nodes(tag_filters=tag_filters)
+        return [
+            k
+            for k, v in nodes.items()
+            if not v["status"].startswith("deallocat") or k in self.terminating_nodes
+        ]
+
+    def is_running(self, node_id):
+        """Return whether the specified node is running."""
+        # always get current status
+        node = self._get_node(node_id=node_id)
+        return node["status"] == "running"
+
+    def is_terminated(self, node_id):
+        """Return whether the specified node is terminated."""
+        # always get current status
+        node = self._get_node(node_id=node_id)
+        return node["status"].startswith("deallocat")
+
+    def node_tags(self, node_id):
+        """Returns the tags of the given node (string dict)."""
+        return self._get_cached_node(node_id=node_id)["tags"]
+
+    def external_ip(self, node_id):
+        """Returns the external ip of the given node."""
+        ip = (
+            self._get_cached_node(node_id=node_id)["external_ip"]
+            or self._get_node(node_id=node_id)["external_ip"]
+        )
+        return ip
+
+    def internal_ip(self, node_id):
+        """Returns the internal ip (Ray ip) of the given node."""
+        ip = (
+            self._get_cached_node(node_id=node_id)["internal_ip"]
+            or self._get_node(node_id=node_id)["internal_ip"]
+        )
+        return ip
+
+    def create_node(self, node_config, tags, count):
+        resource_group = self.provider_config["resource_group"]
+
+        if self.cache_stopped_nodes:
+            VALIDITY_TAGS = [
+                TAG_RAY_CLUSTER_NAME,
+                TAG_RAY_NODE_KIND,
+                TAG_RAY_LAUNCH_CONFIG,
+                TAG_RAY_USER_NODE_TYPE,
+            ]
+            filters = {tag: tags[tag] for tag in VALIDITY_TAGS if tag in tags}
+            reuse_nodes = self.stopped_nodes(filters)[:count]
+            logger.info(
+                f"Reusing nodes {list(reuse_nodes)}. "
+                "To disable reuse, set `cache_stopped_nodes: False` "
+                "under `provider` in the cluster configuration.",
+            )
+            start = get_azure_sdk_function(
+                client=self.compute_client.virtual_machines, function_name="start"
+            )
+            for node_id in reuse_nodes:
+                start(resource_group_name=resource_group, vm_name=node_id).wait()
+                self.set_node_tags(node_id, tags)
+            count -= len(reuse_nodes)
+
+        if count:
+            self._create_node(node_config, tags, count)
+
+    def _create_node(self, node_config, tags, count):
+        """Creates a number of nodes within the namespace."""
+        resource_group = self.provider_config["resource_group"]
+        location = self.provider_config["location"]
+        vm_size = node_config["azure_arm_parameters"]["vmSize"]
+
+        # Determine availability zones with precedence: node-level > provider-level
+        # Check for "availability_zone" field in node config first
+        node_availability_zone = node_config.get("azure_arm_parameters", {}).get(
+            "availability_zone"
+        )
+        # Then check provider-level "availability_zone"
+        provider_availability_zone = self.provider_config.get("availability_zone")
+
+        requested_zones = []
+        zone_source = "default"
+
+        # Precedence: node availability_zone > provider availability_zone
+        if node_availability_zone is not None:
+            requested_zones = self._parse_availability_zones(node_availability_zone)
+            zone_source = "node config availability_zone"
+        elif provider_availability_zone is not None:
+            requested_zones = self._parse_availability_zones(provider_availability_zone)
+            zone_source = "provider availability_zone"
+
+        logger.info(f"Requested zones from {zone_source}: {requested_zones}")
+
+        # Get actually available zones for this VM size
+        available_zones = self._validate_zones_for_node_pool(
+            requested_zones, location, vm_size
+        )
+
+        # Handle explicit zone disabling
+        zones_explicitly_disabled = available_zones is None
+
+        if requested_zones and not zones_explicitly_disabled and not available_zones:
+            raise ValueError(
+                f"No available zones for VM size {vm_size} in {location}. "
+                f"Requested: {requested_zones}, but none are available for this VM size."
+            )
+
+        # load the template file
+        current_path = Path(__file__).parent
+        template_path = current_path.joinpath("azure-vm-template.json")
+        with open(template_path, "r") as template_fp:
+            template = json.load(template_fp)
+
+        # get the tags
+        config_tags = node_config.get("tags", {}).copy()
+        config_tags.update(tags)
+        config_tags[TAG_RAY_CLUSTER_NAME] = self.cluster_name
+
+        deployment_name = "{node}-{unique_id}-{vm_id}".format(
+            node=config_tags.get(TAG_RAY_NODE_NAME, "node"),
+            unique_id=self.provider_config["unique_id"],
+            vm_id=uuid4().hex[:UNIQUE_ID_LEN],
+        )[:VM_NAME_MAX_LEN]
+
+        template_params = node_config["azure_arm_parameters"].copy()
+        # Remove availability_zone from template params since ARM template expects "zones"
+        template_params.pop("availability_zone", None)
+        # Use deployment_name for the vmName template parameter since
+        # the template will append copyIndex() for each VM that gets created
+        # to guarantee uniqueness.
+        template_params["vmName"] = deployment_name
+        # Provision public IP if not using internal IPs or if this is the
+        # head node and use_external_head_ip is True
+        template_params["provisionPublicIp"] = not self.provider_config.get(
+            "use_internal_ips", False
+        ) or (
+            self.provider_config.get("use_external_head_ip", False)
+            and config_tags[TAG_RAY_NODE_KIND] == NODE_KIND_HEAD
+        )
+        template_params["vmTags"] = config_tags
+        template_params["vmCount"] = count
+        template_params["msi"] = self.provider_config["msi"]
+        template_params["nsg"] = self.provider_config["nsg"]
+        template_params["subnet"] = self.provider_config["subnet"]
+
+        # Add zone information based on availability and requested zones
+        if zones_explicitly_disabled:
+            # User explicitly disabled zones with ["None"]
+            template_params["zones"] = []
+            logger.info(
+                f"Creating {count} VMs with zones explicitly disabled (no availability zone)"
+            )
+        elif available_zones:
+            # Pass the list of available zones to the template
+            template_params["zones"] = available_zones
+            logger.info(
+                f"Creating {count} VMs, distributed across availability zones: {available_zones}"
+            )
+        else:
+            # For non-zonal deployments (no zones available), use empty array
+            template_params["zones"] = []
+            logger.info(f"Creating {count} VMs without specific availability zone")
+
+        parameters = {
+            "properties": {
+                "mode": DeploymentMode.incremental,
+                "template": template,
+                "parameters": {
+                    key: {"value": value} for key, value in template_params.items()
+                },
+            }
+        }
+
+        # TODO: we could get the private/public ips back directly
+        create_or_update = get_azure_sdk_function(
+            client=self.resource_client.deployments,
+            function_name="create_or_update",
+        )
+        create_or_update(
+            resource_group_name=resource_group,
+            deployment_name=deployment_name,
+            parameters=parameters,
+        ).wait(timeout=AUTOSCALER_NODE_START_WAIT_S)
+
+    @synchronized
+    def set_node_tags(self, node_id, tags):
+        """Sets the tag values (string dict) for the specified node."""
+        node_tags = self._get_cached_node(node_id)["tags"]
+        node_tags.update(tags)
+        update = get_azure_sdk_function(
+            client=self.compute_client.virtual_machines, function_name="update"
+        )
+        update(
+            resource_group_name=self.provider_config["resource_group"],
+            vm_name=node_id,
+            parameters={"tags": node_tags},
+        )
+        self.cached_nodes[node_id]["tags"] = node_tags
+
+    def terminate_node(self, node_id):
+        """Terminates the specified node. This will delete the VM and
+        associated resources (NIC, IP, Storage) for the specified node."""
+
+        resource_group = self.provider_config["resource_group"]
+
+        if self.cache_stopped_nodes:
+            try:
+                # stop machine and leave all resources
+                logger.info(
+                    f"Stopping instance {node_id}"
+                    "(to fully terminate instead, "
+                    "set `cache_stopped_nodes: False` "
+                    "under `provider` in the cluster configuration)"
+                )
+                stop = get_azure_sdk_function(
+                    client=self.compute_client.virtual_machines,
+                    function_name="deallocate",
+                )
+                stop(resource_group_name=resource_group, vm_name=node_id)
+            except Exception as e:
+                logger.warning("Failed to stop VM: {}".format(e))
+
+        # If node_id is in terminating nodes dict, it's already terminating
+        # Otherwise, kick off termination and add it to the dict
+        elif node_id not in self.terminating_nodes:
+            self.terminating_nodes[node_id] = self.termination_executor.submit(
+                self._delete_node_and_resources, resource_group, node_id
+            )
+
+    def _delete_node_and_resources(self, resource_group, node_id):
+        try:
+            vm = self.compute_client.virtual_machines.get(
+                resource_group_name=resource_group, vm_name=node_id
+            )
+        except ResourceNotFoundError as e:
+            # Node no longer exists
+            logger.warning("Failed to delete VM: {}".format(e))
+            return
+
+        # Gather dependent disks
+        disks = set()
+        if vm.storage_profile is not None and vm.storage_profile.data_disks is not None:
+            for d in vm.storage_profile.data_disks:
+                if d.name is not None:
+                    disks.add(d.name)
+        if (
+            vm.storage_profile is not None
+            and vm.storage_profile.os_disk is not None
+            and vm.storage_profile.os_disk.name is not None
+        ):
+            disks.add(vm.storage_profile.os_disk.name)
+
+        # Gather dependent NICs and public IPs
+        nics = set()
+        ips = set()
+        if (
+            vm.network_profile is not None
+            and vm.network_profile.network_interfaces is not None
+        ):
+            for nint in vm.network_profile.network_interfaces:
+                if nint.id is not None:
+                    nic_name = nint.id.split("/")[-1]
+                    nics.add(nic_name)
+                    # Get public IP if not using internal IPs or if this is the
+                    # head node and use_external_head_ip is True
+                    if not self.provider_config.get("use_internal_ips", False) or (
+                        self.provider_config.get("use_external_head_ip", False)
+                        and vm.tags[TAG_RAY_NODE_KIND] == NODE_KIND_HEAD
+                    ):
+                        nic = self.network_client.network_interfaces.get(
+                            resource_group_name=resource_group,
+                            network_interface_name=nic_name,
+                        )
+                        if nic.ip_configurations is not None:
+                            for ipc in nic.ip_configurations:
+                                if ipc.public_ip_address.id is not None:
+                                    ips.add(ipc.public_ip_address.id.split("/")[-1])
+
+        # Delete VM
+        st = time.monotonic()
+        delete = get_azure_sdk_function(
+            client=self.compute_client.virtual_machines,
+            function_name="delete",
+        )
+        try:
+            delete(resource_group_name=resource_group, vm_name=node_id).wait(
+                timeout=AUTOSCALER_NODE_TERMINATE_WAIT_S
+            )
+        except Exception as e:
+            logger.warning("Failed to delete VM: {}".format(e))
+
+        # Delete disks (no need to wait for these, but gather the LROs for end)
+        disk_lros = []
+        delete = get_azure_sdk_function(
+            client=self.compute_client.disks, function_name="delete"
+        )
+        for d in disks:
+            try:
+                disk_lros.append(
+                    delete(
+                        resource_group_name=resource_group,
+                        disk_name=d,
+                    )
+                )
+            except Exception as e:
+                logger.warning("Failed to delete disk: {}".format(e))
+
+        # Delete NICs
+        nic_lros = []
+        delete = get_azure_sdk_function(
+            client=self.network_client.network_interfaces, function_name="delete"
+        )
+        for n in nics:
+            try:
+                nic_lros.append(
+                    delete(
+                        resource_group_name=resource_group,
+                        network_interface_name=n,
+                    )
+                )
+            except Exception as e:
+                logger.warning("Failed to delete NIC: {}".format(e))
+
+        while (
+            not all(nlro.done() for nlro in nic_lros)
+            and (time.monotonic() - st) < AUTOSCALER_NODE_TERMINATE_WAIT_S
+        ):
+            time.sleep(0.1)
+
+        # Delete Public IPs
+        delete = get_azure_sdk_function(
+            client=self.network_client.public_ip_addresses,
+            function_name="delete",
+        )
+        ip_lros = []
+        for ip in ips:
+            try:
+                ip_lros.append(
+                    delete(
+                        resource_group_name=resource_group,
+                        public_ip_address_name=ip,
+                    )
+                )
+            except Exception as e:
+                logger.warning("Failed to delete public IP: {}".format(e))
+
+        while (
+            not all(dlro.done() for dlro in disk_lros)
+            and (time.monotonic() - st) < AUTOSCALER_NODE_TERMINATE_WAIT_S
+        ):
+            time.sleep(0.1)
+        while (
+            not all(iplro.done() for iplro in ip_lros)
+            and (time.monotonic() - st) < AUTOSCALER_NODE_TERMINATE_WAIT_S
+        ):
+            time.sleep(0.1)
+
+    def cleanup_cluster_resources(self):
+        """Delete shared cluster infrastructure (MSI, NSG, Subnet, VNet)."""
+
+        resource_group = self.provider_config["resource_group"]
+
+        msi_principal_id = self._cleanup_managed_identity(
+            resource_group, self.provider_config.get("msi")
+        )
+
+        subnet_id = self.provider_config.get("subnet")
+        vnet_name = self._cleanup_subnet(resource_group, subnet_id)
+
+        nsg_id = self.provider_config.get("nsg")
+        self._cleanup_nsg(resource_group, nsg_id)
+
+        self._cleanup_vnet(resource_group, subnet_id, vnet_name)
+
+        self._cleanup_role_assignments(resource_group, msi_principal_id)
+
+        self._prune_provider_config_entries()
+        self._cleanup_config_cache()
+
+    @staticmethod
+    def _get_resource_name_from_id(resource_id: Optional[str]) -> Optional[str]:
+        if resource_id:
+            return resource_id.split("/")[-1]
+        return None
+
+    @staticmethod
+    def _retry_delete(delete_fn, max_retries: int = 5, initial_delay: int = 2):
+        """Retry a delete operation with exponential backoff."""
+
+        delay = initial_delay
+        for attempt in range(max_retries):
+            try:
+                return delete_fn()
+            except Exception as exc:  # noqa: BLE001
+                error_msg = str(exc)
+                if "InUse" in error_msg and attempt < max_retries - 1:
+                    logger.info(
+                        "Resource still in use, retrying in %ss (attempt %s/%s)...",
+                        delay,
+                        attempt + 1,
+                        max_retries,
+                    )
+                    time.sleep(delay)
+                    delay *= 2
+                else:
+                    raise
+
+    def _cleanup_managed_identity(
+        self, resource_group: str, msi_id: Optional[str]
+    ) -> Optional[str]:
+        if not msi_id:
+            return None
+
+        msi_name = self._get_resource_name_from_id(msi_id)
+        if not msi_name:
+            return None
+
+        msi_principal_id: Optional[str] = None
+        try:
+            get_identity = get_azure_sdk_function(
+                client=self.resource_client.resources,
+                function_name="get_by_id",
+            )
+            existing_msi = get_identity(msi_id, "2023-01-31")
+            msi_principal_id = getattr(existing_msi, "properties", {}).get(
+                "principalId"
+            )
+        except ResourceNotFoundError:
+            msi_principal_id = None
+        except Exception as exc:  # noqa: BLE001
+            logger.warning(
+                "Failed to query MSI %s for principal ID prior to deletion: %s",
+                msi_name,
+                exc,
+            )
+
+        try:
+            logger.info("Deleting Managed Service Identity: %s", msi_name)
+            delete = get_azure_sdk_function(
+                client=self.resource_client.resources,
+                function_name="delete_by_id",
+            )
+            delete(resource_id=msi_id, api_version="2023-01-31").wait()
+            logger.info("Successfully deleted MSI: %s", msi_name)
+        except ResourceNotFoundError:
+            logger.info("MSI %s not found, may have been already deleted", msi_name)
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("Failed to delete MSI %s: %s", msi_name, exc)
+
+        return msi_principal_id
+
+    def _cleanup_subnet(
+        self, resource_group: str, subnet_id: Optional[str]
+    ) -> Optional[str]:
+        if not subnet_id:
+            return None
+
+        subnet_name = self._get_resource_name_from_id(subnet_id)
+        vnet_name: Optional[str] = None
+
+        if subnet_id and "/virtualNetworks/" in subnet_id:
+            parts = subnet_id.split("/")
+            vnet_idx = parts.index("virtualNetworks")
+            if vnet_idx + 1 < len(parts):
+                vnet_name = parts[vnet_idx + 1]
+
+        if not subnet_name or not vnet_name:
+            return None
+
+        try:
+            logger.info("Deleting Subnet: %s in VNet: %s", subnet_name, vnet_name)
+
+            def delete_subnet():
+                delete = get_azure_sdk_function(
+                    client=self.network_client.subnets, function_name="delete"
+                )
+                delete(
+                    resource_group_name=resource_group,
+                    virtual_network_name=vnet_name,
+                    subnet_name=subnet_name,
+                ).wait()
+
+            self._retry_delete(delete_subnet)
+            logger.info("Successfully deleted Subnet: %s", subnet_name)
+        except ResourceNotFoundError:
+            logger.info(
+                "Subnet %s not found, may have been already deleted", subnet_name
+            )
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("Failed to delete Subnet %s: %s", subnet_name, exc)
+
+        return vnet_name
+
+    def _cleanup_nsg(self, resource_group: str, nsg_id: Optional[str]) -> None:
+        if not nsg_id:
+            return
+
+        nsg_name = self._get_resource_name_from_id(nsg_id)
+        if not nsg_name:
+            return
+
+        try:
+            logger.info("Deleting Network Security Group: %s", nsg_name)
+
+            def delete_nsg():
+                delete = get_azure_sdk_function(
+                    client=self.network_client.network_security_groups,
+                    function_name="delete",
+                )
+                delete(
+                    resource_group_name=resource_group,
+                    network_security_group_name=nsg_name,
+                ).wait()
+
+            self._retry_delete(delete_nsg)
+            logger.info("Successfully deleted NSG: %s", nsg_name)
+        except ResourceNotFoundError:
+            logger.info("NSG %s not found, may have been already deleted", nsg_name)
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("Failed to delete NSG %s: %s", nsg_name, exc)
+
+    def _cleanup_vnet(
+        self,
+        resource_group: str,
+        subnet_id: Optional[str],
+        vnet_name: Optional[str],
+    ) -> None:
+        if not subnet_id or not vnet_name:
+            return
+
+        try:
+            logger.info("Deleting Virtual Network: %s", vnet_name)
+
+            def delete_vnet():
+                delete = get_azure_sdk_function(
+                    client=self.network_client.virtual_networks,
+                    function_name="delete",
+                )
+                delete(
+                    resource_group_name=resource_group,
+                    virtual_network_name=vnet_name,
+                ).wait()
+
+            self._retry_delete(delete_vnet)
+            logger.info("Successfully deleted VNet: %s", vnet_name)
+        except ResourceNotFoundError:
+            logger.info("VNet %s not found, may have been already deleted", vnet_name)
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("Failed to delete VNet %s: %s", vnet_name, exc)
+
+    def _cleanup_role_assignments(
+        self, resource_group: str, msi_principal_id: Optional[str]
+    ) -> None:
+        subscription_id = self.provider_config.get("subscription_id")
+        unique_id = self.provider_config.get("unique_id")
+        if not subscription_id or not unique_id:
+            return
+
+        cluster_id = f"{self.cluster_name}-{unique_id}"
+        role_assignment_name = f"ray-{cluster_id}-ra"
+        role_assignment_guid = _generate_arm_guid(role_assignment_name)
+        role_assignment_id = (
+            f"/subscriptions/{subscription_id}/resourceGroups/{resource_group}/providers"
+            f"/Microsoft.Authorization/roleAssignments/{role_assignment_guid}"
+        )
+
+        if msi_principal_id:
+            _delete_role_assignments_for_principal(
+                self.resource_client, resource_group, msi_principal_id
+            )
+
+        delete_role_assignment = get_azure_sdk_function(
+            client=self.resource_client.resources, function_name="delete_by_id"
+        )
+        try:
+            delete_lro = delete_role_assignment(
+                resource_id=role_assignment_id,
+                api_version="2022-04-01",
+            )
+            if hasattr(delete_lro, "wait"):
+                delete_lro.wait()
+            logger.info(
+                "Deleted role assignment %s for cluster %s",
+                role_assignment_guid,
+                self.cluster_name,
+            )
+        except ResourceNotFoundError:
+            logger.debug(
+                "Role assignment %s not found during cleanup", role_assignment_guid
+            )
+        except Exception as exc:  # noqa: BLE001
+            logger.warning(
+                "Failed to delete role assignment %s: %s",
+                role_assignment_guid,
+                exc,
+            )
+
+    def _prune_provider_config_entries(self) -> None:
+        for key in ("msi", "nsg", "subnet"):
+            self.provider_config.pop(key, None)
+
+    def _cleanup_config_cache(self) -> None:
+        cache_path = self.provider_config.get("_config_cache_path")
+        if not cache_path:
+            return
+
+        try:
+            if os.path.exists(cache_path):
+                os.remove(cache_path)
+                logger.info(
+                    "Deleted cached Ray config at %s after resource cleanup",
+                    cache_path,
+                )
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("Failed to delete cached Ray config %s: %s", cache_path, exc)
+        finally:
+            self.provider_config.pop("_config_cache_path", None)
+
+    def _get_node(self, node_id):
+        self._get_filtered_nodes({})  # Side effect: updates cache
+        return self.cached_nodes[node_id]
+
+    def _get_cached_node(self, node_id):
+        return self.cached_nodes.get(node_id) or self._get_node(node_id=node_id)
+
+    @staticmethod
+    def bootstrap_config(cluster_config):
+        return bootstrap_azure(cluster_config)

lib/python3.12/site-packages/ray/autoscaler/_private/aliyun/config.py

@@ -0,0 +1,116 @@
+import logging
+import os
+import stat
+
+from ray.autoscaler._private.aliyun.utils import AcsClient
+
+# instance status
+PENDING = "Pending"
+RUNNING = "Running"
+STARTING = "Starting"
+STOPPING = "Stopping"
+STOPPED = "Stopped"
+
+logger = logging.getLogger(__name__)
+
+
+def bootstrap_aliyun(config):
+    # print(config["provider"])
+    # create vpc
+    _get_or_create_vpc(config)
+    # create security group id
+    _get_or_create_security_group(config)
+    # create vswitch
+    _get_or_create_vswitch(config)
+    # create key pair
+    _get_or_import_key_pair(config)
+    # print(config["provider"])
+    return config
+
+
+def _client(config):
+    return AcsClient(
+        access_key=config["provider"].get("access_key"),
+        access_key_secret=config["provider"].get("access_key_secret"),
+        region_id=config["provider"]["region"],
+        max_retries=1,
+    )
+
+
+def _get_or_create_security_group(config):
+    cli = _client(config)
+    security_groups = cli.describe_security_groups(vpc_id=config["provider"]["vpc_id"])
+    if security_groups is not None and len(security_groups) > 0:
+        config["provider"]["security_group_id"] = security_groups[0]["SecurityGroupId"]
+        return config
+
+    security_group_id = cli.create_security_group(vpc_id=config["provider"]["vpc_id"])
+
+    for rule in config["provider"].get("security_group_rule", {}):
+        cli.authorize_security_group(
+            security_group_id=security_group_id,
+            port_range=rule["port_range"],
+            source_cidr_ip=rule["source_cidr_ip"],
+            ip_protocol=rule["ip_protocol"],
+        )
+    config["provider"]["security_group_id"] = security_group_id
+    return
+
+
+def _get_or_create_vpc(config):
+    cli = _client(config)
+    vpcs = cli.describe_vpcs()
+    if vpcs is not None and len(vpcs) > 0:
+        config["provider"]["vpc_id"] = vpcs[0].get("VpcId")
+        return
+
+    vpc_id = cli.create_vpc()
+    if vpc_id is not None:
+        config["provider"]["vpc_id"] = vpc_id
+
+
+def _get_or_create_vswitch(config):
+    cli = _client(config)
+    vswitches = cli.describe_v_switches(vpc_id=config["provider"]["vpc_id"])
+    if vswitches is not None and len(vswitches) > 0:
+        config["provider"]["v_switch_id"] = vswitches[0].get("VSwitchId")
+        return
+
+    v_switch_id = cli.create_v_switch(
+        vpc_id=config["provider"]["vpc_id"],
+        zone_id=config["provider"]["zone_id"],
+        cidr_block=config["provider"]["cidr_block"],
+    )
+
+    if v_switch_id is not None:
+        config["provider"]["v_switch_id"] = v_switch_id
+
+
+def _get_or_import_key_pair(config):
+    cli = _client(config)
+    key_name = config["provider"].get("key_name", "ray")
+    key_path = os.path.expanduser("~/.ssh/{}".format(key_name))
+    keypairs = cli.describe_key_pairs(key_pair_name=key_name)
+
+    if keypairs is not None and len(keypairs) > 0:
+        if "ssh_private_key" not in config["auth"]:
+            logger.info(
+                "{} keypair exists, use {} as local ssh key".format(key_name, key_path)
+            )
+            config["auth"]["ssh_private_key"] = key_path
+    else:
+        if "ssh_private_key" not in config["auth"]:
+            # create new keypair
+            resp = cli.create_key_pair(key_pair_name=key_name)
+            if resp is not None:
+                with open(key_path, "w+") as f:
+                    f.write(resp.get("PrivateKeyBody"))
+                os.chmod(key_path, stat.S_IRUSR)
+                config["auth"]["ssh_private_key"] = key_path
+        else:
+            public_key_file = config["auth"]["ssh_private_key"] + ".pub"
+            # create new keypair, from local file
+            with open(public_key_file) as f:
+                public_key = f.readline().strip("\n")
+                cli.import_key_pair(key_pair_name=key_name, public_key_body=public_key)
+                return

lib/python3.12/site-packages/ray/autoscaler/_private/aliyun/node_provider.py

@@ -0,0 +1,324 @@
+import logging
+import random
+import threading
+import time
+from collections import defaultdict
+from typing import Any, Dict, List, Optional
+
+from ray.autoscaler._private.aliyun.config import (
+    PENDING,
+    RUNNING,
+    STOPPED,
+    STOPPING,
+    bootstrap_aliyun,
+)
+from ray.autoscaler._private.aliyun.utils import AcsClient
+from ray.autoscaler._private.cli_logger import cli_logger
+from ray.autoscaler._private.constants import BOTO_MAX_RETRIES
+from ray.autoscaler._private.log_timer import LogTimer
+from ray.autoscaler.node_provider import NodeProvider
+from ray.autoscaler.tags import (
+    TAG_RAY_CLUSTER_NAME,
+    TAG_RAY_LAUNCH_CONFIG,
+    TAG_RAY_NODE_KIND,
+    TAG_RAY_NODE_NAME,
+    TAG_RAY_NODE_STATUS,
+    TAG_RAY_USER_NODE_TYPE,
+)
+
+logger = logging.getLogger(__name__)
+
+TAG_BATCH_DELAY = 1
+STOPPING_NODE_DELAY = 1
+
+
+class AliyunNodeProvider(NodeProvider):
+    def __init__(self, provider_config, cluster_name):
+        NodeProvider.__init__(self, provider_config, cluster_name)
+        self.cache_stopped_nodes = provider_config.get("cache_stopped_nodes", True)
+        self.acs = AcsClient(
+            access_key=provider_config["access_key"],
+            access_key_secret=provider_config["access_key_secret"],
+            region_id=provider_config["region"],
+            max_retries=BOTO_MAX_RETRIES,
+        )
+
+        # Try availability zones round-robin, starting from random offset
+        self.subnet_idx = random.randint(0, 100)
+
+        # Tags that we believe to actually be on the node.
+        self.tag_cache = {}
+        # Tags that we will soon upload.
+        self.tag_cache_pending = defaultdict(dict)
+        # Number of threads waiting for a batched tag update.
+        self.batch_thread_count = 0
+        self.batch_update_done = threading.Event()
+        self.batch_update_done.set()
+        self.ready_for_new_batch = threading.Event()
+        self.ready_for_new_batch.set()
+        self.tag_cache_lock = threading.Lock()
+        self.count_lock = threading.Lock()
+
+        # Cache of node objects from the last nodes() call. This avoids
+        # excessive DescribeInstances requests.
+        self.cached_nodes = {}
+
+    def non_terminated_nodes(self, tag_filters: Dict[str, str]) -> List[str]:
+        tags = [
+            {
+                "Key": TAG_RAY_CLUSTER_NAME,
+                "Value": self.cluster_name,
+            },
+        ]
+        for k, v in tag_filters.items():
+            tags.append(
+                {
+                    "Key": k,
+                    "Value": v,
+                }
+            )
+
+        instances = self.acs.describe_instances(tags=tags)
+        non_terminated_instance = []
+        for instance in instances:
+            if instance.get("Status") == RUNNING or instance.get("Status") == PENDING:
+                non_terminated_instance.append(instance.get("InstanceId"))
+                self.cached_nodes[instance.get("InstanceId")] = instance
+        return non_terminated_instance
+
+    def is_running(self, node_id: str) -> bool:
+        instances = self.acs.describe_instances(instance_ids=[node_id])
+        if instances is not None:
+            instance = instances[0]
+            return instance.get("Status") == "Running"
+        cli_logger.error("Invalid node id: %s", node_id)
+        return False
+
+    def is_terminated(self, node_id: str) -> bool:
+        instances = self.acs.describe_instances(instance_ids=[node_id])
+        if instances is not None:
+            assert len(instances) == 1
+            instance = instances[0]
+            return instance.get("Status") == "Stopped"
+        cli_logger.error("Invalid node id: %s", node_id)
+        return False
+
+    def node_tags(self, node_id: str) -> Dict[str, str]:
+        instances = self.acs.describe_instances(instance_ids=[node_id])
+        if instances is not None:
+            assert len(instances) == 1
+            instance = instances[0]
+            if instance.get("Tags") is not None:
+                node_tags = dict()
+                for tag in instance.get("Tags").get("Tag"):
+                    node_tags[tag.get("TagKey")] = tag.get("TagValue")
+                return node_tags
+        return dict()
+
+    def external_ip(self, node_id: str) -> str:
+        while True:
+            instances = self.acs.describe_instances(instance_ids=[node_id])
+            if instances is not None:
+                assert len(instances)
+                instance = instances[0]
+                if (
+                    instance.get("PublicIpAddress") is not None
+                    and instance.get("PublicIpAddress").get("IpAddress") is not None
+                ):
+                    if len(instance.get("PublicIpAddress").get("IpAddress")) > 0:
+                        return instance.get("PublicIpAddress").get("IpAddress")[0]
+            cli_logger.error("PublicIpAddress attribute is not exist. %s" % instance)
+            time.sleep(STOPPING_NODE_DELAY)
+
+    def internal_ip(self, node_id: str) -> str:
+        while True:
+            instances = self.acs.describe_instances(instance_ids=[node_id])
+            if instances is not None:
+                assert len(instances) == 1
+                instance = instances[0]
+                if (
+                    instance.get("VpcAttributes") is not None
+                    and instance.get("VpcAttributes").get("PrivateIpAddress")
+                    is not None
+                    and len(
+                        instance.get("VpcAttributes")
+                        .get("PrivateIpAddress")
+                        .get("IpAddress")
+                    )
+                    > 0
+                ):
+                    return (
+                        instance.get("VpcAttributes")
+                        .get("PrivateIpAddress")
+                        .get("IpAddress")[0]
+                    )
+            cli_logger.error("InnerIpAddress attribute is not exist. %s" % instance)
+            time.sleep(STOPPING_NODE_DELAY)
+
+    def set_node_tags(self, node_id: str, tags: Dict[str, str]) -> None:
+        is_batching_thread = False
+        with self.tag_cache_lock:
+            if not self.tag_cache_pending:
+                is_batching_thread = True
+                # Wait for threads in the last batch to exit
+                self.ready_for_new_batch.wait()
+                self.ready_for_new_batch.clear()
+                self.batch_update_done.clear()
+            self.tag_cache_pending[node_id].update(tags)
+
+        if is_batching_thread:
+            time.sleep(TAG_BATCH_DELAY)
+            with self.tag_cache_lock:
+                self._update_node_tags()
+                self.batch_update_done.set()
+
+        with self.count_lock:
+            self.batch_thread_count += 1
+        self.batch_update_done.wait()
+
+        with self.count_lock:
+            self.batch_thread_count -= 1
+            if self.batch_thread_count == 0:
+                self.ready_for_new_batch.set()
+
+    def _update_node_tags(self):
+        batch_updates = defaultdict(list)
+
+        for node_id, tags in self.tag_cache_pending.items():
+            for x in tags.items():
+                batch_updates[x].append(node_id)
+            self.tag_cache[node_id] = tags
+
+        self.tag_cache_pending = defaultdict(dict)
+
+        self._create_tags(batch_updates)
+
+    def _create_tags(self, batch_updates):
+
+        for (k, v), node_ids in batch_updates.items():
+            m = "Set tag {}={} on {}".format(k, v, node_ids)
+            with LogTimer("AliyunNodeProvider: {}".format(m)):
+                if k == TAG_RAY_NODE_NAME:
+                    k = "Name"
+
+                self.acs.tag_resource(node_ids, [{"Key": k, "Value": v}])
+
+    def create_node(
+        self, node_config: Dict[str, Any], tags: Dict[str, str], count: int
+    ) -> Optional[Dict[str, Any]]:
+        filter_tags = [
+            {
+                "Key": TAG_RAY_CLUSTER_NAME,
+                "Value": self.cluster_name,
+            },
+            {"Key": TAG_RAY_NODE_KIND, "Value": tags[TAG_RAY_NODE_KIND]},
+            {"Key": TAG_RAY_USER_NODE_TYPE, "Value": tags[TAG_RAY_USER_NODE_TYPE]},
+            {"Key": TAG_RAY_LAUNCH_CONFIG, "Value": tags[TAG_RAY_LAUNCH_CONFIG]},
+            {"Key": TAG_RAY_NODE_NAME, "Value": tags[TAG_RAY_NODE_NAME]},
+        ]
+
+        reused_nodes_dict = {}
+        if self.cache_stopped_nodes:
+            reuse_nodes_candidate = self.acs.describe_instances(tags=filter_tags)
+            if reuse_nodes_candidate:
+                with cli_logger.group("Stopping instances to reuse"):
+                    reuse_node_ids = []
+                    for node in reuse_nodes_candidate:
+                        node_id = node.get("InstanceId")
+                        status = node.get("Status")
+                        if status != STOPPING and status != STOPPED:
+                            continue
+                        if status == STOPPING:
+                            # wait for node stopped
+                            while (
+                                self.acs.describe_instances(instance_ids=[node_id])[
+                                    0
+                                ].get("Status")
+                                == STOPPING
+                            ):
+                                logging.info("wait for %s stop" % node_id)
+                                time.sleep(STOPPING_NODE_DELAY)
+                        # logger.info("reuse %s" % node_id)
+                        reuse_node_ids.append(node_id)
+                        reused_nodes_dict[node.get("InstanceId")] = node
+                        self.acs.start_instance(node_id)
+                        self.tag_cache[node_id] = node.get("Tags")
+                        self.set_node_tags(node_id, tags)
+                        if len(reuse_node_ids) == count:
+                            break
+                count -= len(reuse_node_ids)
+
+        created_nodes_dict = {}
+        if count > 0:
+            filter_tags.append(
+                {"Key": TAG_RAY_NODE_STATUS, "Value": tags[TAG_RAY_NODE_STATUS]}
+            )
+            instance_id_sets = self.acs.run_instances(
+                instance_type=node_config["InstanceType"],
+                image_id=node_config["ImageId"],
+                tags=filter_tags,
+                amount=count,
+                vswitch_id=self.provider_config["v_switch_id"],
+                security_group_id=self.provider_config["security_group_id"],
+                key_pair_name=self.provider_config["key_name"],
+            )
+            instances = self.acs.describe_instances(instance_ids=instance_id_sets)
+
+            if instances is not None:
+                for instance in instances:
+                    created_nodes_dict[instance.get("InstanceId")] = instance
+
+        all_created_nodes = reused_nodes_dict
+        all_created_nodes.update(created_nodes_dict)
+        return all_created_nodes
+
+    def terminate_node(self, node_id: str) -> None:
+        logger.info("terminate node: %s" % node_id)
+        if self.cache_stopped_nodes:
+            logger.info(
+                "Stopping instance {} (to terminate instead, "
+                "set `cache_stopped_nodes: False` "
+                "under `provider` in the cluster configuration)"
+            ).format(node_id)
+            self.acs.stop_instance(node_id)
+        else:
+            self.acs.delete_instance(node_id)
+
+    def terminate_nodes(self, node_ids: List[str]) -> None:
+        if not node_ids:
+            return
+        if self.cache_stopped_nodes:
+            logger.info(
+                "Stopping instances {} (to terminate instead, "
+                "set `cache_stopped_nodes: False` "
+                "under `provider` in the cluster configuration)".format(node_ids)
+            )
+
+            self.acs.stop_instances(node_ids)
+        else:
+            self.acs.delete_instances(node_ids)
+
+    def _get_node(self, node_id):
+        """Refresh and get info for this node, updating the cache."""
+        self.non_terminated_nodes({})  # Side effect: updates cache
+
+        if node_id in self.cached_nodes:
+            return self.cached_nodes[node_id]
+
+        # Node not in {pending, running} -- retry with a point query. This
+        # usually means the node was recently preempted or terminated.
+        matches = self.acs.describe_instances(instance_ids=[node_id])
+
+        assert len(matches) == 1, "Invalid instance id {}".format(node_id)
+        return matches[0]
+
+    def _get_cached_node(self, node_id):
+        """Return node info from cache if possible, otherwise fetches it."""
+        if node_id in self.cached_nodes:
+            return self.cached_nodes[node_id]
+
+        return self._get_node(node_id)
+
+    @staticmethod
+    def bootstrap_config(cluster_config):
+        return bootstrap_aliyun(cluster_config)

lib/python3.12/site-packages/ray/autoscaler/_private/aliyun/utils.py

@@ -0,0 +1,461 @@
+import json
+import logging
+
+from aliyunsdkcore import client
+from aliyunsdkcore.acs_exception.exceptions import ClientException, ServerException
+from aliyunsdkecs.request.v20140526.AllocatePublicIpAddressRequest import (
+    AllocatePublicIpAddressRequest,
+)
+from aliyunsdkecs.request.v20140526.AuthorizeSecurityGroupRequest import (
+    AuthorizeSecurityGroupRequest,
+)
+from aliyunsdkecs.request.v20140526.CreateInstanceRequest import CreateInstanceRequest
+from aliyunsdkecs.request.v20140526.CreateKeyPairRequest import CreateKeyPairRequest
+from aliyunsdkecs.request.v20140526.CreateSecurityGroupRequest import (
+    CreateSecurityGroupRequest,
+)
+from aliyunsdkecs.request.v20140526.CreateVpcRequest import CreateVpcRequest
+from aliyunsdkecs.request.v20140526.CreateVSwitchRequest import CreateVSwitchRequest
+from aliyunsdkecs.request.v20140526.DeleteInstanceRequest import DeleteInstanceRequest
+from aliyunsdkecs.request.v20140526.DeleteInstancesRequest import DeleteInstancesRequest
+from aliyunsdkecs.request.v20140526.DeleteKeyPairsRequest import DeleteKeyPairsRequest
+from aliyunsdkecs.request.v20140526.DescribeInstancesRequest import (
+    DescribeInstancesRequest,
+)
+from aliyunsdkecs.request.v20140526.DescribeKeyPairsRequest import (
+    DescribeKeyPairsRequest,
+)
+from aliyunsdkecs.request.v20140526.DescribeSecurityGroupsRequest import (
+    DescribeSecurityGroupsRequest,
+)
+from aliyunsdkecs.request.v20140526.DescribeVpcsRequest import DescribeVpcsRequest
+from aliyunsdkecs.request.v20140526.DescribeVSwitchesRequest import (
+    DescribeVSwitchesRequest,
+)
+from aliyunsdkecs.request.v20140526.ImportKeyPairRequest import ImportKeyPairRequest
+from aliyunsdkecs.request.v20140526.RunInstancesRequest import RunInstancesRequest
+from aliyunsdkecs.request.v20140526.StartInstanceRequest import StartInstanceRequest
+from aliyunsdkecs.request.v20140526.StopInstanceRequest import StopInstanceRequest
+from aliyunsdkecs.request.v20140526.StopInstancesRequest import StopInstancesRequest
+from aliyunsdkecs.request.v20140526.TagResourcesRequest import TagResourcesRequest
+
+
+class AcsClient:
+    """
+    A wrapper around Aliyun SDK. We use this wrapper in aliyun node provider.
+
+    Parameters:
+        access_key: The AccessKey ID of your aliyun account.
+        access_key_secret: The AccessKey secret of your aliyun account.
+        region_id: A region is a geographic area where a data center resides.
+                   Region_id is the ID of region (e.g., cn-hangzhou,
+                   us-west-1, etc.)
+        max_retries: The maximum number of retries each connection.
+    """
+
+    def __init__(self, access_key, access_key_secret, region_id, max_retries):
+        self.cli = client.AcsClient(
+            ak=access_key,
+            secret=access_key_secret,
+            max_retry_time=max_retries,
+            region_id=region_id,
+        )
+
+    def describe_instances(self, tags=None, instance_ids=None):
+        """Query the details of one or more Elastic Compute Service (ECS) instances.
+
+        :param tags: The tags of the instance.
+        :param instance_ids: The IDs of ECS instances
+        :return: ECS instance list
+        """
+        request = DescribeInstancesRequest()
+        if tags is not None:
+            request.set_Tags(tags)
+        if instance_ids is not None:
+            request.set_InstanceIds(instance_ids)
+        response = self._send_request(request)
+        if response is not None:
+            instance_list = response.get("Instances").get("Instance")
+            return instance_list
+        return None
+
+    def create_instance(
+        self,
+        instance_type,
+        image_id,
+        tags,
+        key_pair_name,
+        optimized="optimized",
+        instance_charge_type="PostPaid",
+        spot_strategy="SpotWithPriceLimit",
+        internet_charge_type="PayByTraffic",
+        internet_max_bandwidth_out=5,
+    ):
+        """Create a subscription or pay-as-you-go ECS instance.
+
+        :param instance_type: The instance type of the ECS.
+        :param image_id: The ID of the image used to create the instance.
+        :param tags: The tags of the instance.
+        :param key_pair_name: The name of the key pair to be bound to
+                              the instance.
+        :param optimized: Specifies whether the instance is I/O optimized
+        :param instance_charge_type: The billing method of the instance.
+                                     Default value: PostPaid.
+        :param spot_strategy: The preemption policy for the pay-as-you-go
+                              instance.
+        :param internet_charge_type: The billing method for network usage.
+                                     Default value: PayByTraffic.
+        :param internet_max_bandwidth_out: The maximum inbound public
+                                           bandwidth. Unit: Mbit/s.
+        :return: The created instance ID.
+        """
+        request = CreateInstanceRequest()
+        request.set_InstanceType(instance_type)
+        request.set_ImageId(image_id)
+        request.set_IoOptimized(optimized)
+        request.set_InstanceChargeType(instance_charge_type)
+        request.set_SpotStrategy(spot_strategy)
+        request.set_InternetChargeType(internet_charge_type)
+        request.set_InternetMaxBandwidthOut(internet_max_bandwidth_out)
+        request.set_KeyPairName(key_pair_name)
+        request.set_Tags(tags)
+
+        response = self._send_request(request)
+        if response is not None:
+            instance_id = response.get("InstanceId")
+            logging.info("instance %s created task submit successfully.", instance_id)
+            return instance_id
+        logging.error("instance created failed.")
+        return None
+
+    def run_instances(
+        self,
+        instance_type,
+        image_id,
+        tags,
+        security_group_id,
+        vswitch_id,
+        key_pair_name,
+        amount=1,
+        optimized="optimized",
+        instance_charge_type="PostPaid",
+        spot_strategy="SpotWithPriceLimit",
+        internet_charge_type="PayByTraffic",
+        internet_max_bandwidth_out=1,
+    ):
+        """Create one or more pay-as-you-go or subscription
+            Elastic Compute Service (ECS) instances
+
+        :param instance_type: The instance type of the ECS.
+        :param image_id: The ID of the image used to create the instance.
+        :param tags: The tags of the instance.
+        :param security_group_id: The ID of the security group to which to
+                                  assign the instance. Instances in the same
+                                  security group can communicate with
+                                  each other.
+        :param vswitch_id: The ID of the vSwitch to which to connect
+                           the instance.
+        :param key_pair_name: The name of the key pair to be bound to
+                              the instance.
+        :param amount: The number of instances that you want to create.
+        :param optimized: Specifies whether the instance is I/O optimized
+        :param instance_charge_type: The billing method of the instance.
+                                     Default value: PostPaid.
+        :param spot_strategy: The preemption policy for the pay-as-you-go
+                              instance.
+        :param internet_charge_type: The billing method for network usage.
+                                     Default value: PayByTraffic.
+        :param internet_max_bandwidth_out: The maximum inbound public
+                                           bandwidth. Unit: Mbit/s.
+        :return: The created instance IDs.
+        """
+        request = RunInstancesRequest()
+        request.set_InstanceType(instance_type)
+        request.set_ImageId(image_id)
+        request.set_IoOptimized(optimized)
+        request.set_InstanceChargeType(instance_charge_type)
+        request.set_SpotStrategy(spot_strategy)
+        request.set_InternetChargeType(internet_charge_type)
+        request.set_InternetMaxBandwidthOut(internet_max_bandwidth_out)
+        request.set_Tags(tags)
+        request.set_Amount(amount)
+        request.set_SecurityGroupId(security_group_id)
+        request.set_VSwitchId(vswitch_id)
+        request.set_KeyPairName(key_pair_name)
+
+        response = self._send_request(request)
+        if response is not None:
+            instance_ids = response.get("InstanceIdSets").get("InstanceIdSet")
+            return instance_ids
+        logging.error("instance created failed.")
+        return None
+
+    def create_security_group(self, vpc_id):
+        """Create a security group
+
+        :param vpc_id: The ID of the VPC in which to create
+                       the security group.
+        :return: The created security group ID.
+        """
+        request = CreateSecurityGroupRequest()
+        request.set_VpcId(vpc_id)
+        response = self._send_request(request)
+        if response is not None:
+            security_group_id = response.get("SecurityGroupId")
+            return security_group_id
+        return None
+
+    def describe_security_groups(self, vpc_id=None, tags=None):
+        """Query basic information of security groups.
+
+        :param vpc_id: The ID of the VPC to which the security group belongs.
+        :param tags: The tags of the security group.
+        :return: Security group list.
+        """
+        request = DescribeSecurityGroupsRequest()
+        if vpc_id is not None:
+            request.set_VpcId(vpc_id)
+        if tags is not None:
+            request.set_Tags(tags)
+        response = self._send_request(request)
+        if response is not None:
+            security_groups = response.get("SecurityGroups").get("SecurityGroup")
+            return security_groups
+        logging.error("describe security group failed.")
+        return None
+
+    def authorize_security_group(
+        self, ip_protocol, port_range, security_group_id, source_cidr_ip
+    ):
+        """Create an inbound security group rule.
+
+        :param ip_protocol: The transport layer protocol.
+        :param port_range: The range of destination ports relevant to
+                           the transport layer protocol.
+        :param security_group_id: The ID of the destination security group.
+        :param source_cidr_ip: The range of source IPv4 addresses.
+                               CIDR blocks and IPv4 addresses are supported.
+        """
+        request = AuthorizeSecurityGroupRequest()
+        request.set_IpProtocol(ip_protocol)
+        request.set_PortRange(port_range)
+        request.set_SecurityGroupId(security_group_id)
+        request.set_SourceCidrIp(source_cidr_ip)
+        self._send_request(request)
+
+    def create_v_switch(self, vpc_id, zone_id, cidr_block):
+        """Create vSwitches to divide the VPC into one or more subnets
+
+        :param vpc_id: The ID of the VPC to which the VSwitch belongs.
+        :param zone_id: The ID of the zone to which
+                        the target VSwitch belongs.
+        :param cidr_block: The CIDR block of the VSwitch.
+        :return:
+        """
+        request = CreateVSwitchRequest()
+        request.set_ZoneId(zone_id)
+        request.set_VpcId(vpc_id)
+        request.set_CidrBlock(cidr_block)
+        response = self._send_request(request)
+        if response is not None:
+            return response.get("VSwitchId")
+        else:
+            logging.error("create_v_switch vpc_id %s failed.", vpc_id)
+        return None
+
+    def create_vpc(self):
+        """Creates a virtual private cloud (VPC).
+
+        :return: The created VPC ID.
+        """
+        request = CreateVpcRequest()
+        response = self._send_request(request)
+        if response is not None:
+            return response.get("VpcId")
+        return None
+
+    def describe_vpcs(self):
+        """Queries one or more VPCs in a region.
+
+        :return: VPC list.
+        """
+        request = DescribeVpcsRequest()
+        response = self._send_request(request)
+        if response is not None:
+            return response.get("Vpcs").get("Vpc")
+        return None
+
+    def tag_resource(self, resource_ids, tags, resource_type="instance"):
+        """Create and bind tags to specified ECS resources.
+
+        :param resource_ids: The IDs of N resources.
+        :param tags: The tags of the resource.
+        :param resource_type: The type of the resource.
+        """
+        request = TagResourcesRequest()
+        request.set_Tags(tags)
+        request.set_ResourceType(resource_type)
+        request.set_ResourceIds(resource_ids)
+        response = self._send_request(request)
+        if response is not None:
+            logging.info("instance %s create tag successfully.", resource_ids)
+        else:
+            logging.error("instance %s create tag failed.", resource_ids)
+
+    def start_instance(self, instance_id):
+        """Start an ECS instance.
+
+        :param instance_id: The Ecs instance ID.
+        """
+        request = StartInstanceRequest()
+        request.set_InstanceId(instance_id)
+        response = self._send_request(request)
+
+        if response is not None:
+            logging.info("instance %s start successfully.", instance_id)
+        else:
+            logging.error("instance %s start failed.", instance_id)
+
+    def stop_instance(self, instance_id, force_stop=False):
+        """Stop an ECS instance that is in the Running state.
+
+        :param instance_id: The Ecs instance ID.
+        :param force_stop: Specifies whether to forcibly stop the instance.
+        :return:
+        """
+        request = StopInstanceRequest()
+        request.set_InstanceId(instance_id)
+        request.set_ForceStop(force_stop)
+        logging.info("Stop %s command submit successfully.", instance_id)
+        self._send_request(request)
+
+    def stop_instances(self, instance_ids, stopped_mode="StopCharging"):
+        """Stop one or more ECS instances that are in the Running state.
+
+        :param instance_ids: The IDs of instances.
+        :param stopped_mode: Specifies whether billing for the instance
+                             continues after the instance is stopped.
+        """
+        request = StopInstancesRequest()
+        request.set_InstanceIds(instance_ids)
+        request.set_StoppedMode(stopped_mode)
+        response = self._send_request(request)
+        if response is None:
+            logging.error("stop_instances failed")
+
+    def delete_instance(self, instance_id):
+        """Release a pay-as-you-go instance or
+            an expired subscription instance.
+
+        :param instance_id: The ID of the instance that you want to release.
+        """
+        request = DeleteInstanceRequest()
+        request.set_InstanceId(instance_id)
+        request.set_Force(True)
+        logging.info("Delete %s command submit successfully", instance_id)
+        self._send_request(request)
+
+    def delete_instances(self, instance_ids):
+        """Release one or more pay-as-you-go instances or
+            expired subscription instances.
+
+        :param instance_ids: The IDs of instances that you want to release.
+        """
+        request = DeleteInstancesRequest()
+        request.set_Force(True)
+        request.set_InstanceIds(instance_ids)
+        self._send_request(request)
+
+    def allocate_public_address(self, instance_id):
+        """Assign a public IP address to an ECS instance.
+
+        :param instance_id: The ID of the instance to which you want to
+                            assign a public IP address.
+        :return: The assigned ip.
+        """
+        request = AllocatePublicIpAddressRequest()
+        request.set_InstanceId(instance_id)
+        response = self._send_request(request)
+        if response is not None:
+            return response.get("IpAddress")
+
+    def create_key_pair(self, key_pair_name):
+        """Create an SSH key pair.
+
+        :param key_pair_name: The name of the key pair.
+        :return: The created keypair data.
+        """
+        request = CreateKeyPairRequest()
+        request.set_KeyPairName(key_pair_name)
+        response = self._send_request(request)
+        if response is not None:
+            logging.info("Create Key Pair %s Successfully", response.get("KeyPairId"))
+            return response
+        else:
+            logging.error("Create Key Pair Failed")
+            return None
+
+    def import_key_pair(self, key_pair_name, public_key_body):
+        """Import the public key of an RSA-encrypted key pair
+            that is generated by a third-party tool.
+
+        :param key_pair_name: The name of the key pair.
+        :param public_key_body: The public key of the key pair.
+        """
+        request = ImportKeyPairRequest()
+        request.set_KeyPairName(key_pair_name)
+        request.set_PublicKeyBody(public_key_body)
+        self._send_request(request)
+
+    def delete_key_pairs(self, key_pair_names):
+        """Delete one or more SSH key pairs.
+
+        :param key_pair_names: The name of the key pair.
+        :return:
+        """
+        request = DeleteKeyPairsRequest()
+        request.set_KeyPairNames(key_pair_names)
+        self._send_request(request)
+
+    def describe_key_pairs(self, key_pair_name=None):
+        """Query one or more key pairs.
+
+        :param key_pair_name: The name of the key pair.
+        :return:
+        """
+        request = DescribeKeyPairsRequest()
+        if key_pair_name is not None:
+            request.set_KeyPairName(key_pair_name)
+        response = self._send_request(request)
+        if response is not None:
+            return response.get("KeyPairs").get("KeyPair")
+        else:
+            return None
+
+    def describe_v_switches(self, vpc_id=None):
+        """Queries one or more VSwitches.
+
+        :param vpc_id: The ID of the VPC to which the VSwitch belongs.
+        :return: VSwitch list.
+        """
+        request = DescribeVSwitchesRequest()
+        if vpc_id is not None:
+            request.set_VpcId(vpc_id)
+        response = self._send_request(request)
+        if response is not None:
+            return response.get("VSwitches").get("VSwitch")
+        else:
+            logging.error("Describe VSwitches Failed.")
+            return None
+
+    def _send_request(self, request):
+        """send open api request"""
+        request.set_accept_format("json")
+        try:
+            response_str = self.cli.do_action_with_exception(request)
+            response_detail = json.loads(response_str)
+            return response_detail
+        except (ClientException, ServerException) as e:
+            logging.error(request.get_action_name())
+            logging.error(e)
+            return None

lib/python3.12/site-packages/ray/autoscaler/_private/autoscaler.py

@@ -0,0 +1,1522 @@
+import copy
+import logging
+import math
+import operator
+import os
+import queue
+import subprocess
+import threading
+import time
+from collections import Counter, defaultdict, namedtuple
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any, Callable, Dict, FrozenSet, List, Optional, Set, Tuple, Union
+
+import yaml
+
+import ray
+from ray._common.utils import PLACEMENT_GROUP_BUNDLE_RESOURCE_NAME
+from ray.autoscaler._private.constants import (
+    AUTOSCALER_HEARTBEAT_TIMEOUT_S,
+    AUTOSCALER_MAX_CONCURRENT_LAUNCHES,
+    AUTOSCALER_MAX_LAUNCH_BATCH,
+    AUTOSCALER_MAX_NUM_FAILURES,
+    AUTOSCALER_STATUS_LOG,
+    AUTOSCALER_UPDATE_INTERVAL_S,
+    DISABLE_LAUNCH_CONFIG_CHECK_KEY,
+    DISABLE_NODE_UPDATERS_KEY,
+    FOREGROUND_NODE_LAUNCH_KEY,
+    WORKER_LIVENESS_CHECK_KEY,
+)
+from ray.autoscaler._private.event_summarizer import EventSummarizer
+from ray.autoscaler._private.legacy_info_string import legacy_log_info_string
+from ray.autoscaler._private.load_metrics import LoadMetrics
+from ray.autoscaler._private.local.node_provider import (
+    LocalNodeProvider,
+    record_local_head_state_if_needed,
+)
+from ray.autoscaler._private.node_launcher import BaseNodeLauncher, NodeLauncher
+from ray.autoscaler._private.node_provider_availability_tracker import (
+    NodeAvailabilitySummary,
+    NodeProviderAvailabilityTracker,
+)
+from ray.autoscaler._private.node_tracker import NodeTracker
+from ray.autoscaler._private.prom_metrics import AutoscalerPrometheusMetrics
+from ray.autoscaler._private.providers import _get_node_provider
+from ray.autoscaler._private.resource_demand_scheduler import (
+    ResourceDemandScheduler,
+    ResourceDict,
+    get_bin_pack_residual,
+    placement_groups_to_resource_demands,
+)
+from ray.autoscaler._private.updater import NodeUpdaterThread
+from ray.autoscaler._private.util import (
+    ConcurrentCounter,
+    NodeCount,
+    NodeID,
+    NodeIP,
+    NodeType,
+    NodeTypeConfigDict,
+    format_info_string,
+    hash_launch_conf,
+    hash_runtime_conf,
+    validate_config,
+    with_head_node_ip,
+)
+from ray.autoscaler.node_provider import NodeProvider
+from ray.autoscaler.tags import (
+    NODE_KIND_HEAD,
+    NODE_KIND_UNMANAGED,
+    NODE_KIND_WORKER,
+    STATUS_UP_TO_DATE,
+    STATUS_UPDATE_FAILED,
+    TAG_RAY_FILE_MOUNTS_CONTENTS,
+    TAG_RAY_LAUNCH_CONFIG,
+    TAG_RAY_NODE_KIND,
+    TAG_RAY_NODE_STATUS,
+    TAG_RAY_RUNTIME_CONFIG,
+    TAG_RAY_USER_NODE_TYPE,
+)
+from ray.exceptions import RpcError
+
+logger = logging.getLogger(__name__)
+
+# Status of a node e.g. "up-to-date", see ray/autoscaler/tags.py
+NodeStatus = str
+
+# Tuple of modified fields for the given node_id returned by should_update
+# that will be passed into a NodeUpdaterThread.
+UpdateInstructions = namedtuple(
+    "UpdateInstructions",
+    ["node_id", "setup_commands", "ray_start_commands", "docker_config"],
+)
+
+NodeLaunchData = Tuple[NodeTypeConfigDict, NodeCount, Optional[NodeType]]
+
+
+@dataclass
+class AutoscalerSummary:
+    active_nodes: Dict[NodeType, int]
+    idle_nodes: Optional[Dict[NodeType, int]]
+    pending_nodes: List[Tuple[NodeIP, NodeType, NodeStatus]]
+    pending_launches: Dict[NodeType, int]
+    failed_nodes: List[Tuple[NodeIP, NodeType]]
+    node_availability_summary: NodeAvailabilitySummary = field(
+        default_factory=lambda: NodeAvailabilitySummary({})
+    )
+    # A dictionary of node IP to a list of reasons the node is not idle.
+    node_activities: Optional[Dict[str, Tuple[NodeIP, List[str]]]] = None
+    pending_resources: Dict[str, int] = field(default_factory=lambda: {})
+    # A mapping from node name (the same key as `usage_by_node`) to node type.
+    # Optional for deployment modes which have the concept of node types and
+    # backwards compatibility.
+    node_type_mapping: Optional[Dict[str, str]] = None
+    # Whether the autoscaler summary is v1 or v2.
+    legacy: bool = False
+
+
+class NonTerminatedNodes:
+    """Class to extract and organize information on non-terminated nodes."""
+
+    def __init__(self, provider: NodeProvider):
+        start_time = time.time()
+        # All non-terminated nodes
+        self.all_node_ids = provider.non_terminated_nodes({})
+
+        # Managed worker nodes (node kind "worker"):
+        self.worker_ids: List[NodeID] = []
+        # The head node (node kind "head")
+        self.head_id: Optional[NodeID] = None
+
+        for node in self.all_node_ids:
+            node_kind = provider.node_tags(node)[TAG_RAY_NODE_KIND]
+            if node_kind == NODE_KIND_WORKER:
+                self.worker_ids.append(node)
+            elif node_kind == NODE_KIND_HEAD:
+                self.head_id = node
+
+        # Note: For typical use-cases, self.all_node_ids == self.worker_ids +
+        # [self.head_id]. The difference being in the case of unmanaged nodes.
+
+        # Record the time of the non_terminated nodes call. This typically
+        # translates to a "describe" or "list" call on most cluster managers
+        # which can be quite expensive. Note that we include the processing
+        # time because on some clients, there may be pagination and the
+        # underlying api calls may be done lazily.
+        self.non_terminated_nodes_time = time.time() - start_time
+        logger.info(
+            f"The autoscaler took {round(self.non_terminated_nodes_time, 3)}"
+            " seconds to fetch the list of non-terminated nodes."
+        )
+
+    def remove_terminating_nodes(self, terminating_nodes: List[NodeID]) -> None:
+        """Remove nodes we're in the process of terminating from internal
+        state."""
+
+        def not_terminating(node):
+            return node not in terminating_nodes
+
+        self.worker_ids = list(filter(not_terminating, self.worker_ids))
+        self.all_node_ids = list(filter(not_terminating, self.all_node_ids))
+
+
+# Whether a worker should be kept based on the min_workers and
+# max_workers constraints.
+#
+# keep: should keep the worker
+# terminate: should terminate the worker
+# decide_later: the worker can be terminated if needed
+KeepOrTerminate = Enum("KeepOrTerminate", "keep terminate decide_later")
+
+
+class StandardAutoscaler:
+    """The autoscaling control loop for a Ray cluster.
+
+    There are two ways to start an autoscaling cluster: manually by running
+    `ray start --head --autoscaling-config=/path/to/config.yaml` on a instance
+    that has permission to launch other instances, or you can also use `ray up
+    /path/to/config.yaml` from your laptop, which will configure the right
+    AWS/Cloud roles automatically. See the Ray documentation
+    (https://docs.ray.io/en/latest/) for a full definition of autoscaling behavior.
+    StandardAutoscaler's `update` method is periodically called in
+    `monitor.py`'s monitoring loop.
+
+    StandardAutoscaler is also used to bootstrap clusters (by adding workers
+    until the cluster size that can handle the resource demand is met).
+    """
+
+    def __init__(
+        self,
+        # TODO(ekl): require config reader to be a callable always.
+        config_reader: Union[str, Callable[[], dict]],
+        load_metrics: LoadMetrics,
+        gcs_client: "ray._raylet.GcsClient",
+        session_name: Optional[str] = None,
+        max_launch_batch: int = AUTOSCALER_MAX_LAUNCH_BATCH,
+        max_concurrent_launches: int = AUTOSCALER_MAX_CONCURRENT_LAUNCHES,
+        max_failures: int = AUTOSCALER_MAX_NUM_FAILURES,
+        process_runner: Any = subprocess,
+        update_interval_s: int = AUTOSCALER_UPDATE_INTERVAL_S,
+        prefix_cluster_info: bool = False,
+        event_summarizer: Optional[EventSummarizer] = None,
+        prom_metrics: Optional[AutoscalerPrometheusMetrics] = None,
+    ):
+        """Create a StandardAutoscaler.
+
+        Args:
+            config_reader: Path to a Ray Autoscaler YAML, or a function to read
+                and return the latest config.
+            load_metrics: Provides metrics for the Ray cluster.
+            session_name: The current Ray session name when this autoscaler
+                is deployed.
+            max_launch_batch: Max number of nodes to launch in one request.
+            max_concurrent_launches: Max number of nodes that can be
+                concurrently launched. This value and `max_launch_batch`
+                determine the number of batches that are used to launch nodes.
+            max_failures: Number of failures that the autoscaler will tolerate
+                before exiting.
+            process_runner: Subproc-like interface used by the CommandRunner.
+            update_interval_s: Seconds between running the autoscaling loop.
+            prefix_cluster_info: Whether to add the cluster name to info strs.
+            event_summarizer: Utility to consolidate duplicated messages.
+            prom_metrics: Prometheus metrics for autoscaler-related operations.
+            gcs_client: client for interactions with the GCS. Used to drain nodes
+                before termination.
+        """
+
+        if isinstance(config_reader, str):
+            # Auto wrap with file reader.
+            def read_fn():
+                with open(config_reader) as f:
+                    new_config = yaml.safe_load(f.read())
+                return new_config
+
+            self.config_reader = read_fn
+        else:
+            self.config_reader = config_reader
+
+        self.node_provider_availability_tracker = NodeProviderAvailabilityTracker()
+        # Prefix each line of info string with cluster name if True
+        self.prefix_cluster_info = prefix_cluster_info
+        # Keep this before self.reset (self.provider needs to be created
+        # exactly once).
+        self.provider = None
+        # Keep this before self.reset (if an exception occurs in reset
+        # then prom_metrics must be instantitiated to increment the
+        # exception counter)
+        self.prom_metrics = prom_metrics or AutoscalerPrometheusMetrics(
+            session_name=session_name
+        )  # noqa
+        self.resource_demand_scheduler = None
+        self.reset(errors_fatal=True)
+        self.load_metrics = load_metrics
+
+        self.max_failures = max_failures
+        self.max_launch_batch = max_launch_batch
+        self.max_concurrent_launches = max_concurrent_launches
+        self.process_runner = process_runner
+        self.event_summarizer = event_summarizer or EventSummarizer()
+
+        # Map from node_id to NodeUpdater threads
+        self.updaters: Dict[NodeID, NodeUpdaterThread] = {}
+        self.num_failed_updates: Dict[NodeID, int] = defaultdict(int)
+        self.num_successful_updates: Dict[NodeID, int] = defaultdict(int)
+        self.num_failures = 0
+        self.last_update_time = 0.0
+        self.update_interval_s = update_interval_s
+
+        # Keeps track of pending and running nodes
+        self.non_terminated_nodes: Optional[NonTerminatedNodes] = None
+
+        # Tracks nodes scheduled for termination
+        self.nodes_to_terminate: List[NodeID] = []
+
+        # Disable NodeUpdater threads if true.
+        # Should be set to true in situations where another component, such as
+        # a Kubernetes operator, is responsible for Ray setup on nodes.
+        self.disable_node_updaters = self.config["provider"].get(
+            DISABLE_NODE_UPDATERS_KEY, False
+        )
+        logger.info(f"{DISABLE_NODE_UPDATERS_KEY}:{self.disable_node_updaters}")
+
+        # Disable launch configuration checking if set to true.
+        # This setting is used in scenarios where there is no meaningful node type
+        # to enforce, such as in fake multinode situations. When this option is enabled,
+        # outdated nodes will not be terminated.
+        self.disable_launch_config_check = self.config["provider"].get(
+            DISABLE_LAUNCH_CONFIG_CHECK_KEY, False
+        )
+        logger.info(
+            f"{DISABLE_LAUNCH_CONFIG_CHECK_KEY}:{self.disable_launch_config_check}"
+        )
+
+        # By default, the autoscaler launches nodes in batches asynchronously in
+        # background threads.
+        # When the following flag is set, that behavior is disabled, so that nodes
+        # are launched in the main thread, all in one batch, blocking until all
+        # NodeProvider.create_node calls have returned.
+        self.foreground_node_launch = self.config["provider"].get(
+            FOREGROUND_NODE_LAUNCH_KEY, False
+        )
+        logger.info(f"{FOREGROUND_NODE_LAUNCH_KEY}:{self.foreground_node_launch}")
+
+        # By default, the autoscaler kills and/or tries to recover
+        # a worker node if it hasn't produced a resource heartbeat in the last 30
+        # seconds. The worker_liveness_check flag allows disabling this behavior in
+        # settings where another component, such as a Kubernetes operator, is
+        # responsible for healthchecks.
+        self.worker_liveness_check = self.config["provider"].get(
+            WORKER_LIVENESS_CHECK_KEY, True
+        )
+        logger.info(f"{WORKER_LIVENESS_CHECK_KEY}:{self.worker_liveness_check}")
+
+        # Node launchers
+        self.foreground_node_launcher: Optional[BaseNodeLauncher] = None
+        self.launch_queue: Optional[queue.Queue[NodeLaunchData]] = None
+        self.pending_launches = ConcurrentCounter()
+        if self.foreground_node_launch:
+            self.foreground_node_launcher = BaseNodeLauncher(
+                provider=self.provider,
+                pending=self.pending_launches,
+                event_summarizer=self.event_summarizer,
+                node_provider_availability_tracker=self.node_provider_availability_tracker,  # noqa: E501 Flake and black disagree how to format this.
+                session_name=session_name,
+                node_types=self.available_node_types,
+                prom_metrics=self.prom_metrics,
+            )
+        else:
+            self.launch_queue = queue.Queue()
+            max_batches = math.ceil(max_concurrent_launches / float(max_launch_batch))
+            for i in range(int(max_batches)):
+                node_launcher = NodeLauncher(
+                    provider=self.provider,
+                    queue=self.launch_queue,
+                    index=i,
+                    pending=self.pending_launches,
+                    event_summarizer=self.event_summarizer,
+                    node_provider_availability_tracker=self.node_provider_availability_tracker,  # noqa: E501 Flake and black disagreee how to format this.
+                    session_name=session_name,
+                    node_types=self.available_node_types,
+                    prom_metrics=self.prom_metrics,
+                )
+                node_launcher.daemon = True
+                node_launcher.start()
+
+        # NodeTracker maintains soft state to track the number of recently
+        # failed nodes. It is best effort only.
+        self.node_tracker = NodeTracker()
+
+        # Expand local file_mounts to allow ~ in the paths. This can't be done
+        # earlier when the config is written since we might be on different
+        # platform and the expansion would result in wrong path.
+        self.config["file_mounts"] = {
+            remote: os.path.expanduser(local)
+            for remote, local in self.config["file_mounts"].items()
+        }
+
+        self.gcs_client = gcs_client
+
+        for local_path in self.config["file_mounts"].values():
+            assert os.path.exists(local_path)
+        logger.info("StandardAutoscaler: {}".format(self.config))
+
+    @property
+    def all_node_types(self) -> Set[str]:
+        return self.config["available_node_types"].keys()
+
+    def update(self):
+        try:
+            self.reset(errors_fatal=False)
+            self._update()
+        except Exception as e:
+            self.prom_metrics.update_loop_exceptions.inc()
+            logger.exception("StandardAutoscaler: Error during autoscaling.")
+            self.num_failures += 1
+            if self.num_failures > self.max_failures:
+                logger.critical("StandardAutoscaler: Too many errors, abort.")
+                raise e
+
+    def _update(self):
+        # For type checking, assert that these objects have been instantitiated.
+        assert self.provider
+        assert self.resource_demand_scheduler
+
+        now = time.time()
+        # Throttle autoscaling updates to this interval to avoid exceeding
+        # rate limits on API calls.
+        if now - self.last_update_time < self.update_interval_s:
+            return
+
+        self.last_update_time = now
+
+        # Query the provider to update the list of non-terminated nodes
+        self.non_terminated_nodes = NonTerminatedNodes(self.provider)
+
+        # Back off the update if the provider says it's not safe to proceed.
+        if not self.provider.safe_to_scale():
+            logger.info(
+                "Backing off of autoscaler update."
+                f" Will try again in {self.update_interval_s} seconds."
+            )
+            return
+
+        # This will accumulate the nodes we need to terminate.
+        self.nodes_to_terminate = []
+
+        # Update status strings
+        if AUTOSCALER_STATUS_LOG:
+            logger.info(self.info_string())
+        legacy_log_info_string(self, self.non_terminated_nodes.worker_ids)
+
+        if not self.provider.is_readonly():
+            self.terminate_nodes_to_enforce_config_constraints(now)
+
+            if self.disable_node_updaters:
+                # Don't handle unhealthy nodes if the liveness check is disabled.
+                # self.worker_liveness_check is True by default.
+                if self.worker_liveness_check:
+                    self.terminate_unhealthy_nodes(now)
+            else:
+                self.process_completed_updates()
+                self.update_nodes()
+                # Don't handle unhealthy nodes if the liveness check is disabled.
+                # self.worker_liveness_check is True by default.
+                if self.worker_liveness_check:
+                    self.attempt_to_recover_unhealthy_nodes(now)
+                self.set_prometheus_updater_data()
+
+        # Update running nodes gauge
+        num_workers = len(self.non_terminated_nodes.worker_ids)
+        self.prom_metrics.running_workers.set(num_workers)
+
+        # Remove IPs from LoadMetrics that are not known to the NodeProvider.
+        self.load_metrics.prune_active_ips(
+            active_ips=[
+                self.provider.internal_ip(node_id)
+                for node_id in self.non_terminated_nodes.all_node_ids
+            ]
+        )
+
+        # Dict[NodeType, int], List[ResourceDict]
+        to_launch, unfulfilled = self.resource_demand_scheduler.get_nodes_to_launch(
+            self.non_terminated_nodes.all_node_ids,
+            self.pending_launches.breakdown(),
+            self.load_metrics.get_resource_demand_vector(),
+            self.load_metrics.get_resource_utilization(),
+            self.load_metrics.get_pending_placement_groups(),
+            self.load_metrics.get_static_node_resources_by_ip(),
+            ensure_min_cluster_size=self.load_metrics.get_resource_requests(),
+            node_availability_summary=self.node_provider_availability_tracker.summary(),
+        )
+        self._report_pending_infeasible(unfulfilled)
+
+        if not self.provider.is_readonly():
+            self.launch_required_nodes(to_launch)
+
+        # Execute optional end-of-update logic.
+        # Keep this method call at the end of autoscaler._update().
+        self.provider.post_process()
+
+        # Record the amount of time the autoscaler took for
+        # this _update() iteration.
+        update_time = time.time() - self.last_update_time
+        logger.info(
+            f"The autoscaler took {round(update_time, 3)}"
+            " seconds to complete the update iteration."
+        )
+        self.prom_metrics.update_time.observe(update_time)
+
+    def terminate_nodes_to_enforce_config_constraints(self, now: float):
+        """Terminates nodes to enforce constraints defined by the autoscaling
+        config.
+
+        (1) Terminates nodes in excess of `max_workers`.
+        (2) Terminates nodes idle for longer than `idle_timeout_minutes`.
+        (3) Terminates outdated nodes,
+                namely nodes whose configs don't match `node_config` for the
+                relevant node type.
+
+        Avoids terminating non-outdated nodes required by
+        autoscaler.sdk.request_resources().
+        """
+        # For type checking, assert that these objects have been instantitiated.
+        assert self.non_terminated_nodes
+        assert self.provider
+
+        last_used = self.load_metrics.ray_nodes_last_used_time_by_ip
+
+        idle_timeout_s = 60 * self.config["idle_timeout_minutes"]
+
+        last_used_cutoff = now - idle_timeout_s
+
+        # Sort based on last used to make sure to keep min_workers that
+        # were most recently used. Otherwise, _keep_min_workers_of_node_type
+        # might keep a node that should be terminated.
+        sorted_node_ids = self._sort_based_on_last_used(
+            self.non_terminated_nodes.worker_ids, last_used
+        )
+
+        # Don't terminate nodes needed by request_resources()
+        nodes_not_allowed_to_terminate = self._get_nodes_needed_for_request_resources(
+            sorted_node_ids
+        )
+
+        # Tracks counts of nodes we intend to keep for each node type.
+        node_type_counts = defaultdict(int)
+
+        def keep_node(node_id: NodeID) -> None:
+            assert self.provider
+            # Update per-type counts.
+            tags = self.provider.node_tags(node_id)
+            if TAG_RAY_USER_NODE_TYPE in tags:
+                node_type = tags[TAG_RAY_USER_NODE_TYPE]
+                node_type_counts[node_type] += 1
+
+        # Nodes that we could terminate, if needed.
+        nodes_we_could_terminate: List[NodeID] = []
+
+        for node_id in sorted_node_ids:
+            # Make sure to not kill idle node types if the number of workers
+            # of that type is lower/equal to the min_workers of that type
+            # or it is needed for request_resources().
+            should_keep_or_terminate, reason = self._keep_worker_of_node_type(
+                node_id, node_type_counts
+            )
+            if should_keep_or_terminate == KeepOrTerminate.terminate:
+                self.schedule_node_termination(node_id, reason, logger.info)
+                continue
+            if (
+                should_keep_or_terminate == KeepOrTerminate.keep
+                or node_id in nodes_not_allowed_to_terminate
+            ) and self.launch_config_ok(node_id):
+                keep_node(node_id)
+                continue
+
+            node_ip = self.provider.internal_ip(node_id)
+
+            if node_ip in last_used and last_used[node_ip] < last_used_cutoff:
+                self.schedule_node_termination(node_id, "idle", logger.info)
+                # Get the local time of the node's last use as a string.
+                formatted_last_used_time = time.asctime(
+                    time.localtime(last_used[node_ip])
+                )
+                logger.info(f"Node last used: {formatted_last_used_time}.")
+                # Note that the current time will appear in the log prefix.
+            elif not self.launch_config_ok(node_id):
+                self.schedule_node_termination(node_id, "outdated", logger.info)
+            else:
+                keep_node(node_id)
+                nodes_we_could_terminate.append(node_id)
+
+        # Terminate nodes if there are too many
+        num_workers = len(self.non_terminated_nodes.worker_ids)
+        num_extra_nodes_to_terminate = (
+            num_workers - len(self.nodes_to_terminate) - self.config["max_workers"]
+        )
+
+        if num_extra_nodes_to_terminate > len(nodes_we_could_terminate):
+            logger.warning(
+                "StandardAutoscaler: trying to terminate "
+                f"{num_extra_nodes_to_terminate} nodes, while only "
+                f"{len(nodes_we_could_terminate)} are safe to terminate."
+                " Inconsistent config is likely."
+            )
+            num_extra_nodes_to_terminate = len(nodes_we_could_terminate)
+
+        # If num_extra_nodes_to_terminate is negative or zero,
+        # we would have less than max_workers nodes after terminating
+        # nodes_to_terminate and we do not need to terminate anything else.
+        if num_extra_nodes_to_terminate > 0:
+            extra_nodes_to_terminate = nodes_we_could_terminate[
+                -num_extra_nodes_to_terminate:
+            ]
+            for node_id in extra_nodes_to_terminate:
+                self.schedule_node_termination(node_id, "max workers", logger.info)
+
+        self.terminate_scheduled_nodes()
+
+    def schedule_node_termination(
+        self, node_id: NodeID, reason_opt: Optional[str], logger_method: Callable
+    ) -> None:
+        # For type checking, assert that this object has been instantitiated.
+        assert self.provider
+
+        if reason_opt is None:
+            raise Exception("reason should be not None.")
+        reason: str = reason_opt
+        node_ip = self.provider.internal_ip(node_id)
+        # Log, record an event, and add node_id to nodes_to_terminate.
+        logger_method(
+            "StandardAutoscaler: "
+            f"Terminating the node with id {node_id}"
+            f" and ip {node_ip}."
+            f" ({reason})"
+        )
+        self.event_summarizer.add(
+            "Removing {} nodes of type "
+            + self._get_node_type(node_id)
+            + " ({}).".format(reason),
+            quantity=1,
+            aggregate=operator.add,
+        )
+        self.nodes_to_terminate.append(node_id)
+
+    def terminate_scheduled_nodes(self):
+        """Terminate scheduled nodes and clean associated autoscaler state."""
+        # For type checking, assert that these objects have been instantitiated.
+        assert self.provider
+        assert self.non_terminated_nodes
+
+        if not self.nodes_to_terminate:
+            return
+
+        # Drain the nodes
+        self.drain_nodes_via_gcs(self.nodes_to_terminate)
+        # Terminate the nodes
+        self.provider.terminate_nodes(self.nodes_to_terminate)
+        for node in self.nodes_to_terminate:
+            self.node_tracker.untrack(node)
+            self.prom_metrics.stopped_nodes.inc()
+
+        # Update internal node lists
+        self.non_terminated_nodes.remove_terminating_nodes(self.nodes_to_terminate)
+
+        self.nodes_to_terminate = []
+
+    def drain_nodes_via_gcs(self, provider_node_ids_to_drain: List[NodeID]):
+        """Send an RPC request to the GCS to drain (prepare for termination)
+        the nodes with the given node provider ids.
+
+        note: The current implementation of DrainNode on the GCS side is to
+        de-register and gracefully shut down the Raylets. In the future,
+        the behavior may change to better reflect the name "Drain."
+        See https://github.com/ray-project/ray/pull/19350.
+        """
+        # For type checking, assert that this object has been instantitiated.
+        assert self.provider
+
+        # The GCS expects Node ids in the request, rather than NodeProvider
+        # ids. To get the Node ids of the nodes to we're draining, we make
+        # the following translations of identifiers:
+        # node provider node id -> ip -> node id
+
+        # Convert node provider node ids to ips.
+        node_ips = set()
+        failed_ip_fetch = False
+        for provider_node_id in provider_node_ids_to_drain:
+            # If the provider's call to fetch ip fails, the exception is not
+            # fatal. Log the exception and proceed.
+            try:
+                ip = self.provider.internal_ip(provider_node_id)
+                node_ips.add(ip)
+            except Exception:
+                logger.exception(
+                    "Failed to get ip of node with id"
+                    f" {provider_node_id} during scale-down."
+                )
+                failed_ip_fetch = True
+        if failed_ip_fetch:
+            self.prom_metrics.drain_node_exceptions.inc()
+
+        # Only attempt to drain connected nodes, i.e. nodes with ips in
+        # LoadMetrics.
+        connected_node_ips = node_ips & self.load_metrics.node_id_by_ip.keys()
+
+        # Convert ips to Node ids.
+        # (The assignment ip->node_id is well-defined under current
+        # assumptions. See "use_node_id_as_ip" in monitor.py)
+        node_ids_to_drain = {
+            self.load_metrics.node_id_by_ip[ip] for ip in connected_node_ips
+        }
+
+        if not node_ids_to_drain:
+            return
+
+        logger.info(f"Draining {len(node_ids_to_drain)} raylet(s).")
+        try:
+            # A successful response indicates that the GCS has marked the
+            # desired nodes as "drained." The cloud provider can then terminate
+            # the nodes without the GCS printing an error.
+            # Check if we succeeded in draining all of the intended nodes by
+            # looking at the RPC response.
+            drained_node_ids = set(
+                self.gcs_client.drain_nodes(node_ids_to_drain, timeout=5)
+            )
+            failed_to_drain = node_ids_to_drain - drained_node_ids
+            if failed_to_drain:
+                self.prom_metrics.drain_node_exceptions.inc()
+                logger.error(f"Failed to drain {len(failed_to_drain)} raylet(s).")
+        # If we get a gRPC error with an UNIMPLEMENTED code, fail silently.
+        # This error indicates that the GCS is using Ray version < 1.8.0,
+        # for which DrainNode is not implemented.
+        except RpcError as e:
+            # If the code is UNIMPLEMENTED, pass.
+            if e.rpc_code == ray._raylet.GRPC_STATUS_CODE_UNIMPLEMENTED:
+                pass
+            # Otherwise, it's a plain old gRPC error and we should log it.
+            else:
+                self.prom_metrics.drain_node_exceptions.inc()
+                logger.exception("Failed to drain Ray nodes. Traceback follows.")
+        except Exception:
+            # We don't need to interrupt the autoscaler update with an
+            # exception, but we should log what went wrong and record the
+            # failure in Prometheus.
+            self.prom_metrics.drain_node_exceptions.inc()
+            logger.exception("Failed to drain Ray nodes. Traceback follows.")
+
+    def launch_required_nodes(self, to_launch: Dict[NodeType, int]) -> None:
+        if to_launch:
+            for node_type, count in to_launch.items():
+                self.launch_new_node(count, node_type=node_type)
+
+    def update_nodes(self):
+        """Run NodeUpdaterThreads to run setup commands, sync files,
+        and/or start Ray.
+        """
+        # Update nodes with out-of-date files.
+        # TODO(edoakes): Spawning these threads directly seems to cause
+        # problems. They should at a minimum be spawned as daemon threads.
+        # See https://github.com/ray-project/ray/pull/5903 for more info.
+        T = []
+        for node_id, setup_commands, ray_start_commands, docker_config in (
+            self.should_update(node_id)
+            for node_id in self.non_terminated_nodes.worker_ids
+        ):
+            if node_id is not None:
+                resources = self._node_resources(node_id)
+                labels = self._node_labels(node_id)
+                logger.debug(f"{node_id}: Starting new thread runner.")
+                T.append(
+                    threading.Thread(
+                        target=self.spawn_updater,
+                        args=(
+                            node_id,
+                            setup_commands,
+                            ray_start_commands,
+                            resources,
+                            labels,
+                            docker_config,
+                        ),
+                    )
+                )
+        for t in T:
+            t.start()
+        for t in T:
+            t.join()
+
+    def process_completed_updates(self):
+        """Clean up completed NodeUpdaterThreads."""
+        completed_nodes = []
+        for node_id, updater in self.updaters.items():
+            if not updater.is_alive():
+                completed_nodes.append(node_id)
+        if completed_nodes:
+            failed_nodes = []
+            for node_id in completed_nodes:
+                updater = self.updaters[node_id]
+                if updater.exitcode == 0:
+                    self.num_successful_updates[node_id] += 1
+                    self.prom_metrics.successful_updates.inc()
+                    if updater.for_recovery:
+                        self.prom_metrics.successful_recoveries.inc()
+                    if updater.update_time:
+                        self.prom_metrics.worker_update_time.observe(
+                            updater.update_time
+                        )
+                    # Mark the node as active to prevent the node recovery
+                    # logic immediately trying to restart Ray on the new node.
+                    self.load_metrics.mark_active(self.provider.internal_ip(node_id))
+                else:
+                    failed_nodes.append(node_id)
+                    self.num_failed_updates[node_id] += 1
+                    self.prom_metrics.failed_updates.inc()
+                    if updater.for_recovery:
+                        self.prom_metrics.failed_recoveries.inc()
+                    self.node_tracker.untrack(node_id)
+                del self.updaters[node_id]
+
+            if failed_nodes:
+                # Some nodes in failed_nodes may already have been terminated
+                # during an update (for being idle after missing a heartbeat).
+
+                # Update the list of non-terminated workers.
+                for node_id in failed_nodes:
+                    # Check if the node has already been terminated.
+                    if node_id in self.non_terminated_nodes.worker_ids:
+                        self.schedule_node_termination(
+                            node_id, "launch failed", logger.error
+                        )
+                    else:
+                        logger.warning(
+                            f"StandardAutoscaler: {node_id}:"
+                            " Failed to update node."
+                            " Node has already been terminated."
+                        )
+                self.terminate_scheduled_nodes()
+
+    def set_prometheus_updater_data(self):
+        """Record total number of active NodeUpdaterThreads and how many of
+        these are being run to recover nodes.
+        """
+        self.prom_metrics.updating_nodes.set(len(self.updaters))
+        num_recovering = 0
+        for updater in self.updaters.values():
+            if updater.for_recovery:
+                num_recovering += 1
+        self.prom_metrics.recovering_nodes.set(num_recovering)
+
+    def _report_pending_infeasible(self, unfulfilled: List[ResourceDict]):
+        """Emit event messages for infeasible or unschedulable tasks.
+
+        This adds messages to the event summarizer for warning on infeasible
+        or "cluster full" resource requests.
+
+        Args:
+            unfulfilled: List of resource demands that would be unfulfilled
+                even after full scale-up.
+        """
+        # For type checking, assert that this object has been instantitiated.
+        assert self.resource_demand_scheduler
+        pending = []
+        infeasible = []
+        for bundle in unfulfilled:
+            placement_group = any(
+                "_group_" in k or k == PLACEMENT_GROUP_BUNDLE_RESOURCE_NAME
+                for k in bundle
+            )
+            if placement_group:
+                continue
+            if self.resource_demand_scheduler.is_feasible(bundle):
+                pending.append(bundle)
+            else:
+                infeasible.append(bundle)
+        if pending:
+            if self.load_metrics.cluster_full_of_actors_detected:
+                for request in pending:
+                    self.event_summarizer.add_once_per_interval(
+                        "Warning: The following resource request cannot be "
+                        "scheduled right now: {}. This is likely due to all "
+                        "cluster resources being claimed by actors. Consider "
+                        "creating fewer actors or adding more nodes "
+                        "to this Ray cluster.".format(request),
+                        key="pending_{}".format(sorted(request.items())),
+                        interval_s=30,
+                    )
+        if infeasible:
+            for request in infeasible:
+                self.event_summarizer.add_once_per_interval(
+                    "Error: No available node types can fulfill resource "
+                    "request {}. Add suitable node types to this cluster to "
+                    "resolve this issue.".format(request),
+                    key="infeasible_{}".format(sorted(request.items())),
+                    interval_s=30,
+                )
+
+    def _sort_based_on_last_used(
+        self, nodes: List[NodeID], last_used: Dict[str, float]
+    ) -> List[NodeID]:
+        """Sort the nodes based on the last time they were used.
+
+        The first item in the return list is the most recently used.
+        """
+        last_used_copy = copy.deepcopy(last_used)
+        # Add the unconnected nodes as the least recently used (the end of
+        # list). This prioritizes connected nodes.
+        least_recently_used = -1
+
+        def last_time_used(node_id: NodeID):
+            assert self.provider
+            node_ip = self.provider.internal_ip(node_id)
+            if node_ip not in last_used_copy:
+                return least_recently_used
+            else:
+                return last_used_copy[node_ip]
+
+        return sorted(nodes, key=last_time_used, reverse=True)
+
+    def _get_nodes_needed_for_request_resources(
+        self, sorted_node_ids: List[NodeID]
+    ) -> FrozenSet[NodeID]:
+        # TODO(ameer): try merging this with resource_demand_scheduler
+        # code responsible for adding nodes for get_resource_requests() and get_pending_placement_groups().
+        """Returns the nodes NOT allowed to terminate due to get_resource_requests() and get_pending_placement_groups().
+
+        Args:
+            sorted_node_ids: the node ids sorted based on last used (LRU last).
+
+        Returns:
+            FrozenSet[NodeID]: a set of nodes (node ids) that
+            we should NOT terminate.
+        """
+        # For type checking, assert that this object has been instantitiated.
+        assert self.provider
+
+        nodes_not_allowed_to_terminate: Set[NodeID] = set()
+
+        resource_demands, strict_spreads = placement_groups_to_resource_demands(
+            self.load_metrics.get_pending_placement_groups()
+        )
+        resource_demands.extend(self.load_metrics.get_resource_requests())
+        if not resource_demands and not strict_spreads:
+            return frozenset(nodes_not_allowed_to_terminate)
+
+        static_node_resources: Dict[
+            NodeIP, ResourceDict
+        ] = self.load_metrics.get_static_node_resources_by_ip()
+
+        head_node_resources: ResourceDict = copy.deepcopy(
+            self.available_node_types[self.config["head_node_type"]]["resources"]
+        )
+        # TODO(ameer): this is somewhat duplicated in
+        # resource_demand_scheduler.py.
+        if not head_node_resources:
+            # Legacy yaml might include {} in the resources field.
+            head_node_ip = self.provider.internal_ip(self.non_terminated_nodes.head_id)
+            head_node_resources = static_node_resources.get(head_node_ip, {})
+
+        node_total_resources: List[ResourceDict] = [head_node_resources]
+        resource_demand_vector_worker_node_ids = []
+        # Get max resources on all the non terminated nodes.
+        for node_id in sorted_node_ids:
+            tags = self.provider.node_tags(node_id)
+            if TAG_RAY_USER_NODE_TYPE in tags:
+                node_type = tags[TAG_RAY_USER_NODE_TYPE]
+                node_resources: ResourceDict = copy.deepcopy(
+                    self.available_node_types[node_type]["resources"]
+                )
+                if not node_resources:
+                    # Legacy yaml might include {} in the resources field.
+                    node_ip = self.provider.internal_ip(node_id)
+                    node_resources = static_node_resources.get(node_ip, {})
+                node_total_resources.append(node_resources)
+                resource_demand_vector_worker_node_ids.append(node_id)
+        # Since it is sorted based on last used, we "keep" nodes that are
+        # most recently used when we binpack. We assume get_bin_pack_residual
+        # is following the given order here.
+        node_remaining_resources = copy.deepcopy(node_total_resources)
+
+        for strict_spread in strict_spreads:
+            unfulfilled, updated_node_remaining_resources = get_bin_pack_residual(
+                node_remaining_resources, strict_spread, strict_spread=True
+            )
+            if unfulfilled:
+                continue
+            node_remaining_resources = updated_node_remaining_resources
+
+        _, node_remaining_resources = get_bin_pack_residual(
+            node_remaining_resources, resource_demands
+        )
+        # Remove the first entry (the head node).
+        node_total_resources.pop(0)
+        # Remove the first entry (the head node).
+        node_remaining_resources.pop(0)
+        for i, node_id in enumerate(resource_demand_vector_worker_node_ids):
+            if (
+                node_remaining_resources[i] == node_total_resources[i]
+                and node_total_resources[i]
+            ):
+                # No resources of the node were needed for request_resources().
+                # node_total_resources[i] is an empty dict for legacy yamls
+                # before the node is connected.
+                pass
+            else:
+                nodes_not_allowed_to_terminate.add(node_id)
+        return frozenset(nodes_not_allowed_to_terminate)
+
+    def _keep_worker_of_node_type(
+        self, node_id: NodeID, node_type_counts: Dict[NodeType, int]
+    ) -> Tuple[KeepOrTerminate, Optional[str]]:
+        """Determines if a worker should be kept based on the min_workers
+        and max_workers constraint of the worker's node_type.
+
+        Returns KeepOrTerminate.keep when both of the following hold:
+        (a) The worker's node_type is present among the keys of the current
+            config's available_node_types dict.
+        (b) Deleting the node would violate the min_workers constraint for that
+            worker's node_type.
+
+        Returns KeepOrTerminate.terminate when both the following hold:
+        (a) The worker's node_type is not present among the keys of the current
+            config's available_node_types dict.
+        (b) Keeping the node would violate the max_workers constraint for that
+            worker's node_type.
+
+        Return KeepOrTerminate.decide_later otherwise.
+
+        Args:
+            node_type_counts(Dict[NodeType, int]): The non_terminated node
+                types counted so far.
+        Returns:
+            KeepOrTerminate: keep if the node should be kept, terminate if the
+            node should be terminated, decide_later if we are allowed
+            to terminate it, but do not have to.
+            Optional[str]: reason for termination. Not None on
+            KeepOrTerminate.terminate, None otherwise.
+        """
+        # For type checking, assert that this object has been instantitiated.
+        assert self.provider
+
+        tags = self.provider.node_tags(node_id)
+        if TAG_RAY_USER_NODE_TYPE in tags:
+            node_type = tags[TAG_RAY_USER_NODE_TYPE]
+
+            min_workers = self.available_node_types.get(node_type, {}).get(
+                "min_workers", 0
+            )
+            max_workers = self.available_node_types.get(node_type, {}).get(
+                "max_workers", 0
+            )
+            if node_type not in self.available_node_types:
+                # The node type has been deleted from the cluster config.
+                # Allow terminating it if needed.
+                available_node_types = list(self.available_node_types.keys())
+                return (
+                    KeepOrTerminate.terminate,
+                    f"not in available_node_types: {available_node_types}",
+                )
+            new_count = node_type_counts[node_type] + 1
+            if new_count <= min(min_workers, max_workers):
+                return KeepOrTerminate.keep, None
+            if new_count > max_workers:
+                return KeepOrTerminate.terminate, "max_workers_per_type"
+
+        return KeepOrTerminate.decide_later, None
+
+    def _node_resources(self, node_id):
+        node_type = self.provider.node_tags(node_id).get(TAG_RAY_USER_NODE_TYPE)
+        if self.available_node_types:
+            return self.available_node_types.get(node_type, {}).get("resources", {})
+        else:
+            return {}
+
+    def _node_labels(self, node_id):
+        node_type = self.provider.node_tags(node_id).get(TAG_RAY_USER_NODE_TYPE)
+        if self.available_node_types:
+            return self.available_node_types.get(node_type, {}).get("labels", {})
+        else:
+            return {}
+
+    def reset(self, errors_fatal=False):
+        sync_continuously = False
+        if hasattr(self, "config"):
+            sync_continuously = self.config.get("file_mounts_sync_continuously", False)
+        try:
+            new_config = self.config_reader()
+            if new_config != getattr(self, "config", None):
+                try:
+                    validate_config(new_config)
+                except Exception as e:
+                    self.prom_metrics.config_validation_exceptions.inc()
+                    logger.debug(
+                        "Cluster config validation failed. The version of "
+                        "the ray CLI you launched this cluster with may "
+                        "be higher than the version of ray being run on "
+                        "the cluster. Some new features may not be "
+                        "available until you upgrade ray on your cluster.",
+                        exc_info=e,
+                    )
+            logger.debug(
+                f"New config after validation: {new_config},"
+                f" of type: {type(new_config)}"
+            )
+            (new_runtime_hash, new_file_mounts_contents_hash) = hash_runtime_conf(
+                new_config["file_mounts"],
+                new_config["cluster_synced_files"],
+                [
+                    new_config["worker_setup_commands"],
+                    new_config["worker_start_ray_commands"],
+                ],
+                generate_file_mounts_contents_hash=sync_continuously,
+            )
+            self.config = new_config
+            self.runtime_hash = new_runtime_hash
+            self.file_mounts_contents_hash = new_file_mounts_contents_hash
+            if not self.provider:
+                self.provider = _get_node_provider(
+                    self.config["provider"], self.config["cluster_name"]
+                )
+
+            # If using the LocalNodeProvider, make sure the head node is marked
+            # non-terminated.
+            if isinstance(self.provider, LocalNodeProvider):
+                record_local_head_state_if_needed(self.provider)
+
+            self.available_node_types = self.config["available_node_types"]
+            upscaling_speed = self.config.get("upscaling_speed")
+            aggressive = self.config.get("autoscaling_mode") == "aggressive"
+            target_utilization_fraction = self.config.get("target_utilization_fraction")
+            if upscaling_speed:
+                upscaling_speed = float(upscaling_speed)
+            # TODO(ameer): consider adding (if users ask) an option of
+            # initial_upscaling_num_workers.
+            elif aggressive:
+                upscaling_speed = 99999
+                logger.warning(
+                    "Legacy aggressive autoscaling mode "
+                    "detected. Replacing it by setting upscaling_speed to "
+                    "99999."
+                )
+            elif target_utilization_fraction:
+                upscaling_speed = 1 / max(target_utilization_fraction, 0.001) - 1
+                logger.warning(
+                    "Legacy target_utilization_fraction config "
+                    "detected. Replacing it by setting upscaling_speed to "
+                    + "1 / target_utilization_fraction - 1."
+                )
+            else:
+                upscaling_speed = 1.0
+            if self.resource_demand_scheduler:
+                # The node types are autofilled internally for legacy yamls,
+                # overwriting the class will remove the inferred node resources
+                # for legacy yamls.
+                self.resource_demand_scheduler.reset_config(
+                    self.provider,
+                    self.available_node_types,
+                    self.config["max_workers"],
+                    self.config["head_node_type"],
+                    upscaling_speed,
+                )
+            else:
+                self.resource_demand_scheduler = ResourceDemandScheduler(
+                    self.provider,
+                    self.available_node_types,
+                    self.config["max_workers"],
+                    self.config["head_node_type"],
+                    upscaling_speed,
+                )
+
+        except Exception as e:
+            self.prom_metrics.reset_exceptions.inc()
+            if errors_fatal:
+                raise e
+            else:
+                logger.exception("StandardAutoscaler: Error parsing config.")
+
+    def launch_config_ok(self, node_id):
+        if self.disable_launch_config_check:
+            return True
+        node_tags = self.provider.node_tags(node_id)
+        tag_launch_conf = node_tags.get(TAG_RAY_LAUNCH_CONFIG)
+        node_type = node_tags.get(TAG_RAY_USER_NODE_TYPE)
+        if node_type not in self.available_node_types:
+            # The node type has been deleted from the cluster config.
+            # Don't keep the node.
+            return False
+
+        # The `worker_nodes` field is deprecated in favor of per-node-type
+        # node_configs. We allow it for backwards-compatibility.
+        launch_config = copy.deepcopy(self.config.get("worker_nodes", {}))
+        if node_type:
+            launch_config.update(
+                self.config["available_node_types"][node_type]["node_config"]
+            )
+        calculated_launch_hash = hash_launch_conf(launch_config, self.config["auth"])
+
+        if calculated_launch_hash != tag_launch_conf:
+            return False
+        return True
+
+    def files_up_to_date(self, node_id):
+        node_tags = self.provider.node_tags(node_id)
+        applied_config_hash = node_tags.get(TAG_RAY_RUNTIME_CONFIG)
+        applied_file_mounts_contents_hash = node_tags.get(TAG_RAY_FILE_MOUNTS_CONTENTS)
+        if applied_config_hash != self.runtime_hash or (
+            self.file_mounts_contents_hash is not None
+            and self.file_mounts_contents_hash != applied_file_mounts_contents_hash
+        ):
+            logger.info(
+                "StandardAutoscaler: "
+                "{}: Runtime state is ({},{}), want ({},{})".format(
+                    node_id,
+                    applied_config_hash,
+                    applied_file_mounts_contents_hash,
+                    self.runtime_hash,
+                    self.file_mounts_contents_hash,
+                )
+            )
+            return False
+        return True
+
+    def heartbeat_on_time(self, node_id: NodeID, now: float) -> bool:
+        """Determine whether we've received a heartbeat from a node within the
+        last AUTOSCALER_HEARTBEAT_TIMEOUT_S seconds.
+        """
+        # For type checking, assert that this object has been instantitiated.
+        assert self.provider
+
+        key = self.provider.internal_ip(node_id)
+
+        if key in self.load_metrics.last_heartbeat_time_by_ip:
+            last_heartbeat_time = self.load_metrics.last_heartbeat_time_by_ip[key]
+            delta = now - last_heartbeat_time
+            if delta < AUTOSCALER_HEARTBEAT_TIMEOUT_S:
+                return True
+        return False
+
+    def terminate_unhealthy_nodes(self, now: float):
+        """Terminated nodes for which we haven't received a heartbeat on time.
+        These nodes are subsequently terminated.
+        """
+        # For type checking, assert that these objects have been instantitiated.
+        assert self.provider
+        assert self.non_terminated_nodes
+
+        for node_id in self.non_terminated_nodes.worker_ids:
+            node_status = self.provider.node_tags(node_id)[TAG_RAY_NODE_STATUS]
+            # We're not responsible for taking down
+            # nodes with pending or failed status:
+            if not node_status == STATUS_UP_TO_DATE:
+                continue
+            # This node is up-to-date. If it hasn't had the chance to produce
+            # a heartbeat, fake the heartbeat now (see logic for completed node
+            # updaters).
+            ip = self.provider.internal_ip(node_id)
+            if ip not in self.load_metrics.last_heartbeat_time_by_ip:
+                self.load_metrics.mark_active(ip)
+            # Heartbeat indicates node is healthy:
+            if self.heartbeat_on_time(node_id, now):
+                continue
+            self.schedule_node_termination(
+                node_id, "lost contact with raylet", logger.warning
+            )
+        self.terminate_scheduled_nodes()
+
+    def attempt_to_recover_unhealthy_nodes(self, now):
+        for node_id in self.non_terminated_nodes.worker_ids:
+            self.recover_if_needed(node_id, now)
+
+    def recover_if_needed(self, node_id, now):
+        if not self.can_update(node_id):
+            return
+        if self.heartbeat_on_time(node_id, now):
+            return
+
+        logger.warning(
+            "StandardAutoscaler: "
+            "{}: No recent heartbeat, "
+            "restarting Ray to recover...".format(node_id)
+        )
+        self.event_summarizer.add(
+            "Restarting {} nodes of type "
+            + self._get_node_type(node_id)
+            + " (lost contact with raylet).",
+            quantity=1,
+            aggregate=operator.add,
+        )
+        head_node_ip = self.provider.internal_ip(self.non_terminated_nodes.head_id)
+        updater = NodeUpdaterThread(
+            node_id=node_id,
+            provider_config=self.config["provider"],
+            provider=self.provider,
+            auth_config=self.config["auth"],
+            cluster_name=self.config["cluster_name"],
+            file_mounts={},
+            initialization_commands=[],
+            setup_commands=[],
+            ray_start_commands=with_head_node_ip(
+                self.config["worker_start_ray_commands"], head_node_ip
+            ),
+            runtime_hash=self.runtime_hash,
+            file_mounts_contents_hash=self.file_mounts_contents_hash,
+            process_runner=self.process_runner,
+            use_internal_ip=True,
+            is_head_node=False,
+            docker_config=self._get_node_specific_docker_config(node_id),
+            node_resources=self._node_resources(node_id),
+            node_labels=self._node_labels(node_id),
+            for_recovery=True,
+        )
+        updater.start()
+        self.updaters[node_id] = updater
+
+    def _get_node_type(self, node_id: str) -> str:
+        # For type checking, assert that this object has been instantitiated.
+        assert self.provider
+
+        node_tags = self.provider.node_tags(node_id)
+        if TAG_RAY_USER_NODE_TYPE in node_tags:
+            return node_tags[TAG_RAY_USER_NODE_TYPE]
+        else:
+            return "unknown_node_type"
+
+    def _get_node_type_specific_fields(self, node_id: str, fields_key: str) -> Any:
+        # For type checking, assert that this object has been instantitiated.
+        assert self.provider
+
+        fields = self.config[fields_key]
+        node_tags = self.provider.node_tags(node_id)
+        if TAG_RAY_USER_NODE_TYPE in node_tags:
+            node_type = node_tags[TAG_RAY_USER_NODE_TYPE]
+            if node_type not in self.available_node_types:
+                raise ValueError(f"Unknown node type tag: {node_type}.")
+            node_specific_config = self.available_node_types[node_type]
+            if fields_key in node_specific_config:
+                fields = node_specific_config[fields_key]
+        return fields
+
+    def _get_node_specific_docker_config(self, node_id):
+        if "docker" not in self.config:
+            return {}
+        docker_config = copy.deepcopy(self.config.get("docker", {}))
+        node_specific_docker = self._get_node_type_specific_fields(node_id, "docker")
+        docker_config.update(node_specific_docker)
+        return docker_config
+
+    def should_update(self, node_id):
+        if not self.can_update(node_id):
+            return UpdateInstructions(None, None, None, None)  # no update
+
+        status = self.provider.node_tags(node_id).get(TAG_RAY_NODE_STATUS)
+        if status == STATUS_UP_TO_DATE and self.files_up_to_date(node_id):
+            return UpdateInstructions(None, None, None, None)  # no update
+
+        successful_updated = self.num_successful_updates.get(node_id, 0) > 0
+        if successful_updated and self.config.get("restart_only", False):
+            setup_commands = []
+            ray_start_commands = self.config["worker_start_ray_commands"]
+        elif successful_updated and self.config.get("no_restart", False):
+            setup_commands = self._get_node_type_specific_fields(
+                node_id, "worker_setup_commands"
+            )
+            ray_start_commands = []
+        else:
+            setup_commands = self._get_node_type_specific_fields(
+                node_id, "worker_setup_commands"
+            )
+            ray_start_commands = self.config["worker_start_ray_commands"]
+
+        docker_config = self._get_node_specific_docker_config(node_id)
+        return UpdateInstructions(
+            node_id=node_id,
+            setup_commands=setup_commands,
+            ray_start_commands=ray_start_commands,
+            docker_config=docker_config,
+        )
+
+    def spawn_updater(
+        self,
+        node_id,
+        setup_commands,
+        ray_start_commands,
+        node_resources,
+        node_labels,
+        docker_config,
+    ):
+        logger.info(
+            f"Creating new (spawn_updater) updater thread for node" f" {node_id}."
+        )
+        ip = self.provider.internal_ip(node_id)
+        node_type = self._get_node_type(node_id)
+        self.node_tracker.track(node_id, ip, node_type)
+        head_node_ip = self.provider.internal_ip(self.non_terminated_nodes.head_id)
+        updater = NodeUpdaterThread(
+            node_id=node_id,
+            provider_config=self.config["provider"],
+            provider=self.provider,
+            auth_config=self.config["auth"],
+            cluster_name=self.config["cluster_name"],
+            file_mounts=self.config["file_mounts"],
+            initialization_commands=with_head_node_ip(
+                self._get_node_type_specific_fields(node_id, "initialization_commands"),
+                head_node_ip,
+            ),
+            setup_commands=with_head_node_ip(setup_commands, head_node_ip),
+            ray_start_commands=with_head_node_ip(ray_start_commands, head_node_ip),
+            runtime_hash=self.runtime_hash,
+            file_mounts_contents_hash=self.file_mounts_contents_hash,
+            is_head_node=False,
+            cluster_synced_files=self.config["cluster_synced_files"],
+            rsync_options={
+                "rsync_exclude": self.config.get("rsync_exclude"),
+                "rsync_filter": self.config.get("rsync_filter"),
+            },
+            process_runner=self.process_runner,
+            use_internal_ip=True,
+            docker_config=docker_config,
+            node_resources=node_resources,
+            node_labels=node_labels,
+        )
+        updater.start()
+        self.updaters[node_id] = updater
+
+    def can_update(self, node_id):
+        if self.disable_node_updaters:
+            return False
+        if node_id in self.updaters:
+            return False
+        if not self.launch_config_ok(node_id):
+            return False
+        if self.num_failed_updates.get(node_id, 0) > 0:  # TODO(ekl) retry?
+            return False
+        logger.debug(
+            f"{node_id} is not being updated and "
+            "passes config check (can_update=True)."
+        )
+        return True
+
+    def launch_new_node(self, count: int, node_type: str) -> None:
+        logger.info("StandardAutoscaler: Queue {} new nodes for launch".format(count))
+        self.pending_launches.inc(node_type, count)
+        config = copy.deepcopy(self.config)
+        if self.foreground_node_launch:
+            assert self.foreground_node_launcher is not None
+            # Launch in the main thread and block.
+            self.foreground_node_launcher.launch_node(config, count, node_type)
+        else:
+            assert self.launch_queue is not None
+            # Split into individual launch requests of the max batch size.
+            while count > 0:
+                # Enqueue launch data for the background NodeUpdater threads.
+                self.launch_queue.put(
+                    (config, min(count, self.max_launch_batch), node_type)
+                )
+                count -= self.max_launch_batch
+
+    def kill_workers(self):
+        logger.error("StandardAutoscaler: kill_workers triggered")
+        nodes = self.workers()
+        if nodes:
+            self.provider.terminate_nodes(nodes)
+            for node in nodes:
+                self.node_tracker.untrack(node)
+                self.prom_metrics.stopped_nodes.inc()
+        logger.error("StandardAutoscaler: terminated {} node(s)".format(len(nodes)))
+
+    def summary(self) -> Optional[AutoscalerSummary]:
+        """Summarizes the active, pending, and failed node launches.
+
+        An active node is a node whose raylet is actively reporting heartbeats.
+        A pending node is non-active node whose node tag is uninitialized,
+        waiting for ssh, syncing files, or setting up.
+        If a node is not pending or active, it is failed.
+
+        Returns:
+            AutoscalerSummary: The summary.
+        """
+        # For type checking, assert that this object has been instantitiated.
+        assert self.provider
+
+        if not self.non_terminated_nodes:
+            return None
+        active_nodes: Dict[NodeType, int] = Counter()
+        pending_nodes = []
+        failed_nodes = []
+        non_failed = set()
+
+        node_type_mapping = {}
+        now = time.time()
+        for node_id in self.non_terminated_nodes.all_node_ids:
+            ip = self.provider.internal_ip(node_id)
+            node_tags = self.provider.node_tags(node_id)
+
+            if not all(
+                tag in node_tags
+                for tag in (
+                    TAG_RAY_NODE_KIND,
+                    TAG_RAY_USER_NODE_TYPE,
+                    TAG_RAY_NODE_STATUS,
+                )
+            ):
+                # In some node providers, creation of a node and tags is not
+                # atomic, so just skip it.
+                continue
+
+            if node_tags[TAG_RAY_NODE_KIND] == NODE_KIND_UNMANAGED:
+                continue
+            node_type = node_tags[TAG_RAY_USER_NODE_TYPE]
+
+            node_type_mapping[ip] = node_type
+
+            is_active = self.heartbeat_on_time(node_id, now)
+            if is_active:
+                active_nodes[node_type] += 1
+                non_failed.add(node_id)
+            else:
+                status = node_tags[TAG_RAY_NODE_STATUS]
+                completed_states = [STATUS_UP_TO_DATE, STATUS_UPDATE_FAILED]
+                is_pending = status not in completed_states
+                if is_pending:
+                    pending_nodes.append((node_id, ip, node_type, status))
+                    non_failed.add(node_id)
+
+        failed_nodes = self.node_tracker.get_all_failed_node_info(non_failed)
+
+        # The concurrent counter leaves some 0 counts in, so we need to
+        # manually filter those out.
+        pending_launches = {}
+        for node_type, count in self.pending_launches.breakdown().items():
+            if count:
+                pending_launches[node_type] = count
+
+        pending_resources = {}
+        for node_resources in self.resource_demand_scheduler.calculate_node_resources(
+            nodes=[node_id for node_id, _, _, _ in pending_nodes],
+            pending_nodes=pending_launches,
+            # We don't fill this field out because we're intentionally only
+            # passing pending nodes (which aren't tracked by load metrics
+            # anyways).
+            unused_resources_by_ip={},
+        )[0]:
+            for key, value in node_resources.items():
+                pending_resources[key] = value + pending_resources.get(key, 0)
+
+        return AutoscalerSummary(
+            # Convert active_nodes from counter to dict for later serialization
+            active_nodes=dict(active_nodes),
+            idle_nodes=None,
+            pending_nodes=[
+                (ip, node_type, status) for _, ip, node_type, status in pending_nodes
+            ],
+            pending_launches=pending_launches,
+            failed_nodes=failed_nodes,
+            node_availability_summary=self.node_provider_availability_tracker.summary(),
+            pending_resources=pending_resources,
+            node_type_mapping=node_type_mapping,
+            legacy=True,
+        )
+
+    def info_string(self):
+        lm_summary = self.load_metrics.summary()
+        autoscaler_summary = self.summary()
+        assert autoscaler_summary
+        return "\n" + format_info_string(lm_summary, autoscaler_summary)