Duplicate from aws-neuron/optimum-neuron-cache

Co-authored-by: Michael Benayoun <michaelbenayoun@users.noreply.huggingface.co>

README.md

@@ -0,0 +1,27 @@
+---
+license: apache-2.0
+---
+# AWS Neuron optimum model cache
+
+This repository contains cached neuron compilation artifacts for the most popular models on the Hugging Face Hub.
+
+## Inference
+
+### LLM models 
+
+The transparent caching mechanism included in `optimum-neuron` and `NeuronX TGI`, makes it easier to export and deploy cached models to Neuron platforms such as Trainium and Inferentia.
+
+To deploy directly any cached model to SageMaker:
+- go to the model page,
+- select "Deploy" in the top right corner,
+- select "AWS SageMaker" in the drop-down,
+- select the "AWS Inferentia & Trainium" tab,
+- copy the code snippet.
+
+You can now paste the code snippet in your deployment script or notebook, following the instructions in the comment.
+
+To export a model to Neuron and save it locally, please follow the instructions in the `optimum-neuron` [documentation](https://huggingface.co/docs/optimum-neuron/guides/export_model).
+
+For a list of the cached models and configurations, please refer to the inference cache [configuration files](https://huggingface.co/aws-neuron/optimum-neuron-cache/tree/main/inference-cache-config).
+
+Alternatively, you can use the `optimum-cli neuron cache lookup` command to look for a specific model and see the cached configurations.

inference-cache-config/granite.json

@@ -0,0 +1,66 @@
+{
+   "ibm-granite/granite-3.1-2b-instruct": [
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "num_cores": 2,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 4,
+      "sequence_length": 4096,
+      "num_cores": 2,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 8,
+      "sequence_length": 4096,
+      "num_cores": 2,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "num_cores": 8,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 32,
+      "sequence_length": 4096,
+      "num_cores": 8,
+      "auto_cast_type": "bf16"
+    }
+  ],
+  "ibm-granite/granite-3.1-8b-instruct": [
+   {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "num_cores": 2,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 4,
+      "sequence_length": 4096,
+      "num_cores": 2,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 8,
+      "sequence_length": 4096,
+      "num_cores": 2,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "num_cores": 8,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 32,
+      "sequence_length": 4096,
+      "num_cores": 8,
+      "auto_cast_type": "bf16"
+    }
+  ]
+}

inference-cache-config/llama-variants.json

@@ -0,0 +1,88 @@
+{
+  "unsloth/Llama-3.2-1B": [
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "num_cores": 2,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 4,
+      "sequence_length": 4096,
+      "num_cores": 2,
+      "auto_cast_type": "bf16"
+    }
+  ],
+  "deepseek-ai/DeepSeek-R1-Distill-Llama-8B": [
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "num_cores": 2,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 4,
+      "sequence_length": 4096,
+      "num_cores": 2,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 8,
+      "sequence_length": 4096,
+      "num_cores": 2,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 4,
+      "sequence_length": 4096,
+      "num_cores": 8,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 8,
+      "sequence_length": 4096,
+      "num_cores": 8,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 16,
+      "sequence_length": 4096,
+      "num_cores": 8,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 32,
+      "sequence_length": 4096,
+      "num_cores": 8,
+      "auto_cast_type": "bf16"
+    }
+  ],
+  "princeton-nlp/Sheared-LLaMA-1.3B": [
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "num_cores": 2,
+      "auto_cast_type": "fp16"
+    },
+    {
+      "batch_size": 4,
+      "sequence_length": 4096,
+      "num_cores": 2,
+      "auto_cast_type": "fp16"
+    }
+  ],
+  "lmsys/vicuna-7b-v1.5": [
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "num_cores": 2,
+      "auto_cast_type": "fp16"
+    },
+    {
+      "batch_size": 4,
+      "sequence_length": 4096,
+      "num_cores": 2,
+      "auto_cast_type": "fp16"
+    }
+  ]
+}

inference-cache-config/llama.json

@@ -0,0 +1,106 @@
+{
+   "meta-llama/Meta-Llama-3.1-8B": [
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "num_cores": 2,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 4,
+      "sequence_length": 4096,
+      "num_cores": 2,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 8,
+      "sequence_length": 4096,
+      "num_cores": 2,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 4,
+      "sequence_length": 4096,
+      "num_cores": 8,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 8,
+      "sequence_length": 4096,
+      "num_cores": 8,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 16,
+      "sequence_length": 4096,
+      "num_cores": 8,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 32,
+      "sequence_length": 4096,
+      "num_cores": 8,
+      "auto_cast_type": "bf16"
+    }
+  ],
+  "meta-llama/Llama-2-7b-hf": [
+    {
+      "batch_size": 1,
+      "sequence_length": 2048,
+      "num_cores": 2,
+      "auto_cast_type": "bf16"
+    }
+  ],
+  "meta-llama/Llama-2-13b-hf": [
+    {
+      "batch_size": 1,
+      "sequence_length": 2048,
+      "num_cores": 8,
+      "auto_cast_type": "bf16"
+    }
+  ],
+   "meta-llama/Meta-Llama-3-8B": [
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "num_cores": 2,
+      "auto_cast_type": "bf16"
+    }
+  ],
+   "meta-llama/Llama-3.2-1B": [
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "num_cores": 2,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 4,
+      "sequence_length": 4096,
+      "num_cores": 2,
+      "auto_cast_type": "bf16"
+    }
+  ],
+   "meta-llama/Llama-3.2-3B": [
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "num_cores": 2,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 4,
+      "sequence_length": 4096,
+      "num_cores": 2,
+      "auto_cast_type": "bf16"
+    }
+  ],
+     "TinyLlama/TinyLlama-1.1B-Chat-v1.0": [
+    {
+      "batch_size": 1,
+      "sequence_length": 2048,
+      "num_cores": 2,
+      "auto_cast_type": "bf16"
+    }
+  ]
+}

inference-cache-config/llama2-70b.json

@@ -0,0 +1,16 @@
+{
+  "meta-llama/Llama-2-70b-chat-hf": [
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "num_cores": 24,
+      "auto_cast_type": "fp16"
+    },
+    {
+      "batch_size": 4,
+      "sequence_length": 4096,
+      "num_cores": 24,
+      "auto_cast_type": "fp16"
+    }
+  ]
+}

inference-cache-config/llama3-70b.json

@@ -0,0 +1,30 @@
+{
+  "deepseek-ai/DeepSeek-R1-Distill-Llama-70B": [
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "num_cores": 24,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 4,
+      "sequence_length": 4096,
+      "num_cores": 24,
+      "auto_cast_type": "bf16"
+    }
+  ],
+  "meta-llama/Llama-3.3-70B-Instruct": [
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "num_cores": 24,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 4,
+      "sequence_length": 4096,
+      "num_cores": 24,
+      "auto_cast_type": "bf16"
+    }
+  ]
+}

inference-cache-config/llama3.1-70b.json

@@ -0,0 +1,16 @@
+{
+  "meta-llama/Llama-3.1-70B-Instruct": [
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "num_cores": 24,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 4,
+      "sequence_length": 4096,
+      "num_cores": 24,
+      "auto_cast_type": "bf16"
+    }
+  ]
+}

inference-cache-config/llama4.json

@@ -0,0 +1,126 @@
+{
+  "meta-llama/Llama-4-Scout-17B-16E-Instruct": [
+    {
+      "task": "text-generation",
+      "instance_type": "trn1",
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "num_cores": 16,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "task": "text-generation",
+      "instance_type": "trn1",
+      "batch_size": 4,
+      "sequence_length": 4096,
+      "num_cores": 16,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "task": "text-generation",
+      "instance_type": "trn1",
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "num_cores": 32,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "task": "text-generation",
+      "instance_type": "trn1",
+      "batch_size": 4,
+      "sequence_length": 4096,
+      "num_cores": 32,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "task": "text-generation",
+      "instance_type": "trn1",
+      "batch_size": 8,
+      "sequence_length": 4096,
+      "num_cores": 32,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "task": "text-generation",
+      "instance_type": "trn2",
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "num_cores": 32,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "task": "text-generation",
+      "instance_type": "trn2",
+      "batch_size": 4,
+      "sequence_length": 4096,
+      "num_cores": 32,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "task": "text-generation",
+      "instance_type": "trn2",
+      "batch_size": 8,
+      "sequence_length": 4096,
+      "num_cores": 32,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "task": "text-generation",
+      "instance_type": "trn2",
+      "batch_size": 16,
+      "sequence_length": 4096,
+      "num_cores": 32,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "task": "text-generation",
+      "instance_type": "trn2",
+      "batch_size": 32,
+      "sequence_length": 4096,
+      "num_cores": 32,
+      "auto_cast_type": "bf16"
+    }
+  ],
+  "meta-llama/Llama-4-Maverick-17B-128E-Instruct": [
+    {
+      "task": "text-generation",
+      "instance_type": "trn2",
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "num_cores": 64,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "task": "text-generation",
+      "instance_type": "trn2",
+      "batch_size": 4,
+      "sequence_length": 4096,
+      "num_cores": 64,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "task": "text-generation",
+      "instance_type": "trn1",
+      "batch_size": 8,
+      "sequence_length": 4096,
+      "num_cores": 64,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "task": "text-generation",
+      "instance_type": "trn1",
+      "batch_size": 16,
+      "sequence_length": 4096,
+      "num_cores": 64,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "task": "text-generation",
+      "instance_type": "trn1",
+      "batch_size": 32,
+      "sequence_length": 4096,
+      "num_cores": 64,
+      "auto_cast_type": "bf16"
+    }
+  ]
+}

inference-cache-config/mixtral.json

@@ -0,0 +1,30 @@
+{
+  "mistralai/Mixtral-8x7B-Instruct-v0.1": [
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "num_cores": 24,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 4,
+      "sequence_length": 4096,
+      "num_cores": 24,
+      "auto_cast_type": "bf16"
+    }
+  ],
+  "mistralai/Mixtral-8x22B-Instruct-v0.1": [
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "num_cores": 24,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 4,
+      "sequence_length": 4096,
+      "num_cores": 24,
+      "auto_cast_type": "bf16"
+    }
+  ]
+}

inference-cache-config/phi4.json

@@ -0,0 +1,30 @@
+{
+   "microsoft/Phi-3-mini-4k-instruct": [
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "num_cores": 2,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 4,
+      "sequence_length": 4096,
+      "num_cores": 2,
+      "auto_cast_type": "bf16"
+    }
+  ],
+  "microsoft/phi-4": [
+   {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "num_cores": 10,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 16,
+      "sequence_length": 4096,
+      "num_cores": 10,
+      "auto_cast_type": "bf16"
+    }
+  ]
+}

inference-cache-config/pixart-alpha-xl-512x512.json

@@ -0,0 +1,5 @@
+{
+    "PixArt-alpha/PixArt-XL-2-512x512": [
+        {  "task": "pixart", "batch_size": 1, "height": 512, "width": 512, "num_images_per_prompt": 1, "torch_dtype": "bfloat16", "sequence_length": 120 }
+    ]
+}

inference-cache-config/pixart-sigma-xl-512x512.json

@@ -0,0 +1,5 @@
+{
+    "Jingya/pixart_sigma_pipe_xl_2_512_ms": [
+        {  "task": "pixart", "batch_size": 1, "height": 512, "width": 512, "num_images_per_prompt": 1, "torch_dtype": "bfloat16", "sequence_length": 120 }
+    ]
+}

inference-cache-config/qwen-moe.json

@@ -0,0 +1,28 @@
+{
+   "Qwen/Qwen3-30B-A3B-Instruct-2507": [
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "num_cores": 8,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 4,
+      "sequence_length": 4096,
+      "num_cores": 8,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 8,
+      "sequence_length": 4096,
+      "num_cores": 16,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 8,
+      "sequence_length": 4096,
+      "num_cores": 32,
+      "auto_cast_type": "bf16"
+    }
+   ]
+}

inference-cache-config/qwen.json

@@ -0,0 +1,104 @@
+{
+   "Qwen/Qwen3-8B": [
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "num_cores": 2,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 4,
+      "sequence_length": 4096,
+      "num_cores": 2,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 8,
+      "sequence_length": 4096,
+      "num_cores": 2,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "num_cores": 8,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 32,
+      "sequence_length": 4096,
+      "num_cores": 8,
+      "auto_cast_type": "bf16"
+    }
+  ],
+  "Qwen/Qwen3-0.6B": [
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "num_cores": 2,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 4,
+      "sequence_length": 4096,
+      "num_cores": 2,
+      "auto_cast_type": "bf16"
+    }
+  ],
+  "Qwen/Qwen3-1.7B": [
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "num_cores": 2,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 4,
+      "sequence_length": 4096,
+      "num_cores": 2,
+      "auto_cast_type": "bf16"
+    }
+  ],
+  "Qwen/Qwen3-4B": [
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "num_cores": 2,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 4,
+      "sequence_length": 4096,
+      "num_cores": 2,
+      "auto_cast_type": "bf16"
+    }
+  ],
+  "Qwen/Qwen3-14B": [
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "num_cores": 8,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 16,
+      "sequence_length": 4096,
+      "num_cores": 8,
+      "auto_cast_type": "bf16"
+    }
+  ],
+  "Qwen/Qwen3-32B": [
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "num_cores": 8,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 16,
+      "sequence_length": 4096,
+      "num_cores": 8,
+      "auto_cast_type": "bf16"
+    }
+  ]
+}

inference-cache-config/qwen2.5-large.json

@@ -0,0 +1,44 @@
+{
+  "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B": [
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "num_cores": 8,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 8,
+      "sequence_length": 4096,
+      "num_cores": 8,
+      "auto_cast_type": "bf16"
+    }
+  ],
+  "Qwen/Qwen2.5-32B-Instruct": [
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "num_cores": 8,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 8,
+      "sequence_length": 4096,
+      "num_cores": 8,
+      "auto_cast_type": "bf16"
+    }
+  ],
+  "Qwen/Qwen2.5-72B-Instruct": [
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "num_cores": 2,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 4,
+      "sequence_length": 4096,
+      "num_cores": 24,
+      "auto_cast_type": "bf16"
+    }
+  ]
+}

inference-cache-config/qwen2.5.json

@@ -0,0 +1,136 @@
+{
+   "Qwen/Qwen2.5-7B-Instruct": [
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "num_cores": 2,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 4,
+      "sequence_length": 4096,
+      "num_cores": 2,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 8,
+      "sequence_length": 4096,
+      "num_cores": 2,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "num_cores": 8,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 32,
+      "sequence_length": 4096,
+      "num_cores": 8,
+      "auto_cast_type": "bf16"
+    }
+  ],
+   "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B": [
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "num_cores": 2,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 4,
+      "sequence_length": 4096,
+      "num_cores": 2,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 8,
+      "sequence_length": 4096,
+      "num_cores": 2,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "num_cores": 8,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 32,
+      "sequence_length": 4096,
+      "num_cores": 8,
+      "auto_cast_type": "bf16"
+    }
+  ],
+  "Qwen/Qwen2.5-0.5B": [
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "num_cores": 2,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 4,
+      "sequence_length": 4096,
+      "num_cores": 2,
+      "auto_cast_type": "bf16"
+    }
+  ],
+  "Qwen/Qwen2.5-1.5B": [
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "num_cores": 2,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 4,
+      "sequence_length": 4096,
+      "num_cores": 2,
+      "auto_cast_type": "bf16"
+    }
+  ],
+  "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B": [
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "num_cores": 2,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 4,
+      "sequence_length": 4096,
+      "num_cores": 2,
+      "auto_cast_type": "bf16"
+    }
+  ],
+  "Qwen/Qwen2.5-14B": [
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "num_cores": 8,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 16,
+      "sequence_length": 4096,
+      "num_cores": 8,
+      "auto_cast_type": "bf16"
+    }
+  ],
+  "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B": [
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "num_cores": 8,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 16,
+      "sequence_length": 4096,
+      "num_cores": 8,
+      "auto_cast_type": "bf16"
+    }
+  ]
+}

inference-cache-config/sdxl-turbo.json

@@ -0,0 +1,6 @@
+{
+    "stabilityai/sdxl-turbo": [
+        {  "task": "stable-diffusion", "batch_size": 1, "height": 768, "width": 768, "num_images_per_prompt": 1, "auto_cast": "matmul", "auto_cast_type": "bf16" },
+        {  "task": "stable-diffusion", "batch_size": 1, "height": 1024, "width": 1024, "num_images_per_prompt": 1, "auto_cast": "matmul", "auto_cast_type": "bf16" }
+    ]
+}

inference-cache-config/smollm3.json

@@ -0,0 +1,22 @@
+{
+ "HuggingFaceTB/SmolLM3-3B": [
+   {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "num_cores": 2,
+      "auto_cast_type": "bf16"
+    },
+   {
+      "batch_size": 4,
+      "sequence_length": 4096,
+      "num_cores": 2,
+      "auto_cast_type": "bf16"
+    },
+    {
+      "batch_size": 16,
+      "sequence_length": 4096,
+      "num_cores": 2,
+      "auto_cast_type": "bf16"
+    }
+  ]
+}

inference-cache-config/stable-diffusion-2-1.json

@@ -0,0 +1,6 @@
+{
+    "stabilityai/stable-diffusion-2-1": [
+        {  "task": "stable-diffusion", "batch_size": 1, "height": 512, "width": 512, "num_images_per_prompt": 1, "auto_cast": "matmul", "auto_cast_type": "bf16" },
+        {  "task": "stable-diffusion", "batch_size": 1, "height": 768, "width": 768, "num_images_per_prompt": 1, "auto_cast": "matmul", "auto_cast_type": "bf16" }
+    ]
+}

inference-cache-config/stable-diffusion-v1-5.json

@@ -0,0 +1,6 @@
+{
+    "jyoung105/stable-diffusion-v1-5": [
+        {  "task": "stable-diffusion", "batch_size": 1, "height": 512, "width": 512, "num_images_per_prompt": 1, "auto_cast": "matmul", "auto_cast_type": "bf16" },
+        {  "task": "stable-diffusion", "batch_size": 1, "height": 768, "width": 768, "num_images_per_prompt": 1, "auto_cast": "matmul", "auto_cast_type": "bf16" }
+    ]
+}

inference-cache-config/stable-diffusion-xl-base-1.0.json

@@ -0,0 +1,6 @@
+{
+    "stabilityai/stable-diffusion-xl-base-1.0": [
+        {  "task": "stable-diffusion", "batch_size": 1, "height": 768, "width": 768, "num_images_per_prompt": 1, "auto_cast": "matmul", "auto_cast_type": "bf16" },
+        {  "task": "stable-diffusion", "batch_size": 1, "height": 1024, "width": 1024, "num_images_per_prompt": 1, "auto_cast": "matmul", "auto_cast_type": "bf16" }
+    ]
+}

inference-cache-config/stable-diffusion-xl-refiner-1.0.json

@@ -0,0 +1,6 @@
+{
+    "stabilityai/stable-diffusion-xl-refiner-1.0": [
+        {  "task": "stable-diffusion", "batch_size": 1, "height": 768, "width": 768, "num_images_per_prompt": 1, "auto_cast": "matmul", "auto_cast_type": "bf16" },
+        {  "task": "stable-diffusion", "batch_size": 1, "height": 1024, "width": 1024, "num_images_per_prompt": 1, "auto_cast": "matmul", "auto_cast_type": "bf16" }
+    ]
+}

inference-cache-config/trn1/granite.json

@@ -0,0 +1,74 @@
+{
+   "ibm-granite/granite-3.1-2b-instruct": [
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 2,
+      "instance_type" : "trn1"
+    },
+    {
+      "batch_size": 4,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 2,
+      "instance_type" : "trn1"
+    },
+    {
+      "batch_size": 8,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 2,
+      "instance_type" : "trn1"
+    },
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 8,
+      "instance_type" : "trn1"
+    },
+    {
+      "batch_size": 32,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 8,
+      "instance_type" : "trn1"
+    }
+  ],
+  "ibm-granite/granite-3.1-8b-instruct": [
+   {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 2,
+      "instance_type" : "trn1"
+    },
+    {
+      "batch_size": 4,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 2,
+      "instance_type" : "trn1"
+    },
+    {
+      "batch_size": 8,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 2,
+      "instance_type" : "trn1"
+    },
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 8,
+      "instance_type" : "trn1"
+    },
+    {
+      "batch_size": 32,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 8,
+      "instance_type" : "trn1"
+    }
+  ],
+  "ibm-granite/granite-3.3-8b-instruct": [
+   {
+      "batch_size": 1,
+      "sequence_length": 16384,
+      "tensor_parallel_size": 8,
+      "instance_type" : "trn1"
+   }
+  ]
+}

inference-cache-config/trn1/llama3.json

@@ -0,0 +1,156 @@
+{
+   "meta-llama/Meta-Llama-3.1-8B": [
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 2,
+      "instance_type" : "trn1"
+    },
+    {
+      "batch_size": 1,
+      "sequence_length": 16384,
+      "tensor_parallel_size": 2,
+      "instance_type" : "trn1"
+    },
+    {
+      "batch_size": 4,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 2,
+      "instance_type" : "trn1"
+    },
+    {
+      "batch_size": 8,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 2,
+      "instance_type" : "trn1"
+    },
+    {
+      "batch_size": 1,
+      "sequence_length": 16384,
+      "tensor_parallel_size": 8,
+      "instance_type" : "trn1"
+    },
+    {
+      "batch_size": 4,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 8,
+      "instance_type" : "trn1"
+    },
+    {
+      "batch_size": 8,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 8,
+      "instance_type" : "trn1"
+    },
+    {
+      "batch_size": 16,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 8,
+      "instance_type" : "trn1"
+    },
+    {
+      "batch_size": 32,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 8,
+      "instance_type" : "trn1"
+    }
+  ],
+  "meta-llama/Llama-2-7b-hf": [
+    {
+      "batch_size": 1,
+      "sequence_length": 2048,
+      "tensor_parallel_size": 2,
+      "instance_type" : "trn1"
+    }
+  ],
+  "meta-llama/Llama-2-13b-hf": [
+    {
+      "batch_size": 1,
+      "sequence_length": 2048,
+      "tensor_parallel_size": 8,
+      "instance_type" : "trn1"
+    }
+  ],
+   "meta-llama/Meta-Llama-3-8B": [
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 2,
+      "instance_type" : "trn1"
+    }
+  ],
+   "meta-llama/Llama-3.2-1B": [
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 2,
+      "instance_type" : "trn1"
+    },
+    {
+      "batch_size": 4,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 2,
+      "instance_type" : "trn1"
+    }
+  ],
+   "meta-llama/Llama-3.2-3B": [
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 2,
+      "instance_type" : "trn1"
+    },
+    {
+      "batch_size": 4,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 2,
+      "instance_type" : "trn1"
+    }
+  ],
+     "TinyLlama/TinyLlama-1.1B-Chat-v1.0": [
+    {
+      "batch_size": 1,
+      "sequence_length": 2048,
+      "tensor_parallel_size": 2,
+      "instance_type" : "trn1"
+    }
+  ],
+  "meta-llama/Llama-3.3-70B-Instruct": [
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 24,
+      "instance_type" : "trn1"
+    },
+    {
+      "batch_size": 1,
+      "sequence_length": 16384,
+      "tensor_parallel_size": 24,
+      "instance_type" : "trn1"
+    },
+    {
+      "batch_size": 8,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 24,
+      "instance_type" : "trn1"
+    },
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 32,
+      "instance_type" : "trn1"
+    },
+    {
+      "batch_size": 1,
+      "sequence_length": 16384,
+      "tensor_parallel_size": 32,
+      "instance_type" : "trn1"
+    },
+    {
+      "batch_size": 8,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 32,
+      "instance_type" : "trn1"
+    }
+  ]
+}

inference-cache-config/trn1/llama4.json

@@ -0,0 +1,39 @@
+{
+  "meta-llama/Llama-4-Scout-17B-16E-Instruct": [
+    {
+      "task": "text-generation",
+      "instance_type": "trn1",
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 16
+    },
+    {
+      "task": "text-generation",
+      "instance_type": "trn1",
+      "batch_size": 4,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 16
+    },
+    {
+      "task": "text-generation",
+      "instance_type": "trn1",
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 32
+    },
+    {
+      "task": "text-generation",
+      "instance_type": "trn1",
+      "batch_size": 4,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 32
+    },
+    {
+      "task": "text-generation",
+      "instance_type": "trn1",
+      "batch_size": 8,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 32
+    }
+  ]
+}

inference-cache-config/trn1/mixtral.json

@@ -0,0 +1,34 @@
+{
+  "mistralai/Mixtral-8x7B-Instruct-v0.1": [
+    {
+      "task": "text-generation",
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 32,
+      "instance_type" : "trn1"
+    },
+    {
+      "task": "text-generation",
+      "batch_size": 4,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 32,
+      "instance_type" : "trn1"
+    }
+  ],
+  "mistralai/Mixtral-8x22B-Instruct-v0.1": [
+    {
+      "task": "text-generation",
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 32,
+      "instance_type" : "trn1"
+    },
+    {
+      "task": "text-generation",
+      "batch_size": 4,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 32,
+      "instance_type" : "trn1"
+    }
+  ]
+}

inference-cache-config/trn1/phi4.json

@@ -0,0 +1,30 @@
+{
+   "microsoft/Phi-3-mini-4k-instruct": [
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 2,
+      "instance_type" : "trn1"
+    },
+    {
+      "batch_size": 4,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 2,
+      "instance_type" : "trn1"
+    }
+  ],
+  "microsoft/phi-4": [
+   {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 10,
+      "instance_type" : "trn1"
+    },
+    {
+      "batch_size": 16,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 10,
+      "instance_type" : "trn1"
+    }
+  ]
+}

inference-cache-config/trn1/qwen3-moe.json

@@ -0,0 +1,28 @@
+{
+   "Qwen/Qwen3-30B-A3B-Instruct-2507": [
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 8,
+      "instance_type" : "trn1"
+    },
+    {
+      "batch_size": 4,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 8,
+      "instance_type" : "trn1"
+    },
+    {
+      "batch_size": 8,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 16,
+      "instance_type" : "trn1"
+    },
+    {
+      "batch_size": 8,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 32,
+      "instance_type" : "trn1"
+    }
+   ]
+}

inference-cache-config/trn1/qwen3.json

@@ -0,0 +1,104 @@
+{
+   "Qwen/Qwen3-8B": [
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 2,
+      "instance_type" : "trn1"
+    },
+    {
+      "batch_size": 4,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 2,
+      "instance_type" : "trn1"
+    },
+    {
+      "batch_size": 8,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 2,
+      "instance_type" : "trn1"
+    },
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 8,
+      "instance_type" : "trn1"
+    },
+    {
+      "batch_size": 32,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 8,
+      "instance_type" : "trn1"
+    }
+  ],
+  "Qwen/Qwen3-0.6B": [
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 2,
+      "instance_type" : "trn1"
+    },
+    {
+      "batch_size": 4,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 2,
+      "instance_type" : "trn1"
+    }
+  ],
+  "Qwen/Qwen3-1.7B": [
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 2,
+      "instance_type" : "trn1"
+    },
+    {
+      "batch_size": 4,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 2,
+      "instance_type" : "trn1"
+    }
+  ],
+  "Qwen/Qwen3-4B": [
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 2,
+      "instance_type" : "trn1"
+    },
+    {
+      "batch_size": 4,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 2,
+      "instance_type" : "trn1"
+    }
+  ],
+  "Qwen/Qwen3-14B": [
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 8,
+      "instance_type" : "trn1"
+    },
+    {
+      "batch_size": 16,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 8,
+      "instance_type" : "trn1"
+    }
+  ],
+  "Qwen/Qwen3-32B": [
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 8,
+      "instance_type" : "trn1"
+    },
+    {
+      "batch_size": 16,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 8,
+      "instance_type" : "trn1"
+    }
+  ]
+}

inference-cache-config/trn1/smollm3.json

@@ -0,0 +1,22 @@
+{
+ "HuggingFaceTB/SmolLM3-3B": [
+   {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 2,
+      "instance_type" : "trn1"
+    },
+   {
+      "batch_size": 4,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 2,
+      "instance_type" : "trn1"
+    },
+    {
+      "batch_size": 16,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 2,
+      "instance_type" : "trn1"
+    }
+  ]
+}

inference-cache-config/trn2/llama3.json

@@ -0,0 +1,120 @@
+{
+  "meta-llama/Llama-3.1-8B": [
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 4,
+      "instance_type" : "trn2"
+    },
+    {
+      "batch_size": 1,
+      "sequence_length": 16384,
+      "tensor_parallel_size": 4,
+      "instance_type" : "trn2"
+    },
+    {
+      "batch_size": 8,
+      "sequence_length": 4096,
+      "tensor_parallel_size" : 4,
+      "instance_type" : "trn2"
+    },
+    {
+      "batch_size": 32,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 4,
+      "instance_type" : "trn2"
+    },
+    {
+      "batch_size": 64,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 4,
+      "instance_type" : "trn2"
+    }
+  ],
+  "meta-llama/Llama-3.2-1B": [
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "tensor_parallel_size" : 4,
+      "instance_type" : "trn2"
+    },
+    {
+      "batch_size": 1,
+      "sequence_length": 16384,
+      "tensor_parallel_size" : 4,
+      "instance_type" : "trn2"
+    },
+    {
+      "batch_size": 4,
+      "sequence_length": 4096,
+      "tensor_parallel_size" : 4,
+      "instance_type" : "trn2"
+    },
+    {
+      "batch_size": 64,
+      "sequence_length": 4096,
+      "tensor_parallel_size" : 4,
+      "instance_type" : "trn2"
+    }
+  ],
+  "meta-llama/Llama-3.2-3B": [
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "tensor_parallel_size" : 4,
+      "instance_type" : "trn2"
+    },
+    {
+      "batch_size": 1,
+      "sequence_length": 16384,
+      "tensor_parallel_size" : 4,
+      "instance_type" : "trn2"
+    },
+    {
+      "batch_size": 64,
+      "sequence_length": 4096,
+      "tensor_parallel_size" : 4,
+      "instance_type" : "trn2"
+    }
+  ],
+  "TinyLlama/TinyLlama-1.1B-Chat-v1.0": [
+    {
+      "batch_size": 1,
+      "sequence_length": 2048,
+      "tensor_parallel_size" : 4,
+      "instance_type" : "trn2"
+    }
+  ],
+  "meta-llama/Llama-3.3-70B-Instruct": [
+    {
+      "batch_size": 32,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 32,
+      "instance_type" : "trn2"
+    },
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 64,
+      "instance_type" : "trn2"
+    },
+    {
+      "batch_size": 1,
+      "sequence_length": 16384,
+      "tensor_parallel_size": 64,
+      "instance_type" : "trn2"
+    },
+    {
+      "batch_size": 32,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 64,
+      "instance_type" : "trn2"
+    },
+    {
+      "batch_size": 64,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 64,
+      "instance_type" : "trn2"
+    }
+  ]
+}

inference-cache-config/trn2/llama4.json

@@ -0,0 +1,48 @@
+{
+  "meta-llama/Llama-4-Scout-17B-16E-Instruct": [
+    {
+      "task": "text-generation",
+      "instance_type": "trn2",
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 32
+    },
+    {
+      "task": "text-generation",
+      "instance_type": "trn2",
+      "batch_size": 8,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 32
+    },
+    {
+      "task": "text-generation",
+      "instance_type": "trn2",
+      "batch_size": 32,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 32
+    }
+  ],
+  "meta-llama/Llama-4-Maverick-17B-128E-Instruct": [
+    {
+      "task": "text-generation",
+      "instance_type": "trn2",
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 64
+    },
+    {
+      "task": "text-generation",
+      "instance_type": "trn1",
+      "batch_size": 8,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 64
+    },
+    {
+      "task": "text-generation",
+      "instance_type": "trn2",
+      "batch_size": 32,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 64
+    }
+  ]
+}

inference-cache-config/trn2/qwen3-moe.json

@@ -0,0 +1,36 @@
+{
+  "Qwen/Qwen3-30B-A3B-Instruct-2507": [
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 4,
+      "instance_type" : "trn2"
+    },
+    {
+      "batch_size": 8,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 4,
+      "instance_type" : "trn2"
+    },
+    {
+      "batch_size": 16,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 4,
+      "instance_type" : "trn2"
+    }
+  ],
+  "Qwen/Qwen3-235B-A22B-Instruct-2507": [
+    {
+      "batch_size": 1,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 64,
+      "instance_type" : "trn2"
+    },
+    {
+      "batch_size": 4,
+      "sequence_length": 4096,
+      "tensor_parallel_size": 64,
+      "instance_type" : "trn2"
+    }
+  ]
+}

neuronxcc-2.19.8089.0+8ab9f450/0_REGISTRY/0.3.0.dev2/granite/hf-internal-testing/tiny-random-GraniteForCausalLM/4bd0c6ed2a9764168cb6.json

@@ -0,0 +1,73 @@
+{
+  "_entry_class": "SingleModelCacheEntry",
+  "_model_id": "hf-internal-testing/tiny-random-GraniteForCausalLM",
+  "_task": "text-generation",
+  "architectures": [
+    "GraniteForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "attention_multiplier": 1.0,
+  "embedding_multiplier": 1.0,
+  "hidden_act": "silu",
+  "hidden_size": 32,
+  "initializer_range": 0.02,
+  "intermediate_size": 64,
+  "logits_scaling": 1.0,
+  "max_position_embeddings": 2048,
+  "mlp_bias": false,
+  "model_type": "granite",
+  "neuron": {
+    "_serialized_key": "NxDNeuronConfig",
+    "async_mode": false,
+    "attn_kernel_enabled": false,
+    "batch_size": 1,
+    "capacity_factor": null,
+    "cc_pipeline_tiling_factor": 2,
+    "checkpoint_id": "hf-internal-testing/tiny-random-GraniteForCausalLM",
+    "checkpoint_revision": "c3074ebc0ac2fe545305f5e5f6cce2cc9b2aa0c5",
+    "continuous_batching": false,
+    "enable_bucketing": false,
+    "ep_degree": 1,
+    "flash_decoding_enabled": false,
+    "fused_qkv": true,
+    "glu_mlp": true,
+    "is_chunked_prefill": false,
+    "local_ranks_size": 2,
+    "logical_nc_config": 1,
+    "max_batch_size": 1,
+    "max_context_length": 100,
+    "max_topk": 256,
+    "mlp_kernel_enabled": false,
+    "mlp_kernel_fuse_residual_add": false,
+    "n_active_tokens": 100,
+    "neuronxcc_version": "2.19.8089.0+8ab9f450",
+    "num_cores_per_group": 1,
+    "on_device_sampling": true,
+    "optimum_neuron_version": "0.3.0.dev2",
+    "output_logits": false,
+    "padding_side": "right",
+    "pp_degree": 1,
+    "qk_layernorm": false,
+    "qkv_kernel_enabled": false,
+    "rpl_reduce_dtype": "float16",
+    "sequence_length": 100,
+    "sequence_parallel_enabled": false,
+    "speculation_length": 0,
+    "start_rank_id": 0,
+    "target": null,
+    "torch_dtype": "float16",
+    "tp_degree": 2,
+    "vocab_parallel": false
+  },
+  "num_attention_heads": 4,
+  "num_hidden_layers": 2,
+  "num_key_value_heads": 4,
+  "residual_multiplier": 1.0,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": false,
+  "use_cache": true,
+  "vocab_size": 49152
+}

neuronxcc-2.19.8089.0+8ab9f450/0_REGISTRY/0.3.0.dev2/granite/hf-internal-testing/tiny-random-GraniteForCausalLM/58bb3ae501d4d87ab565.json

@@ -0,0 +1,73 @@
+{
+  "_entry_class": "SingleModelCacheEntry",
+  "_model_id": "hf-internal-testing/tiny-random-GraniteForCausalLM",
+  "_task": "text-generation",
+  "architectures": [
+    "GraniteForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "attention_multiplier": 1.0,
+  "embedding_multiplier": 1.0,
+  "hidden_act": "silu",
+  "hidden_size": 32,
+  "initializer_range": 0.02,
+  "intermediate_size": 64,
+  "logits_scaling": 1.0,
+  "max_position_embeddings": 2048,
+  "mlp_bias": false,
+  "model_type": "granite",
+  "neuron": {
+    "_serialized_key": "NxDNeuronConfig",
+    "async_mode": false,
+    "attn_kernel_enabled": false,
+    "batch_size": 2,
+    "capacity_factor": null,
+    "cc_pipeline_tiling_factor": 2,
+    "checkpoint_id": "hf-internal-testing/tiny-random-GraniteForCausalLM",
+    "checkpoint_revision": "c3074ebc0ac2fe545305f5e5f6cce2cc9b2aa0c5",
+    "continuous_batching": true,
+    "enable_bucketing": false,
+    "ep_degree": 1,
+    "flash_decoding_enabled": false,
+    "fused_qkv": true,
+    "glu_mlp": true,
+    "is_chunked_prefill": false,
+    "local_ranks_size": 2,
+    "logical_nc_config": 1,
+    "max_batch_size": 2,
+    "max_context_length": 100,
+    "max_topk": 256,
+    "mlp_kernel_enabled": false,
+    "mlp_kernel_fuse_residual_add": false,
+    "n_active_tokens": 100,
+    "neuronxcc_version": "2.19.8089.0+8ab9f450",
+    "num_cores_per_group": 1,
+    "on_device_sampling": false,
+    "optimum_neuron_version": "0.3.0.dev2",
+    "output_logits": false,
+    "padding_side": "right",
+    "pp_degree": 1,
+    "qk_layernorm": false,
+    "qkv_kernel_enabled": false,
+    "rpl_reduce_dtype": "float16",
+    "sequence_length": 100,
+    "sequence_parallel_enabled": false,
+    "speculation_length": 0,
+    "start_rank_id": 0,
+    "target": null,
+    "torch_dtype": "float16",
+    "tp_degree": 2,
+    "vocab_parallel": false
+  },
+  "num_attention_heads": 4,
+  "num_hidden_layers": 2,
+  "num_key_value_heads": 4,
+  "residual_multiplier": 1.0,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": false,
+  "use_cache": true,
+  "vocab_size": 49152
+}

neuronxcc-2.19.8089.0+8ab9f450/0_REGISTRY/0.3.0.dev2/granite/hf-internal-testing/tiny-random-GraniteForCausalLM/f5771202abb1a7ae2611.json

@@ -0,0 +1,73 @@
+{
+  "_entry_class": "SingleModelCacheEntry",
+  "_model_id": "hf-internal-testing/tiny-random-GraniteForCausalLM",
+  "_task": "text-generation",
+  "architectures": [
+    "GraniteForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "attention_multiplier": 1.0,
+  "embedding_multiplier": 1.0,
+  "hidden_act": "silu",
+  "hidden_size": 32,
+  "initializer_range": 0.02,
+  "intermediate_size": 64,
+  "logits_scaling": 1.0,
+  "max_position_embeddings": 2048,
+  "mlp_bias": false,
+  "model_type": "granite",
+  "neuron": {
+    "_serialized_key": "NxDNeuronConfig",
+    "async_mode": false,
+    "attn_kernel_enabled": false,
+    "batch_size": 1,
+    "capacity_factor": null,
+    "cc_pipeline_tiling_factor": 2,
+    "checkpoint_id": "hf-internal-testing/tiny-random-GraniteForCausalLM",
+    "checkpoint_revision": "c3074ebc0ac2fe545305f5e5f6cce2cc9b2aa0c5",
+    "continuous_batching": false,
+    "enable_bucketing": false,
+    "ep_degree": 1,
+    "flash_decoding_enabled": false,
+    "fused_qkv": true,
+    "glu_mlp": true,
+    "is_chunked_prefill": false,
+    "local_ranks_size": 2,
+    "logical_nc_config": 1,
+    "max_batch_size": 1,
+    "max_context_length": 100,
+    "max_topk": 256,
+    "mlp_kernel_enabled": false,
+    "mlp_kernel_fuse_residual_add": false,
+    "n_active_tokens": 100,
+    "neuronxcc_version": "2.19.8089.0+8ab9f450",
+    "num_cores_per_group": 1,
+    "on_device_sampling": true,
+    "optimum_neuron_version": "0.3.0.dev2",
+    "output_logits": false,
+    "padding_side": "right",
+    "pp_degree": 1,
+    "qk_layernorm": false,
+    "qkv_kernel_enabled": false,
+    "rpl_reduce_dtype": "bfloat16",
+    "sequence_length": 100,
+    "sequence_parallel_enabled": false,
+    "speculation_length": 0,
+    "start_rank_id": 0,
+    "target": null,
+    "torch_dtype": "bfloat16",
+    "tp_degree": 2,
+    "vocab_parallel": false
+  },
+  "num_attention_heads": 4,
+  "num_hidden_layers": 2,
+  "num_key_value_heads": 4,
+  "residual_multiplier": 1.0,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": false,
+  "use_cache": true,
+  "vocab_size": 49152
+}

neuronxcc-2.19.8089.0+8ab9f450/0_REGISTRY/0.3.0.dev2/granite/ibm-granite/granite-3.1-2b-instruct/af1749224d7631fdb38a.json

@@ -0,0 +1,73 @@
+{
+  "_entry_class": "SingleModelCacheEntry",
+  "_model_id": "ibm-granite/granite-3.1-2b-instruct",
+  "_task": "text-generation",
+  "architectures": [
+    "GraniteForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.1,
+  "attention_multiplier": 0.015625,
+  "embedding_multiplier": 12.0,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "logits_scaling": 8.0,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "granite",
+  "neuron": {
+    "_serialized_key": "NxDNeuronConfig",
+    "async_mode": false,
+    "attn_kernel_enabled": false,
+    "batch_size": 4,
+    "capacity_factor": null,
+    "cc_pipeline_tiling_factor": 2,
+    "checkpoint_id": "ibm-granite/granite-3.1-2b-instruct",
+    "checkpoint_revision": "bbc2aed595bd38bd770263dc3ab831db9794441d",
+    "continuous_batching": true,
+    "enable_bucketing": false,
+    "ep_degree": 1,
+    "flash_decoding_enabled": false,
+    "fused_qkv": true,
+    "glu_mlp": true,
+    "is_chunked_prefill": false,
+    "local_ranks_size": 2,
+    "logical_nc_config": 1,
+    "max_batch_size": 4,
+    "max_context_length": 4096,
+    "max_topk": 256,
+    "mlp_kernel_enabled": false,
+    "mlp_kernel_fuse_residual_add": false,
+    "n_active_tokens": 4096,
+    "neuronxcc_version": "2.19.8089.0+8ab9f450",
+    "num_cores_per_group": 1,
+    "on_device_sampling": false,
+    "optimum_neuron_version": "0.3.0.dev2",
+    "output_logits": false,
+    "padding_side": "right",
+    "pp_degree": 1,
+    "qk_layernorm": false,
+    "qkv_kernel_enabled": false,
+    "rpl_reduce_dtype": "bfloat16",
+    "sequence_length": 4096,
+    "sequence_parallel_enabled": false,
+    "speculation_length": 0,
+    "start_rank_id": 0,
+    "target": null,
+    "torch_dtype": "bfloat16",
+    "tp_degree": 2,
+    "vocab_parallel": false
+  },
+  "num_attention_heads": 32,
+  "num_hidden_layers": 40,
+  "num_key_value_heads": 8,
+  "residual_multiplier": 0.22,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 5000000.0,
+  "tie_word_embeddings": true,
+  "use_cache": true,
+  "vocab_size": 49155
+}

neuronxcc-2.19.8089.0+8ab9f450/0_REGISTRY/0.3.0.dev2/llama/deepseek-ai/DeepSeek-R1-Distill-Llama-70B/38357c754f2cbd1d382e.json

@@ -0,0 +1,77 @@
+{
+  "_entry_class": "SingleModelCacheEntry",
+  "_model_id": "deepseek-ai/DeepSeek-R1-Distill-Llama-70B",
+  "_task": "text-generation",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 8192,
+  "initializer_range": 0.02,
+  "intermediate_size": 28672,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "neuron": {
+    "_serialized_key": "NxDNeuronConfig",
+    "async_mode": false,
+    "attn_kernel_enabled": false,
+    "batch_size": 8,
+    "capacity_factor": null,
+    "cc_pipeline_tiling_factor": 2,
+    "checkpoint_id": "deepseek-ai/DeepSeek-R1-Distill-Llama-70B",
+    "checkpoint_revision": "b1c0b44b4369b597ad119a196caf79a9c40e141e",
+    "continuous_batching": true,
+    "enable_bucketing": false,
+    "ep_degree": 1,
+    "flash_decoding_enabled": false,
+    "fused_qkv": true,
+    "glu_mlp": true,
+    "is_chunked_prefill": false,
+    "local_ranks_size": 24,
+    "logical_nc_config": 1,
+    "max_batch_size": 8,
+    "max_context_length": 4096,
+    "max_topk": 256,
+    "mlp_kernel_enabled": false,
+    "mlp_kernel_fuse_residual_add": false,
+    "n_active_tokens": 4096,
+    "neuronxcc_version": "2.19.8089.0+8ab9f450",
+    "num_cores_per_group": 1,
+    "on_device_sampling": true,
+    "optimum_neuron_version": "0.3.0.dev2",
+    "output_logits": false,
+    "padding_side": "right",
+    "pp_degree": 1,
+    "qk_layernorm": false,
+    "qkv_kernel_enabled": false,
+    "rpl_reduce_dtype": "bfloat16",
+    "sequence_length": 4096,
+    "sequence_parallel_enabled": false,
+    "speculation_length": 0,
+    "start_rank_id": 0,
+    "target": null,
+    "torch_dtype": "bfloat16",
+    "tp_degree": 24,
+    "vocab_parallel": false
+  },
+  "num_attention_heads": 64,
+  "num_hidden_layers": 80,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 8.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "use_cache": true,
+  "vocab_size": 128256
+}

neuronxcc-2.19.8089.0+8ab9f450/0_REGISTRY/0.3.0.dev2/llama/deepseek-ai/DeepSeek-R1-Distill-Llama-70B/89acac1543511de63934.json

@@ -0,0 +1,77 @@
+{
+  "_entry_class": "SingleModelCacheEntry",
+  "_model_id": "deepseek-ai/DeepSeek-R1-Distill-Llama-70B",
+  "_task": "text-generation",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 8192,
+  "initializer_range": 0.02,
+  "intermediate_size": 28672,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "neuron": {
+    "_serialized_key": "NxDNeuronConfig",
+    "async_mode": false,
+    "attn_kernel_enabled": false,
+    "batch_size": 4,
+    "capacity_factor": null,
+    "cc_pipeline_tiling_factor": 2,
+    "checkpoint_id": "deepseek-ai/DeepSeek-R1-Distill-Llama-70B",
+    "checkpoint_revision": "b1c0b44b4369b597ad119a196caf79a9c40e141e",
+    "continuous_batching": true,
+    "enable_bucketing": false,
+    "ep_degree": 1,
+    "flash_decoding_enabled": false,
+    "fused_qkv": true,
+    "glu_mlp": true,
+    "is_chunked_prefill": false,
+    "local_ranks_size": 24,
+    "logical_nc_config": 1,
+    "max_batch_size": 4,
+    "max_context_length": 4096,
+    "max_topk": 256,
+    "mlp_kernel_enabled": false,
+    "mlp_kernel_fuse_residual_add": false,
+    "n_active_tokens": 4096,
+    "neuronxcc_version": "2.19.8089.0+8ab9f450",
+    "num_cores_per_group": 1,
+    "on_device_sampling": true,
+    "optimum_neuron_version": "0.3.0.dev2",
+    "output_logits": false,
+    "padding_side": "right",
+    "pp_degree": 1,
+    "qk_layernorm": false,
+    "qkv_kernel_enabled": false,
+    "rpl_reduce_dtype": "bfloat16",
+    "sequence_length": 4096,
+    "sequence_parallel_enabled": false,
+    "speculation_length": 0,
+    "start_rank_id": 0,
+    "target": null,
+    "torch_dtype": "bfloat16",
+    "tp_degree": 24,
+    "vocab_parallel": false
+  },
+  "num_attention_heads": 64,
+  "num_hidden_layers": 80,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 8.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "use_cache": true,
+  "vocab_size": 128256
+}

neuronxcc-2.19.8089.0+8ab9f450/0_REGISTRY/0.3.0.dev2/llama/deepseek-ai/DeepSeek-R1-Distill-Llama-70B/a91139c3aee28b0f6c94.json

@@ -0,0 +1,77 @@
+{
+  "_entry_class": "SingleModelCacheEntry",
+  "_model_id": "deepseek-ai/DeepSeek-R1-Distill-Llama-70B",
+  "_task": "text-generation",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 8192,
+  "initializer_range": 0.02,
+  "intermediate_size": 28672,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "neuron": {
+    "_serialized_key": "NxDNeuronConfig",
+    "async_mode": false,
+    "attn_kernel_enabled": false,
+    "batch_size": 1,
+    "capacity_factor": null,
+    "cc_pipeline_tiling_factor": 2,
+    "checkpoint_id": "deepseek-ai/DeepSeek-R1-Distill-Llama-70B",
+    "checkpoint_revision": "b1c0b44b4369b597ad119a196caf79a9c40e141e",
+    "continuous_batching": false,
+    "enable_bucketing": false,
+    "ep_degree": 1,
+    "flash_decoding_enabled": false,
+    "fused_qkv": true,
+    "glu_mlp": true,
+    "is_chunked_prefill": false,
+    "local_ranks_size": 24,
+    "logical_nc_config": 1,
+    "max_batch_size": 1,
+    "max_context_length": 4096,
+    "max_topk": 256,
+    "mlp_kernel_enabled": false,
+    "mlp_kernel_fuse_residual_add": false,
+    "n_active_tokens": 4096,
+    "neuronxcc_version": "2.19.8089.0+8ab9f450",
+    "num_cores_per_group": 1,
+    "on_device_sampling": true,
+    "optimum_neuron_version": "0.3.0.dev2",
+    "output_logits": false,
+    "padding_side": "right",
+    "pp_degree": 1,
+    "qk_layernorm": false,
+    "qkv_kernel_enabled": false,
+    "rpl_reduce_dtype": "bfloat16",
+    "sequence_length": 4096,
+    "sequence_parallel_enabled": false,
+    "speculation_length": 0,
+    "start_rank_id": 0,
+    "target": null,
+    "torch_dtype": "bfloat16",
+    "tp_degree": 24,
+    "vocab_parallel": false
+  },
+  "num_attention_heads": 64,
+  "num_hidden_layers": 80,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 8.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "use_cache": true,
+  "vocab_size": 128256
+}

neuronxcc-2.19.8089.0+8ab9f450/0_REGISTRY/0.3.0.dev2/llama/llamafactory/tiny-random-Llama-3/3ed9abae6f7f67d07da0.json

@@ -0,0 +1,77 @@
+{
+  "_entry_class": "SingleModelCacheEntry",
+  "_model_id": "llamafactory/tiny-random-Llama-3",
+  "_task": "text-generation",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "head_dim": 4,
+  "hidden_act": "silu",
+  "hidden_size": 16,
+  "initializer_range": 0.02,
+  "intermediate_size": 64,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "neuron": {
+    "_serialized_key": "NxDNeuronConfig",
+    "async_mode": false,
+    "attn_kernel_enabled": false,
+    "batch_size": 2,
+    "capacity_factor": null,
+    "cc_pipeline_tiling_factor": 2,
+    "checkpoint_id": "llamafactory/tiny-random-Llama-3",
+    "checkpoint_revision": "bf2a2e3bf199ad2ee96f02a3c00246c608db22a8",
+    "continuous_batching": true,
+    "enable_bucketing": false,
+    "ep_degree": 1,
+    "flash_decoding_enabled": false,
+    "fused_qkv": true,
+    "glu_mlp": true,
+    "is_chunked_prefill": false,
+    "local_ranks_size": 2,
+    "logical_nc_config": 1,
+    "max_batch_size": 2,
+    "max_context_length": 100,
+    "max_topk": 256,
+    "mlp_kernel_enabled": false,
+    "mlp_kernel_fuse_residual_add": false,
+    "n_active_tokens": 100,
+    "neuronxcc_version": "2.19.8089.0+8ab9f450",
+    "num_cores_per_group": 1,
+    "on_device_sampling": false,
+    "optimum_neuron_version": "0.3.0.dev2",
+    "output_logits": false,
+    "padding_side": "right",
+    "pp_degree": 1,
+    "qk_layernorm": false,
+    "qkv_kernel_enabled": false,
+    "rpl_reduce_dtype": "float16",
+    "sequence_length": 100,
+    "sequence_parallel_enabled": false,
+    "speculation_length": 0,
+    "start_rank_id": 0,
+    "target": null,
+    "torch_dtype": "float16",
+    "tp_degree": 2,
+    "vocab_parallel": false
+  },
+  "num_attention_heads": 4,
+  "num_hidden_layers": 2,
+  "num_key_value_heads": 4,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 8.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "use_cache": true,
+  "vocab_size": 128256
+}

neuronxcc-2.19.8089.0+8ab9f450/0_REGISTRY/0.3.0.dev2/llama/llamafactory/tiny-random-Llama-3/42b299016a0208fd74b0.json

@@ -0,0 +1,77 @@
+{
+  "_entry_class": "SingleModelCacheEntry",
+  "_model_id": "llamafactory/tiny-random-Llama-3",
+  "_task": "text-generation",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "head_dim": 4,
+  "hidden_act": "silu",
+  "hidden_size": 16,
+  "initializer_range": 0.02,
+  "intermediate_size": 64,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "neuron": {
+    "_serialized_key": "NxDNeuronConfig",
+    "async_mode": false,
+    "attn_kernel_enabled": false,
+    "batch_size": 1,
+    "capacity_factor": null,
+    "cc_pipeline_tiling_factor": 2,
+    "checkpoint_id": "llamafactory/tiny-random-Llama-3",
+    "checkpoint_revision": "bf2a2e3bf199ad2ee96f02a3c00246c608db22a8",
+    "continuous_batching": false,
+    "enable_bucketing": false,
+    "ep_degree": 1,
+    "flash_decoding_enabled": false,
+    "fused_qkv": true,
+    "glu_mlp": true,
+    "is_chunked_prefill": false,
+    "local_ranks_size": 2,
+    "logical_nc_config": 1,
+    "max_batch_size": 1,
+    "max_context_length": 100,
+    "max_topk": 256,
+    "mlp_kernel_enabled": false,
+    "mlp_kernel_fuse_residual_add": false,
+    "n_active_tokens": 100,
+    "neuronxcc_version": "2.19.8089.0+8ab9f450",
+    "num_cores_per_group": 1,
+    "on_device_sampling": true,
+    "optimum_neuron_version": "0.3.0.dev2",
+    "output_logits": false,
+    "padding_side": "right",
+    "pp_degree": 1,
+    "qk_layernorm": false,
+    "qkv_kernel_enabled": false,
+    "rpl_reduce_dtype": "float16",
+    "sequence_length": 100,
+    "sequence_parallel_enabled": false,
+    "speculation_length": 0,
+    "start_rank_id": 0,
+    "target": null,
+    "torch_dtype": "float16",
+    "tp_degree": 2,
+    "vocab_parallel": false
+  },
+  "num_attention_heads": 4,
+  "num_hidden_layers": 2,
+  "num_key_value_heads": 4,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 8.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "use_cache": true,
+  "vocab_size": 128256
+}

neuronxcc-2.19.8089.0+8ab9f450/0_REGISTRY/0.3.0.dev2/llama/llamafactory/tiny-random-Llama-3/7b749096ec9bec32a9c5.json

@@ -0,0 +1,77 @@
+{
+  "_entry_class": "SingleModelCacheEntry",
+  "_model_id": "llamafactory/tiny-random-Llama-3",
+  "_task": "text-generation",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "head_dim": 4,
+  "hidden_act": "silu",
+  "hidden_size": 16,
+  "initializer_range": 0.02,
+  "intermediate_size": 64,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "neuron": {
+    "_serialized_key": "NxDNeuronConfig",
+    "async_mode": false,
+    "attn_kernel_enabled": false,
+    "batch_size": 1,
+    "capacity_factor": null,
+    "cc_pipeline_tiling_factor": 2,
+    "checkpoint_id": "llamafactory/tiny-random-Llama-3",
+    "checkpoint_revision": "bf2a2e3bf199ad2ee96f02a3c00246c608db22a8",
+    "continuous_batching": false,
+    "enable_bucketing": false,
+    "ep_degree": 1,
+    "flash_decoding_enabled": false,
+    "fused_qkv": true,
+    "glu_mlp": true,
+    "is_chunked_prefill": false,
+    "local_ranks_size": 2,
+    "logical_nc_config": 1,
+    "max_batch_size": 1,
+    "max_context_length": 131072,
+    "max_topk": 256,
+    "mlp_kernel_enabled": false,
+    "mlp_kernel_fuse_residual_add": false,
+    "n_active_tokens": 131072,
+    "neuronxcc_version": "2.19.8089.0+8ab9f450",
+    "num_cores_per_group": 1,
+    "on_device_sampling": true,
+    "optimum_neuron_version": "0.3.0.dev2",
+    "output_logits": false,
+    "padding_side": "right",
+    "pp_degree": 1,
+    "qk_layernorm": false,
+    "qkv_kernel_enabled": false,
+    "rpl_reduce_dtype": "bfloat16",
+    "sequence_length": 131072,
+    "sequence_parallel_enabled": false,
+    "speculation_length": 0,
+    "start_rank_id": 0,
+    "target": null,
+    "torch_dtype": "bfloat16",
+    "tp_degree": 2,
+    "vocab_parallel": false
+  },
+  "num_attention_heads": 4,
+  "num_hidden_layers": 2,
+  "num_key_value_heads": 4,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 8.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "use_cache": true,
+  "vocab_size": 128256
+}

neuronxcc-2.19.8089.0+8ab9f450/0_REGISTRY/0.3.0.dev2/llama/llamafactory/tiny-random-Llama-3/c36bfaee43f6f840c055.json

@@ -0,0 +1,77 @@
+{
+  "_entry_class": "SingleModelCacheEntry",
+  "_model_id": "llamafactory/tiny-random-Llama-3",
+  "_task": "text-generation",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "head_dim": 4,
+  "hidden_act": "silu",
+  "hidden_size": 16,
+  "initializer_range": 0.02,
+  "intermediate_size": 64,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "neuron": {
+    "_serialized_key": "NxDNeuronConfig",
+    "async_mode": false,
+    "attn_kernel_enabled": false,
+    "batch_size": 1,
+    "capacity_factor": null,
+    "cc_pipeline_tiling_factor": 2,
+    "checkpoint_id": "llamafactory/tiny-random-Llama-3",
+    "checkpoint_revision": "bf2a2e3bf199ad2ee96f02a3c00246c608db22a8",
+    "continuous_batching": false,
+    "enable_bucketing": false,
+    "ep_degree": 1,
+    "flash_decoding_enabled": false,
+    "fused_qkv": true,
+    "glu_mlp": true,
+    "is_chunked_prefill": false,
+    "local_ranks_size": 2,
+    "logical_nc_config": 1,
+    "max_batch_size": 1,
+    "max_context_length": 100,
+    "max_topk": 256,
+    "mlp_kernel_enabled": false,
+    "mlp_kernel_fuse_residual_add": false,
+    "n_active_tokens": 100,
+    "neuronxcc_version": "2.19.8089.0+8ab9f450",
+    "num_cores_per_group": 1,
+    "on_device_sampling": true,
+    "optimum_neuron_version": "0.3.0.dev2",
+    "output_logits": false,
+    "padding_side": "right",
+    "pp_degree": 1,
+    "qk_layernorm": false,
+    "qkv_kernel_enabled": false,
+    "rpl_reduce_dtype": "bfloat16",
+    "sequence_length": 100,
+    "sequence_parallel_enabled": false,
+    "speculation_length": 0,
+    "start_rank_id": 0,
+    "target": null,
+    "torch_dtype": "bfloat16",
+    "tp_degree": 2,
+    "vocab_parallel": false
+  },
+  "num_attention_heads": 4,
+  "num_hidden_layers": 2,
+  "num_key_value_heads": 4,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 8.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "use_cache": true,
+  "vocab_size": 128256
+}

neuronxcc-2.19.8089.0+8ab9f450/0_REGISTRY/0.3.0.dev2/llama/unsloth/Llama-3.1-8B-Instruct/29f3c7ef0e6a87f35788.json

@@ -0,0 +1,78 @@
+{
+  "_entry_class": "SingleModelCacheEntry",
+  "_model_id": "unsloth/Llama-3.1-8B-Instruct",
+  "_task": "text-generation",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "neuron": {
+    "_serialized_key": "NxDNeuronConfig",
+    "async_mode": false,
+    "attn_kernel_enabled": false,
+    "batch_size": 48,
+    "capacity_factor": null,
+    "cc_pipeline_tiling_factor": 2,
+    "checkpoint_id": "unsloth/Llama-3.1-8B-Instruct",
+    "checkpoint_revision": "4699cc75b550f9c6f3173fb80f4703b62d946aa5",
+    "continuous_batching": true,
+    "enable_bucketing": false,
+    "ep_degree": 1,
+    "flash_decoding_enabled": false,
+    "fused_qkv": true,
+    "glu_mlp": true,
+    "is_chunked_prefill": false,
+    "local_ranks_size": 8,
+    "logical_nc_config": 1,
+    "max_batch_size": 48,
+    "max_context_length": 4096,
+    "max_topk": 256,
+    "mlp_kernel_enabled": false,
+    "mlp_kernel_fuse_residual_add": false,
+    "n_active_tokens": 4096,
+    "neuronxcc_version": "2.19.8089.0+8ab9f450",
+    "num_cores_per_group": 1,
+    "on_device_sampling": true,
+    "optimum_neuron_version": "0.3.0.dev2",
+    "output_logits": false,
+    "padding_side": "right",
+    "pp_degree": 1,
+    "qk_layernorm": false,
+    "qkv_kernel_enabled": false,
+    "rpl_reduce_dtype": "bfloat16",
+    "sequence_length": 4096,
+    "sequence_parallel_enabled": false,
+    "speculation_length": 0,
+    "start_rank_id": 0,
+    "target": null,
+    "torch_dtype": "bfloat16",
+    "tp_degree": 8,
+    "vocab_parallel": false
+  },
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 8.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "unsloth_fixed": true,
+  "use_cache": true,
+  "vocab_size": 128256
+}

neuronxcc-2.19.8089.0+8ab9f450/0_REGISTRY/0.3.0.dev2/llama/unsloth/Llama-3.1-8B-Instruct/6d3aa041c08b8396ee41.json

@@ -0,0 +1,78 @@
+{
+  "_entry_class": "SingleModelCacheEntry",
+  "_model_id": "unsloth/Llama-3.1-8B-Instruct",
+  "_task": "text-generation",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "neuron": {
+    "_serialized_key": "NxDNeuronConfig",
+    "async_mode": false,
+    "attn_kernel_enabled": false,
+    "batch_size": 16,
+    "capacity_factor": null,
+    "cc_pipeline_tiling_factor": 2,
+    "checkpoint_id": "unsloth/Llama-3.1-8B-Instruct",
+    "checkpoint_revision": "4699cc75b550f9c6f3173fb80f4703b62d946aa5",
+    "continuous_batching": true,
+    "enable_bucketing": false,
+    "ep_degree": 1,
+    "flash_decoding_enabled": false,
+    "fused_qkv": true,
+    "glu_mlp": true,
+    "is_chunked_prefill": false,
+    "local_ranks_size": 8,
+    "logical_nc_config": 1,
+    "max_batch_size": 16,
+    "max_context_length": 4096,
+    "max_topk": 256,
+    "mlp_kernel_enabled": false,
+    "mlp_kernel_fuse_residual_add": false,
+    "n_active_tokens": 4096,
+    "neuronxcc_version": "2.19.8089.0+8ab9f450",
+    "num_cores_per_group": 1,
+    "on_device_sampling": true,
+    "optimum_neuron_version": "0.3.0.dev2",
+    "output_logits": false,
+    "padding_side": "right",
+    "pp_degree": 1,
+    "qk_layernorm": false,
+    "qkv_kernel_enabled": false,
+    "rpl_reduce_dtype": "bfloat16",
+    "sequence_length": 4096,
+    "sequence_parallel_enabled": false,
+    "speculation_length": 0,
+    "start_rank_id": 0,
+    "target": null,
+    "torch_dtype": "bfloat16",
+    "tp_degree": 8,
+    "vocab_parallel": false
+  },
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 8.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "unsloth_fixed": true,
+  "use_cache": true,
+  "vocab_size": 128256
+}

neuronxcc-2.19.8089.0+8ab9f450/0_REGISTRY/0.3.0.dev2/llama/unsloth/Llama-3.1-8B-Instruct/7523661354aea5f66d6c.json

@@ -0,0 +1,78 @@
+{
+  "_entry_class": "SingleModelCacheEntry",
+  "_model_id": "unsloth/Llama-3.1-8B-Instruct",
+  "_task": "text-generation",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "neuron": {
+    "_serialized_key": "NxDNeuronConfig",
+    "async_mode": false,
+    "attn_kernel_enabled": false,
+    "batch_size": 4,
+    "capacity_factor": null,
+    "cc_pipeline_tiling_factor": 2,
+    "checkpoint_id": "unsloth/Llama-3.1-8B-Instruct",
+    "checkpoint_revision": "4699cc75b550f9c6f3173fb80f4703b62d946aa5",
+    "continuous_batching": true,
+    "enable_bucketing": false,
+    "ep_degree": 1,
+    "flash_decoding_enabled": false,
+    "fused_qkv": true,
+    "glu_mlp": true,
+    "is_chunked_prefill": false,
+    "local_ranks_size": 8,
+    "logical_nc_config": 1,
+    "max_batch_size": 4,
+    "max_context_length": 4096,
+    "max_topk": 256,
+    "mlp_kernel_enabled": false,
+    "mlp_kernel_fuse_residual_add": false,
+    "n_active_tokens": 4096,
+    "neuronxcc_version": "2.19.8089.0+8ab9f450",
+    "num_cores_per_group": 1,
+    "on_device_sampling": true,
+    "optimum_neuron_version": "0.3.0.dev2",
+    "output_logits": false,
+    "padding_side": "right",
+    "pp_degree": 1,
+    "qk_layernorm": false,
+    "qkv_kernel_enabled": false,
+    "rpl_reduce_dtype": "bfloat16",
+    "sequence_length": 4096,
+    "sequence_parallel_enabled": false,
+    "speculation_length": 0,
+    "start_rank_id": 0,
+    "target": null,
+    "torch_dtype": "bfloat16",
+    "tp_degree": 8,
+    "vocab_parallel": false
+  },
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 8.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "unsloth_fixed": true,
+  "use_cache": true,
+  "vocab_size": 128256
+}

neuronxcc-2.19.8089.0+8ab9f450/0_REGISTRY/0.3.0.dev2/llama/unsloth/Llama-3.1-8B-Instruct/7c2b4b0f296eeda524b6.json

@@ -0,0 +1,78 @@
+{
+  "_entry_class": "SingleModelCacheEntry",
+  "_model_id": "unsloth/Llama-3.1-8B-Instruct",
+  "_task": "text-generation",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "neuron": {
+    "_serialized_key": "NxDNeuronConfig",
+    "async_mode": false,
+    "attn_kernel_enabled": false,
+    "batch_size": 1,
+    "capacity_factor": null,
+    "cc_pipeline_tiling_factor": 2,
+    "checkpoint_id": "unsloth/Llama-3.1-8B-Instruct",
+    "checkpoint_revision": "4699cc75b550f9c6f3173fb80f4703b62d946aa5",
+    "continuous_batching": false,
+    "enable_bucketing": false,
+    "ep_degree": 1,
+    "flash_decoding_enabled": false,
+    "fused_qkv": true,
+    "glu_mlp": true,
+    "is_chunked_prefill": false,
+    "local_ranks_size": 8,
+    "logical_nc_config": 1,
+    "max_batch_size": 1,
+    "max_context_length": 4096,
+    "max_topk": 256,
+    "mlp_kernel_enabled": false,
+    "mlp_kernel_fuse_residual_add": false,
+    "n_active_tokens": 4096,
+    "neuronxcc_version": "2.19.8089.0+8ab9f450",
+    "num_cores_per_group": 1,
+    "on_device_sampling": true,
+    "optimum_neuron_version": "0.3.0.dev2",
+    "output_logits": false,
+    "padding_side": "right",
+    "pp_degree": 1,
+    "qk_layernorm": false,
+    "qkv_kernel_enabled": false,
+    "rpl_reduce_dtype": "bfloat16",
+    "sequence_length": 4096,
+    "sequence_parallel_enabled": false,
+    "speculation_length": 0,
+    "start_rank_id": 0,
+    "target": null,
+    "torch_dtype": "bfloat16",
+    "tp_degree": 8,
+    "vocab_parallel": false
+  },
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 8.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "unsloth_fixed": true,
+  "use_cache": true,
+  "vocab_size": 128256
+}

neuronxcc-2.19.8089.0+8ab9f450/0_REGISTRY/0.3.0.dev2/llama/unsloth/Llama-3.1-8B-Instruct/9932c1d05bd91a781c3a.json

@@ -0,0 +1,78 @@
+{
+  "_entry_class": "SingleModelCacheEntry",
+  "_model_id": "unsloth/Llama-3.1-8B-Instruct",
+  "_task": "text-generation",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "neuron": {
+    "_serialized_key": "NxDNeuronConfig",
+    "async_mode": false,
+    "attn_kernel_enabled": false,
+    "batch_size": 8,
+    "capacity_factor": null,
+    "cc_pipeline_tiling_factor": 2,
+    "checkpoint_id": "unsloth/Llama-3.1-8B-Instruct",
+    "checkpoint_revision": "4699cc75b550f9c6f3173fb80f4703b62d946aa5",
+    "continuous_batching": true,
+    "enable_bucketing": false,
+    "ep_degree": 1,
+    "flash_decoding_enabled": false,
+    "fused_qkv": true,
+    "glu_mlp": true,
+    "is_chunked_prefill": false,
+    "local_ranks_size": 8,
+    "logical_nc_config": 1,
+    "max_batch_size": 8,
+    "max_context_length": 4096,
+    "max_topk": 256,
+    "mlp_kernel_enabled": false,
+    "mlp_kernel_fuse_residual_add": false,
+    "n_active_tokens": 4096,
+    "neuronxcc_version": "2.19.8089.0+8ab9f450",
+    "num_cores_per_group": 1,
+    "on_device_sampling": true,
+    "optimum_neuron_version": "0.3.0.dev2",
+    "output_logits": false,
+    "padding_side": "right",
+    "pp_degree": 1,
+    "qk_layernorm": false,
+    "qkv_kernel_enabled": false,
+    "rpl_reduce_dtype": "bfloat16",
+    "sequence_length": 4096,
+    "sequence_parallel_enabled": false,
+    "speculation_length": 0,
+    "start_rank_id": 0,
+    "target": null,
+    "torch_dtype": "bfloat16",
+    "tp_degree": 8,
+    "vocab_parallel": false
+  },
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 8.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "unsloth_fixed": true,
+  "use_cache": true,
+  "vocab_size": 128256
+}