| { |
| "deployments": [ |
| { |
| "spec": { |
| "model_label": "Llama-3.1-8B", |
| "model_id": "meta-llama/Llama-3.1-8B-Instruct", |
| "gpus_per_replica": 1, |
| "num_replicas": 720, |
| "initial_batch_size": 128, |
| "itl_deadline_s": 0.08, |
| "feasible_batch_sizes": [1, 2, 3, 4, 5, 6, 7, 8] |
| }, |
| "vllm_base_url": "http://node1:8000", |
| "gpu_endpoints": [ |
| {"host": "node1", "port": 4938, "gpu_indices": [0, 1, 2, 3]}, |
| {"host": "node1", "port": 4938, "gpu_indices": [4, 5, 6, 7]} |
| ] |
| }, |
| { |
| "spec": { |
| "model_label": "Llama-3.1-70B", |
| "model_id": "meta-llama/Llama-3.1-70B-Instruct", |
| "gpus_per_replica": 4, |
| "num_replicas": 180, |
| "initial_batch_size": 128, |
| "itl_deadline_s": 0.10, |
| "feasible_batch_sizes": [1, 2, 3, 4, 5, 6, 7, 8] |
| }, |
| "vllm_base_url": "http://node2:8000", |
| "gpu_endpoints": [ |
| {"host": "node2", "port": 4938, "gpu_indices": [0, 1, 2, 3]}, |
| {"host": "node2", "port": 4938, "gpu_indices": [4, 5, 6, 7]} |
| ] |
| } |
| ], |
| "requests": { |
| "dataset": "lm-arena-chat", |
| "num_requests": 1000, |
| "max_completion_tokens": 512, |
| "seed": 0 |
| }, |
| "requests_dir": null, |
| "ieee_case_dir": "examples/ieee13", |
| "data_dir": null, |
| "data_sources": [ |
| {"model_label": "Llama-3.1-8B", "task": "lm-arena-chat", "gpu": "H100", "batch_sizes": [8, 16, 32, 64, 96, 128, 192, 256, 384, 512, 768, 1024]}, |
| {"model_label": "Llama-3.1-70B", "task": "lm-arena-chat", "gpu": "H100", "batch_sizes": [8, 16, 32, 64, 96, 128, 192, 256, 384, 512, 768, 1024, 1536, 2048]} |
| ], |
| "mlenergy_data_dir": null |
| } |
|
|
