Hugging Face's logo

aws-neuron
/
optimum-neuron-cache

Model card Files
xet
Community
643
dacorvo HF Staff commited on
Commit
e64396b
·
verified ·
1 Parent(s): 76344ba

Update inference-cache-config/trn1/mixtral.json

Browse files
Files changed (1) hide show
  1. inference-cache-config/trn1/mixtral.json +8 -4
inference-cache-config/trn1/mixtral.json CHANGED
@@ -1,29 +1,33 @@
1
  {
2
  "mistralai/Mixtral-8x7B-Instruct-v0.1": [
3
  {
 
4
  "batch_size": 1,
5
  "sequence_length": 4096,
6
- "tensor_parallel_size": 24,
7
  "instance_type" : "trn1"
8
  },
9
  {
 
10
  "batch_size": 4,
11
  "sequence_length": 4096,
12
- "tensor_parallel_size": 24,
13
  "instance_type" : "trn1"
14
  }
15
  ],
16
  "mistralai/Mixtral-8x22B-Instruct-v0.1": [
17
  {
 
18
  "batch_size": 1,
19
  "sequence_length": 4096,
20
- "tensor_parallel_size": 24,
21
  "instance_type" : "trn1"
22
  },
23
  {
 
24
  "batch_size": 4,
25
  "sequence_length": 4096,
26
- "tensor_parallel_size": 24,
27
  "instance_type" : "trn1"
28
  }
29
  ]
 
1
  {
2
  "mistralai/Mixtral-8x7B-Instruct-v0.1": [
3
  {
4
+ "task: "text-generation",
5
  "batch_size": 1,
6
  "sequence_length": 4096,
7
+ "tensor_parallel_size": 32,
8
  "instance_type" : "trn1"
9
  },
10
  {
11
+ "task: "text-generation",
12
  "batch_size": 4,
13
  "sequence_length": 4096,
14
+ "tensor_parallel_size": 32,
15
  "instance_type" : "trn1"
16
  }
17
  ],
18
  "mistralai/Mixtral-8x22B-Instruct-v0.1": [
19
  {
20
+ "task: "text-generation",
21
  "batch_size": 1,
22
  "sequence_length": 4096,
23
+ "tensor_parallel_size": 32,
24
  "instance_type" : "trn1"
25
  },
26
  {
27
+ "task: "text-generation",
28
  "batch_size": 4,
29
  "sequence_length": 4096,
30
+ "tensor_parallel_size": 32,
31
  "instance_type" : "trn1"
32
  }
33
  ]