| dtype: bfloat16 | |
| merge_method: linear | |
| slices: | |
| - sources: | |
| - layer_range: [0, 32] # Assuming the first half of the model is more general and can be reduced more | |
| model: codellama/CodeLlama-7b-Instruct-hf | |
| parameters: | |
| weight: 0.25 # Reduce the weight of the first half to make room for the second half | |
| - layer_range: [0, 32] # Assuming the second half of the model is more specialized and can be reduced less | |
| model: codellama/CodeLlama-7b-Instruct-hf | |
| parameters: | |
| weight: 0.25 # Maintain the weight of the second half | |