remove old routing stuff
Browse files
myolmoe/modeling_myolmoe.py
CHANGED
|
@@ -453,8 +453,6 @@ class OlmoeSparseMoeBlock(nn.Module):
|
|
| 453 |
self.num_experts = config.num_experts
|
| 454 |
self.top_k = config.num_experts_per_tok
|
| 455 |
self.norm_topk_prob = config.norm_topk_prob
|
| 456 |
-
self.routing_type = getattr(config, "routing_type", "topk") # default to topk
|
| 457 |
-
self.n_step = getattr(config, "nth_step", 2) # used in nth-descending
|
| 458 |
self.gate = nn.Linear(config.hidden_size, self.num_experts, bias=False)
|
| 459 |
self.experts = nn.ModuleList([OlmoeMLP(config) for _ in range(self.num_experts)])
|
| 460 |
|
|
|
|
| 453 |
self.num_experts = config.num_experts
|
| 454 |
self.top_k = config.num_experts_per_tok
|
| 455 |
self.norm_topk_prob = config.norm_topk_prob
|
|
|
|
|
|
|
| 456 |
self.gate = nn.Linear(config.hidden_size, self.num_experts, bias=False)
|
| 457 |
self.experts = nn.ModuleList([OlmoeMLP(config) for _ in range(self.num_experts)])
|
| 458 |
|