File size: 653 Bytes
d91766b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 | from __future__ import annotations
from abc import ABC, abstractmethod
import torch
import torch.nn as nn
from diffulex.moe.topk.output import TopKOutput
class TopKRouter(nn.Module, ABC):
"""Top-k expert selection for MoE inference."""
def __init__(
self,
top_k: int,
*,
renormalize: bool = True,
scoring_func: str = "softmax",
) -> None:
super().__init__()
self.top_k = top_k
self.renormalize = renormalize
self.scoring_func = scoring_func
@abstractmethod
def forward(self, router_logits: torch.Tensor) -> TopKOutput:
raise NotImplementedError
|