Rename core MiniMindBlock to Block for modality-agnostic naming
Browse filesThe transformer block is a generic primitive reused by lm/vlm/vam
backbones, so rename MiniMindBlock -> Block in core/block.py and update
all references (core/__init__, models/__init__, models/lm/model.py,
models/vam/model.py, lora comment).
Co-Authored-By: opencode <noreply@opencode.ai>
- src/omni/core/__init__.py +2 -2
- src/omni/core/block.py +1 -1
- src/omni/models/__init__.py +2 -3
- src/omni/models/lm/lora.py +1 -1
- src/omni/models/lm/model.py +2 -2
- src/omni/models/vam/model.py +2 -2
src/omni/core/__init__.py
CHANGED
|
@@ -2,7 +2,7 @@ from omni.core.norm import RMSNorm
|
|
| 2 |
from omni.core.rope import precompute_freqs_cis, apply_rotary_pos_emb, repeat_kv
|
| 3 |
from omni.core.attention import Attention
|
| 4 |
from omni.core.mlp import FeedForward, MOEFeedForward
|
| 5 |
-
from omni.core.block import
|
| 6 |
|
| 7 |
__all__ = [
|
| 8 |
"RMSNorm",
|
|
@@ -12,5 +12,5 @@ __all__ = [
|
|
| 12 |
"Attention",
|
| 13 |
"FeedForward",
|
| 14 |
"MOEFeedForward",
|
| 15 |
-
"
|
| 16 |
]
|
|
|
|
| 2 |
from omni.core.rope import precompute_freqs_cis, apply_rotary_pos_emb, repeat_kv
|
| 3 |
from omni.core.attention import Attention
|
| 4 |
from omni.core.mlp import FeedForward, MOEFeedForward
|
| 5 |
+
from omni.core.block import Block
|
| 6 |
|
| 7 |
__all__ = [
|
| 8 |
"RMSNorm",
|
|
|
|
| 12 |
"Attention",
|
| 13 |
"FeedForward",
|
| 14 |
"MOEFeedForward",
|
| 15 |
+
"Block",
|
| 16 |
]
|
src/omni/core/block.py
CHANGED
|
@@ -5,7 +5,7 @@ from omni.core.attention import Attention
|
|
| 5 |
from omni.core.mlp import FeedForward, MOEFeedForward
|
| 6 |
|
| 7 |
|
| 8 |
-
class
|
| 9 |
def __init__(self, layer_id: int, config: "MiniMindConfig"):
|
| 10 |
super().__init__()
|
| 11 |
self.self_attn = Attention(config)
|
|
|
|
| 5 |
from omni.core.mlp import FeedForward, MOEFeedForward
|
| 6 |
|
| 7 |
|
| 8 |
+
class Block(nn.Module):
|
| 9 |
def __init__(self, layer_id: int, config: "MiniMindConfig"):
|
| 10 |
super().__init__()
|
| 11 |
self.self_attn = Attention(config)
|
src/omni/models/__init__.py
CHANGED
|
@@ -16,7 +16,7 @@ from omni.core import (
|
|
| 16 |
Attention,
|
| 17 |
FeedForward,
|
| 18 |
MOEFeedForward,
|
| 19 |
-
|
| 20 |
precompute_freqs_cis,
|
| 21 |
apply_rotary_pos_emb,
|
| 22 |
)
|
|
@@ -38,8 +38,7 @@ __all__ = [
|
|
| 38 |
"Attention",
|
| 39 |
"FeedForward",
|
| 40 |
"MOEFeedForward",
|
| 41 |
-
"
|
| 42 |
-
"MiniMindModel",
|
| 43 |
"precompute_freqs_cis",
|
| 44 |
"apply_rotary_pos_emb",
|
| 45 |
]
|
|
|
|
| 16 |
Attention,
|
| 17 |
FeedForward,
|
| 18 |
MOEFeedForward,
|
| 19 |
+
Block,
|
| 20 |
precompute_freqs_cis,
|
| 21 |
apply_rotary_pos_emb,
|
| 22 |
)
|
|
|
|
| 38 |
"Attention",
|
| 39 |
"FeedForward",
|
| 40 |
"MOEFeedForward",
|
| 41 |
+
"Block",
|
|
|
|
| 42 |
"precompute_freqs_cis",
|
| 43 |
"apply_rotary_pos_emb",
|
| 44 |
]
|
src/omni/models/lm/lora.py
CHANGED
|
@@ -3,7 +3,7 @@ from torch import optim, nn
|
|
| 3 |
|
| 4 |
# LoRA for MiniMind-series backbones.
|
| 5 |
# Applies to every nn.Linear whose in_features == out_features, i.e. the
|
| 6 |
-
# q/k/v/o_proj and gate/up/down_proj inside
|
| 7 |
# Because VLM/Omni reuse the same MiniMind Attention/MLP for their LLM
|
| 8 |
# (thinker)主干, apply_lora also works on MiniMindVLM / MiniMindOmni, but only
|
| 9 |
# touches the shared LLM layers -- vision/audio projectors and the speech
|
|
|
|
| 3 |
|
| 4 |
# LoRA for MiniMind-series backbones.
|
| 5 |
# Applies to every nn.Linear whose in_features == out_features, i.e. the
|
| 6 |
+
# q/k/v/o_proj and gate/up/down_proj inside Block's Attention/MLP.
|
| 7 |
# Because VLM/Omni reuse the same MiniMind Attention/MLP for their LLM
|
| 8 |
# (thinker)主干, apply_lora also works on MiniMindVLM / MiniMindOmni, but only
|
| 9 |
# touches the shared LLM layers -- vision/audio projectors and the speech
|
src/omni/models/lm/model.py
CHANGED
|
@@ -4,7 +4,7 @@ from torch import nn
|
|
| 4 |
from transformers import PreTrainedModel, GenerationMixin
|
| 5 |
from transformers.modeling_outputs import MoeCausalLMOutputWithPast
|
| 6 |
|
| 7 |
-
from omni.core import RMSNorm, precompute_freqs_cis,
|
| 8 |
from omni.models.lm.config import MiniMindConfig
|
| 9 |
|
| 10 |
|
|
@@ -15,7 +15,7 @@ class MiniMindModel(nn.Module):
|
|
| 15 |
self.vocab_size, self.num_hidden_layers = config.vocab_size, config.num_hidden_layers
|
| 16 |
self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size)
|
| 17 |
self.dropout = nn.Dropout(config.dropout)
|
| 18 |
-
self.layers = nn.ModuleList([
|
| 19 |
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
| 20 |
freqs_cos, freqs_sin = precompute_freqs_cis(dim=config.head_dim, end=config.max_position_embeddings, rope_base=config.rope_theta, rope_scaling=config.rope_scaling)
|
| 21 |
self.register_buffer("freqs_cos", freqs_cos, persistent=False)
|
|
|
|
| 4 |
from transformers import PreTrainedModel, GenerationMixin
|
| 5 |
from transformers.modeling_outputs import MoeCausalLMOutputWithPast
|
| 6 |
|
| 7 |
+
from omni.core import RMSNorm, precompute_freqs_cis, Block, MOEFeedForward
|
| 8 |
from omni.models.lm.config import MiniMindConfig
|
| 9 |
|
| 10 |
|
|
|
|
| 15 |
self.vocab_size, self.num_hidden_layers = config.vocab_size, config.num_hidden_layers
|
| 16 |
self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size)
|
| 17 |
self.dropout = nn.Dropout(config.dropout)
|
| 18 |
+
self.layers = nn.ModuleList([Block(l, config) for l in range(self.num_hidden_layers)])
|
| 19 |
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
| 20 |
freqs_cos, freqs_sin = precompute_freqs_cis(dim=config.head_dim, end=config.max_position_embeddings, rope_base=config.rope_theta, rope_scaling=config.rope_scaling)
|
| 21 |
self.register_buffer("freqs_cos", freqs_cos, persistent=False)
|
src/omni/models/vam/model.py
CHANGED
|
@@ -10,7 +10,7 @@ from torch.nn import functional as F
|
|
| 10 |
from transformers.modeling_outputs import MoeCausalLMOutputWithPast
|
| 11 |
from transformers import SiglipVisionModel, SiglipImageProcessor, logging as hf_logging
|
| 12 |
|
| 13 |
-
from omni.core import RMSNorm, precompute_freqs_cis,
|
| 14 |
from omni.models.lm.config import MiniMindConfig
|
| 15 |
from omni.models.lm.model import MiniMindForCausalLM
|
| 16 |
from omni.models.vam.config import OmniConfig
|
|
@@ -53,7 +53,7 @@ class TalkerModule(nn.Module):
|
|
| 53 |
def __init__(self, config: OmniConfig):
|
| 54 |
super().__init__()
|
| 55 |
self.talker_config = MiniMindConfig(hidden_size=config.talker_hidden_size, use_moe=config.use_moe)
|
| 56 |
-
self.layers = nn.ModuleList([
|
| 57 |
self.norm = RMSNorm(config.talker_hidden_size, eps=config.rms_norm_eps)
|
| 58 |
self.lm_head = TalkerHead(config.talker_hidden_size, config.audio_vocab_size)
|
| 59 |
self.embed_tokens = TalkerEmbedding(config.audio_vocab_size, config.talker_hidden_size)
|
|
|
|
| 10 |
from transformers.modeling_outputs import MoeCausalLMOutputWithPast
|
| 11 |
from transformers import SiglipVisionModel, SiglipImageProcessor, logging as hf_logging
|
| 12 |
|
| 13 |
+
from omni.core import RMSNorm, precompute_freqs_cis, Block, MOEFeedForward
|
| 14 |
from omni.models.lm.config import MiniMindConfig
|
| 15 |
from omni.models.lm.model import MiniMindForCausalLM
|
| 16 |
from omni.models.vam.config import OmniConfig
|
|
|
|
| 53 |
def __init__(self, config: OmniConfig):
|
| 54 |
super().__init__()
|
| 55 |
self.talker_config = MiniMindConfig(hidden_size=config.talker_hidden_size, use_moe=config.use_moe)
|
| 56 |
+
self.layers = nn.ModuleList([Block(l, self.talker_config) for l in range(config.num_talker_hidden_layers)])
|
| 57 |
self.norm = RMSNorm(config.talker_hidden_size, eps=config.rms_norm_eps)
|
| 58 |
self.lm_head = TalkerHead(config.talker_hidden_size, config.audio_vocab_size)
|
| 59 |
self.embed_tokens = TalkerEmbedding(config.audio_vocab_size, config.talker_hidden_size)
|