src/models/config_openmind.py

"""
OpenMind Model Configuration.

Hugging Face compatible configuration class for the OpenMind transformer model.
"""

import json
import os
from dataclasses import dataclass, field, asdict
from typing import Optional


@dataclass
class OpenMindConfig:
    """Configuration for the OpenMind decoder-only transformer model."""

    # Model architecture
    vocab_size: int = 32000
    max_seq_len: int = 2048
    dim: int = 768
    n_layers: int = 12
    n_heads: int = 12
    n_kv_heads: int = 12           # Set < n_heads for Grouped Query Attention
    intermediate_dim: int = 2048    # SwiGLU intermediate dimension
    dropout: float = 0.0
    tie_embeddings: bool = True
    rope_theta: float = 10000.0

    # Derived
    head_dim: int = 0  # Will be computed as dim // n_heads

    # Model metadata
    model_type: str = "openmind"
    architectures: list = field(default_factory=lambda: ["OpenMindForCausalLM"])

    def __post_init__(self):
        self.head_dim = self.dim // self.n_heads
        assert self.dim % self.n_heads == 0, "dim must be divisible by n_heads"
        assert self.n_heads % self.n_kv_heads == 0, "n_heads must be divisible by n_kv_heads"

    def save_pretrained(self, output_dir: str) -> None:
        """Save config to directory as config.json."""
        os.makedirs(output_dir, exist_ok=True)
        config_path = os.path.join(output_dir, "config.json")
        with open(config_path, "w", encoding="utf-8") as f:
            json.dump(asdict(self), f, indent=2)
        print(f"Config saved to {config_path}")

    @classmethod
    def from_pretrained(cls, model_dir: str) -> "OpenMindConfig":
        """Load config from a directory containing config.json."""
        config_path = os.path.join(model_dir, "config.json")
        with open(config_path, "r", encoding="utf-8") as f:
            config_dict = json.load(f)
        # Remove derived fields that will be recomputed
        config_dict.pop("head_dim", None)
        return cls(**config_dict)

    @classmethod
    def from_yaml(cls, yaml_path: str) -> "OpenMindConfig":
        """Load config from a YAML training config file."""
        import yaml
        with open(yaml_path, "r", encoding="utf-8") as f:
            data = yaml.safe_load(f)
        model_cfg = data.get("model", {})
        # Map YAML keys to dataclass fields
        return cls(
            vocab_size=model_cfg.get("vocab_size", 32000),
            max_seq_len=model_cfg.get("max_seq_len", 2048),
            dim=model_cfg.get("dim", 768),
            n_layers=model_cfg.get("n_layers", 12),
            n_heads=model_cfg.get("n_heads", 12),
            n_kv_heads=model_cfg.get("n_kv_heads", 12),
            intermediate_dim=model_cfg.get("intermediate_dim", 2048),
            dropout=model_cfg.get("dropout", 0.0),
            tie_embeddings=model_cfg.get("tie_embeddings", True),
            rope_theta=model_cfg.get("rope_theta", 10000.0),
            model_type=model_cfg.get("name", "openmind"),
        )

    def to_dict(self) -> dict:
        return asdict(self)

    def __repr__(self) -> str:
        params = ", ".join(f"{k}={v}" for k, v in asdict(self).items())
        return f"OpenMindConfig({params})"


# Predefined model sizes
CONFIGS = {
    "openmind-125m": OpenMindConfig(
        dim=768, n_layers=12, n_heads=12, n_kv_heads=12, intermediate_dim=2048,
    ),
    "openmind-350m": OpenMindConfig(
        dim=1024, n_layers=24, n_heads=16, n_kv_heads=4, intermediate_dim=2730,
    ),
    "openmind-760m": OpenMindConfig(
        dim=1536, n_layers=24, n_heads=16, n_kv_heads=4, intermediate_dim=4096,
    ),
    "openmind-1.3b": OpenMindConfig(
        dim=2048, n_layers=24, n_heads=16, n_kv_heads=4, intermediate_dim=5460,
    ),
}