Upload logic_mini_config.py with huggingface_hub

logic_mini_config.py

@@ -0,0 +1,302 @@
+#!/usr/bin/env python3
+"""
+Logic Mini Model Configuration
+Optimized for scientific reasoning and multi-domain expertise
+Based on MiniMind framework, customized for CrowLogic ecosystem
+"""
+
+from dataclasses import dataclass, field
+from typing import Optional, Dict, List
+import json
+
+
+@dataclass
+class LogicMiniConfig:
+    """
+    Configuration for Logic Mini - Deep Scientific Reasoning Model
+
+    Designed for:
+    - Multi-domain expertise (mycology, drug discovery, AI systems, business)
+    - Chain-of-thought reasoning with <think></think> tags
+    - Prologic framework integration (intercept-annotate-correlate)
+    - Vertical application deployment
+    """
+
+    # Model Architecture
+    hidden_size: int = 768  # Larger than base MiniMind for complex reasoning
+    num_hidden_layers: int = 16  # Deeper for multi-step logic
+    num_attention_heads: int = 12
+    num_key_value_heads: int = 4  # Multi-query attention for efficiency
+    intermediate_size: Optional[int] = None  # Will be computed as 8/3 * hidden_size
+
+    # Vocabulary
+    vocab_size: int = 6400  # Can expand if needed for scientific terms
+    max_position_embeddings: int = 8192  # Extended context for research papers
+
+    # Mixture of Experts (Domain Specialization)
+    use_moe: bool = True  # Enable for multi-domain expertise
+    n_routed_experts: int = 8  # One per major domain
+    n_shared_experts: int = 1
+    num_experts_per_tok: int = 2  # Hybrid reasoning across domains
+    scoring_func: str = 'softmax'
+    aux_loss_alpha: float = 0.01  # Load balancing
+    seq_aux: bool = True
+
+    # Positional Encoding
+    rope_theta: float = 1e6  # RoPE base frequency
+    rope_scaling: Optional[Dict] = None  # YaRN scaling for extended context
+
+    # Normalization
+    rms_norm_eps: float = 1e-5
+
+    # Activation
+    hidden_act: str = "silu"  # SwiGLU-style activation
+
+    # Training
+    initializer_range: float = 0.02
+    use_cache: bool = True
+    pad_token_id: int = 0
+    bos_token_id: int = 1
+    eos_token_id: int = 2
+    tie_word_embeddings: bool = False
+    flash_attn: bool = True
+    dropout: float = 0.0
+
+    # Logic Mini Specific
+    reasoning_token_weight: float = 10.0  # Emphasize <think> tag learning
+    domain_experts: List[str] = field(default_factory=list)
+
+    def __post_init__(self):
+        """Post-initialization setup"""
+        if self.intermediate_size is None:
+            self.intermediate_size = int(8 * self.hidden_size / 3)
+            # Round to nearest multiple of 64 for efficiency
+            self.intermediate_size = ((self.intermediate_size + 63) // 64) * 64
+
+        if self.rope_scaling is None:
+            # Default YaRN scaling for 4x context extension
+            self.rope_scaling = {
+                "type": "yarn",
+                "factor": 4.0,
+                "original_max_position_embeddings": 2048,
+                "beta_fast": 4,
+                "beta_slow": 1
+            }
+
+        if not self.domain_experts:
+            # Define domain expert specializations
+            self.domain_experts = [
+                "mycology_cultivation",
+                "drug_discovery_chemistry",
+                "ai_systems_architecture",
+                "prologic_methodology",
+                "business_strategy",
+                "scientific_reasoning",
+                "technical_debugging",
+                "general_knowledge"
+            ]
+
+    def to_dict(self) -> Dict:
+        """Convert config to dictionary"""
+        return {
+            "model_type": "minimind",
+            "hidden_size": self.hidden_size,
+            "num_hidden_layers": self.num_hidden_layers,
+            "num_attention_heads": self.num_attention_heads,
+            "num_key_value_heads": self.num_key_value_heads,
+            "intermediate_size": self.intermediate_size,
+            "vocab_size": self.vocab_size,
+            "max_position_embeddings": self.max_position_embeddings,
+            "use_moe": self.use_moe,
+            "n_routed_experts": self.n_routed_experts,
+            "n_shared_experts": self.n_shared_experts,
+            "num_experts_per_tok": self.num_experts_per_tok,
+            "rope_theta": self.rope_theta,
+            "rope_scaling": self.rope_scaling,
+            "rms_norm_eps": self.rms_norm_eps,
+            "hidden_act": self.hidden_act,
+            "flash_attn": self.flash_attn,
+            "dropout": self.dropout,
+            "bos_token_id": self.bos_token_id,
+            "eos_token_id": self.eos_token_id,
+            "pad_token_id": self.pad_token_id,
+            "reasoning_token_weight": self.reasoning_token_weight,
+            "domain_experts": self.domain_experts,
+            "scoring_func": self.scoring_func,
+            "aux_loss_alpha": self.aux_loss_alpha,
+            "seq_aux": self.seq_aux,
+            "norm_topk_prob": True,
+            "inference_rope_scaling": False
+        }
+
+    def save(self, path: str):
+        """Save configuration to JSON file"""
+        with open(path, 'w') as f:
+            json.dump(self.to_dict(), f, indent=2)
+
+    @classmethod
+    def from_dict(cls, config_dict: Dict):
+        """Load configuration from dictionary"""
+        # Filter out keys that aren't in the dataclass
+        valid_keys = {f.name for f in cls.__dataclass_fields__.values()}
+        filtered_dict = {k: v for k, v in config_dict.items() if k in valid_keys}
+        return cls(**filtered_dict)
+
+    @classmethod
+    def load(cls, path: str):
+        """Load configuration from JSON file"""
+        with open(path, 'r') as f:
+            config_dict = json.load(f)
+        return cls.from_dict(config_dict)
+
+
+# Predefined configurations for different use cases
+LOGIC_MINI_TINY = LogicMiniConfig(
+    hidden_size=512,
+    num_hidden_layers=8,
+    num_attention_heads=8,
+    num_key_value_heads=2,
+    use_moe=False,
+    max_position_embeddings=2048
+)
+
+LOGIC_MINI_SMALL = LogicMiniConfig(
+    hidden_size=768,
+    num_hidden_layers=12,
+    num_attention_heads=12,
+    num_key_value_heads=4,
+    use_moe=True,
+    n_routed_experts=4,
+    max_position_embeddings=4096
+)
+
+LOGIC_MINI_MEDIUM = LogicMiniConfig(
+    hidden_size=1024,
+    num_hidden_layers=16,
+    num_attention_heads=16,
+    num_key_value_heads=4,
+    use_moe=True,
+    n_routed_experts=8,
+    max_position_embeddings=8192
+)
+
+LOGIC_MINI_LARGE = LogicMiniConfig(
+    hidden_size=1536,
+    num_hidden_layers=24,
+    num_attention_heads=24,
+    num_key_value_heads=6,
+    use_moe=True,
+    n_routed_experts=16,
+    max_position_embeddings=8192
+)
+
+
+def get_config(size: str = "medium") -> LogicMiniConfig:
+    """
+    Get predefined configuration by size
+
+    Args:
+        size: One of "tiny", "small", "medium", "large"
+
+    Returns:
+        LogicMiniConfig instance
+    """
+    configs = {
+        "tiny": LOGIC_MINI_TINY,
+        "small": LOGIC_MINI_SMALL,
+        "medium": LOGIC_MINI_MEDIUM,
+        "large": LOGIC_MINI_LARGE
+    }
+
+    if size not in configs:
+        raise ValueError(f"Unknown size: {size}. Choose from {list(configs.keys())}")
+
+    return configs[size]
+
+
+def print_model_info(config: LogicMiniConfig):
+    """Print detailed model information"""
+
+    # Calculate approximate parameter count
+    embedding_params = config.vocab_size * config.hidden_size
+
+    # Per layer
+    attention_params = (
+        4 * config.hidden_size * config.hidden_size +  # QKV + O projections
+        config.hidden_size  # Bias terms
+    )
+
+    if config.use_moe:
+        # MoE FFN
+        expert_params = (
+            config.hidden_size * config.intermediate_size +
+            config.intermediate_size * config.hidden_size
+        ) * (config.n_routed_experts + config.n_shared_experts)
+
+        gate_params = config.hidden_size * config.n_routed_experts
+        ffn_params = expert_params + gate_params
+    else:
+        # Standard FFN
+        ffn_params = (
+            config.hidden_size * config.intermediate_size * 2 +  # Gate + Up
+            config.intermediate_size * config.hidden_size  # Down
+        )
+
+    layer_params = attention_params + ffn_params + config.hidden_size * 4  # Norms
+    total_layer_params = layer_params * config.num_hidden_layers
+
+    lm_head_params = config.vocab_size * config.hidden_size
+
+    total_params = embedding_params + total_layer_params + lm_head_params
+
+    print("=" * 70)
+    print("LOGIC MINI MODEL CONFIGURATION")
+    print("=" * 70)
+    print(f"\nArchitecture:")
+    print(f"  Hidden Size:          {config.hidden_size:,}")
+    print(f"  Layers:               {config.num_hidden_layers}")
+    print(f"  Attention Heads:      {config.num_attention_heads}")
+    print(f"  KV Heads:             {config.num_key_value_heads}")
+    print(f"  Intermediate Size:    {config.intermediate_size:,}")
+    print(f"  Vocabulary Size:      {config.vocab_size:,}")
+    print(f"  Max Context:          {config.max_position_embeddings:,} tokens")
+
+    print(f"\nMixture of Experts:")
+    print(f"  Enabled:              {config.use_moe}")
+    if config.use_moe:
+        print(f"  Routed Experts:       {config.n_routed_experts}")
+        print(f"  Shared Experts:       {config.n_shared_experts}")
+        print(f"  Experts per Token:    {config.num_experts_per_tok}")
+        print(f"  Domain Specializations:")
+        for i, domain in enumerate(config.domain_experts, 1):
+            print(f"    {i}. {domain.replace('_', ' ').title()}")
+
+    print(f"\nCapabilities:")
+    print(f"  Chain-of-Thought:     ✓ (<think></think> tags)")
+    print(f"  Prologic Framework:   ✓ (intercept-annotate-correlate)")
+    print(f"  Extended Context:     ✓ (YaRN scaling, 4x extension)")
+    print(f"  Multi-Domain:         ✓ ({len(config.domain_experts)} specializations)")
+
+    print(f"\nParameter Count:")
+    print(f"  Embeddings:           {embedding_params:,} ({embedding_params/1e6:.1f}M)")
+    print(f"  Transformer Layers:   {total_layer_params:,} ({total_layer_params/1e6:.1f}M)")
+    print(f"  LM Head:              {lm_head_params:,} ({lm_head_params/1e6:.1f}M)")
+    print(f"  TOTAL:                {total_params:,} ({total_params/1e6:.1f}M)")
+
+    print(f"\nTraining Estimates:")
+    print(f"  GPU Memory (bf16):    ~{total_params * 2 / 1e9:.1f} GB")
+    print(f"  Training Time:        ~{total_params / 25.8e6 * 2:.1f} hours (on consumer GPU)")
+    print(f"  Dataset Needed:       ~{total_params / 1e6 * 100:.0f}M tokens minimum")
+    print("=" * 70)
+
+
+if __name__ == "__main__":
+    # Example usage
+    print("\n🚀 Logic Mini Configuration Examples\n")
+
+    for size in ["tiny", "small", "medium", "large"]:
+        config = get_config(size)
+        print(f"\n{'='*70}")
+        print(f"LOGIC MINI - {size.upper()}")
+        print_model_info(config)
+        print()