Upload 7 files

config.json

@@ -0,0 +1,199 @@
+{
+  "SELECTED_MODEL": ["model_Custm.py", "model_PrTr.py", "model_Combn.py"],
+  "MODEL_NAME": "Wildnerve-tlm01",
+  "BASE_DATA_DIR": "data",
+  "FILE_FORMATS": ["csv", "json", "txt"],
+  "MAX_SEQ_LENGTH": 512,
+  "SIMILARITY_THRESHOLD": 0.85,
+  "DATASET_PATHS": {
+    "general": ["data/general.json"],
+    "programming_software_dev": ["data/programming_software_dev.json"],
+    "other_information": ["data/other_information.json"]
+  },
+  "LAZY_LOADING_ENABLED": true,
+  "MAX_INITIAL_SPECIALIZATIONS": 2,
+  "train_file_path": "data/computer_advanced_debugging.json",
+  "NUM_EPOCHS": 50,
+  "LEARNING_RATE": 0.0001,
+  "INPUT_SIZE": 768,
+  "OUTPUT_SIZE": 768,
+  "SPECIALIZATIONS": [
+    "general",
+    "programming_software_dev"
+  ],
+  "ALL_SPECIALIZATIONS": [
+    "general",
+    "mbpp", 
+    "programming_software_dev",
+    "machine_learning_ai_data_science",
+    "industrial_engineering",
+    "science_engineering",
+    "mathematics",
+    "healthcare_and_lifesciences",
+    "chemistry",
+    "hardware_devops_cloud",
+    "cyber_security",
+    "business_legal_finance",
+    "other_information"
+  ],
+  "PREPROCESSING": {
+    "LOWERCASE": true,
+    "REMOVE_SPECIAL_CHARACTERS": true,
+    "REPLACE_MULTIPLE_SPACES": true
+  },
+  "STDP_CONFIG": {
+    "WEIGHT_THRESHOLD": 0.5,
+    "ACTIVATION_THRESHOLD": 0.2,
+    "USE_SNN": true,
+    "ALPHA": 0.1,
+    "BETA": 0.2,
+    "BASE_DIR": "checkpoints",
+    "SNN_FILENAME_FORMAT": "snn_model_{specialization}_{epoch}.pt",
+    "STDPLearningRate": 0.01,
+    "STDPMemDecay": 0.9,
+    "SpikeThreshold": 0.5,
+    "firing_rate": 10,
+    "MAX_SEQ_LENGTH": 2048,
+    "STDP_PRETRAIN_EPOCHS": 5,
+    "STDP_FINETUNE_EPOCHS": 3,
+    "BATCH_SIZE_PRETRAIN": 32,
+    "BATCH_SIZE_FINETUNE": 16,
+    "NUM_NEURONS": 1024,
+    "MAX_RATE": 100
+  },
+  "TRAINING_CONFIG": {
+    "PATIENCE": 3,
+    "DELTA": 0.001,
+    "VERBOSE": true,
+    "NUM_EPOCHS": 10,
+    "LEARNING_RATE": 0.0001,
+    "TRANSFORMER_LEARNING_RATE": 5e-5,
+    "TRANSFORMER_NUM_EPOCHS": 5
+  },
+  "CHECKPOINT_CONFIG": {
+    "PATH": "checkpoints",
+    "BASE_DIR": "checkpoints",
+    "TRANSFORMER_FILENAME_FORMAT": "transformer_model_{specialization}_{epoch}.pt",
+    "SNN_FILENAME_FORMAT": "snn_model_{specialization}_{epoch}.pt"
+  },
+  "GENERATION_CONFIG": {
+    "temperature": 0.7,
+    "top_p": 0.9,
+    "num_return_sequences": 1
+  },
+  "TOKENIZER_CONFIG": {
+    "MODEL_NAME": "bert-base-uncased",
+    "MAX_SEQ_LENGTH": 512,
+    "POOLING_MODE": "mean"
+  },
+  "DATA_LOADER_CONFIG": {
+    "BATCH_SIZE": 32,
+    "NUM_WORKERS": 0,
+    "SHUFFLE": true,
+    "INCLUDE_CRAWL": true
+  },
+  "ATTENTION_CONFIG": {
+    "WINDOW_SIZE": 256,
+    "STRIDE": 128,
+    "MEMORY_SIZE": 64,
+    "NUM_HEADS": 8,
+    "ATTENTION_DROPOUT": 0.1,
+    "ATTENTION_TYPES": {
+      "SLIDING": true,
+      "HIERARCHICAL": true,
+      "GLOBAL": true
+    },
+    "PROMPT_THRESHOLDS": {
+      "LENGTH_THRESHOLD": 500,
+      "COMPLEXITY_THRESHOLD": 0.7,
+      "PERPLEXITY_THRESHOLD": 50
+    },
+    "ATTENTION_WEIGHTS": {
+      "SHORT_COMPLEX": {
+        "SLIDING": 0.4,
+        "HIERARCHICAL": 0.6
+      },
+      "LONG_CONTEXT": {
+        "SLIDING": 0.3,
+        "HIERARCHICAL": 0.4,
+        "GLOBAL": 0.3
+      }
+    }
+  },
+  "TRANSFORMER_CONFIG": {
+    "TEST_MODE": false,
+    "LOGGING_LEVEL": "INFO",
+    "LOG_FILE": "logs/training.log",
+    "SAVE_CHECKPOINTS": true,
+    "BASE_DIR": "checkpoints",
+    "TRANSFORMER_FILENAME_FORMAT": "transformer_model_{specialization}_{epoch}.pt",
+    "MODEL_NAME": "Wildnerve-tlm01-0.05Bx12",
+    "MAX_SEQ_LENGTH": 512,
+    "NUM_EPOCHS": 10,
+    "LEARNING_RATE": 5e-5,
+    "BATCH_SIZE": 32,
+    "EMBEDDING_DIM": 768,
+    "NUM_HEADS": 12,
+    "HIDDEN_DIM": 768,
+    "NUM_LAYERS": 12,
+    "DROPOUT": 0.1,
+    "specialization1": "cpp",
+    "specialization2": "java",
+    "specialization3": "go",
+    "specialization4": "javascript",
+    "specialization5": "nim",
+    "specialization6": "python",
+    "specialization7": "rust",
+    "specialization8": "solidity",
+    "specialization9": "computer",
+    "specialization10": "mathematics",
+    "specialization11": "physics",
+    "specialization12": "other_information", 
+    "DATASET_PATH": "data/cpp_ai_language_model.json",
+    "OUTPUT_SIZE": 768,
+    "POOLING_MODE": "mean",
+    "VOCAB_SIZE": 30522,
+    "MAX_RATE": 100,
+    "MODE": "pretrained",
+    "MODE2": "custom",
+    "SHUFFLE": true,
+    "SIMILARITY_THRESHOLD": 0.85,
+    "USE_PRETRAINED_ENCODER": true,
+    "ATTENTION_MECHANISM": {
+      "TYPE": "hybrid",
+      "WINDOW_SIZE": 1024,
+      "STRIDE": 512,
+      "USE_MEMORY": true
+    },
+    "SPECIALIZATIONS": {
+      "mbpp": "mbpp",
+      "programming_software_dev": "programming_software_dev",
+      "machine_learning_ai_data_science": "machine_learning_ai_data_science",
+      "industrial_engineering": "industrial_engineering",
+      "science_engineering": "science_engineering",
+      "mathematics": "mathematics",
+      "healthcare_and_lifesciences": "healthcare_and_lifesciences",
+      "chemistry": "chemistry",
+      "hardware_devops_cloud": "hardware_devops_cloud",
+      "cyber_security": "cyber_security",
+      "business_legal_finance": "business_legal_finance",
+      "other_information": "other_information"
+    }
+  },
+  "DUAL_ENCODER_CONFIG": {
+    "USE_PRETRAINED_ENCODER": true,
+    "USE_CUSTOM_ENCODER": true,
+    "DEBUG": false
+  },
+  "PROMPT_ANALYZER_CONFIG": {
+    "MODEL_NAME": "gpt2",
+    "DATASET_PATH": null,
+    "SPECIALIZATION": null,
+    "HIDDEN_DIM": 768,
+    "MAX_CACHE_SIZE": 10
+  },
+  "MAX_ACTIVE_MODELS": 5,
+  "MODEL_IDLE_THRESHOLD": 600,
+  "MAX_MEMORY_USAGE": 0.8,
+  "TOP_K": 3
+}

config.py

@@ -0,0 +1,397 @@
+# config.py - 21/02/2025, cleaned up version, 5:14pm, C:\Users\User\OneDrive\Documents\tlm\config.py
+import os
+import json
+import logging
+import argparse
+from pathlib import Path
+from typing import Optional, Dict, List, Literal, Any
+from pydantic import BaseModel, Field, ValidationError, ConfigDict
+#from types import SimpleNamespace
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+)
+logger = logging.getLogger(__name__)
+
+
+
+class PathConfig:
+    """Handle path configurations"""
+    @staticmethod
+    def get_project_root() -> Path:
+        return Path(__file__).resolve().parent
+
+    @staticmethod
+    def get_data_dir() -> Path:
+        """Get writable data directory, falling back to temp if needed"""
+        # First try in project directory
+        project_dir = PathConfig.get_project_root()
+        data_dir = project_dir / "data"
+        
+        # Check if we can write to this location
+        try:
+            if not data_dir.exists():
+                data_dir.mkdir(parents=True, exist_ok=True)
+            # Test write access with a small file
+            test_file = data_dir / ".write_test"
+            test_file.touch()
+            test_file.unlink()
+            return data_dir
+        except (PermissionError, IOError):
+            # Fall back to temp directory
+            import tempfile
+            tmp_dir = Path(tempfile.gettempdir()) / "wildnerve_data"
+            tmp_dir.mkdir(parents=True, exist_ok=True)
+            logger.info("Using temporary directory for data: %s", tmp_dir)
+            return tmp_dir
+
+    @staticmethod
+    def get_checkpoint_dir() -> Path:
+        # First try in project directory
+        project_dir = PathConfig.get_project_root()
+        checkpoint_dir = project_dir / "checkpoints"
+        
+        # Check if we can write to this directory
+        if os.access(project_dir, os.W_OK):
+            return checkpoint_dir
+            
+        # If not writable, fallback to temp directory
+        import tempfile
+        tmp_dir = Path(tempfile.gettempdir()) / "wildnerve_checkpoints"
+        return tmp_dir
+
+# Replace the current directory setup with:
+BASE_DIR = PathConfig.get_project_root()
+DATA_DIR = PathConfig.get_data_dir()
+CHECKPOINT_DIR = PathConfig.get_checkpoint_dir()
+
+# Add these model architecture parameters
+INPUT_SIZE = 768  # BERT base hidden size
+OUTPUT_SIZE = 768  # Output embedding size
+HIDDEN_SIZE = 768  # Hidden layer size
+
+# Add SPECIALIZATIONS list
+SPECIALIZATIONS = [
+    "cpp",
+    "java",
+    "go",
+    "javascript",
+    "nim",
+    "python",
+    "rust",
+    "solidity",
+    "computer",
+    "mathematics",
+    "physics",
+    "other_information"
+]
+
+# Define DATASET_PATHS so that each specialization is a string or a list of strings
+DATASET_PATHS = {
+    "computer": [
+        List[str(DATA_DIR / "data" / "computer_advanced_debugging.json")],
+        List[str(DATA_DIR / "data" / "computer_agenticAI.json")],
+        List[str(DATA_DIR / "data" / "computer_architecture.json")],
+        List[str(DATA_DIR / "data" / "computer_cloud_security.json")],
+        List[str(DATA_DIR / "data" / "computer_creativity.json")],
+        List[str(DATA_DIR / "data" / "computer_crossplatform.json")],
+        List[str(DATA_DIR / "data" / "computer_cybersecurity.json")],
+        List[str(DATA_DIR / "data" / "computer_error_handling_examples.json")],
+        List[str(DATA_DIR / "data" / "computer_gitInstruct.json")]
+    ],
+
+    "cpp": [
+        List[str(DATA_DIR / "data" / "cpp_advanced_debugging.json")],
+        List[str(DATA_DIR / "data" / "cpp_blockchain.json")],
+        List[str(DATA_DIR / "data" / "cpp_mbcppp.json")],
+        List[str(DATA_DIR / "data" / "cpp_programming.json")]
+    ],
+
+    "java": [
+        List[str(DATA_DIR / "data" / "java_ai_language_model.json")],
+        List[str(DATA_DIR / "data" / "java_blockchain.json")],
+        List[str(DATA_DIR / "data" / "java_mbjp.json")],
+        List[str(DATA_DIR / "data" / "java_programming.json")],
+        List[str(DATA_DIR / "data" / "java_transformer_language_model.json")],
+    ],
+
+    "go": [
+        List[str(DATA_DIR / "data" / "golang_ai_language_model.json")],
+        List[str(DATA_DIR / "data" / "golang_mbgp.json")],
+        List[str(DATA_DIR / "data" / "golang_programming.json")]
+    ],
+
+    "javascript": [
+        List[str(DATA_DIR / "data" / "javascript_chatbot.json")],
+        List[str(DATA_DIR / "data" / "javascript_n_Typescript_frontend.json")],
+        List[str(DATA_DIR / "data" / "javascript_n_Typescript_backend.json")],
+        List[str(DATA_DIR / "data" / "javascript_programming.json")]
+    ],
+
+    "nim": [
+        List[str(DATA_DIR / "data" / "nim_ai_language_model.json")],
+        List[str(DATA_DIR / "data" / "nim_blockchain.json")],
+        List[str(DATA_DIR / "data" / "nim_chatbot.json")],
+        List[str(DATA_DIR / "data" / "nim_mbnp.json")],
+        List[str(DATA_DIR / "data" / "nim_programming.json")]
+    ],
+
+    "python": [
+        List[str(DATA_DIR / "data" / "python_chatbot_guide.json")],
+        List[str(DATA_DIR / "data" / "python_mbpp.json")],
+        List[str(DATA_DIR / "data" / "python_programming.json")],
+        List[str(DATA_DIR / "data" / "python_transformer_model.json")]
+    ],
+
+    "rust": [
+        List[str(DATA_DIR / "data" / "rust_ai_language_model.json")],
+        List[str(DATA_DIR / "data" / "rust_blockchain.json")],
+        List[str(DATA_DIR / "data" / "rust_mbrp.json")],
+        List[str(DATA_DIR / "data" / "rust_programming.json")]
+    ],
+
+    "solidity": [
+        List[str(DATA_DIR / "data" / "solidity_programming.json")]
+    ],
+
+    "mathematics": [
+        List[str(DATA_DIR / "data" / "mathematics.json")],
+        List[str(DATA_DIR / "data" / "mathematics_training.json")]
+    ],
+
+    "physics": [
+        List[str(DATA_DIR / "data" / "physics_n_engineering.json")],
+        List[str(DATA_DIR / "data" / "physics_n_engineering_applied.json")]
+    ],
+
+    "other_information": [
+        List[str(DATA_DIR / "data" / "other_information.json")]
+    ]
+}
+
+# Nested configuration models
+class TrainingConfig(BaseModel):
+    PATIENCE: int = Field(..., description="Early stopping patience")
+    DELTA: float = Field(..., description="Minimum change in the monitored value")
+    VERBOSE: bool = Field(..., description="Verbosity of training logs")
+    NUM_EPOCHS: int = Field(..., description="Number of training epochs")
+    LEARNING_RATE: float = Field(..., description="Learning rate for optimizer")
+    TRANSFORMER_LEARNING_RATE: float = Field(..., description="Learning rate for transformer")
+    TRANSFORMER_NUM_EPOCHS: int = Field(..., description="Transformer training epochs")
+    
+    model_config = ConfigDict(
+        validate_assignment=True,
+        extra="allow"
+    )
+
+class CheckpointConfig(BaseModel):
+    PATH: str = Field(..., description="Checkpoint saving folder")
+    BASE_DIR: str = Field(..., description="Base directory for checkpoints")
+    TRANSFORMER_FILENAME_FORMAT: str = Field(..., description="Transformer checkpoint filename format")
+    SNN_FILENAME_FORMAT: str = Field(..., description="SNN checkpoint filename format")
+    
+    model_config = ConfigDict(
+        validate_assignment=True,
+        extra="allow"
+    )
+
+class TokenizerConfig(BaseModel):
+    MODEL_NAME: str = Field(..., description="Name of the tokenizer model")
+    MAX_SEQ_LENGTH: int = Field(..., description="Maximum length the tokenizer handles")
+    POOLING_MODE: str = Field(..., description="Pooling mode for embeddings")
+    
+    model_config = ConfigDict(
+        validate_assignment=True,
+        extra="allow"
+    )
+
+class DataLoaderConfig(BaseModel):
+    SHUFFLE: bool = Field(..., description="Whether to Shuffle the dataset")
+    BATCH_SIZE: int = Field(..., description="Batch size for dataloader")
+    NUM_WORKERS: int = Field(..., description="Number of workers for dataloader")
+    INCLUDE_CRAWL: bool = Field(..., description="Include crawl parameter")
+    
+    model_config = ConfigDict(
+        validate_assignment=True,
+        extra="allow"
+    )
+
+class GenerationConfig(BaseModel):
+    temperature: float = Field(0.7, description="Decoding temperature.")
+    top_p: float = Field(0.9, description="Nucleus sampling probability.")
+    num_return_sequences: int = Field(1, description="Number of sequences to generate.")
+    
+    model_config = ConfigDict(
+        validate_assignment=True,
+        extra="allow"
+    )
+
+class PretrainedLimitsConfig(BaseModel):
+    GPT2: int = Field(1024, description="Maximum sequence length for GPT-2")
+    BERT: int = Field(512, description="Maximum sequence length for BERT")
+    
+    model_config = ConfigDict(
+        validate_assignment=True,
+        extra="allow"
+    )
+
+class CustomWindowsConfig(BaseModel):
+    MAX_SEQ_LENGTH: int = Field(2048, description="Maximum sequence length for custom models")
+    WINDOW_SIZE: int = Field(1024, description="Window size for sliding window attention")
+    STRIDE: int = Field(512, description="Stride for sliding window attention")
+    
+    model_config = ConfigDict(
+        validate_assignment=True,
+        extra="allow"
+    )
+
+class AttentionConfig(BaseModel):
+    PRETRAINED_LIMITS: PretrainedLimitsConfig = Field(default_factory=PretrainedLimitsConfig)
+    CUSTOM_WINDOWS: CustomWindowsConfig = Field(default_factory=CustomWindowsConfig)
+    
+    model_config = ConfigDict(
+        validate_assignment=True,
+        extra="allow"
+    )
+
+class TransformerConfig(BaseModel):
+    ATTENTION_MECHANISM: Dict[str, Any] = Field(
+        default={
+            "TYPE": "hybrid",
+            "WINDOW_SIZE": 1024,
+            "STRIDE": 512,
+            "USE_MEMORY": True,
+            "ATTENTION_TYPES": {
+                "SLIDING": True,
+                "HIERARCHICAL": True,
+                "GLOBAL": True
+            }
+        },
+        description="Attention mechanism configuration"
+    )
+    
+    BASE_DIR: str = Field(..., description="Base directory for transformer checkpoints")
+    TRANSFORMER_FILENAME_FORMAT: str = Field(..., description="Filename format for transformer checkpoints")
+    MODEL_NAME: str = Field("bert-base-uncased", description="Name of the primary model from Hugging Face")  # Changed from Wildnerve-tlm01
+    NUM_EPOCHS: int = Field(30, description="Number of epochs for transformer training")  # Increased from whatever value was here before
+    LEARNING_RATE: float = Field(..., description="Learning rate for transformer")
+    BATCH_SIZE: int = Field(..., description="Batch size for transformer training")
+    EMBEDDING_DIM: int = Field(..., description="Embedding dimension")
+    NUM_HEADS: int = Field(..., description="Number of attention heads")
+    HIDDEN_DIM: int = Field(..., description="Hidden dimension")
+    NUM_LAYERS: int = Field(..., description="Number of layers")
+    DROPOUT: float = Field(..., description="Dropout rate")
+    specialization: str = Field(..., description="Specialization type")
+    DATASET_PATH: str = Field(..., description="Path to the dataset")
+    OUTPUT_SIZE: int = Field(..., description="Size of the output (usually vocab size)")
+    MAX_SEQ_LENGTH: int = Field(..., description="Maximum sequence length")
+    POOLING_MODE: str = Field(..., description="Pooling mode")
+    VOCAB_SIZE: int = Field(..., description="Vocabulary size")
+    MAX_RATE: int = Field(..., description="Maximum rate")
+    MODE: str = Field(..., description="Model mode")
+    MODE2: str = Field(..., description="Secondary mode")
+    SHUFFLE: bool = Field(..., description="Shuffle flag for transformer")
+    SIMILARITY_THRESHOLD: float = Field(..., description="Similarity threshold for weight sharing")
+    USE_PRETRAINED_ENCODER: bool = Field(..., description="Enable pretrained encoder branch")
+    
+    model_config = ConfigDict(
+        validate_assignment=True,
+        extra="allow"
+    )
+
+class PreprocessingConfig(BaseModel):
+    LOWERCASE: bool = Field(True, description="Convert text to lowercase")
+    REMOVE_SPECIAL_CHARACTERS: bool = Field(True, description="Remove special characters from text")
+    REPLACE_MULTIPLE_SPACES: bool = Field(True, description="Replace multiple spaces with a single space")
+    
+    model_config = ConfigDict(
+        validate_assignment=True,
+        extra="allow"
+    )
+
+class STDPConfig(BaseModel):
+    WEIGHT_THRESHOLD: float = Field(..., description="Threshold for STDP weight update")
+    ACTIVATION_THRESHOLD: float = Field(..., description="Threshold for STDP activation")
+    USE_SNN: bool = Field(..., description="Use spiking neural network")
+    ALPHA: float = Field(..., description="STDP alpha parameter")
+    BETA: float = Field(..., description="STDP beta parameter")
+    BASE_DIR: str = Field(..., description="Directory for STDP checkpoints")
+    SNN_FILENAME_FORMAT: str = Field(..., description="Filename format for SNN checkpoints")
+    STDPLearningRate: float = Field(..., description="STDP learning rate")
+    STDPMemDecay: float = Field(..., description="STDP memory decay factor")
+    SpikeThreshold: float = Field(..., description="Spike threshold")
+    firing_rate: int = Field(..., description="Firing rate")
+    MAX_SEQ_LENGTH: int = Field(..., description="Maximum sequence length")
+    STDP_PRETRAIN_EPOCHS: int = Field(..., description="Pre-training epochs for STDP")
+    STDP_FINETUNE_EPOCHS: int = Field(..., description="Fine-tuning epochs for STDP")
+    BATCH_SIZE_PRETRAIN: int = Field(..., description="Batch size during STDP pre-training")
+    BATCH_SIZE_FINETUNE: int = Field(..., description="Batch size during STDP fine-tuning")
+    NUM_NEURONS: int = Field(..., description="Number of neurons in the STDP model")
+    MAX_RATE: int = Field(..., description="Maximum rate for STDP")
+    
+    model_config = ConfigDict(
+        validate_assignment=True,
+        extra="allow"
+    )
+
+class AppConfig(BaseModel):
+    DATA_DIR: str = Field(default="/tmp/tlm_data")
+    MODEL_DIR: str = Field(default="/tmp/tlm_data/models")
+    # Change the type from dict to TransformerConfig so that attributes can be accessed:
+    TRANSFORMER_CONFIG: TransformerConfig = Field(default_factory=TransformerConfig)
+    SIMILARITY_THRESHOLD: float = Field(default=0.85)
+    TOP_K: int = Field(default=3)
+    # ... add other expected fields here ...
+
+import json
+import logging
+import os
+logger = logging.getLogger(__name__)
+
+def load_config():
+    config_path = os.path.join(os.path.dirname(__file__), "config.json")
+    logger.info(f"Attempting to load config from: {config_path}")
+    try:
+        with open(config_path, "r") as f:
+            config = json.load(f)
+        logger.info(f"Config loaded successfully: {config}")
+        return config
+    except Exception as e:
+        logger.error(f"Failed to load config: {e}")
+        return {}
+
+def load_config():
+    config_path = os.path.join(os.path.dirname(__file__), "config.json")
+    try:
+        with open(config_path, "r") as f:
+            config = json.load(f)
+    except Exception as e:
+        raise RuntimeError(f"Failed to load config file: {e}")
+    # Ensure keys exist and are of the expected type:
+    config["DATA_DIR"] = config.get("DATA_DIR", "/tmp/tlm_data")
+    config["DATASET_PATHS"] = config.get("DATASET_PATHS", {})
+    if not isinstance(config["DATASET_PATHS"], dict):
+        config["DATASET_PATHS"] = {}
+    config["TRANSFORMER_CONFIG"] = config.get("TRANSFORMER_CONFIG", {})
+    if not isinstance(config["TRANSFORMER_CONFIG"], dict):
+        config["TRANSFORMER_CONFIG"] = {}
+    config["SIMILARITY_THRESHOLD"] = float(config.get("SIMILARITY_THRESHOLD", 0.85))
+    config["TOP_K"] = int(config.get("TOP_K", 3))
+    config["MAX_ACTIVE_MODELS"] = int(config.get("MAX_ACTIVE_MODELS", 2))
+    config["MODEL_IDLE_THRESHOLD"] = int(config.get("MODEL_IDLE_THRESHOLD", 600))
+    # Also fix MAX_SEQ_LENGTH if provided at root level; fallback to TRANSFORMER_CONFIG
+    if "MAX_SEQ_LENGTH" in config:
+        config["MAX_SEQ_LENGTH"] = int(config["MAX_SEQ_LENGTH"])
+    else:
+        config["MAX_SEQ_LENGTH"] = int(config["TRANSFORMER_CONFIG"].get("MAX_SEQ_LENGTH", 512))
+    return config
+
+# Load config on import
+app_config = load_config()
+
+if __name__ == "__main__":
+    args = argparse.ArgumentParser(description="Tiny Language Model Configuration").parse_args()
+    print("Configuration loaded:")
+    print(app_config)

model_Combn.py

@@ -0,0 +1,387 @@
+import os, sys, math, torch, logging, importlib
+import torch.nn as nn
+import numpy as np
+from config import load_config
+from service_registry import registry, MODEL, TOKENIZER
+from transformers import AutoTokenizer, AutoModel
+from typing import Optional, List, Dict, Any, Union, Tuple
+from utils.smartHybridAttention import SmartHybridAttention, get_hybrid_attention_config
+from base_interfaces.common_types import *
+from base_interfaces.model_interface import AbstractModel
+
+app_config = load_config()
+logger = logging.getLogger(__name__)
+
+class PositionalEncoding(nn.Module):
+    def __init__(self, d_model: int, max_len: int = app_config.TRANSFORMER_CONFIG.MAX_SEQ_LENGTH):
+        super().__init__()
+        pe = torch.zeros(max_len, d_model)
+        position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)
+        div_term = torch.exp(torch.arange(0, d_model, 2, dtype=torch.float)*( -math.log(10000.0)/d_model))
+        pe[:, 0::2] = torch.sin(position*div_term)
+        pe[:, 1::2] = torch.cos(position*div_term)
+        pe = pe.unsqueeze(1)
+        self.register_buffer("pe", pe)
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        seq_len = x.size(0)
+        return x + self.pe[:seq_len]
+
+class Wildnerve_tlm01(nn.Module, AbstractModel):
+    def __init__(self, vocab_size: int, specialization: str, dataset_path: str, model_name: str, embedding_dim: int,
+                 num_heads: int, hidden_dim: int, num_layers: int, output_size: int, dropout: float,
+                 max_seq_length: int, pooling_mode: str, use_pretrained_encoder: bool = False, use_custom_encoder: bool = True, debug: bool = False) -> None:
+        super(Wildnerve_tlm01, self).__init__()
+        self.specialization = specialization
+        self.dataset_path = dataset_path
+        self.model_name = model_name
+        self.pooling_mode = pooling_mode
+        self.embedding_dim = embedding_dim
+        self.vocab_size = vocab_size
+        self.max_seq_length = max_seq_length
+        self.num_heads = num_heads
+        self.hidden_dim = hidden_dim
+        self.num_layers = num_layers
+        self.output_size = output_size
+        self.dropout = dropout
+        self.use_pretrained_encoder = use_pretrained_encoder
+        self.use_custom_encoder = use_custom_encoder
+        self.debug = debug
+        if use_pretrained_encoder:
+            try:
+                from transformers import AutoTokenizer, AutoModel
+                self.tokenizer = AutoTokenizer.from_pretrained("gpt2")
+                self.pretrained_encoder = AutoModel.from_pretrained("gpt2")
+                logger.info("Loaded GPT-2 for pretrained encoder")
+            except Exception as e:
+                logger.warning(f"GPT-2 load failed: {e} - falling back to bert-base-uncased")
+                from transformers import AutoTokenizer, AutoModel
+                self.tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
+                self.pretrained_encoder = AutoModel.from_pretrained("bert-base-uncased")
+            # Projection layer to convert pretrained output (assumed 768) to embedding_dim
+            self.pretrained_projection = nn.Linear(768, embedding_dim)
+        else:
+            self.tokenizer = None
+            self.pretrained_encoder = None
+        if use_custom_encoder:
+            self.embedding = nn.Embedding(vocab_size, embedding_dim)
+            self.pos_encoder = PositionalEncoding(embedding_dim, max_len=max_seq_length)
+            self.token_type_embeddings = nn.Embedding(2, embedding_dim)
+            encoder_layer = nn.TransformerEncoderLayer(d_model=embedding_dim, nhead=num_heads,
+                                                        dim_feedforward=hidden_dim, dropout=dropout, batch_first=True)
+            self.transformer_encoder = nn.TransformerEncoder(encoder_layer, num_layers=num_layers)
+            attention_config = get_hybrid_attention_config()
+            attention_config["NUM_HEADS"] = num_heads
+            attention_config["WINDOW_SIZE"] = max(256, max_seq_length//4)
+            self.hybrid_attention = SmartHybridAttention(attention_config)
+        self.tgt_embedding = nn.Embedding(vocab_size, embedding_dim)
+        self.pos_decoder = PositionalEncoding(embedding_dim, max_len=max_seq_length)
+        decoder_layer = nn.TransformerDecoderLayer(d_model=embedding_dim, nhead=num_heads,
+                                                    dim_feedforward=hidden_dim, dropout=dropout, batch_first=True)
+        self.transformer_decoder = nn.TransformerDecoder(decoder_layer, num_layers=num_layers)
+        self.adapter = nn.Sequential(nn.Linear(embedding_dim, hidden_dim), nn.ReLU(), nn.Linear(hidden_dim, embedding_dim))
+        self.classifier = nn.Linear(embedding_dim, vocab_size)
+        self.dropout_layer = nn.Dropout(dropout)
+        self.init_weights()
+        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+
+    def init_weights(self) -> None:
+        initrange = 0.1
+        with torch.no_grad():
+            self.embedding.weight.uniform_(-initrange, initrange)
+            self.tgt_embedding.weight.uniform_(-initrange, initrange)
+            self.classifier.weight.uniform_(-initrange, initrange)
+            self.classifier.bias.zero_()
+            for layer in self.adapter:
+                if isinstance(layer, nn.Linear):
+                    layer.weight.uniform_(-initrange, initrange)
+                    if layer.bias is not None:
+                        layer.bias.zero_()
+                        
+    def forward(self, src: torch.Tensor, tgt: Optional[torch.Tensor] = None, token_type_ids: Optional[torch.Tensor] = None,
+                src_mask: Optional[torch.Tensor] = None, tgt_mask: Optional[torch.Tensor] = None,
+                src_key_padding_mask: Optional[torch.Tensor] = None, tgt_key_padding_mask: Optional[torch.Tensor] = None,
+                return_sequence: bool = False, input_ids: Optional[torch.Tensor] = None, attention_mask: Optional[torch.Tensor] = None) -> torch.Tensor:
+        if src is None and input_ids is not None:
+            src = input_ids
+        if src_key_padding_mask is None and attention_mask is not None:
+            src_key_padding_mask = attention_mask
+        if src.dim() == 2:
+            pass
+        elif src.dim() == 3 and src.size(0) > src.size(1):
+            src = src.transpose(0, 1)
+        src_emb = self.embedding(src)*math.sqrt(self.embedding_dim)
+        src_emb = self.pos_encoder(src_emb.transpose(0, 1)).transpose(0, 1)
+        if src.size(1) > 256 and hasattr(self, "hybrid_attention"):
+            query = src_emb.transpose(0, 1)
+            key = query
+            value = query
+            attended, _ = self.hybrid_attention(query=query, key=key, value=value, key_padding_mask=src_key_padding_mask, attn_mask=src_mask, prompt_length=src.size(1), prompt_complexity=0.5)
+            encoded_src = attended.transpose(0, 1)
+        else:
+            encoded_src = self.transformer_encoder(src_emb, mask=src_mask, src_key_padding_mask=src_key_padding_mask)
+        adapted = self.adapter(encoded_src)
+        if tgt is not None:
+            if tgt.dim() == 3 and tgt.size(0)>tgt.size(1):
+                tgt = tgt.transpose(0,1)
+            tgt_emb = self.tgt_embedding(tgt)*math.sqrt(self.embedding_dim)
+            tgt_emb = self.pos_decoder(tgt_emb.transpose(0,1)).transpose(0,1)
+            decoded = self.transformer_decoder(tgt_emb, adapted, tgt_mask=tgt_mask, memory_key_padding_mask=src_key_padding_mask, tgt_key_padding_mask=tgt_key_padding_mask)
+            output = self.classifier(decoded)
+            if not return_sequence:
+                output = output.mean(dim=1)
+        else:
+            if self.pooling_mode=="mean":
+                pooled = adapted.mean(dim=1)
+            elif self.pooling_mode=="max":
+                pooled = torch.max(adapted, dim=1)[0]
+            elif self.pooling_mode=="cls":
+                pooled = adapted[:, 0]
+            else:
+                pooled = adapted.mean(dim=1)
+            pooled = self.dropout_layer(pooled)
+            output = self.classifier(pooled)
+        return output
+
+    def encode_sentences(self, sentences, batch_size=32, normalize_embeddings=True):
+        self.eval()
+        from torch.utils.data import DataLoader, Dataset
+        if isinstance(sentences, str):
+            sentences = [sentences]
+        class SentencesDataset(Dataset):
+            def __init__(self, sentences, tokenizer, max_length):
+                self.sentences = sentences
+                self.tokenizer = tokenizer
+                self.max_length = max_length
+            def __len__(self): return len(self.sentences)
+            def __getitem__(self, idx):
+                return self.tokenizer(self.sentences[idx], padding="max_length", truncation=True, max_length=self.max_length, return_tensors="pt")
+        dataset = SentencesDataset(sentences, self.tokenizer, self.max_seq_length)
+        dataloader = DataLoader(dataset, batch_size=batch_size)
+        all_emb = []
+        device = next(self.parameters()).device
+        with torch.no_grad():
+            for batch in dataloader:
+                inputs = {k: v.squeeze(1).to(device) for k,v in batch.items()}
+                outputs = self(inputs["input_ids"], src_key_padding_mask=inputs.get("attention_mask"))
+                if normalize_embeddings:
+                    outputs = torch.nn.functional.normalize(outputs, p=2, dim=1)
+                all_emb.append(outputs.cpu().numpy())
+        return np.vstack(all_emb)
+
+    def similarity(self, sentence1: str, sentence2: str) -> float:
+        emb = self.encode_sentences([sentence1, sentence2])
+        return np.dot(emb[0], emb[1])/(np.linalg.norm(emb[0])*np.linalg.norm(emb[1]))
+
+    def generate(self, input_ids: torch.Tensor, max_length: int = app_config.TRANSFORMER_CONFIG.MAX_SEQ_LENGTH, device: str = "cpu", temperature: float = 1.0, start_token_id: Optional[int] = None) -> List[List[int]]:
+        self.eval()
+        batch_size = input_ids.shape[0]
+        start_token_id = start_token_id or (input_ids[0,0].item() if input_ids.numel()>0 else 0)
+        generated = [[start_token_id] for _ in range(batch_size)]
+        src = input_ids.transpose(0,1)
+        src_emb = self.embedding(src)*math.sqrt(self.embedding_dim)
+        src_emb = self.pos_encoder(src_emb)
+        encoded_src = self.transformer_encoder(src_emb)
+        encoded_src = self.adapter(encoded_src)
+        for _ in range(max_length -1):
+            current_tgt = torch.tensor(generated, dtype=torch.long, device=device).transpose(0,1)
+            tgt_emb = self.tgt_embedding(current_tgt)*math.sqrt(self.embedding_dim)
+            tgt_emb = self.pos_decoder(tgt_emb)
+            current_seq_length = current_tgt.size(0)
+            tgt_mask = nn.Transformer.generate_square_subsequent_mask(current_seq_length).to(device)
+            decoded = self.transformer_decoder(tgt_emb, encoded_src, tgt_mask=tgt_mask)
+            logits = self.classifier(decoded[-1, :, :])
+            if temperature==0:
+                next_tokens = torch.argmax(logits, dim=-1)
+            else:
+                probs = torch.softmax(logits/temperature, dim=-1)
+                next_tokens = torch.multinomial(probs, num_samples=1).squeeze(-1)
+            next_tokens = next_tokens.cpu().tolist()
+            for i, token in enumerate(next_tokens):
+                generated[i].append(token)
+        return generated
+
+    def decode_tokens(self, token_ids: List[int]) -> str:
+        try:
+            return self.tokenizer.decode(token_ids, skip_special_tokens=True, clean_up_tokenization_spaces=True)
+        except Exception as e:
+            logger.error(f"Decoding error: {e}")
+            return str(e)
+
+    def generate_with_decoding(self, input_ids: torch.Tensor, max_length: int = app_config.TRANSFORMER_CONFIG.MAX_SEQ_LENGTH, device: str = "cpu", temperature: float = 1.0, start_token_id: Optional[int] = None) -> str:
+        generated_sequences = self.generate(input_ids, max_length, device, temperature, start_token_id)
+        if generated_sequences:
+            return self.decode_tokens(generated_sequences[0])
+        return ""
+
+    def generate_streaming(self, prompt, **kwargs):
+        inputs = self.tokenizer(prompt, return_tensors="pt", padding="max_length", truncation=True, max_length=self.max_seq_length).to(self.device)
+        with torch.no_grad():
+            outputs = self(inputs.input_ids)
+            next_token_logits = outputs[:, -1, :]
+            if "temperature" in kwargs and kwargs["temperature"] > 0:
+                next_token_logits /= kwargs["temperature"]
+            probs = torch.softmax(next_token_logits, dim=-1)
+            next_token = torch.multinomial(probs, num_samples=1).squeeze(1)
+            generated_ids = next_token
+            token_text = self.tokenizer.decode([next_token.item()], skip_special_tokens=True)
+            yield token_text
+            max_length = kwargs.get("max_length", 100)
+            for _ in range(max_length-1):
+                context_ids = torch.cat([inputs.input_ids, generated_ids.unsqueeze(0)], dim=1)
+                outputs = self(context_ids)
+                next_token_logits = outputs[:, -1, :]
+                if "temperature" in kwargs and kwargs["temperature"] > 0:
+                    next_token_logits /= kwargs["temperature"]
+                probs = torch.softmax(next_token_logits, dim=-1)
+                next_token = torch.multinomial(probs, num_samples=1).squeeze(1)
+                generated_ids = torch.cat([generated_ids, next_token.unsqueeze(0)], dim=0)
+                token_text = self.tokenizer.decode([next_token.item()], skip_special_tokens=True)
+                if next_token.item() == self.tokenizer.eos_token_id:
+                    break
+                yield token_text
+
+    def forward_with_custom_embeddings(self, embeddings: torch.Tensor) -> torch.Tensor:
+        try:
+            device = next(self.parameters()).device
+            embeddings = embeddings.to(device)
+            batch_first = True
+            if not batch_first and embeddings.shape[0] <= embeddings.shape[1]:
+                embeddings = embeddings.transpose(0,1)
+            if hasattr(self, "pos_encoder"):
+                if batch_first:
+                    embeddings = self.pos_encoder(embeddings)
+                else:
+                    embeddings = self.pos_encoder(embeddings.transpose(0,1)).transpose(0,1)
+            encoded = self.transformer_encoder(embeddings)
+            if hasattr(self, "adapter"):
+                encoded = self.adapter(encoded)
+            if self.pooling_mode=="mean":
+                pooled = encoded.mean(dim=1)
+            elif self.pooling_mode=="max":
+                pooled = torch.max(encoded, dim=1)[0]
+            elif self.pooling_mode=="cls":
+                pooled = encoded[:,0]
+            else:
+                pooled = encoded.mean(dim=1)
+            pooled = self.dropout_layer(pooled)
+            output = self.classifier(pooled)
+            return output
+        except Exception as e:
+            logger.error(f"Custom embeddings forward error: {e}")
+            batch_size = embeddings.size(0)
+            return torch.zeros((batch_size, self.output_size), device=device)
+
+    def forward_with_error_handling(self, input_ids: Optional[torch.Tensor] = None, attention_mask: Optional[torch.Tensor] = None, token_type_ids: Optional[torch.Tensor] = None, **kwargs) -> Union[torch.Tensor, Dict[str, torch.Tensor]]:
+        try:
+            return self.forward(src=input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids, **kwargs)
+        except RuntimeError as e:
+            if "shape" in str(e):
+                logger.warning(f"Shape error: {e}")
+                try:
+                    embedded = self.embedding(input_ids)
+                    if hasattr(self, "pos_encoder"):
+                        embedded = self.pos_encoder(embedded)
+                    encoder_output = self.transformer_encoder(embedded)
+                    if self.pooling_mode=="mean":
+                        pooled = encoder_output.mean(dim=1)
+                    elif self.pooling_mode=="max":
+                        pooled = torch.max(encoder_output, dim=1)[0]
+                    elif self.pooling_mode=="cls":
+                        pooled = encoder_output[:,0]
+                    else:
+                        pooled = encoder_output.mean(dim=1)
+                    pooled = self.dropout_layer(pooled)
+                    return self.classifier(pooled)
+                except Exception as inner_e:
+                    logger.error(f"Error adapting input: {inner_e}")
+                    batch_size = input_ids.size(0) if input_ids is not None else 1
+                    return torch.zeros((batch_size, self.output_size), device=self.device)
+            raise
+        except Exception as e:
+            logger.error(f"Unhandled error: {e}")
+            batch_size = input_ids.size(0) if input_ids is not None else 1
+            return torch.zeros((batch_size, self.output_size), device=self.device)
+
+    def train_with_emissions_tracking(self, dataloader, optimizer, criterion, num_epochs=1):
+        from codecarbon import EmissionsTracker
+        tracker = EmissionsTracker()
+        tracker.start()
+        self.train()
+        for epoch in range(num_epochs):
+            for batch in dataloader:
+                inputs, labels = batch
+                inputs, labels = inputs.to(self.device), labels.to(self.device)
+                optimizer.zero_grad()
+                outputs = self(inputs)
+                loss = criterion(outputs, labels)
+                loss.backward()
+                optimizer.step()
+            logger.info(f"Epoch {epoch+1} completed.")
+        emissions = tracker.stop()
+        logger.info(f"Training emissions: {emissions:.4f} kg CO2")
+
+    def infer_with_emissions_tracking(self, input_ids):
+        from codecarbon import EmissionsTracker
+        tracker = EmissionsTracker()
+        tracker.start()
+        self.eval()
+        with torch.no_grad():
+            outputs = self(input_ids)
+        emissions = tracker.stop()
+        logger.info(f"Inference emissions: {emissions:.4f} kg CO2")
+        return outputs
+
+    def decode_tokens(self, token_ids: List[int]) -> str:
+        try:
+            return self.tokenizer.decode(token_ids, skip_special_tokens=True, clean_up_tokenization_spaces=True)
+        except Exception as e:
+            logger.error(f"Decoding error: {e}")
+            return "Error decoding tokens"
+
+    def generate_with_decoding(self, input_ids, max_length=100, **kwargs) -> str:
+        generated_ids = self.generate(input_ids, max_length=max_length, **kwargs)
+        if generated_ids and len(generated_ids)>0:
+            return self.decode_tokens(generated_ids[0])
+        return ""
+
+    def generate_streaming(self, **kwargs):
+        device = next(self.parameters()).device
+        input_ids = kwargs.get("input_ids")
+        prompt = kwargs.get("prompt")
+        if prompt and not input_ids and self.tokenizer:
+            input_ids = self.tokenizer(prompt, return_tensors="pt", padding=True, truncation=True, max_length=512).input_ids.to(device)
+        if input_ids is None:
+            raise ValueError("Input must be provided")
+        max_length = kwargs.get("max_length", 100)
+        generated_ids = None
+        with torch.no_grad():
+            outputs = self(input_ids)
+            next_token_logits = outputs[:, -1, :]
+            if kwargs.get("temperature", 1.0) > 0:
+                next_token_logits /= kwargs["temperature"]
+            probs = torch.softmax(next_token_logits, dim=-1)
+            next_token = torch.multinomial(probs, num_samples=1).squeeze(1)
+            generated_ids = next_token
+            token_text = self.tokenizer.decode([next_token.item()], skip_special_tokens=True)
+            yield token_text
+            for _ in range(max_length-1):
+                context_ids = torch.cat([input_ids, generated_ids.unsqueeze(0)], dim=1)
+                outputs = self(context_ids)
+                next_token_logits = outputs[:, -1, :]
+                if kwargs.get("temperature", 1.0) > 0:
+                    next_token_logits /= kwargs["temperature"]
+                probs = torch.softmax(next_token_logits, dim=-1)
+                next_token = torch.multinomial(probs, num_samples=1).squeeze(1)
+                generated_ids = torch.cat([generated_ids, next_token.unsqueeze(0)], dim=0)
+                token_text = self.tokenizer.decode([next_token.item()], skip_special_tokens=True)
+                if next_token.item() == self.tokenizer.eos_token_id:
+                    break
+                yield token_text
+
+# Register CombinedModel in registry
+registry.register("model_class_combined", Wildnerve_tlm01)
+
+def initialize_combined_model():
+    # For now, simply call the constructor with a sample config.
+    config = {"EMBEDDING_DIM":768, "OUTPUT_SIZE":768, "MODEL_NAME":"bert-base-uncased", "MAX_SEQ_LENGTH":512}
+    return Wildnerve_tlm01(**config)

model_Custm.py

@@ -0,0 +1,702 @@
+# model_Custm.py
+import os
+import sys
+import math
+import torch
+import logging
+import numpy as np
+import torch.nn as nn
+from typing import Optional, List, Dict, Union
+
+# Import the carbon tracking early - before transformers
+from codecarbon import EmissionsTracker  # Import EmissionsTracker
+
+# Apply patches before importing transformers
+import transformer_patches
+
+# Now we can safely import transformers
+import transformers
+
+# Continue with standard imports
+from service_registry import registry, MODEL, TOKENIZER
+from utils.transformer_utils import get_tokenizer
+from utils.smartHybridAttention import SmartHybridAttention, get_hybrid_attention_config
+
+# Import base interfaces
+from base_interfaces.common_types import *
+from base_interfaces.model_interface import AbstractModel
+
+logger = logging.getLogger(__name__)
+
+# Check if transformers integrations has CodeCarbonCallback
+if hasattr(transformers, 'integrations') and hasattr(transformers.integrations, 'CodeCarbonCallback'):
+    logger.info("transformers.integrations.CodeCarbonCallback is available")
+
+# Check if we're using our proxy or the real implementation
+if hasattr(transformers.integrations, 'CodeCarbonCallback'):
+    callback_module = transformers.integrations.CodeCarbonCallback.__module__
+    if callback_module == 'carbon_tracking':
+        logger.info("Using our clean architecture implementation for CodeCarbonCallback")
+    else:
+        logger.info(f"Using original implementation for CodeCarbonCallback from {callback_module}")
+
+# Continue with existing code
+try:
+    if 'TLM_DATA_DIR' in os.environ:
+        data_dir = os.environ.get('TLM_DATA_DIR', '/tmp/tlm_data')
+        model_dir = os.path.join(data_dir, "models")
+        logging.info(f"Using data directory from environment: {data_dir}")
+        from types import SimpleNamespace
+        app_config = SimpleNamespace()
+        app_config.DATA_DIR = data_dir
+        app_config.MODEL_DIR = model_dir
+        app_config.TRANSFORMER_CONFIG = SimpleNamespace()
+        app_config.TRANSFORMER_CONFIG.MAX_SEQ_LENGTH = 512
+        try:
+            from config import load_config, app_config as config_app_config
+            app_config = load_config() if not hasattr(config_app_config, 'DATA_DIR') else config_app_config
+        except Exception as config_error:
+            logging.warning(f"Using minimal config due to error: {config_error}")
+    else:
+        from config import load_config, app_config as config_app_config
+        app_config = load_config() if not hasattr(config_app_config, 'DATA_DIR') else config_app_config
+except Exception as e:
+    logging.warning(f"Error importing config: {e}")
+    from types import SimpleNamespace
+    app_config = SimpleNamespace()
+    app_config.DATA_DIR = '/tmp/tlm_data'
+    app_config.MODEL_DIR = '/tmp/tlm_data/models'
+    app_config.TRANSFORMER_CONFIG = SimpleNamespace()
+    app_config.TRANSFORMER_CONFIG.MAX_SEQ_LENGTH = 512
+
+# Ensure the necessary directories exist, but don't fail if they can't be created
+try:
+    os.makedirs(getattr(app_config, "DATA_DIR", "/tmp/tlm_data"), exist_ok=True)
+    os.makedirs(getattr(app_config, "MODEL_DIR", "/tmp/tlm_data/models"), exist_ok=True)
+except Exception as e:
+    logging.warning(f"Could not create directories: {e}")
+
+# Configure logging and suppress TensorFlow warnings
+os.environ["TF_ENABLE_ONEDNN_OPTS"] = "0"
+os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"
+
+# ----------------------------
+# Positional Encoding Module
+# ----------------------------
+class PositionalEncoding(nn.Module):
+    def __init__(self, d_model: int, max_len: int = app_config.TRANSFORMER_CONFIG.MAX_SEQ_LENGTH):
+        super().__init__()
+        pe = torch.zeros(max_len, d_model)
+        position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)
+        div_term = torch.exp(torch.arange(0, d_model, 2, dtype=torch.float) * (-math.log(10000.0) / d_model))
+        pe[:, 0::2] = torch.sin(position * div_term)
+        pe[:, 1::2] = torch.cos(position * div_term)
+        pe = pe.unsqueeze(1)  # shape: (max_len, 1, d_model)
+        self.register_buffer("pe", pe)
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        # x shape: (seq_len, batch_size, d_model)
+        seq_len = x.size(0)
+        x = x + self.pe[:seq_len]
+        return x
+
+# ----------------------------
+# Wildnerve-tlm01 using Only Custom Encoder/Decoder
+# ----------------------------
+class Wildnerve_tlm01(nn.Module, AbstractModel):
+    """A Transformer-based Tiny Language Model that uses:
+      - A custom built encoder & decoder (embedding, positional encoding, and TransformerEncoder)
+      - An adapter and classifier for post-processing
+      - The AutoTokenizer for consistent tokenization and decoding
+      - SmartHybridAttention for better context handling"""
+    def __init__(
+        self,
+        vocab_size=30522,  # Default BERT vocab size
+        specialization="general",
+        dataset_path=None,
+        model_name="Wildnerve-tlm01-0.05Bx12",  # Primary model name
+        embedding_dim=768,
+        num_heads=12,
+        hidden_dim=768,
+        num_layers=6,
+        output_size=768,
+        dropout=0.1,
+        max_seq_length=512,
+        pooling_mode="mean",
+        tokenizer=None,  # Accept tokenizer as parameter
+        **kwargs  # Accept additional kwargs for compatibility
+    ) -> None:
+        super().__init__()
+        # Set device once at the start
+        object.__setattr__(self, "device", torch.device('cuda' if torch.cuda.is_available() else 'cpu'))
+        self.specialization = specialization
+        self.dataset_path = dataset_path
+        self.model_name = model_name
+        self.pooling_mode = pooling_mode
+        self.embedding_dim = embedding_dim
+        self.vocab_size = vocab_size
+        self.max_seq_length = max_seq_length
+        self.num_heads = num_heads
+        self.hidden_dim = hidden_dim
+        self.num_layers = num_layers
+        self.output_size = output_size
+        self.dropout = dropout
+
+        # Optionally track model usage
+        self.model_last_used = {}  
+        
+        # Unified tokenizer initialization:
+        if tokenizer is not None:
+            self.tokenizer = tokenizer
+        else:
+            if registry.has(TOKENIZER):
+                self.tokenizer = registry.get(TOKENIZER)
+            else:
+                try:
+                    from transformers import AutoTokenizer
+                    self.tokenizer = AutoTokenizer.from_pretrained("Wildnerve-tlm01-0.05Bx12")
+                    logger.info("Loaded primary tokenizer: Wildnerve-tlm01-0.05Bx12")
+                except Exception as e:
+                    logger.warning(f"Primary tokenizer load failed: {e}")
+                    try:
+                        from transformers import BertTokenizer
+                        self.tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")
+                        logger.info("Loaded fallback tokenizer: bert-base-uncased")
+                    except Exception as e2:
+                        logger.error(f"Fallback tokenizer load failed: {e2}")
+                        self.tokenizer = None
+        registry.register(TOKENIZER, self.tokenizer, overwrite=True)
+
+        # Register this model instance in the registry by specialization
+        model_registry_key = f"model_{specialization}"
+        registry.register(model_registry_key, self)
+        
+        # Also register as default model if it's the primary specialization
+        if specialization == "general":
+            registry.register(MODEL, self)
+
+        # ----------------------------
+        # Encoder Components (Custom)
+        # ----------------------------
+        self.embedding = nn.Embedding(vocab_size, embedding_dim)
+        self.pos_encoder = PositionalEncoding(embedding_dim, max_len=max_seq_length)
+
+        # ----------------------------
+        # Decoder Components (Custom)
+        # ----------------------------
+        self.tgt_embedding = nn.Embedding(vocab_size, embedding_dim)
+        self.pos_decoder = PositionalEncoding(embedding_dim, max_len=max_seq_length)
+
+        # ----------------------------
+        # Transformer Encoder and Decoder (Custom)
+        # Always create with batch_first=True for better performance
+        # ----------------------------
+        encoder_layer = nn.TransformerEncoderLayer(
+            d_model=embedding_dim,
+            nhead=num_heads,
+            dim_feedforward=hidden_dim,
+            dropout=dropout,
+            batch_first=True  # Fixed to use batch_first=True
+        )
+        self.transformer_encoder = nn.TransformerEncoder(encoder_layer, num_layers=num_layers)
+
+        decoder_layer = nn.TransformerDecoderLayer(
+            d_model=embedding_dim,
+            nhead=num_heads,
+            dim_feedforward=hidden_dim,
+            dropout=dropout,
+            batch_first=True  # Fixed to use batch_first=True
+        )
+        self.transformer_decoder = nn.TransformerDecoder(decoder_layer, num_layers=num_layers)
+
+        # Initialize the smart hybrid attention
+        attention_config = get_hybrid_attention_config()
+        attention_config['NUM_HEADS'] = num_heads
+        attention_config['WINDOW_SIZE'] = max(256, max_seq_length // 4)
+        self.hybrid_attention = SmartHybridAttention(attention_config)
+
+        # ----------------------------
+        # Adapter & Output Layers
+        # ----------------------------
+        self.adapter = nn.Sequential(
+            nn.Linear(embedding_dim, hidden_dim),
+            nn.ReLU(),
+            nn.Linear(hidden_dim, embedding_dim)
+        )
+        self.classifier = nn.Linear(embedding_dim, self.vocab_size)
+        self.dropout_layer = nn.Dropout(dropout)
+
+        self.init_weights()
+
+    def init_weights(self) -> None:
+        initrange = 0.1
+        with torch.no_grad():
+            self.embedding.weight.uniform_(-initrange, initrange)
+            self.tgt_embedding.weight.uniform_(-initrange, initrange)
+            self.classifier.weight.uniform_(-initrange, initrange)
+            self.classifier.bias.zero_()
+            for layer in self.adapter:
+                if isinstance(layer, nn.Linear):
+                    layer.weight.uniform_(-initrange, initrange)
+                    if layer.bias is not None:
+                        layer.bias.zero_()
+    def forward(
+        self,
+        src: torch.Tensor = None,
+        tgt: Optional[torch.Tensor] = None,
+        token_type_ids: Optional[torch.Tensor] = None,  # Not used in this implementation
+        src_mask: Optional[torch.Tensor] = None,
+        tgt_mask: Optional[torch.Tensor] = None,
+        src_key_padding_mask: Optional[torch.Tensor] = None,
+        tgt_key_padding_mask: Optional[torch.Tensor] = None,
+        return_sequence: bool = False,
+        # Add Hugging Face compatibility parameters
+        input_ids: Optional[torch.Tensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+    ) -> torch.Tensor:
+        # Use Hugging Face parameters if provided
+        if src is None and input_ids is not None:
+            src = input_ids
+        if src_key_padding_mask is None and attention_mask is not None:
+            src_key_padding_mask = attention_mask
+
+        # Handle input shape - our layers expect batch_first=True format
+        if src.dim() == 2:
+            # src is already [batch_size, seq_len]
+            pass
+        elif src.dim() == 3 and src.size(0) > src.size(1):
+            # src is [seq_len, batch_size, dim] - need to transpose
+            src = src.transpose(0, 1)
+            
+        # ----------------------------
+        # Encoder: Custom processing of source
+        # ----------------------------
+        src_emb = self.embedding(src) * math.sqrt(self.embedding_dim)
+        src_emb = self.pos_encoder(src_emb.transpose(0, 1)).transpose(0, 1)  # Apply positional encoding
+
+        # Use hybrid attention if sequence length is above the threshold
+        if src.size(1) > 256 and hasattr(self, 'hybrid_attention'):
+            # Prepare inputs for hybrid attention
+            query = src_emb.transpose(0, 1)  # Ensure shape is [seq_len, batch, dim]
+            key = query
+            value = query
+            
+            # Apply smart hybrid attention
+            attended_output, _ = self.hybrid_attention(
+                query=query,
+                key=key,
+                value=value,
+                key_padding_mask=src_key_padding_mask,
+                attn_mask=src_mask,
+                prompt_length=src.size(1),
+                prompt_complexity=0.5  # Default value, can be computed based on input
+            )
+            
+            # Convert back to expected format
+            encoded_src = attended_output.transpose(0, 1)
+        else:
+            # Use standard transformer encoder for shorter sequences
+            encoded_src = self.transformer_encoder(src_emb, mask=src_mask, src_key_padding_mask=src_key_padding_mask)
+        
+        # Process through adapter layer
+        adapted = self.adapter(encoded_src)
+
+        # ----------------------------
+        # Decoder / Output
+        # ----------------------------
+        if tgt is not None:
+            # Handle tgt shape for batch_first format
+            if tgt.dim() == 2:
+                # tgt is already [batch_size, seq_len]
+                pass
+            elif tgt.dim() == 3 and tgt.size(0) > tgt.size(1):
+                # tgt is [seq_len, batch_size, dim] - need to transpose
+                tgt = tgt.transpose(0, 1)
+                
+            tgt_emb = self.tgt_embedding(tgt) * math.sqrt(self.embedding_dim)
+            tgt_emb = self.pos_decoder(tgt_emb.transpose(0, 1)).transpose(0, 1)  # Apply positional encoding
+            
+            decoded = self.transformer_decoder(
+                tgt_emb,
+                adapted,
+                tgt_mask=tgt_mask,
+                memory_key_padding_mask=src_key_padding_mask,
+                tgt_key_padding_mask=tgt_key_padding_mask
+            )
+            
+            output = self.classifier(decoded)  # [batch_size, seq_len, output_size]
+                
+            if not return_sequence:
+                output = output.mean(dim=1)  # Average over sequence dimension
+        else:
+            # For encoder-only tasks (e.g., classification)
+            if self.pooling_mode == "mean":
+                pooled = encoded_src.mean(dim=1)
+            elif self.pooling_mode == "max":
+                pooled = torch.max(encoded_src, dim=1)[0]
+            elif self.pooling_mode == "cls":
+                pooled = encoded_src[:, 0]  # Use first token (CLS) - batch_first format
+            else:
+                pooled = encoded_src.mean(dim=1)
+            pooled = self.dropout_layer(pooled)
+            output = self.classifier(pooled)
+        
+        return output
+        
+    # Add sentence transformer methods
+    def encode_sentences(self, sentences, batch_size=32, normalize_embeddings=True):
+        """Encode sentences into vectors (sentence transformer functionality)"""
+        self.eval()
+        from torch.utils.data import DataLoader, Dataset
+        
+        # Handle single sentence
+        if isinstance(sentences, str):
+            sentences = [sentences]
+            
+        class SentencesDataset(Dataset):
+            def __init__(self, sentences, tokenizer, max_length):
+                self.sentences = sentences
+                self.tokenizer = tokenizer
+                self.max_length = max_length
+                
+            def __len__(self):
+                return len(self.sentences)
+                
+            def __getitem__(self, idx):
+                return self.tokenizer(self.sentences[idx], 
+                                     padding='max_length',
+                                     truncation=True,
+                                     max_length=self.max_length,
+                                     return_tensors='pt')
+        
+        # Create dataset and dataloader
+        dataset = SentencesDataset(sentences, self.tokenizer, self.max_seq_length)
+        dataloader = DataLoader(dataset, batch_size=batch_size)
+        
+        all_embeddings = []
+        device = next(self.parameters()).device
+        
+        with torch.no_grad():
+            for batch in dataloader:
+                inputs = {k: v.squeeze(1).to(device) for k, v in batch.items()}
+                outputs = self(inputs['input_ids'], src_key_padding_mask=inputs.get('attention_mask'))
+                
+                if normalize_embeddings:
+                    outputs = torch.nn.functional.normalize(outputs, p=2, dim=1)
+                    
+                all_embeddings.append(outputs.cpu().numpy())
+                
+        return np.vstack(all_embeddings)
+    
+    def similarity(self, sentence1: str, sentence2: str) -> float:
+        """Compute cosine similarity between two sentences"""
+        embeddings = self.encode_sentences([sentence1, sentence2])
+        return np.dot(embeddings[0], embeddings[1]) / (np.linalg.norm(embeddings[0]) * np.linalg.norm(embeddings[1]))
+
+    def generate(
+        self,
+        input_ids: torch.Tensor,
+        max_length: int = app_config.TRANSFORMER_CONFIG.MAX_SEQ_LENGTH,
+        device: str = 'cpu',
+        temperature: float = 1.0,
+        start_token_id: Optional[int] = None
+    ) -> List[List[int]]:
+        """Generates a sequence of token IDs using the encoder-decoder architecture."""
+        self.eval()
+        batch_size = input_ids.shape[0]
+        if start_token_id is None:
+            start_token_id = input_ids[0, 0].item()
+        generated = [[start_token_id] for _ in range(batch_size)]
+        
+        # Encode source input using the custom encoder.
+        src = input_ids.transpose(0, 1)
+        src_emb = self.embedding(src) * math.sqrt(self.embedding_dim)
+        src_emb = self.pos_encoder(src_emb)
+        encoded_src = self.transformer_encoder(src_emb)
+        encoded_src = self.adapter(encoded_src)
+        
+        for _ in range(max_length - 1):
+            current_tgt = torch.tensor(generated, dtype=torch.long, device=device)
+            current_tgt = current_tgt.transpose(0, 1)
+            tgt_emb = self.tgt_embedding(current_tgt) * math.sqrt(self.embedding_dim)
+            tgt_emb = self.pos_decoder(tgt_emb)
+            current_seq_length = current_tgt.size(0)
+            
+            # Create causal mask for the decoder.
+            tgt_mask = nn.Transformer.generate_square_subsequent_mask(current_seq_length).to(device)
+            decoded = self.transformer_decoder(tgt_emb, encoded_src, tgt_mask=tgt_mask)
+            logits = self.classifier(decoded[-1, :, :])
+            
+            if temperature == 0:
+                next_tokens = torch.argmax(logits, dim=-1)
+            else:
+                probs = torch.softmax(logits / temperature, dim=-1)
+                next_tokens = torch.multinomial(probs, num_samples=1).squeeze(-1)
+            
+            next_tokens = next_tokens.cpu().tolist()
+            for i, token in enumerate(next_tokens):
+                generated[i].append(token)
+                
+        return generated
+
+    def decode_tokens(self, token_ids: List[int]) -> str:
+        """Decodes a list of token IDs into a human-readable string."""
+        try:
+            return self.tokenizer.decode(token_ids, skip_special_tokens=True, clean_up_tokenization_spaces=True)
+        except Exception as e:
+            logger.error(f"Error decoding tokens: {e}")
+            return str(e)
+    def generate_with_decoding(
+        self,
+        input_ids: torch.Tensor,
+        max_length: int = app_config.TRANSFORMER_CONFIG.MAX_SEQ_LENGTH,
+        device: str = 'cpu',
+        temperature: float = 1.0,
+        start_token_id: Optional[int] = None
+    ) -> str:
+        """Generates a sequence and returns the decoded text."""
+        generated_sequences = self.generate(input_ids, max_length, device, temperature, start_token_id)
+        if generated_sequences:
+            return self.decode_tokens(generated_sequences[0])
+        return ""
+
+    def generate_streaming(self, prompt, **kwargs):
+        """Generate a response token-by-token from the model"""
+        # Prepare input
+        inputs = self.tokenizer(
+            prompt, 
+            return_tensors="pt",
+            padding="max_length",
+            truncation=True,
+            max_length=self.max_seq_length
+        ).to(self.device)
+        
+        # Generate initial token
+        # This is a simplified implementation - a real one would use beam search or sampling
+        with torch.no_grad():
+            # Get initial logits from the model
+            outputs = self(inputs.input_ids)
+            next_token_logits = outputs[:, -1, :]
+            
+            # Choose next token (using temperature if specified)
+            if "temperature" in kwargs and kwargs["temperature"] > 0:
+                # Apply temperature
+                next_token_logits = next_token_logits / kwargs["temperature"]
+                
+            # Sample from the distribution
+            probs = torch.softmax(next_token_logits, dim=-1)
+            next_token = torch.multinomial(probs, num_samples=1).squeeze(1)
+            
+            # Create the sequence with the new token
+            generated_ids = next_token
+            
+            # Decode and yield the first token
+            token_text = self.tokenizer.decode([next_token.item()], skip_special_tokens=True)
+            yield token_text
+            
+            # Generate remaining tokens up to max_length
+            max_length = kwargs.get("max_length", 100)
+            
+            for _ in range(max_length - 1):
+                # Create input with context plus generated tokens
+                context_ids = torch.cat([inputs.input_ids, generated_ids.unsqueeze(0)], dim=1)
+                
+                # Get next token prediction
+                outputs = self(context_ids)
+                next_token_logits = outputs[:, -1, :]
+                
+                # Apply temperature if specified
+                if "temperature" in kwargs and kwargs["temperature"] > 0:
+                    next_token_logits = next_token_logits / kwargs["temperature"]
+                    
+                # Sample from the distribution
+                probs = torch.softmax(next_token_logits, dim=-1)
+                next_token = torch.multinomial(probs, num_samples=1).squeeze(1)
+                
+                # Append to generated sequence
+                generated_ids = torch.cat([generated_ids, next_token.unsqueeze(0)], dim=0)
+                
+                # Decode and yield the next token
+                token_text = self.tokenizer.decode([next_token.item()], skip_special_tokens=True)
+                
+                # Check for end of sequence token
+                if next_token.item() == self.tokenizer.eos_token_id:
+                    break
+                yield token_text
+
+    def forward_with_custom_embeddings(self, embeddings: torch.Tensor) -> torch.Tensor:
+        """Forward pass that accepts pre-calculated embeddings to bypass shape errors."""
+        try:
+            # Get device
+            device = next(self.parameters()).device
+            embeddings = embeddings.to(device)
+            
+            # Process through transformer encoder - bypassing the embedding layer
+            # Check if embeddings need to be transposed for batch_first format
+            batch_first = getattr(self.transformer_encoder, 'batch_first', False)
+            
+            if batch_first and embeddings.shape[0] <= embeddings.shape[1]:
+                # First dimension is smaller than second, likely needs transpose
+                # from [seq_len, batch, dim] to [batch, seq_len, dim]
+                embeddings = embeddings.transpose(0, 1)
+                
+            # Apply position encoding if needed
+            if hasattr(self, 'pos_encoder'):
+                # Check if position encoder expects seq_first or batch_first
+                if not batch_first:
+                    # Ensure shape is [seq_len, batch, dim]
+                    if embeddings.shape[0] > embeddings.shape[1]:
+                        # Already in correct format
+                        embeddings = self.pos_encoder(embeddings)
+                    else:
+                        # Need to transpose first
+                        embeddings = embeddings.transpose(0, 1)
+                        embeddings = self.pos_encoder(embeddings)
+                        embeddings = embeddings.transpose(0, 1)
+                else:
+                    # With batch_first, no need to transpose
+                    embeddings = self.pos_encoder(embeddings)
+            
+            # Process through encoder
+            encoded = self.transformer_encoder(embeddings)
+            
+            # Process through adapter
+            if hasattr(self, 'adapter'):
+                encoded = self.adapter(encoded)
+            
+            # Apply pooling for output
+            if self.pooling_mode == "mean":
+                pooled = encoded.mean(dim=1)
+            elif self.pooling_mode == "max":
+                pooled = torch.max(encoded, dim=1)[0]
+            elif self.pooling_mode == "cls":
+                # Use first token (CLS token) for classification
+                pooled = encoded[:, 0]
+            else:
+                pooled = encoded.mean(dim=1)
+            
+            # Final dropout and classification
+            pooled = self.dropout_layer(pooled)
+            output = self.classifier(pooled)
+            
+            return output
+        except Exception as e:
+            logger.error(f"Error in custom embeddings forward pass: {e}")
+            # Return a tensor of the right shape to prevent further errors
+            return torch.zeros(1, self.output_size, device=device)
+
+    def forward_with_error_handling(
+        self,
+        input_ids: Optional[torch.Tensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        token_type_ids: Optional[torch.Tensor] = None,
+        **kwargs
+    ) -> Union[torch.Tensor, Dict[str, torch.Tensor]]:
+        """Forward pass with enhanced error handling for shape mismatches"""
+        try:
+            # Try standard forward pass first
+            return self.forward(
+                src=input_ids, 
+                attention_mask=attention_mask,
+                token_type_ids=token_type_ids,
+                **kwargs
+            )
+        except RuntimeError as e:
+            # Check if this is a shape error
+            if "shape" in str(e):
+                logger.warning(f"Shape mismatch detected: {e}")
+                if input_ids.dim() == 3 and input_ids.size(0) > input_ids.size(1):
+                    input_ids = input_ids.transpose(0, 1)  # Adjust shape as needed
+                # Retry the forward pass using adapted input
+                try:
+                    embedded = self.embedding(input_ids)
+                    if hasattr(self, 'pos_encoder'):
+                        embedded = self.pos_encoder(embedded)
+                    encoder_out = self.transformer_encoder(embedded)
+                    pooled = encoder_out.mean(dim=1)
+                    pooled = self.dropout_layer(pooled)
+                    return self.classifier(pooled)
+                except Exception as inner_e:
+                    logger.error(f"Adaptation failed: {inner_e}")
+                    batch_size = input_ids.size(0) if input_ids is not None else 1
+                    return torch.zeros((batch_size, self.output_size), device=self.device)
+            # Re-raise the exception if not handled
+            raise
+        except Exception as e:
+            logger.error(f"Unhandled error in forward_with_error_handling: {e}")
+            batch_size = input_ids.size(0) if input_ids is not None else 1
+            return torch.zeros((batch_size, self.output_size), device=self.device)
+
+    def train_with_emissions_tracking(self, dataloader, optimizer, criterion, num_epochs=1):
+        """
+        Train the model while tracking carbon emissions using CodeCarbon.
+        """
+        tracker = EmissionsTracker()
+        tracker.start()  # Start tracking emissions
+
+        self.train()  # Set model to training mode
+        for epoch in range(num_epochs):
+            for batch in dataloader:
+                inputs, labels = batch
+                inputs, labels = inputs.to(self.device), labels.to(self.device)
+
+                optimizer.zero_grad()
+                outputs = self(inputs)
+                loss = criterion(outputs, labels)
+                loss.backward()
+                optimizer.step()
+
+            logging.info(f"Epoch {epoch + 1}/{num_epochs} completed.")
+
+        emissions = tracker.stop()  # Stop tracking emissions
+        logging.info(f"Training completed. Carbon emissions: {emissions:.4f} kg CO2")
+
+    def infer_with_emissions_tracking(self, input_ids):
+        """
+        Perform inference while tracking carbon emissions using CodeCarbon.
+        """
+        tracker = EmissionsTracker()
+        tracker.start()  # Start tracking emissions
+
+        self.eval()  # Set model to evaluation mode
+        with torch.no_grad():
+            outputs = self(input_ids)
+
+        emissions = tracker.stop()  # Stop tracking emissions
+        logging.info(f"Inference completed. Carbon emissions: {emissions:.4f} kg CO2")
+        return outputs
+
+# Register the model class in registry for discovery
+registry.register("model_class_custom", Wildnerve_tlm01)
+
+# Check if tokenizer is initialized properly.
+def initialize_tokenizer():
+    """
+    Fallback function to initialize the tokenizer.
+    Tries up to 5 times and logs debug messages on each attempt.
+    """
+    from transformers import BertTokenizer, AutoTokenizer
+    max_attempts = 5
+    for attempt in range(1, max_attempts + 1):
+        try:
+            # Attempt to get tokenizer from the registry
+            from service_registry import registry, TOKENIZER
+            if registry.has(TOKENIZER):
+                tokenizer = registry.get(TOKENIZER)
+                if tokenizer is not None:
+                    logger.debug(f"Attempt {attempt}: Successfully retrieved tokenizer from registry.")
+                    return tokenizer
+            # Fallback: load tokenizer directly
+            tokenizer = AutoTokenizer.from_pretrained("Wildnerve-tlm01-0.05Bx12")
+            logger.debug(f"Attempt {attempt}: Successfully loaded tokenizer from pretrained model.")
+            # Register it for future use
+            registry.register(TOKENIZER, tokenizer)
+            return tokenizer
+        except Exception as e:
+            logger.debug(f"Attempt {attempt}: Failed to initialize tokenizer due to: {e}")
+    logger.error("Tokenizer initialization failed after 5 attempts. Using default BertTokenizer.")
+    try:
+        tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")
+    except Exception as e:
+        logger.error(f"Default tokenizer initialization failed: {e}")
+        tokenizer = None
+    return tokenizer

model_List.py

@@ -0,0 +1,138 @@
+# model_List.py - Model selection and analysis component
+import logging
+import time
+import math
+import torch
+import importlib.util
+import os
+import re
+import logging
+from typing import List, Tuple, Dict
+import torch
+import numpy as np
+from sklearn.metrics.pairwise import cosine_similarity
+import nltk
+try:
+    nltk.data.find('tokenizers/punkt')
+except LookupError:
+    nltk.download("punkt")
+from transformers import AutoTokenizer, AutoModel
+from utils.smartHybridAttention import SmartHybridAttention, get_hybrid_attention_config
+from service_registry import registry, TOKENIZER, MODEL
+
+logger = logging.getLogger(__name__)
+
+class PromptAnalyzer:
+    """
+    A complete prompt analyzer that:
+      - Loads a lightweight Transformer encoder (DistilBERT)
+      - Applies SmartHybridAttention to refine token embeddings
+      - Compares the resulting prompt embedding against predefined topic embeddings
+      - Determines a primary topic and subtopics
+      - Provides candidate model identifiers or a single best match.
+    """
+    def __init__(self):
+        # Predefined topics with keyword sets for topic understanding
+        self.predefined_topics: Dict[str, List[str]] = {
+            "general": ["general", "overview", "basic", "introduction"],
+            "programming": ["code", "programming", "debug", "software", "algorithm", "bug"],
+            "science": ["research", "experiment", "science", "physics", "biology", "chemistry"],
+            "history": ["history", "ancient", "modern", "civilization", "war"],
+            "mathematics": ["math", "algebra", "calculus", "geometry", "statistics"]
+        }
+        # Initialize a lightweight transformer encoder for embeddings
+        self.tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased")
+        self.encoder = AutoModel.from_pretrained("distilbert-base-uncased")
+        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        self.encoder.to(self.device)
+        # Initialize SmartHybridAttention for refined representations
+        attention_config = get_hybrid_attention_config()
+        self.attention = SmartHybridAttention(attention_config)
+        self.attention.to(self.device)
+        logger.info("PromptAnalyzer initialized with DistilBERT and SmartHybridAttention.")
+
+    def _encode_text(self, text: str) -> np.ndarray:
+        """
+        Encode text into an embedding vector.
+        First, obtain token embeddings using DistilBERT.
+        Then refine these embeddings with SmartHybridAttention.
+        Finally, average-pool to produce a single vector.
+        """
+        inputs = self.tokenizer(text, return_tensors="pt", truncation=True, max_length=128)
+        inputs = {k: v.to(self.device) for k, v in inputs.items()}
+        with torch.no_grad():
+            outputs = self.encoder(**inputs)  # shape: [batch, seq_len, dim]
+        token_embeds = outputs.last_hidden_state  # [1, seq_len, dim]
+        # Transpose for attention: [seq_len, batch, dim]
+        token_embeds = token_embeds.transpose(0, 1)
+        attended, _ = self.attention(query=token_embeds, key=token_embeds, value=token_embeds)
+        # Transpose back and pool over tokens: [batch, seq_len, dim] -> [batch, dim]
+        attended = attended.transpose(0, 1)
+        pooled = attended.mean(dim=1)
+        return pooled.squeeze().cpu().numpy()
+
+    def analyze_prompt(self, prompt: str) -> Tuple[str, List[str]]:
+        """
+        Analyze the given prompt:
+          - Compute its refined embedding.
+          - For each predefined topic, encode its keyword string.
+          - Compute cosine similarity between prompt and topic embeddings.
+          - Return the primary topic (highest similarity) and any subtopics
+            with similarity above 80% of the top score.
+        """
+        prompt_embedding = self._encode_text(prompt)
+        topic_scores = {}
+        for topic, keywords in self.predefined_topics.items():
+            topic_text = " ".join(keywords)
+            topic_embedding = self._encode_text(topic_text)
+            similarity = cosine_similarity(
+                prompt_embedding.reshape(1, -1),
+                topic_embedding.reshape(1, -1)
+            )[0][0]
+            topic_scores[topic] = similarity
+        sorted_topics = sorted(topic_scores.items(), key=lambda x: x[1], reverse=True)
+        primary_topic = sorted_topics[0][0] if sorted_topics else "general"
+        threshold = sorted_topics[0][1] * 0.8 if sorted_topics else 0.0
+        subtopics = [topic for topic, score in sorted_topics if score >= threshold and topic != primary_topic]
+        logger.debug(f"Prompt analyzed (first 30 chars): '{prompt[:30]}...' -> Primary: {primary_topic}, Subtopics: {subtopics}")
+        return primary_topic, subtopics
+
+    def get_selected_models(self) -> List[str]:
+        """
+        Return candidate model identifiers.
+        For example, if the prompt is technical (programming) the custom model might be top.
+        This method can later be expanded to select multiple or weighted candidates.
+        """
+        # Here we return our primary custom model and a fallback general model.
+        return ["Wildnerve-tlm01-0.05Bx12", "bert-base-uncased"]
+
+    def choose_model(self, prompt: str) -> str:
+        """
+        Based on the analyzed prompt, select the most appropriate model identifier.
+        For instance, if 'programming' is detected, return the custom model.
+        Otherwise, return a general/pretrained model or a combination indicator.
+        """
+        primary_topic, _ = self.analyze_prompt(prompt)
+        if primary_topic == "programming":
+            return "Wildnerve-tlm01-0.05Bx12"
+        elif primary_topic in ["science", "mathematics", "history"]:
+            return "model_Combn.py"
+        else:
+            return "bert-base-uncased"
+
+# Optionally, additional helper methods could be added here for richer topic decomposition.
+
+# Register the PromptAnalyzer in the service registry to resolve dependencies.
+registry.register("prompt_analyzer", PromptAnalyzer())
+
+# If additional functions or initialization code is needed, include here:
+def main():
+    # For testing purposes; in production, model_manager will retrieve the analyzer.
+    analyzer = registry.get("prompt_analyzer")
+    sample_prompt = "I'm having trouble debugging my Python code for a sorting algorithm."
+    primary_topic, subtopics = analyzer.analyze_prompt(sample_prompt)
+    selected = analyzer.choose_model(sample_prompt)
+    logger.info(f"Sample prompt analysis:\nPrimary Topic: {primary_topic}\nSubtopics: {subtopics}\nSelected Model: {selected}")
+
+if __name__ == "__main__":
+    main()

model_PrTr.py

@@ -0,0 +1,482 @@
+# model_prtr.py
+import os
+import sys
+import math
+import torch
+import logging
+import importlib
+import torch.nn as nn
+from config import load_config
+from transformers import AutoTokenizer, AutoModel
+from typing import Optional, List, Dict, Any, Union
+from sentence_transformers import SentenceTransformer
+# Import service registry
+from service_registry import registry, MODEL, TOKENIZER
+# First import base interfaces
+from base_interfaces.common_types import *
+from base_interfaces.model_interface import AbstractModel
+
+# Import environment setup first to ensure config is available
+from model_env_setup import app_config
+
+app_config = load_config()
+logger = logging.getLogger(__name__)
+
+# ----------------------------
+# Positional Encoding Module (for decoder)
+# ----------------------------
+class PositionalEncoding(nn.Module):
+    def __init__(self, d_model: int, max_len: int = app_config.MAX_SEQ_LENGTH):
+        super().__init__()
+        pe = torch.zeros(max_len, d_model)
+        position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)
+        div_term = torch.exp(torch.arange(0, d_model, 2, dtype=torch.float) * (-math.log(10000.0) / d_model))
+        pe[:, 0::2] = torch.sin(position * div_term)
+        pe[:, 1::2] = torch.cos(position * div_term)
+        pe = pe.unsqueeze(1)  # shape: (max_len, 1, d_model)
+        self.register_buffer('pe', pe)
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        # x shape: (seq_len, batch_size, d_model)
+        seq_len = x.size(0)
+        x = x + self.pe[:seq_len]
+        return x
+
+# ----------------------------
+# Wildnerve-tlm01 using Only Pretrained Encoder
+# ----------------------------
+class Wildnerve_tlm01(nn.Module, AbstractModel):
+    """A Transformer-based language model that uses:
+      - A pretrained encoder (via AutoModel)
+      - A custom decoder stack
+    The model uses the AutoTokenizer for consistent tokenization."""
+    def __init__(
+        self,
+        vocab_size: int,
+        specialization: str,
+        dataset_path: str,
+        model_name: str,
+        embedding_dim: int,
+        num_heads: int,
+        hidden_dim: int,
+        num_layers: int,
+        output_size: int,
+        dropout: float,
+        max_seq_length: int,
+        pooling_mode: str,
+        tokenizer=None  # Accept tokenizer as parameter
+    ) -> None:
+        super().__init__()
+        self.specialization = specialization
+        self.dataset_path = dataset_path
+        self.model_name = model_name
+        self.pooling_mode = pooling_mode
+        self.vocab_size = vocab_size
+        self.max_seq_length = max_seq_length
+        self.embedding_dim = embedding_dim
+        self.num_heads = num_heads
+        self.hidden_dim = hidden_dim
+        self.num_layers = num_layers
+        self.output_size = output_size
+        self.dropout = dropout
+
+      # Add dimension projection layer for pretrained model output
+        self.pretrained_projection = nn.Linear(768, embedding_dim)  # 768 → 256
+        
+        # Initialize projection layer
+        nn.init.xavier_uniform_(self.pretrained_projection.weight)
+        nn.init.zeros_(self.pretrained_projection.bias)
+
+        # Use tokenizer from params, registry, or create new
+        if tokenizer is not None:
+            self.tokenizer = tokenizer
+        else:
+            # Try to get tokenizer from registry first
+            if registry.has(TOKENIZER):
+                self.tokenizer = registry.get(TOKENIZER)
+                logger.info("Using tokenizer from registry")
+            else:
+                # Load a new tokenizer
+                if tokenizer is None:
+                    try:
+                        self.tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
+                        logger.info("Loaded primary pretrained tokenizer: bert-base-uncased")
+                    except Exception as e:
+                        logger.warning(f"Bert tokenizer load failed: {e}")
+                        try:
+                            self.tokenizer = AutoTokenizer.from_pretrained("gpt2")
+                            logger.info("Loaded fallback tokenizer: GPT2")
+                        except Exception as e2:
+                            logger.error(f"GPT2 tokenizer load failed: {e2}")
+                            self.tokenizer = None
+
+        # Register this model instance in the registry by specialization
+        model_registry_key = f"model_{specialization}"
+        registry.register(model_registry_key, self)
+
+        # ----------------------------
+        # Decoder (Target) Components
+        # ----------------------------
+        self.tgt_embedding = nn.Embedding(vocab_size, embedding_dim)
+        self.pos_decoder = PositionalEncoding(embedding_dim, max_len=max_seq_length)
+        decoder_layer = nn.TransformerDecoderLayer(
+            d_model=embedding_dim,
+            nhead=num_heads,
+            dim_feedforward=hidden_dim,
+            dropout=dropout,
+            batch_first=False
+        )
+        self.transformer_decoder = nn.TransformerDecoder(decoder_layer, num_layers=num_layers)
+        
+        # ----------------------------
+        # Adapter & Output Components
+        # ----------------------------
+        self.adapter = nn.Sequential(
+            nn.Linear(embedding_dim, hidden_dim),
+            nn.ReLU(),
+            nn.Linear(hidden_dim, embedding_dim)
+        )
+        self.classifier = nn.Linear(embedding_dim, output_size)
+        self.dropout_layer = nn.Dropout(dropout)
+
+        self.init_weights()
+
+    def init_weights(self) -> None:
+        """Initialize weights for decoder, adapter and classifier."""
+        initrange = 0.1
+        with torch.no_grad():
+            self.tgt_embedding.weight.uniform_(-initrange, initrange)
+            self.classifier.weight.uniform_(-initrange, initrange)
+            self.classifier.bias.zero_()
+            for layer in self.adapter:
+                if isinstance(layer, nn.Linear):
+                    layer.weight.uniform_(-initrange, initrange)
+                    if layer.bias is not None:
+                        layer.bias.zero_()
+
+    def forward(self, src: torch.Tensor, tgt: Optional[torch.Tensor] = None, 
+           src_key_padding_mask: Optional[torch.Tensor] = None,
+           tgt_key_padding_mask: Optional[torch.Tensor] = None,
+           return_sequence: bool = False,
+           **kwargs) -> torch.Tensor:
+        try:
+            # Pretrained encoder expects input shape: (batch_size, seq_length)
+            encoded_output = self.pretrained_encoder(src)[0]  # (batch_size, seq_length, embedding_dim)
+           
+            # Project from 768 to 256
+            encoded_output = self.pretrained_projection(encoded_output)
+                    
+            # Transpose to (seq_length, batch_size, embedding_dim)
+            encoded_output = encoded_output.transpose(0, 1)
+        
+            # Process through adapter layer
+            adapted = self.adapter(encoded_output)
+        
+            # If a target sequence is provided, run the decoder
+            if tgt is not None:
+                tgt = tgt.transpose(0, 1)  # (seq_length, batch_size)
+                tgt_emb = self.tgt_embedding(tgt) * math.sqrt(self.embedding_dim)
+                tgt_emb = self.pos_decoder(tgt_emb)
+                tgt_mask = nn.Transformer.generate_square_subsequent_mask(tgt.size(0)).to(src.device)
+                decoded = self.transformer_decoder(
+                    tgt_emb,
+                    adapted,
+                    tgt_mask=tgt_mask,
+                    memory_key_padding_mask=src_key_padding_mask,
+                    tgt_key_padding_mask=tgt_key_padding_mask
+                )
+                output = self.classifier(decoded.transpose(0, 1))
+                if not return_sequence:
+                    output = output.mean(dim=1)
+            else:
+                if self.pooling_mode == "mean":
+                    output = adapted.mean(dim=0)
+                elif self.pooling_mode == "max":
+                    output = torch.max(adapted, dim=0)[0]
+                else:
+                    output = adapted.mean(dim=0)
+                    
+                output = self.dropout_layer(output)
+                output = self.classifier(output)
+                
+            return output
+            
+        except Exception as e:
+            logger.error(f"Error during forward pass: {e}")
+            raise
+
+    @staticmethod
+    def generate_square_subsequent_mask(sz: int) -> torch.Tensor:
+        """Generate square subsequent mask for transformer."""
+        mask = (torch.triu(torch.ones(sz, sz)) == 1).transpose(0, 1)
+        mask = mask.float().masked_fill(mask == 0, float('-inf')).masked_fill(mask == 1, float(0.0))
+        return mask
+
+    def generate(
+        self,
+        input_ids: torch.Tensor,
+        max_length: int = app_config.TRANSFORMER_CONFIG.MAX_SEQ_LENGTH,
+        device: str = 'cpu',
+        temperature: float = 1.0,
+        start_token_id: Optional[int] = None
+    ) -> List[List[int]]:
+        """Generates token ID sequences using the pretrained encoder and custom decoder."""
+        self.eval()
+        batch_size = input_ids.shape[0]
+        if start_token_id is None:
+            start_token_id = input_ids[0, 0].item()
+        generated = [[start_token_id] for _ in range(batch_size)]
+        
+        # Use pretrained encoder to encode source input.
+        encoded_output = self.pretrained_encoder(input_ids)[0]  # (batch_size, seq_length, embedding_dim)
+        encoded_output = encoded_output.transpose(0, 1)  # (seq_length, batch_size, embedding_dim)
+        adapted = self.adapter(encoded_output)
+        
+        for _ in range(max_length - 1):
+            current_tgt = torch.tensor(generated, dtype=torch.long, device=device)
+            current_tgt = current_tgt.transpose(0, 1)
+            tgt_emb = self.tgt_embedding(current_tgt) * math.sqrt(self.embedding_dim)
+            tgt_emb = self.pos_decoder(tgt_emb)
+            current_seq_length = current_tgt.size(0)
+            
+            tgt_mask = nn.Transformer.generate_square_subsequent_mask(current_seq_length).to(device)
+            decoded = self.transformer_decoder(tgt_emb, adapted, tgt_mask=tgt_mask)
+            logits = self.classifier(decoded[-1, :, :])
+            
+            if temperature == 0:
+                next_tokens = torch.argmax(logits, dim=-1)
+            else:
+                probs = torch.softmax(logits / temperature, dim=-1)
+                next_tokens = torch.multinomial(probs, num_samples=1).squeeze(-1)
+            
+            next_tokens = next_tokens.cpu().tolist()
+            for i, token in enumerate(next_tokens):
+                generated[i].append(token)
+                
+        return generated
+
+    def decode_tokens(self, token_ids: List[int]) -> str:
+        """Decodes a list of token IDs into text."""
+        try:
+            return self.tokenizer.decode(token_ids, skip_special_tokens=True, clean_up_tokenization_spaces=True)
+        except Exception as e:
+            logger.error(f"Error decoding tokens: {e}")
+            return str(e)
+
+    def generate_with_decoding(
+        self,
+        input_ids: torch.Tensor,
+        max_length: int = 20,
+        device: str = 'cpu',
+        temperature: float = 1.0,
+        start_token_id: Optional[int] = None
+    ) -> str:
+        """Generates a sequence and decodes it into text."""
+        generated_sequences = self.generate(input_ids, max_length, device, temperature, start_token_id)
+        if generated_sequences:
+            return self.decode_tokens(generated_sequences[0])
+        return ""
+
+    def generate_streaming(self, prompt=None, input_ids=None, attention_mask=None, **kwargs):
+        """Generate a response token-by-token from the model"""
+        # Consistent device handling
+        device = next(self.parameters()).device
+        
+        # Handle either text prompt or tokenized input
+        if prompt is not None and input_ids is None:
+            inputs = self.tokenizer(
+                prompt, 
+                return_tensors="pt",
+                padding="max_length",
+                truncation=True,
+                max_length=self.max_seq_length
+            )
+            input_ids = inputs["input_ids"].to(device)
+            attention_mask = inputs.get("attention_mask", None)
+            if attention_mask is not None:
+                attention_mask = attention_mask.to(device)
+        
+        # Ensure input_ids is valid
+        if input_ids is None:
+            raise ValueError("Either prompt or input_ids must be provided")
+            
+        # Use pretrained encoder to encode source input
+        encoded_output = self.pretrained_encoder(input_ids)[0]  # (batch_size, seq_length, embedding_dim)
+        encoded_output = self.pretrained_projection(encoded_output)
+        encoded_output = encoded_output.transpose(0, 1)  # (seq_length, batch_size, embedding_dim)
+        adapted = self.adapter(encoded_output)
+            
+        # Get generation config params
+        max_length = kwargs.get('max_length', 100)
+        temperature = kwargs.get('temperature', 0.7)
+            
+        # Generate first token
+        with torch.no_grad():
+            # Initialize with start token (could be from input or specified)
+            start_token_id = kwargs.get('start_token_id', input_ids[0, 0].item())
+            current_tgt = torch.tensor([[start_token_id]], dtype=torch.long, device=device)
+            current_tgt = current_tgt.transpose(0, 1)  # (1, batch_size=1)
+            
+            # Process first token
+            tgt_emb = self.tgt_embedding(current_tgt) * math.sqrt(self.embedding_dim)
+            tgt_emb = self.pos_decoder(tgt_emb)
+            tgt_mask = nn.Transformer.generate_square_subsequent_mask(1).to(device)
+            decoded = self.transformer_decoder(tgt_emb, adapted, tgt_mask=tgt_mask)
+            logits = self.classifier(decoded[-1, :, :])
+            
+            # Sample from distribution
+            if temperature == 0:
+                next_token = torch.argmax(logits, dim=-1)
+            else:
+                probs = torch.softmax(logits / temperature, dim=-1)
+                next_token = torch.multinomial(probs, num_samples=1).squeeze(-1)
+            
+            # Convert to token text and yield
+            token_text = self.tokenizer.decode([next_token.item()], skip_special_tokens=True)
+            yield token_text
+            
+            # Generate rest of sequence
+            generated_ids = [next_token.item()]
+            
+            for _ in range(max_length - 1):
+                # Update target sequence
+                current_tgt = torch.tensor([generated_ids], dtype=torch.long, device=device)
+                current_tgt = current_tgt.transpose(0, 1)  # (seq_len, batch=1)
+                
+                # Process next token
+                tgt_emb = self.tgt_embedding(current_tgt) * math.sqrt(self.embedding_dim)
+                tgt_emb = self.pos_decoder(tgt_emb)
+                current_seq_length = current_tgt.size(0)
+                tgt_mask = nn.Transformer.generate_square_subsequent_mask(current_seq_length).to(device)
+                decoded = self.transformer_decoder(tgt_emb, adapted, tgt_mask=tgt_mask)
+                logits = self.classifier(decoded[-1, :, :])
+                
+                # Sample next token
+                if temperature == 0:
+                    next_token = torch.argmax(logits, dim=-1)
+                else:
+                    probs = torch.softmax(logits / temperature, dim=-1)
+                    next_token = torch.multinomial(probs, num_samples=1).squeeze(-1)
+                
+                # Add to generated sequence
+                next_token_id = next_token.item()
+                generated_ids.append(next_token_id)
+                
+                # Decode and yield the token
+                token_text = self.tokenizer.decode([next_token_id], skip_special_tokens=True)
+                
+                # Check for EOS token
+                if next_token_id == self.tokenizer.eos_token_id:
+                    break
+                
+                yield token_text
+
+#-------Pretrained Transformer Model-------------
+class PretrainedTransformer(nn.Module, AbstractModel):
+    """A simple wrapper around a pretrained Hugging Face transformer model."""
+    def __init__(
+        self,
+        vocab_size=30522,
+        specialization="general",
+        dataset_path=None,
+        model_name="bert-base-uncased",  # Primary model name for pretrained transformer
+        embedding_dim=768,
+        num_heads=12,
+        hidden_dim=768,
+        num_layers=6,
+        output_size=768,
+        dropout=0.1,
+        max_seq_length=512,
+        pooling_mode="mean",
+        tokenizer=None,
+        **kwargs
+    ) -> None:
+        super().__init__()
+        
+        # Optionally track model usage
+        self.model_last_used = {}  
+        
+        # Unified tokenizer initialization:
+        # Primary: Load tokenizer for "bert-base-uncased"
+        # Fallback: if it fails, try GPT2 tokenizer
+        if tokenizer is not None:
+            self.tokenizer = tokenizer
+        else:
+            from transformers import AutoTokenizer, BertTokenizer
+            if registry.has(TOKENIZER):
+                self.tokenizer = registry.get(TOKENIZER)
+            else:
+                try:
+                    self.tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")
+                    logger.info("Loaded primary tokenizer: bert-base-uncased")
+                except Exception as e:
+                    logger.warning(f"Primary tokenizer load failed: {e}")
+                    try:
+                        self.tokenizer = AutoTokenizer.from_pretrained("gpt2")
+                        logger.info("Loaded fallback tokenizer: GPT2")
+                    except Exception as e2:
+                        logger.error(f"Fallback tokenizer load failed: {e2}")
+                        self.tokenizer = None
+        registry.register(TOKENIZER, self.tokenizer)
+        
+        # Set model names for fallback chain explicitly
+        self.model_name = model_name  # Should be "bert-base-uncased"
+        self.fallback_model = "gpt2"    # Fallback tokenization/model if needed
+        
+        self.model = AutoModel.from_pretrained(model_name)
+        try:
+            self.tokenizer = AutoTokenizer.from_pretrained(model_name)
+        except Exception as e:
+            logger.error(f"Failed to load tokenizer for {model_name}: {e}")
+            self.tokenizer = None
+        
+    def forward(self, input_ids, attention_mask=None):
+        outputs = self.model(input_ids=input_ids, attention_mask=attention_mask)
+        return outputs.last_hidden_state
+
+    def encode(self, text: str):
+        if not self.tokenizer:
+            raise ValueError("Tokenizer not available")
+        inputs = self.tokenizer(text, return_tensors="pt", truncation=True, padding=True)
+        with torch.no_grad():
+            outputs = self.forward(inputs.input_ids, inputs.get("attention_mask"))
+        # Pool by averaging the token embeddings
+        return outputs.mean(dim=1)
+    
+    def generate(self, input_ids, max_length=100, **kwargs):
+        # Use generate method from model if available, else fallback.
+        if hasattr(self.model, "generate"):
+            return self.model.generate(input_ids=input_ids, max_length=max_length, **kwargs)
+        else:
+            # Simple fallback: return input_ids as is
+            return input_ids
+
+# Register model classes in registry
+registry.register("model_class_pretrained", Wildnerve_tlm01)
+registry.register("pretrained_transformer_class", PretrainedTransformer)
+
+# Check if pretrained transformers are properly initialized.
+def initialize_pretrained_model():
+    """Attempt to initialize a pretrained tokenizer with a fallback mechanism.
+    Tries to load 'bert-base-uncased' first; if that fails, attempts to load 'gpt2'.
+    If the fallback is used, then reattempts loading 'bert-base-uncased' on subsequent tries.
+    Repeats up to 5 attempts in total.
+    Returns:
+        The initialized tokenizer instance if successful, otherwise None."""
+    max_attempts = 5
+    for attempt in range(1, max_attempts + 1):
+        try:
+            tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
+            logger.info(f"Attempt {attempt}: Successfully loaded bert-base-uncased.")
+            return tokenizer
+        except Exception as e:
+            logger.warning(f"Attempt {attempt}: Loading bert-base-uncased failed: {e}")
+            try:
+                tokenizer = AutoTokenizer.from_pretrained("gpt2")
+                logger.info(f"Attempt {attempt}: Successfully loaded gpt2 as fallback.")
+                return tokenizer
+            except Exception as e2:
+                logger.warning(f"Attempt {attempt}: Loading gpt2 failed as fallback: {e2}")
+        logger.info("Retrying tokenizer initialization...")
+    logger.error("Failed to initialize pretrained model tokenizer after 5 attempts.")
+    return None

model_manager.py

@@ -0,0 +1,735 @@
+import gc, os, sys, time, torch, logging, inspect, numpy as np, pandas as pd, importlib.util
+from pathlib import Path
+from threading import Lock
+from collections import OrderedDict
+from nltk.stem import WordNetLemmatizer
+from typing import List, Dict, Any, Tuple, Optional, TYPE_CHECKING
+from sklearn.metrics.pairwise import cosine_similarity
+from config import app_config
+from dataset import TensorDataset
+from utils.transformer_utils import get_sentence_transformer
+from utils.smartHybridAttention import SmartHybridAttention, get_hybrid_attention_config
+if TYPE_CHECKING:
+    from service_registry import registry
+from service_registry import registry, MODEL, TOKENIZER, MODEL_MANAGER, COMMUNICATOR
+
+logger = logging.getLogger(__name__)
+
+try:
+    import psutil
+    PSUTIL_AVAILABLE = True
+except ImportError:
+    logger.warning("psutil not available")
+    PSUTIL_AVAILABLE = False
+    class DummyProcess:
+        def __init__(self, pid=None): self.pid = pid or 1
+        def memory_info(self):
+            class MemInfo:
+                def __init__(self): self.rss = 1e6; self.vms = 1e6
+            return MemInfo()
+        def memory_percent(self): return 1.0
+    class DummyPsutil:
+        @staticmethod
+        def Process(pid=None): return DummyProcess(pid)
+    psutil = DummyPsutil()
+
+def safe_get_config(config_obj, key, default=None):
+    if isinstance(config_obj, dict):
+        return config_obj.get(key, default)
+    elif hasattr(config_obj, key):
+        return getattr(config_obj, key, default)
+    return default
+
+def safe_get_config_value(config_obj, key, default=None):
+    try:
+        if isinstance(config_obj, dict):
+            return config_obj.get(key, default)
+        elif hasattr(config_obj, key):
+            return getattr(config_obj, key, default)
+        elif isinstance(config_obj, (int, float, str, bool)):
+            return config_obj
+        return default
+    except:
+        return default
+
+class DatasetManager:
+    def __init__(self):
+        self.datasets: Dict[str, Any] = {}
+        self.lock = Lock()
+        
+    def load_dataset(self, path: str, specialization: str) -> Any:
+        with self.lock:
+            if specialization in self.datasets:
+                logger.info(f"Using cached dataset for {specialization}")
+                return self.datasets[specialization]
+            dataset = self._load_and_process_dataset(path, specialization)
+            self.datasets[specialization] = dataset
+            return dataset
+        
+    def _load_and_process_dataset(self, path: str, specialization: str) -> TensorDataset:
+        if not os.path.exists(path):
+            raise FileNotFoundError(f"Dataset {path} not found.")
+        logger.info(f"Loading dataset: {specialization}")
+        data = pd.read_csv(path)
+        if "label" not in data.columns:
+            raise ValueError("Dataset must have a 'label' column.")
+        features = data.drop("label", axis=1).values
+        labels = data["label"].values
+        features_tensor = torch.tensor(features, dtype=torch.float32)
+        labels_tensor = torch.tensor(labels, dtype=torch.long)
+        return TensorDataset(features_tensor, labels_tensor)
+        
+    def get_status(self) -> Dict[str, Any]:
+        return {"loaded_datasets": list(self.datasets.keys()), "cache_size": len(self.datasets)}
+        
+    def clear_cache(self):
+        with self.lock:
+            self.datasets.clear()
+
+class ModelManager:
+    def __init__(self, tokenizer=None, max_active_models=5, model_idle_threshold=600):
+        self.models = {}
+        self.lock = Lock()
+        self.model_pool = OrderedDict()
+        self.max_active_models = max_active_models if isinstance(max_active_models, int) and max_active_models > 0 else 2
+        self.model_idle_threshold = model_idle_threshold if isinstance(model_idle_threshold, int) and model_idle_threshold > 0 else 600
+        self.tokenizer = tokenizer
+        dataset_paths = safe_get_config(app_config, "DATASET_PATHS", {})
+        self.specializations = list(dataset_paths.keys()) if isinstance(dataset_paths, dict) else ["default"]
+        self._performance_metrics = {}
+        attention_config = get_hybrid_attention_config()
+        self.smart_attention = SmartHybridAttention(
+            dim=attention_config["DIM"],
+            num_heads=attention_config["NUM_HEADS"],
+            window_size=attention_config["WINDOW_SIZE"],
+            use_sliding=attention_config["USE_SLIDING"],
+            use_global=attention_config["USE_GLOBAL"],
+            use_hierarchical=attention_config["USE_HIERARCHICAL"],
+            global_token_ratio=attention_config["GLOBAL_TOKEN_RATIO"],
+            memory_tokens=attention_config["MEMORY_TOKENS"]
+        )
+        self.dataset_manager = DatasetManager()
+        transformer_config = safe_get_config(app_config, "TRANSFORMER_CONFIG", {})
+        # Force use of our custom model with no fallback
+        model_name = safe_get_config(transformer_config, "MODEL_NAME", "Wildnerve-tlm01-0.05Bx12")
+        self.embedding_model = get_sentence_transformer(model_name)
+        self.similarity_threshold = safe_get_config(app_config, "SIMILARITY_THRESHOLD", 0.85)
+        self.top_k = safe_get_config(app_config, "TOP_K", 3)
+        self.prompt_analyzer = None
+        self.selected_models = self._get_selected_models()
+        logger.info(f"ModelManager initialized with {len(self.specializations)} specializations")
+        self._load_models()
+    
+    def _get_selected_models(self) -> List[str]:
+        model_files = safe_get_config(app_config, "SELECTED_MODEL", ["model_Custm.py"])
+        return model_files if model_files else ["model_Custm.py"]
+    
+    def _import_model_class(self, model_key: str):
+        try:
+            abs_path = f"{os.path.dirname(__file__)}/{model_key}.py"
+            if os.path.exists(abs_path):
+                spec = importlib.util.spec_from_file_location(model_key, abs_path)
+                module = importlib.util.module_from_spec(spec)
+                spec.loader.exec_module(module)
+            elif os.path.exists(f"{model_key}.py"):
+                spec = importlib.util.spec_from_file_location(model_key, f"{model_key}.py")
+                module = importlib.util.module_from_spec(spec)
+                spec.loader.exec_module(module)
+            else:
+                module = importlib.import_module(model_key)
+            if module and hasattr(module, "Wildnerve_tlm01"):
+                return getattr(module, "Wildnerve_tlm01")
+            else:
+                logger.warning(f"Module {model_key} missing Wildnerve_tlm01 class")
+                return None
+        except Exception as e:
+            logger.error(f"Failed to import {model_key}: {e}")
+            return None
+    
+    def _load_models(self):
+        """Initialize models with lazy loading and limited initial specializations"""
+        # Define all specializations but only load a minimal subset at startup
+        all_specializations = [
+            "mbpp",
+            "programming_software_dev",
+            "machine_learning_ai_data_science", 
+            "industrial_engineering",
+            "science_engineering", 
+            "mathematics",
+            "healthcare_and_lifesciences", 
+            "chemistry", 
+            "hardware_devops_cloud",
+            "cyber_security", 
+            "business_legal_finance", 
+            "other_information"
+        ]
+        
+        # Only load 2 specializations at startup to prevent resource exhaustion
+        initial_specializations = ["general", "programming_software_dev"]
+        self.all_specializations = all_specializations  # Store all for later lazy loading
+        self.models = {}
+        
+        # Set up data directory
+        data_dir = os.environ.get("TLM_DATA_DIR", "/tmp/tlm_data")
+        os.makedirs(data_dir, exist_ok=True)
+        
+        # Only initialize the minimal subset at startup
+        for spec in initial_specializations:
+            try:
+                self._initialize_model_for_specialization(spec, data_dir)
+                logger.info(f"Initialized model for {spec}")
+            except Exception as e:
+                logger.error(f"Error initializing model for {spec}: {e}")
+        
+        logger.info(f"Loaded {len(self.models)} initial models, {len(all_specializations)} total available")
+        return True
+
+    def _initialize_model_for_specialization(self, spec, data_dir):
+        """Initialize a single model with proper error handling and timeouts"""
+        # Get dataset path with fallbacks
+        dataset_path = None
+        try:
+            if isinstance(app_config, dict) and "DATASET_PATHS" in app_config:
+                dataset_path = app_config["DATASET_PATHS"].get(spec)
+            elif hasattr(app_config, "DATASET_PATHS"):
+                dataset_path = getattr(app_config.DATASET_PATHS, spec, None)
+        except Exception as e:
+            logger.warning(f"Error getting dataset path: {e}")
+        
+        # Use default path if not provided
+        if not dataset_path:
+            dataset_path = os.path.join(data_dir, f"{spec}.csv")
+            
+        # Create minimal dataset if needed
+        if not os.path.exists(dataset_path):
+            try:
+                with open(dataset_path, "w") as f:
+                    f.write("text,label\n")
+                    f.write(f"sample {spec} text,0\n")
+                logger.info(f"Created minimal dataset for {spec}")
+            except Exception as e:
+                logger.error(f"Error creating dataset for {spec}: {e}")
+        
+        # Create model with timeout protection
+        start_time = time.time()
+        timeout = 30  # 30 second timeout for model creation
+        
+        try:
+            # Import with timeout check to avoid hanging
+            from model_Custm import Wildnerve_tlm01
+            
+            # Initialize model with appropriate parameters
+            model = Wildnerve_tlm01(
+                vocab_size=30522,
+                specialization=spec,
+                dataset_path=dataset_path,
+                model_name="Wildnerve-tlm01-0.05Bx12",
+                embedding_dim=768,
+                num_heads=12,
+                hidden_dim=768,
+                num_layers=6,
+                output_size=768,
+                dropout=0.1,
+                max_seq_length=512,
+                pooling_mode="mean",
+                tokenizer=self.tokenizer
+            )
+            
+            # Add model to the pool
+            self.models[spec] = model
+            self.model_pool[spec] = None
+            self._performance_metrics[spec] = {
+                "inference_time": 0.0, 
+                "memory_usage": 0.0, 
+                "last_accessed": time.time(), 
+                "num_inferences": 0
+            }
+            
+            # Check for timeout
+            if time.time() - start_time > timeout:
+                logger.warning(f"Model creation for {spec} took longer than {timeout}s!")
+                
+        except Exception as e:
+            logger.error(f"Error creating model for {spec}: {e}")
+            raise
+
+    def get_or_create_model(self, specialization: str) -> Any:
+        """Get an existing model or create it on demand if not already loaded"""
+        with self.lock:
+            # Check if model already exists
+            model = self.get_model(specialization)
+            if model:
+                logger.info(f"Using existing model for {specialization}")
+                return model
+                
+            # Check if it's a valid specialization
+            if specialization not in self.all_specializations and specialization != "general":
+                logger.warning(f"Unknown specialization: {specialization}, using general")
+                specialization = "general"
+            
+            # Create model if needed
+            logger.info(f"Lazily loading model for {specialization}")
+            
+            # Remove least recently used model if needed
+            if len(self.models) >= self.max_active_models:
+                lru_specialization = next(iter(self.model_pool))
+                self.remove_model_instance(lru_specialization)
+                
+            # Initialize the requested model
+            data_dir = os.environ.get("TLM_DATA_DIR", "/tmp/tlm_data")
+            try:
+                self._initialize_model_for_specialization(specialization, data_dir)
+                return self.models.get(specialization)
+            except Exception as e:
+                logger.error(f"Error initializing model: {e}")
+                
+                # Fallback to general model
+                if specialization != "general" and "general" in self.models:
+                    return self.models["general"]
+                
+                # Last resort - create a minimal model
+                return self._create_minimal_model()
+
+    def _create_minimal_model(self):
+        """Create a minimal fallback model for emergencies"""
+        try:
+            from model_Custm import Wildnerve_tlm01
+            model = Wildnerve_tlm01(
+                vocab_size=30522,
+                specialization="minimal",
+                dataset_path=None,
+                model_name="bert-base-uncased",  # Use simpler base model
+                embedding_dim=768,
+                num_heads=12,
+                hidden_dim=768,
+                num_layers=2,  # Reduced layers
+                output_size=768,
+                dropout=0.1,
+                max_seq_length=128,  # Reduced sequence length
+                pooling_mode="mean",
+                tokenizer=self.tokenizer
+            )
+            model._is_minimal = True  # Mark as minimal model
+            return model
+        except Exception as e:
+            logger.error(f"Failed to create minimal model: {e}")
+            return None
+
+    def get_model(self, specialization: str) -> Any:
+        with self.lock:
+            model = self.models.get(specialization)
+            if model:
+                self.model_pool.move_to_end(specialization)
+                if specialization in self._performance_metrics:
+                    self._performance_metrics[specialization]["last_accessed"] = time.time()
+            return model
+    
+    def route_input(self, input_text: str) -> dict:
+        input_embedding = self.embedding_model.encode(input_text)
+        similarities = {}
+        for spec in self.specializations:
+            model = self.get_model(spec)
+            if model and hasattr(model, "embedding"):
+                sim = cosine_similarity(input_embedding.reshape(1, -1), model.embedding.reshape(1, -1))[0][0]
+                similarities[spec] = sim
+        if similarities:
+            best_match = max(similarities.items(), key=lambda x: x[1])
+            return {"matched_specialization": best_match[0], "confidence": best_match[1], "all_scores": similarities}
+        return {"matched_specialization": self.specializations[0], "confidence": 0.0, "all_scores": similarities}
+    
+    def get_model_for_prompt(self, prompt: str) -> Tuple[Any, str]:
+        try:
+            routing_result = self.route_input(prompt)
+            specialization = routing_result.get("matched_specialization", self.specializations[0])
+            model = self.get_or_create_model(specialization)
+            start_time = time.time()
+            def update_metrics():
+                if specialization in self._performance_metrics:
+                    m = self._performance_metrics[specialization]
+                    elapsed = time.time() - start_time
+                    n = m.get("num_inferences", 0) + 1
+                    m["inference_time"] = ((m.get("inference_time", 0) * (n-1)) + elapsed) / n
+                    m["num_inferences"] = n
+                    m["last_accessed"] = time.time()
+                    if hasattr(model, "get_memory_usage"):
+                        m["memory_usage"] = model.get_memory_usage()
+            update_metrics()
+            return model, specialization
+        except Exception as e:
+            logger.error(f"Error selecting model: {e}")
+            if self.models:
+                default_key = list(self.models.keys())[0]
+                return self.models[default_key], default_key
+            else:
+                logger.error("No models available for routing")
+                return None, "none"
+    
+    def generate(self, prompt: str, **kwargs):
+        self.validate_input(prompt)
+        model, specialization = self.get_model_for_prompt(prompt)
+        start_time = time.time()
+        try:
+            result = model.generate(prompt=prompt, **kwargs)
+            elapsed = time.time() - start_time
+            if specialization in self._performance_metrics:
+                m = self._performance_metrics[specialization]
+                n = m.get("num_inferences", 0) + 1
+                m["inference_time"] = ((m.get("inference_time", 0) * (n-1)) + elapsed) / n
+                m["num_inferences"] = n
+                m["last_accessed"] = time.time()
+            return result
+        except Exception as e:
+            logger.error(f"Error generating with {specialization}: {e}")
+            default_spec = self.specializations[0]
+            default_model = self.get_or_create_model(default_spec)
+            return default_model.generate(prompt=prompt, **kwargs)
+    
+    def generate_streaming(self, prompt: str, **kwargs):
+        self.validate_input(prompt)
+        model, specialization = self.get_model_for_prompt(prompt)
+        start_time = time.time()
+        try:
+            if hasattr(model, "generate_streaming"):
+                for token in model.generate_streaming(prompt=prompt, **kwargs):
+                    yield token
+            else:
+                logger.info("Simulating streaming generation")
+                result = model.generate(prompt=prompt, **kwargs)
+                for word in result.split():
+                    yield word + " "
+            elapsed = time.time() - start_time
+            if specialization in self._performance_metrics:
+                m = self._performance_metrics[specialization]
+                n = m.get("num_inferences", 0) + 1
+                m["inference_time"] = ((m.get("inference_time", 0) * (n-1)) + elapsed) / n
+                m["num_inferences"] = n
+                m["last_accessed"] = time.time()
+        except Exception as e:
+            logger.error(f"Error in streaming generation: {e}")
+            default_spec = self.specializations[0]
+            default_model = self.get_or_create_model(default_spec)
+            if hasattr(default_model, "generate_streaming"):
+                for token in default_model.generate_streaming(prompt=prompt, **kwargs):
+                    yield token
+            else:
+                fallback_result = default_model.generate(prompt=prompt, **kwargs)
+                for word in fallback_result.split():
+                    yield word + " "
+    
+    def remove_model_instance(self, specialization: str) -> bool:
+        with self.lock:
+            if specialization in self.models:
+                del self.models[specialization]
+                self.model_pool.pop(specialization, None)
+                gc.collect()
+                if torch.cuda.is_available():
+                    torch.cuda.empty_cache()
+                logger.info(f"Removed model for {specialization}")
+                return True
+            return False
+    
+    def validate_input(self, input_text: str) -> bool:
+        if not input_text or len(input_text.strip()) == 0:
+            raise ValueError("Empty input text")
+        max_length = safe_get_config(app_config, "MAX_INPUT_LENGTH", safe_get_config(app_config, "MAX_SEQ_LENGTH", 128))
+        if len(input_text) > max_length:
+            raise ValueError(f"Input exceeds maximum length of {max_length}")
+        return True
+    
+    def get_health_status(self) -> Dict[str, Any]:
+        with self.lock:
+            process = psutil.Process(os.getpid())
+            mem_info = process.memory_info()
+            return {
+                "active_models": len(self.models),
+                "memory_usage": {
+                    "rss_mb": mem_info.rss / (1024 * 1024),
+                    "vms_mb": mem_info.vms / (1024 * 1024),
+                    "percent": process.memory_percent()
+                },
+                "model_performance": self._get_model_metrics(),
+                "dataset_status": self.dataset_manager.get_status(),
+                "cache_efficiency": len(self.model_pool) / max(1, self.max_active_models)
+            }
+    
+    def _get_model_metrics(self) -> Dict[str, Dict[str, Any]]:
+        metrics = {}
+        for spec, model in self.models.items():
+            base = self._performance_metrics.get(spec, {})
+            mem_usage = 0
+            if hasattr(model, "get_memory_usage"):
+                mem_usage = model.get_memory_usage()
+            elif hasattr(model, "parameters"):
+                mem_usage = sum(p.numel() * p.element_size() for p in model.parameters()) / (1024 * 1024)
+            metrics[spec] = {
+                "inference_time": base.get("inference_time", 0),
+                "memory_usage_mb": mem_usage,
+                "last_accessed": base.get("last_accessed", 0),
+                "num_inferences": base.get("num_inferences", 0),
+                "model_type": model.__class__.__name__
+            }
+        return metrics
+    
+    def get_available_models(self) -> Dict[str, Any]:
+        with self.lock:
+            return dict(self.models)
+    
+    def shutdown(self):
+        try:
+            logger.info("Initiating shutdown")
+            for spec in list(self.models.keys()):
+                self.remove_model_instance(spec)
+            self.dataset_manager.clear_cache()
+            logger.info("Shutdown complete")
+        except Exception as e:
+            logger.error(f"Error during shutdown: {e}")
+    
+    def manage_model_cache(self):
+        try:
+            current = time.time()
+            with self.lock:
+                while len(self.models) > self.max_active_models:
+                    oldest = next(iter(self.model_pool))
+                    self.remove_model_instance(oldest)
+                    logger.info(f"Removed LRU model: {oldest}")
+                for spec, last in list(self.model_pool.items()):
+                    m = self._performance_metrics.get(spec, {})
+                    if m.get("last_accessed", 0) and (current - m["last_accessed"] > self.model_idle_threshold):
+                        self.remove_model_instance(spec)
+                        logger.info(f"Removed idle model: {spec}")
+                sorted_models = sorted(self.model_pool.items(), key=lambda x: self._performance_metrics.get(x[0], {}).get("last_accessed", 0), reverse=True)
+                self.model_pool = OrderedDict(sorted_models)
+        except Exception as e:
+            logger.error(f"Error in cache management: {e}")
+    
+    def set_tokenizer(self, tokenizer):
+        self.tokenizer = tokenizer
+        with self.lock:
+            for name, model in self.models.items():
+                if hasattr(model, "set_tokenizer"):
+                    try:
+                        model.tokenizer = tokenizer
+                        logger.debug(f"Updated tokenizer for {name}")
+                    except Exception as ex:
+                        logger.warning(f"Failed to set tokenizer for {name}: {ex}")
+        logger.info("Tokenizer updated for models")
+        return self
+
+    def initialize_models(self):
+        try:
+            logger.info("Initializing models from weights")
+            prompt_analyzer = registry.get("prompt_analyzer")
+            if not prompt_analyzer:
+                try:
+                    from pathlib import Path
+                    model_list_path = Path(__file__).parent / "model_List.py"
+                    if model_list_path.exists():
+                        spec = importlib.util.find_spec("model_List")
+                        if spec:
+                            model_list = importlib.util.module_from_spec(spec)
+                            spec.loader.exec_module(model_list)
+                            if hasattr(model_list, "PromptAnalyzer"):
+                                prompt_analyzer = model_list.PromptAnalyzer()
+                                registry.register("prompt_analyzer", prompt_analyzer)
+                                logger.info("Imported PromptAnalyzer")
+                except Exception as e:
+                    logger.error(f"Error importing PromptAnalyzer: {e}")
+            self.prompt_analyzer = prompt_analyzer
+            selected_models_list = prompt_analyzer.get_selected_models() if prompt_analyzer and hasattr(prompt_analyzer, "get_selected_models") else ["model_Custm.py"]
+            logger.info(f"Selected model types: {selected_models_list}")
+            specializations = ["general", "programming", "science", "history", "mathematics"]
+            for spec in specializations:
+                try:
+                    model_name = selected_models_list[0].replace(".py", "")
+                    from pathlib import Path
+                    model_path = Path(__file__).parent / f"{model_name}.py"
+                    if model_path.exists():
+                        spec_obj = importlib.util.find_spec(model_name)
+                        if spec_obj:
+                            model_module = importlib.util.module_from_spec(spec_obj)
+                            spec_obj.loader.exec_module(model_module)
+                            if hasattr(model_module, "Wildnerve_tlm01"):
+                                model_class = getattr(model_module, "Wildnerve_tlm01")
+                                embedding_dim = 768
+                                num_heads = 12 if embedding_dim % 12 == 0 else 1
+                                model_instance = model_class(
+                                    vocab_size=30522,
+                                    specialization=spec,
+                                    dataset_path=None,
+                                    model_name="bert-base-uncased",
+                                    embedding_dim=embedding_dim,
+                                    num_heads=num_heads,
+                                    hidden_dim=768,
+                                    num_layers=2,
+                                    output_size=768,
+                                    dropout=0.1,
+                                    max_seq_length=128,
+                                    pooling_mode="mean"
+                                )
+                                self.models[spec] = model_instance
+                                logger.info(f"Created model for {spec}")
+                except Exception as e:
+                    logger.error(f"Error creating model for {spec}: {e}")
+            if not self.models:
+                logger.error("No models created")
+                return False
+            try:
+                import os
+                attention_config_path = os.path.join(app_config.DATA_DIR, "attention_configuration.json")
+                from utils.attention_connector import get_attention_connector
+                attention_connector = get_attention_connector()
+                if hasattr(attention_connector, "config_path"):
+                    attention_connector.config_path = attention_config_path
+                    attention_connector._init_profile_selector()
+                logger.info(f"Initialized attention connector with config: {attention_config_path}")
+            except Exception as e:
+                logger.warning(f"Failed to initialize attention connector: {e}")
+            logger.info(f"Successfully initialized {len(self.models)} models")
+            return True
+        except Exception as e:
+            logger.error(f"Error initializing models: {e}", exc_info=True)
+            return False
+
+    def get_alternative_model_for_prompt(self, prompt: str, current_model=None) -> any:
+        try:
+            if self.prompt_analyzer and hasattr(self.prompt_analyzer, "choose_model"):
+                model_type = self.prompt_analyzer.choose_model(prompt)
+                if model_type:
+                    try:
+                        alt_model = model_type(
+                            vocab_size=30522,
+                            specialization="general",
+                            dataset_path=None,
+                            model_name="bert-base-uncased",
+                            embedding_dim=768,
+                            num_heads=12,
+                            hidden_dim=768,
+                            num_layers=6,
+                            output_size=768,
+                            dropout=0.1,
+                            max_seq_length=512,
+                            pooling_mode="mean",
+                            tokenizer=self.tokenizer
+                        )
+                        if alt_model != current_model:
+                            logger.info("Found alternative model via prompt_analyzer")
+                            return alt_model
+                    except Exception as e:
+                        logger.error(f"Error initializing alternative model: {e}")
+            for name, model in self.get_available_models().items():
+                if model != current_model:
+                    logger.info(f"Using alternative model: {name}")
+                    return model
+            try:
+                from model_Custm import Wildnerve_tlm01
+                fallback_model = Wildnerve_tlm01(
+                    vocab_size=30522,
+                    specialization="general",
+                    model_name="bert-base-uncased",
+                    embedding_dim=768,
+                    num_heads=12,
+                    hidden_dim=768,
+                    num_layers=6,
+                    output_size=30522,
+                    dropout=0.1,
+                    max_seq_length=512,
+                    pooling_mode="mean",
+                    tokenizer=self.tokenizer
+                )
+                logger.info("Created fallback model")
+                return fallback_model
+            except Exception as e:
+                logger.error(f"Error creating fallback model: {e}")
+            return None
+        except Exception as e:
+            logger.error(f"Error getting alternative model: {e}")
+            return None
+
+    def prepare_model_input(self, text: str, model) -> dict:
+        device = next(model.parameters()).device
+        try:
+            tokenizer = getattr(model, "tokenizer", None)
+            if tokenizer:
+                inputs = tokenizer(
+                    text,
+                    return_tensors="pt",
+                    padding=True,
+                    truncation=True,
+                    max_length=safe_get_config_value(app_config, "MAX_SEQ_LENGTH", 512)
+                )
+                input_ids = inputs["input_ids"].to(device)
+                return {"input_ids": input_ids, "max_length": safe_get_config_value(app_config, "MAX_SEQ_LENGTH", 512), "device": device, "temperature": getattr(self, "generation_config", {}).get("temperature", 0.7)}
+            else:
+                logger.warning("No tokenizer in model; using basic input")
+                return {"input_text": text, "max_length": safe_get_config_value(app_config, "MAX_SEQ_LENGTH", 512)}
+        except Exception as e:
+            logger.error(f"Error preparing model input: {e}")
+            return {"input_text": text}
+
+    def process_with_context(self, input_text: str, context: Optional[dict] = None) -> dict:
+        conversation_context = self.get_conversation_context(window_size=3)
+        contextualized_prompt = input_text
+        if conversation_context:
+            max_seq_length = safe_get_config_value(app_config, "MAX_SEQ_LENGTH", 512)
+            max_seq_length = int(max_seq_length) if isinstance(max_seq_length, (int, float)) else 512
+            contextualized_prompt = f"Previous conversation:\n{conversation_context}\n\nCurrent question: {input_text}"
+        result = self.process_input(contextualized_prompt, context)
+        if isinstance(result, dict):
+            result["original_query"] = input_text
+        return result
+
+    def get_conversation_context(self, window_size: int = 3) -> str:
+        if not hasattr(self, "conversation_history"):
+            self.conversation_history = []
+        recent = self.conversation_history[-window_size*2:]
+        lines = []
+        for entry in recent:
+            prefix = "User: " if entry["role"]=="user" else "Assistant: "
+            lines.append(f"{prefix}{entry['content']}")
+        return "\n".join(lines)
+
+# Factory methods for model manager creation
+def create_model_manager(tokenizer=None) -> ModelManager:
+    try:
+        max_active_models = safe_get_config_value(app_config, "MAX_ACTIVE_MODELS", 2)
+        model_idle_threshold = safe_get_config_value(app_config, "MODEL_IDLE_THRESHOLD", 600)
+        manager = ModelManager(tokenizer=tokenizer, max_active_models=max_active_models, model_idle_threshold=model_idle_threshold)
+        if tokenizer:
+            manager.set_tokenizer(tokenizer)
+        elif registry.has(TOKENIZER):
+            manager.set_tokenizer(registry.get(TOKENIZER))
+        registry.register(MODEL_MANAGER, manager)
+        return manager
+    except Exception as e:
+        logger.error(f"Error creating ModelManager: {e}")
+        minimal_manager = ModelManager(tokenizer=tokenizer, max_active_models=1)
+        registry.register(MODEL_MANAGER, minimal_manager)
+        return minimal_manager
+
+def create_model_manager_with_tokenizer(tokenizer):
+    try:
+        max_active_models = safe_get_config_value(app_config, "MAX_ACTIVE_MODELS", 2)
+        model_idle_threshold = safe_get_config_value(app_config, "MODEL_IDLE_THRESHOLD", 600)
+        manager = ModelManager(max_active_models=max_active_models, model_idle_threshold=model_idle_threshold)
+        manager.tokenizer = tokenizer
+        manager.initialize_models()
+        registry.register(MODEL_MANAGER, manager)
+        return manager
+    except Exception as e:
+        logger.error(f"Error creating ModelManager with tokenizer: {e}")
+        minimal_manager = ModelManager(max_active_models=1)
+        minimal_manager.tokenizer = tokenizer
+        registry.register(MODEL_MANAGER, minimal_manager)
+        return minimal_manager
+
+if __name__ == "__main__":
+    tokenizer = registry.get(TOKENIZER)
+    if not tokenizer:
+        from utils.transformer_utils import get_tokenizer
+        tokenizer = get_tokenizer("bert-base-uncased")
+        registry.register(TOKENIZER, tokenizer)
+    model_manager = create_model_manager(tokenizer)
+    logger.info(f"Model Manager initialized with {len(model_manager.models)} models")
+else:
+    model_manager = None
+    logger.info("ModelManager module imported; initialization deferred")