Upload 3 files

config.json

@@ -10,7 +10,74 @@
   "DATASET_PATHS": {
     "general": ["data/general.json"],
     "programming_software_dev": ["data/programming_software_dev.json"],
-    "other_information": ["data/other_information.json"]
+    "other_information": ["data/other_information.json"],
+    "python": [
+      "data/python_mbpp.json",
+      "data/python_programming.json",
+      "data/python_transformer_model.json"
+    ],
+    "rust": [
+      "data/rust_ai_language_model.json",
+      "data/rust_blockchain.json",
+      "data/rust_mbrp.json",
+      "data/rust_programming.json"
+    ],
+    "solidity": ["data/solidity_programming.json"],
+    "computer": [
+      "data/computer_advanced_debugging.json",
+      "data/computer_agenticAI.json",
+      "data/computer_architecture.json",
+      "data/computer_cloud_security.json",
+      "data/computer_cloudCI-CD.json",
+      "data/computer_creativity.json",
+      "data/computer_crossplatform.json",
+      "data/computer_cybersecurity.json",
+      "data/computer_error_handling_examples.json",
+      "data/computer_gitInstruct.json"
+    ],
+    "cpp": [
+      "data/cpp_ai_language_model.json",
+      "data/cpp_blockchain.json",
+      "data/cpp_mbcppp.json",
+      "data/cpp_programming.json"
+    ],
+    "go": [
+      "data/golang_ai_language_model.json",
+      "data/golang_mbgp.json",
+      "data/golang_programming.json"
+    ],
+    "java": [
+      "data/java_ai_language_model.json",
+      "data/java_blockchain.json",
+      "data/java_mbjp.json",
+      "data/java_programming.json",
+      "data/java_transformer_language_model.json"
+    ],
+    "javascript": [
+      "data/javascript_chatbot.json",
+      "data/javascript_n_Typescript_backend.json",
+      "data/javascript_n_Typescript_frontend.json",
+      "data/javascript_n_Typescript_programming.json"
+    ],
+    "mathematics": [
+      "data/mathematics.json",
+      "data/mathematics_training.json"
+    ],
+    "nim": [
+      "data/nim_ai_language_model.json",
+      "data/nim_blockchain.json",
+      "data/nim_chatbot.json",
+      "data/nim_conversation.json",
+      "data/nim_mbnp.json",
+      "data/nim_programming.json"
+    ],
+    "other_information": ["data/other_information.json"],
+    "physics": [
+      "data/physics_n_engineering.json",
+      "data/physics_n_engineering_applied.json",
+      "data/project_structure.json",
+      "data/python_chatbot_guide.json"
+    ]
   },
   "LAZY_LOADING_ENABLED": true,
   "MAX_INITIAL_SPECIALIZATIONS": 2,
@@ -20,23 +87,18 @@
   "INPUT_SIZE": 768,
   "OUTPUT_SIZE": 768,
   "SPECIALIZATIONS": [
-    "general",
-    "programming_software_dev"
-  ],
-  "ALL_SPECIALIZATIONS": [
-    "general",
-    "mbpp", 
-    "programming_software_dev",
-    "machine_learning_ai_data_science",
-    "industrial_engineering",
-    "science_engineering",
+    "python",
+    "rust",
+    "solidity",
+    "computer",
+    "cpp",
+    "go",
+    "java",
+    "javascript",
     "mathematics",
-    "healthcare_and_lifesciences",
-    "chemistry",
-    "hardware_devops_cloud",
-    "cyber_security",
-    "business_legal_finance",
-    "other_information"
+    "nim",
+    "other_information",
+    "physics"
   ],
   "PREPROCESSING": {
     "LOWERCASE": true,

config.py

@@ -94,99 +94,92 @@ HIDDEN_SIZE = 768  # Hidden layer size
 
 # Add SPECIALIZATIONS list
 SPECIALIZATIONS = [
-    "cpp",
-    "java",
-    "go",
-    "javascript",
-    "nim",
     "python",
     "rust",
     "solidity",
     "computer",
+    "cpp",
+    "go",
+    "java",
+    "javascript",
     "mathematics",
-    "physics",
-    "other_information"
+    "nim",
+    "other_information",
+    "physics"
 ]
 
-# Define DATASET_PATHS so that each specialization is a string or a list of strings
+# Define DATASET_PATHS so that each specialization maps to its JSON files
 DATASET_PATHS = {
+    "python": [
+        str(DATA_DIR / "data" / "python_mbpp.json"),
+        str(DATA_DIR / "data" / "python_programming.json"),
+        str(DATA_DIR / "data" / "python_transformer_model.json")
+    ],
+    "rust": [
+        str(DATA_DIR / "data" / "rust_ai_language_model.json"),
+        str(DATA_DIR / "data" / "rust_blockchain.json"),
+        str(DATA_DIR / "data" / "rust_mbrp.json"),
+        str(DATA_DIR / "data" / "rust_programming.json")
+    ],
+    "solidity": [
+        str(DATA_DIR / "data" / "solidity_programming.json")
+    ],
     "computer": [
         str(DATA_DIR / "data" / "computer_advanced_debugging.json"),
         str(DATA_DIR / "data" / "computer_agenticAI.json"),
         str(DATA_DIR / "data" / "computer_architecture.json"),
         str(DATA_DIR / "data" / "computer_cloud_security.json"),
+        str(DATA_DIR / "data" / "computer_cloudCI-CD.json"),
         str(DATA_DIR / "data" / "computer_creativity.json"),
         str(DATA_DIR / "data" / "computer_crossplatform.json"),
         str(DATA_DIR / "data" / "computer_cybersecurity.json"),
         str(DATA_DIR / "data" / "computer_error_handling_examples.json"),
         str(DATA_DIR / "data" / "computer_gitInstruct.json")
     ],
-
     "cpp": [
-        str(DATA_DIR / "data" / "cpp_advanced_debugging.json"),
+        str(DATA_DIR / "data" / "cpp_ai_language_model.json"),
         str(DATA_DIR / "data" / "cpp_blockchain.json"),
         str(DATA_DIR / "data" / "cpp_mbcppp.json"),
         str(DATA_DIR / "data" / "cpp_programming.json")
     ],
-
-    "java": [
-        str(DATA_DIR / "data" / "java_ai_language_model.json"),
-        str(DATA_DIR / "data" / "java_blockchain.json"),
-        str(DATA_DIR / "data" / "java_mbjp.json"),
-        str(DATA_DIR / "data" / "java_transformer_language_model.json"),
-    ],
-
     "go": [
         str(DATA_DIR / "data" / "golang_ai_language_model.json"),
         str(DATA_DIR / "data" / "golang_mbgp.json"),
         str(DATA_DIR / "data" / "golang_programming.json")
     ],
-
+    "java": [
+        str(DATA_DIR / "data" / "java_ai_language_model.json"),
+        str(DATA_DIR / "data" / "java_blockchain.json"),
+        str(DATA_DIR / "data" / "java_mbjp.json"),
+        str(DATA_DIR / "data" / "java_programming.json"),
+        str(DATA_DIR / "data" / "java_transformer_language_model.json")
+    ],
     "javascript": [
         str(DATA_DIR / "data" / "javascript_chatbot.json"),
-        str(DATA_DIR / "data" / "javascript_n_Typescript_frontend.json"),
         str(DATA_DIR / "data" / "javascript_n_Typescript_backend.json"),
-        str(DATA_DIR / "data" / "javascript_programming.json")
+        str(DATA_DIR / "data" / "javascript_n_Typescript_frontend.json"),
+        str(DATA_DIR / "data" / "javascript_n_Typescript_programming.json")
+    ],
+    "mathematics": [
+        str(DATA_DIR / "data" / "mathematics.json"),
+        str(DATA_DIR / "data" / "mathematics_training.json")
     ],
-
     "nim": [
         str(DATA_DIR / "data" / "nim_ai_language_model.json"),
         str(DATA_DIR / "data" / "nim_blockchain.json"),
         str(DATA_DIR / "data" / "nim_chatbot.json"),
+        str(DATA_DIR / "data" / "nim_conversation.json"),
         str(DATA_DIR / "data" / "nim_mbnp.json"),
         str(DATA_DIR / "data" / "nim_programming.json")
     ],
-
-    "python": [
-        str(DATA_DIR / "data" / "python_chatbot_guide.json"),
-        str(DATA_DIR / "data" / "python_mbpp.json"),
-        str(DATA_DIR / "data" / "python_programming.json"),
-        str(DATA_DIR / "data" / "python_transformer_model.json")
-    ],
-
-    "rust": [
-        str(DATA_DIR / "data" / "rust_ai_language_model.json"),
-        str(DATA_DIR / "data" / "rust_blockchain.json"),
-        str(DATA_DIR / "data" / "rust_mbrp.json"),
-        str(DATA_DIR / "data" / "rust_programming.json")
-    ],
-
-    "solidity": [
-        str(DATA_DIR / "data" / "solidity_programming.json")
-    ],
-
-    "mathematics": [
-        str(DATA_DIR / "data" / "mathematics.json"),
-        str(DATA_DIR / "data" / "mathematics_training.json")
+    "other_information": [
+        str(DATA_DIR / "data" / "other_information.json")
     ],
-
     "physics": [
         str(DATA_DIR / "data" / "physics_n_engineering.json"),
-        str(DATA_DIR / "data" / "physics_n_engineering_applied.json")
-    ],
-
-    "other_information": [
-        str(DATA_DIR / "data" / "other_information.json")
+        str(DATA_DIR / "data" / "physics_n_engineering_applied.json"),
+        str(DATA_DIR / "data" / "project_structure.json"),
+        str(DATA_DIR / "data" / "python_chatbot_guide.json")
     ]
 }
 

model_manager.py

@@ -3,16 +3,13 @@ from pathlib import Path
 from threading import Lock
 from collections import OrderedDict
 from nltk.stem import WordNetLemmatizer
-from typing import List, Dict, Any, Tuple, Optional, TYPE_CHECKING
+from typing import List, Dict, Any, Tuple, Optional
 from sklearn.metrics.pairwise import cosine_similarity
 from config import app_config
 from dataset import TensorDataset
 from utils.transformer_utils import get_sentence_transformer
 from utils.smartHybridAttention import SmartHybridAttention, get_hybrid_attention_config
 from transformers import AutoModelForCausalLM, AutoTokenizer
-
-if TYPE_CHECKING:
-    from service_registry import registry
 from service_registry import registry, MODEL, TOKENIZER, MODEL_MANAGER, COMMUNICATOR
 
 logger = logging.getLogger(__name__)
@@ -236,22 +233,8 @@ class ModelManager:
         }
 
     def _load_models(self):
-        """Load a small initial set of specs, fail loudly on errors."""
-        all_specs = [
-            "mbpp",
-            "programming_software_dev",
-            "machine_learning_ai_data_science", 
-            "industrial_engineering",
-            "science_engineering", 
-            "mathematics",
-            "healthcare_and_lifesciences", 
-            "chemistry", 
-            "hardware_devops_cloud",
-            "cyber_security", 
-            "business_legal_finance", 
-            "other_information"
-        ]
-        initial = ["general", "programming_software_dev"]
+        """Load initial specializations—now pulled from config.DATASET_PATHS keys."""
+        initial = ["general"]  # only preload the 'general' specialization
         data_dir = os.environ.get("TLM_DATA_DIR", "/tmp/tlm_data")
         os.makedirs(data_dir, exist_ok=True)
 
@@ -264,7 +247,7 @@ class ModelManager:
                 logger.error(f"Failed to load model for '{spec}'", exc_info=True)
                 raise
 
-        logger.info(f"{len(self.models)} models loaded at startup (of {len(all_specs)} total)")
+        logger.info(f"{len(self.models)} models loaded at startup (of {len(self.specializations)} total)")
 
     def get_or_create_model(self, specialization: str) -> Any:
         """Get an existing model or create it on demand if not already loaded"""