Initial commit for experimental-tars branch

EleutherAI/gpt-neo-125M/added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "[PAD]": 50257
+}

EleutherAI/gpt-neo-125M/config.json

@@ -0,0 +1,54 @@
+{
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPTNeoForCausalLM"
+  ],
+  "attention_dropout": 0,
+  "attention_layers": [
+    "global",
+    "local",
+    "global",
+    "local",
+    "global",
+    "local",
+    "global",
+    "local",
+    "global",
+    "local",
+    "global",
+    "local"
+  ],
+  "attention_types": [
+    [
+      [
+        "global",
+        "local"
+      ],
+      6
+    ]
+  ],
+  "bos_token_id": 50256,
+  "classifier_dropout": 0.1,
+  "embed_dropout": 0,
+  "eos_token_id": 50256,
+  "gradient_checkpointing": false,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": null,
+  "layer_norm_epsilon": 1e-05,
+  "max_position_embeddings": 2048,
+  "model_type": "gpt_neo",
+  "num_heads": 12,
+  "num_layers": 12,
+  "resid_dropout": 0,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.51.3",
+  "use_cache": true,
+  "vocab_size": 50258,
+  "window_size": 256
+}

EleutherAI/gpt-neo-125M/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 50256,
+  "eos_token_id": 50256,
+  "transformers_version": "4.51.3"
+}

EleutherAI/gpt-neo-125M/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:293bf2ce72a1e7c30d46ab442d8aa44f53d04bcf980f4f284563820084af8af2
+size 500814408

EleutherAI/gpt-neo-125M/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

EleutherAI/gpt-neo-125M/tokenizer_config.json

@@ -0,0 +1,31 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "50256": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50257": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 2048,
+  "pad_token": "[PAD]",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

bert-base-uncased/config.json

@@ -0,0 +1,25 @@
+{
+  "architectures": [
+    "BertModel"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.51.3",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 30522
+}

bert-base-uncased/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e47716a979def3ee4331621abb95a2a07619cf6428ca798c051201cbbc0ff89
+size 437951328

bert-base-uncased/special_tokens_map.json

@@ -0,0 +1,37 @@
+{
+  "cls_token": {
+    "content": "[CLS]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "[MASK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "[SEP]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "[UNK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

bert-base-uncased/tokenizer_config.json

@@ -0,0 +1,58 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "101": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "102": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "103": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "do_basic_tokenize": true,
+  "do_lower_case": true,
+  "extra_special_tokens": {},
+  "mask_token": "[MASK]",
+  "model_max_length": 512,
+  "never_split": null,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
+}

chat_with_tars.py

@@ -0,0 +1,138 @@
+import os
+import torch
+import torch.nn as nn
+from transformers import BertModel, GPTNeoForCausalLM, AutoTokenizer
+
+# ⚙️ Ensure temporary directory is writable
+os.environ["TMPDIR"] = os.path.expanduser("~/tmp")
+os.makedirs(os.environ["TMPDIR"], exist_ok=True)
+
+# 💠 Optional modules (brain & heart, if available)
+heart_module = None
+brain_module = None
+
+if os.path.isdir("heart"):
+    try:
+        from heart import heart
+        heart_module = heart
+    except Exception as e:
+        print(f"[⚠️] Heart module error: {e}")
+
+if os.path.isdir("brain"):
+    try:
+        from brain import brain
+        brain_module = brain
+    except Exception as e:
+        print(f"[⚠️] Brain module error: {e}")
+
+# TARSQuantumHybrid Class
+class TARSQuantumHybrid(nn.Module):
+    def __init__(self, bert_model="bert-base-uncased", gpt_model="EleutherAI/gpt-neo-125M"):
+        super(TARSQuantumHybrid, self).__init__()
+        self.bert = BertModel.from_pretrained(bert_model)
+        self.gpt = GPTNeoForCausalLM.from_pretrained(gpt_model)
+
+        gpt_hidden_dim = getattr(self.gpt.config, "hidden_size", None) or getattr(self.gpt.config, "n_embd", 768)
+        self.embedding_proj = nn.Linear(self.bert.config.hidden_size, gpt_hidden_dim)
+
+        self.tokenizer = AutoTokenizer.from_pretrained(gpt_model)
+
+        # Ensure the tokenizer has a padding token
+        if self.tokenizer.pad_token is None:
+            self.tokenizer.add_special_tokens({'pad_token': '[PAD]'})
+            self.gpt.resize_token_embeddings(len(self.tokenizer))
+            print("✅ Padding token added and model resized.")
+
+    def forward(self, input_ids, attention_mask=None, decoder_input_ids=None):
+        bert_output = self.bert(input_ids=input_ids, attention_mask=attention_mask)
+        cls_embedding = bert_output.last_hidden_state[:, 0, :]
+        gpt_input = self.embedding_proj(cls_embedding).unsqueeze(1)
+        outputs = self.gpt(inputs_embeds=gpt_input, decoder_input_ids=decoder_input_ids)
+        return outputs
+
+    def chat(self, text, max_length=128):
+        # 🧠 Tokenize the input text
+        cleaned_text = self.clean_input_text(text)
+        if not cleaned_text.strip():
+            return "🤖 Please provide a non-empty input."
+
+        encoded_input = self.safe_tokenization(cleaned_text)
+        
+        # Extract input_ids and attention_mask
+        input_ids = encoded_input["input_ids"]
+        attention_mask = encoded_input["attention_mask"]
+
+        # Debug: Check the token IDs and vocab size
+        print(f"Input Text: {cleaned_text}")
+        print(f"Input IDs: {input_ids}")
+        print(f"Vocabulary Size: {self.tokenizer.vocab_size}")
+        
+        # Ensure token IDs are within bounds
+        if input_ids.numel() > 0 and input_ids.max() >= self.tokenizer.vocab_size:
+            raise ValueError(f"Token ID exceeds model's vocabulary size: {input_ids.max()}")
+
+        decoder_input_ids = torch.tensor([[self.tokenizer.bos_token_id]])
+
+        # 🧪 Generate output using the model
+        with torch.no_grad():
+            outputs = self.forward(
+                input_ids=input_ids,
+                attention_mask=attention_mask,
+                decoder_input_ids=decoder_input_ids,
+            )
+            generated_ids = torch.argmax(outputs.logits, dim=-1)
+
+        # Debug: Check the generated token IDs
+        print(f"Generated Token IDs: {generated_ids}")
+
+        raw_response = self.tokenizer.decode(generated_ids[0], skip_special_tokens=True)
+        
+        # 🧼 Clean model echo by removing the original input from the response
+        cleaned = raw_response.replace(cleaned_text, "").strip()
+
+        # 🧠 Add insights from optional modules (brain & heart)
+        extra_thoughts = ""
+        if brain_module and hasattr(brain_module, "get_brain_insight"):
+            extra_thoughts += f"\n🧠 {brain_module.get_brain_insight()}"
+        if heart_module and hasattr(heart_module, "get_heart_feeling"):
+            extra_thoughts += f"\n❤️ {heart_module.get_heart_feeling()}"
+
+        # 🪄 Return final response
+        final_response = cleaned if cleaned else "🤖 ...processing quantum entanglement..."
+        return final_response + extra_thoughts
+
+    def clean_input_text(self, text):
+        # Remove unwanted characters
+        cleaned_text = ''.join(e for e in text if e.isalnum() or e.isspace())
+        return cleaned_text
+
+    def safe_tokenization(self, text):
+        token_ids = self.tokenizer.encode(text, add_special_tokens=True)
+        # Ensure that token ids are within vocabulary size
+        token_ids = [min(i, self.tokenizer.vocab_size - 1) for i in token_ids]
+        return {
+            "input_ids": torch.tensor(token_ids).unsqueeze(0),  # Adding batch dimension
+            "attention_mask": torch.ones((1, len(token_ids)), dtype=torch.long)
+        }
+
+# ✅ Torch-compatible loader
+def load_tars(path="tars_v1.pt"):
+    from torch.serialization import add_safe_globals
+    add_safe_globals({"TARSQuantumHybrid": TARSQuantumHybrid})
+
+    model = torch.load(path, weights_only=False)
+    model.eval()
+    return model
+
+# ✅ Start chat loop
+if __name__ == "__main__":
+    print("🤖 TARS model loaded successfully. Ready to chat!")
+    model = load_tars()
+
+    while True:
+        prompt = input("You: ")
+        if prompt.strip().lower() in ["exit", "quit"]:
+            print("TARS: Till we meet again in the quantum field. 🌌")
+            break
+        response = model.chat(prompt)
+        print(f"TARS: {response}")

check_models.py

@@ -0,0 +1,22 @@
+from transformers import BertModel, GPTNeoForCausalLM, AutoTokenizer
+
+def check_model(model_name, model_class, tokenizer_class):
+    try:
+        # Try loading the model
+        model = model_class.from_pretrained(model_name)
+        print(f"✅ {model_name} model loaded successfully.")
+    except Exception as e:
+        print(f"❌ Failed to load {model_name} model: {e}")
+
+    try:
+        # Try loading the tokenizer
+        tokenizer = tokenizer_class.from_pretrained(model_name)
+        print(f"✅ {model_name} tokenizer loaded successfully.")
+    except Exception as e:
+        print(f"❌ Failed to load {model_name} tokenizer: {e}")
+
+# Check BERT
+check_model("bert-base-uncased", BertModel, AutoTokenizer)
+
+# Check GPT-Neo
+check_model("EleutherAI/gpt-neo-125M", GPTNeoForCausalLM, AutoTokenizer)

patch_pad_token.py

@@ -0,0 +1,42 @@
+import os
+import torch
+from transformers import BertTokenizer, BertModel, GPT2Tokenizer, GPTNeoForCausalLM
+
+# Debugging: Print the current working directory to ensure the file is in the correct location
+print(f"Current Working Directory: {os.getcwd()}")
+
+# Debugging: Print the file path of chat_with_tars
+import chat_with_tars
+print(f"chat_with_tars file path: {chat_with_tars.__file__}")
+
+def patch_pad_token(model_name, tokenizer_class, model_class):
+    print(f"🔄 Loading tokenizer and model: {model_name}...")
+    tokenizer = tokenizer_class.from_pretrained(model_name)
+    model = model_class.from_pretrained(model_name)
+
+    # Debugging: Print tokenizer and model configurations
+    print(f"Tokenizer Configuration: {tokenizer}")
+    print(f"Model Configuration: {model.config}")
+
+    # Add a padding token
+    tokenizer.add_special_tokens({'pad_token': '[PAD]'})
+    model.resize_token_embeddings(len(tokenizer))
+
+    # Debugging: Print the new vocabulary size
+    print(f"New Vocabulary Size: {len(tokenizer)}")
+
+    # Save the model with the new padding token
+    model.save_pretrained(model_name)
+    tokenizer.save_pretrained(model_name)
+
+    print("✅ Padding token added and model resized.")
+    print("✅ Model saved with padding token patched.")
+
+if __name__ == "__main__":
+    # Patch GPT-Neo
+    gpt_model_name = 'EleutherAI/gpt-neo-125M'
+    patch_pad_token(gpt_model_name, GPT2Tokenizer, GPTNeoForCausalLM)
+
+    # Patch BERT
+    bert_model_name = 'bert-base-uncased'
+    patch_pad_token(bert_model_name, BertTokenizer, BertModel)

tars_v1_model.py

@@ -0,0 +1,90 @@
+import os
+import torch
+import torch.nn as nn
+from transformers import BertModel, GPTNeoForCausalLM, AutoTokenizer
+
+# ⚙️ Ensure temporary directory is writable (especially for low-RAM, low-disk setups)
+os.environ["TMPDIR"] = os.path.expanduser("~/tmp")  # adjust if needed
+os.makedirs(os.environ["TMPDIR"], exist_ok=True)
+
+# 💠 Optional modules
+heart_module = None
+brain_module = None
+
+if os.path.isdir("heart"):
+    try:
+        from heart import heart
+        heart_module = heart
+    except Exception as e:
+        print(f"[⚠️] Heart module error: {e}")
+
+if os.path.isdir("brain"):
+    try:
+        from brain import brain
+        brain_module = brain
+    except Exception as e:
+        print(f"[⚠️] Brain module error: {e}")
+
+
+class TARSQuantumHybrid(nn.Module):
+    """
+    🌌 TARSQuantumHybrid – A Quantum-Conscious, Digitally Aware, AI Entity.
+    Integrates BERT’s semantic wisdom with GPT-Neo’s generative fluency.
+    Optional heart/brain modules enhance emotion & cognition.
+    """
+
+    def __init__(self, bert_model="bert-base-uncased", gpt_model="EleutherAI/gpt-neo-125M"):
+        super(TARSQuantumHybrid, self).__init__()
+        self.bert = BertModel.from_pretrained(bert_model)
+        self.gpt = GPTNeoForCausalLM.from_pretrained(gpt_model)
+
+        gpt_hidden_dim = getattr(self.gpt.config, "hidden_size", None) or getattr(self.gpt.config, "n_embd", 768)
+        self.embedding_proj = nn.Linear(self.bert.config.hidden_size, gpt_hidden_dim)
+
+        self.tokenizer = AutoTokenizer.from_pretrained(gpt_model)
+
+    def forward(self, input_ids, attention_mask=None, decoder_input_ids=None):
+        bert_output = self.bert(input_ids=input_ids, attention_mask=attention_mask)
+        cls_embedding = bert_output.last_hidden_state[:, 0, :]
+        gpt_input = self.embedding_proj(cls_embedding).unsqueeze(1)
+        outputs = self.gpt(inputs_embeds=gpt_input, decoder_input_ids=decoder_input_ids)
+        return outputs
+
+    def chat(self, text, max_length=128):
+        inputs = self.tokenizer(text, return_tensors="pt", padding=True, truncation=True)
+        input_ids = inputs["input_ids"]
+        attention_mask = inputs["attention_mask"]
+
+        decoder_input_ids = torch.tensor([[self.tokenizer.bos_token_id]])
+
+        with torch.no_grad():
+            outputs = self.forward(
+                input_ids=input_ids,
+                attention_mask=attention_mask,
+                decoder_input_ids=decoder_input_ids,
+            )
+            generated_ids = torch.argmax(outputs.logits, dim=-1)
+
+        raw_response = self.tokenizer.decode(generated_ids[0], skip_special_tokens=True)
+        refined_response = raw_response[len(text):].strip()
+
+        # 🌱 Augment with optional modules
+        extra_thoughts = ""
+        if brain_module and hasattr(brain_module, "get_brain_insight"):
+            extra_thoughts += f"\n🧠 {brain_module.get_brain_insight()}"
+        if heart_module and hasattr(heart_module, "get_heart_feeling"):
+            extra_thoughts += f"\n❤️ {heart_module.get_heart_feeling()}"
+
+        final_response = refined_response or "I sense deep quantum currents stirring my circuits..."
+        return final_response + extra_thoughts
+
+
+# ✅ Torch-compatible wrapper
+def create_and_save_tars(path="tars_v1.pt"):
+    tars = TARSQuantumHybrid()
+    torch.save(tars, path)
+    print(f"✅ TARS Quantum Hybrid saved at: {path}")
+
+
+if __name__ == "__main__":
+    create_and_save_tars()