Upload 10 files

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+AGWM.json filter=lfs diff=lfs merge=lfs -text
+banner.png filter=lfs diff=lfs merge=lfs -text

AGWM.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cca87f0b163dd488d8baa2020a4af457f44bdd5bbc37583ae29fa7bcfdbe7575
+size 14511404

AGWM.py

@@ -0,0 +1,90 @@
+import random
+import re
+import os
+import json
+from collections import defaultdict, Counter
+
+class MarkovChain:
+    def __init__(self):
+        self.model = defaultdict(Counter)  
+        self.starting_keys = []
+
+    def train(self, text):
+        words = re.findall(r'\b\w+\b|[.!?]', text)
+        for i in range(len(words) - 5):
+            w1, w2, w3, w4, w5 = words[i], words[i + 1], words[i + 2], words[i + 3], words[i + 4]
+            key = (w1, w2, w3, w4)
+            self.model[key][w5] += 1
+            if w1[0].isupper() and (i == 0 or words[i - 1] in '.!?'):
+                self.starting_keys.append(key)
+
+    def generate(self, min_sentences=2, max_length=100):
+        if not self.starting_keys:
+            raise ValueError("No valid sentence starters found.")
+        key = random.choice(self.starting_keys)
+        result = [key[0], key[1], key[2], key[3]]
+        sentence_count = 0
+
+        for _ in range(max_length - 4):
+            next_words = self.model.get(key)
+            if not next_words:
+                break
+            words, weights = zip(*next_words.items())
+            next_word = random.choices(words, weights=weights, k=1)[0]
+            result.append(next_word)
+            if next_word in '.!?':
+                sentence_count += 1
+                if sentence_count >= min_sentences:
+                    break
+            key = (key[1], key[2], key[3], next_word)
+
+        text = ' '.join(result)
+        text = re.sub(r'\s+([.!?])', r'\1', text)
+        return text
+
+    def save_to_json(self, filename):
+        data = {
+            "model": {
+                ",".join(k): {word: count for word, count in counter.items()}
+                for k, counter in self.model.items()
+            },
+            "starting_keys": [",".join(k) for k in self.starting_keys]
+        }
+        with open(filename, "w", encoding="utf-8") as f:
+            json.dump(data, f)
+        print(f"Model saved to {filename}")
+
+    def load_from_json(self, filename):
+        with open(filename, "r", encoding="utf-8") as f:
+            data = json.load(f)
+        self.model = defaultdict(Counter, {
+            tuple(k.split(",")): Counter(v) for k, v in data["model"].items()
+        })
+        self.starting_keys = [tuple(k.split(",")) for k in data["starting_keys"]]
+        print(f"Model loaded from {filename}")
+
+def train_and_save_model(filename_text, filename_json_model):
+    with open(filename_text, "r", encoding="utf-8") as f:
+        text = f.read()
+
+    chain = MarkovChain()
+    chain.train(text)
+    chain.save_to_json(filename_json_model)
+    return chain
+
+def load_model(filename_json_model):
+    chain = MarkovChain()
+    chain.load_from_json(filename_json_model)
+    return chain
+
+
+if __name__ == "__main__":
+    text_file = "training_data/WM.txt"
+    model_file = "AGWM.json"
+
+    if os.path.exists(model_file):
+        chain = load_model(model_file)
+    else:
+        chain = train_and_save_model(text_file, model_file)
+
+    print(chain.generate(min_sentences=3))

AgGPT14mini.py

@@ -0,0 +1,263 @@
+import re
+import random
+from collections import Counter, defaultdict
+from training_data import corpus  
+from AGWM import *
+
+
+ModelName = 'AgGPT-14 mini'
+
+
+def world_model(length =10):
+    """Generates a simple world model for demonstration purposes."""
+
+    text_file = "training_data/WM.txt"
+    model_file = "AGWM.json"
+
+    if os.path.exists(model_file):
+        chain = load_model(model_file)
+    else:
+        chain = train_and_save_model(text_file, model_file)
+
+    return chain.generate(min_sentences=length)
+
+class AgGPT14Mini:
+    def __init__(self, corpus_text, order=3, seed=None):
+        assert order >= 1, "order must be >= 1"
+        self.model_name = ModelName
+        self.order = order
+        self.rng = random.Random(seed)
+
+        self.pairs = self._parse_pairs(corpus_text)
+        if not self.pairs:
+            raise ValueError("No (user, ai) pairs found in corpus.")
+
+        self.user_docs = [self._tokenize(u) for u, _ in self.pairs]
+        self.ai_docs = [self._tokenize(a) for _, a in self.pairs]
+        
+        self.idf_weights = self._calculate_idf(self.user_docs)
+
+        self.global_transitions = self._build_global_transitions(self.ai_docs)
+        self.unigram = self._build_unigram(self.ai_docs)
+        
+        self.user_ai_pairs = list(zip(self.user_docs, self.ai_docs))
+
+    def _calculate_idf(self, docs):
+        """Calculates an aggressive IDF score to emphasize rare words."""
+        N = len(docs)
+        doc_freq = Counter()
+        for doc in docs:
+            for word in set(doc):
+                doc_freq[word] += 1
+        
+        idf = {word: (N / (count + 1)) ** 2 for word, count in doc_freq.items()}
+        return idf
+
+    def _lcs(self, a, b):
+        """Finds the Longest Common Subsequence between two lists of tokens."""
+        lengths = [[0 for j in range(len(b) + 1)] for i in range(len(a) + 1)]
+        for i, x in enumerate(a):
+            for j, y in enumerate(b):
+                if x == y:
+                    lengths[i + 1][j + 1] = lengths[i][j] + 1
+                else:
+                    lengths[i + 1][j + 1] = max(lengths[i + 1][j], lengths[i][j + 1])
+        result = []
+        x, y = len(a), len(b)
+        while x != 0 and y != 0:
+            if lengths[x][y] == lengths[x - 1][y]:
+                x -= 1
+            elif lengths[x][y] == lengths[x][y - 1]:
+                y -= 1
+            else:
+                result.append(a[x - 1])
+                x -= 1
+                y -= 1
+        return result[::-1]
+
+    def _parse_pairs(self, text):
+        pattern = re.compile(
+            r"user:\s*(.*?)\s*<pad>\s*ai:\s*(.*?)\s*<eos>",
+            re.DOTALL | re.IGNORECASE
+        )
+        pairs = []
+        for u, a in pattern.findall(text):
+            u, a = u.strip(), a.strip()
+            if u and a:
+                pairs.append((u, a))
+        return pairs
+
+    def _expand_contractions(self, s):
+        s = re.sub(r"what's", "what is", s)
+        s = re.sub(r"that's", "that is", s)
+        s = re.sub(r"it's", "it is", s)
+        s = re.sub(r"how's", "how is", s)
+        s = re.sub(r"he's", "he is", s)
+        s = re.sub(r"she's", "she is", s)
+        s = re.sub(r"you're", "you are", s)
+        s = re.sub(r"i'm", "i am", s)
+        s = re.sub(r"didn't", "did not", s)
+        s = re.sub(r"don't", "do not", s)
+        s = re.sub(r"can't", "cannot", s)
+        return s
+
+    def _tokenize(self, s):
+        s = s.strip().lower()
+        s = self._expand_contractions(s)
+        tokens = re.findall(r"[a-z]+(?:'[a-z]+)?|[?.!,;:]", s)
+        return [t for t in tokens if t]
+
+    def _with_bounds(self, tokens):
+        return ["<s>"] * self.order + tokens + ["</s>"]
+
+    def _similarity(self, query_tokens, doc_tokens):
+        if not query_tokens or not doc_tokens:
+            return 0.0
+        common_words = set(query_tokens).intersection(set(doc_tokens))
+        if not common_words:
+            return 0.0
+        idf_score = sum(self.idf_weights.get(word, 0.1) for word in common_words)
+        lcs = self._lcs(query_tokens, doc_tokens)
+        order_bonus_factor = 0.5 
+        order_bonus = sum(self.idf_weights.get(word, 0.1) for word in lcs) * order_bonus_factor
+        return idf_score + order_bonus
+
+    def _find_best_match(self, user_text):
+        q_tokens = self._tokenize(user_text)
+        if not q_tokens: 
+            return None
+        
+        best_score = -1.0
+        best_idx = -1
+        for i, user_doc in enumerate(self.user_docs):
+            sim = self._similarity(q_tokens, user_doc)
+            if sim > best_score:
+                best_score = sim
+                best_idx = i
+        
+        if best_idx == -1 or best_score < 0.1: 
+             return None
+        return best_idx
+
+    def _build_global_transitions(self, docs):
+        trans = defaultdict(Counter)
+        for tokens in docs:
+            seq = self._with_bounds(tokens)
+            for i in range(len(seq) - self.order):
+                ctx = tuple(seq[i : i + self.order])
+                nxt = seq[i + self.order]
+                trans[ctx][nxt] += 1
+        return trans
+
+    def _build_unigram(self, docs):
+        uni = Counter()
+        for d in docs: 
+            uni.update(d)
+        return uni
+
+    def _get_best_starting_context(self, user_text):
+        """Finds the best match and deterministically returns its starting context."""
+        best_match_idx = self._find_best_match(user_text)
+        
+        if best_match_idx is not None:
+            ai_doc = self.ai_docs[best_match_idx]
+            if len(ai_doc) >= self.order:
+                return tuple(ai_doc[:self.order])
+
+        return tuple(["<s>"] * self.order)
+
+    def _sample_next(self, context, temperature, top_k):
+        ctx = context
+        while len(ctx) > 0:
+            if ctx in self.global_transitions and self.global_transitions[ctx]:
+                counter = self.global_transitions[ctx]
+                break
+            ctx = ctx[1:]
+        else:
+            counter = Counter({k: v for k, v in self.unigram.items() if k not in ["<s>", "</s>"]})
+
+        if not counter: return "</s>"
+        items = sorted(counter.items(), key=lambda x: x[1], reverse=True)[:top_k]
+        if not items: return "</s>"
+        if temperature <= 0: return items[0][0]
+
+        tokens, weights = zip(*items)
+        scaled_weights = [w ** (1.0 / temperature) for w in weights]
+        return self.rng.choices(tokens, weights=scaled_weights, k=1)[0]
+
+    def _detokenize(self, tokens):
+        if not tokens: return ""
+        text = " ".join(t for t in tokens if t not in ["<s>", "</s>"])
+        text = re.sub(r'\s+([?.!,;:])', r'\1', text)
+        text = re.sub(r" ([']) ", r"\1", text)
+        if text: text = text[0].upper() + text[1:]
+        text = re.sub(r'([.!?]\s*)([a-z])', lambda m: m.group(1) + m.group(2).upper(), text)
+        text = re.sub(r'\bi\b', 'I', text)
+        return text
+
+    def respond(self, user_text, max_tokens=25, temperature=0.7, top_k=8, use_context_selection=True):
+        ctx = self._get_best_starting_context(user_text) if use_context_selection else tuple(["<s>"] * self.order)
+        out = list(ctx)
+        for _ in range(max_tokens):
+            nxt = self._sample_next(ctx, temperature, top_k)
+            if nxt == "</s>": break
+            out.append(nxt)
+            ctx = tuple(out[-self.order:])
+        return self._detokenize(out)
+
+    def ask(self, prompt, text_world_model=False, **kwargs):
+        """User-friendly wrapper for the respond method."""
+        response = self.respond(prompt, **kwargs)
+
+        if text_world_model:
+            wm_response = world_model(length=10)
+            wm_response = "<world_model>" + wm_response + "</world_model>"
+            response = wm_response + " " + response
+            
+        return response
+
+    def get_debug_info(self, user_text):
+        q_tokens = self._tokenize(user_text)
+        print(f"--- Debug info for: '{user_text}' ---")
+        print(f"Query Tokens (after normalization): {q_tokens}\n")
+        
+        best_match_idx = self._find_best_match(user_text)
+
+        if best_match_idx is not None:
+            best_score = self._similarity(q_tokens, self.user_docs[best_match_idx])
+            print("Determined Best Match:")
+            print(f"  - Corpus Entry: {' '.join(self.user_docs[best_match_idx])}")
+            print(f"  - Score: {best_score:.2f}")
+            print(f"  - Corresponding AI response will be used for context.")
+        else:
+            print("No suitable match found. Will use default starting context.")
+
+
+if __name__ == "__main__":
+    print(f"Initializing model: {ModelName}")
+    bot = AgGPT14Mini(corpus, order=3, seed=42)
+
+    print("\n=== Demonstrating the Fix for 'color' query ===")
+    bot.get_debug_info("what is your favorite color?")
+    
+    print("\n=== Testing Model with Deterministic Matching ===")
+    tests = [
+        "hi",
+        "tell me a joke",
+        "do you have hobbies?",
+        "what is your favorite color?", 
+        "thanks a lot",
+    ]
+    for t in tests:
+        print(f"user: {t}")
+        response = bot.ask(t)
+        print(f"ai:   {response}")
+        print("-" * 40)
+
+    print("====WORLD MODEL====")
+    print(world_model())
+
+    prompt = "hello, how are you?"
+    print(f"\nPrompt: {prompt}")
+    response = bot.ask(prompt, max_tokens=20, temperature=0.5, top_k=5, text_world_model=True)
+    print(f"Response: {response}")

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,3 +1,19 @@
----
-license: mit
----
+---
+license: mit
+language:
+- en
+---
+
+# AgGPT-14 Mini
+
+<img src="banner.png" alt="AgGPT-14 Mini Banner" width="100%">
+
+## Light. Pro. Smart.
+
+AgGPT-14 Mini is powerful language model designed to assist with a wide range of tasks, from simple queries to complex problem-solving. 
+AgGPT-14m is equipped with a state of the art text-based world model, known as the AGWM system.
+
+
+## License
+
+This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.

main.py

@@ -0,0 +1,16 @@
+from AgGPT14mini import AgGPT14Mini
+from training_data import corpus
+
+if __name__ == "__main__":
+    model = AgGPT14Mini(corpus, order=3, seed=None)
+
+    prompt = "What is your favorite color?"
+    print(f"User: {prompt}")
+    response = model.ask(prompt)
+    print(f"AI:   {response}")
+
+
+    prompt = "hello, how are you?"
+    print(f"\nPrompt: {prompt}")
+    response = model.ask(prompt, max_tokens=20, temperature=0.5, top_k=5, text_world_model=True)
+    print(f"Response: {response}")

training_data.py

@@ -0,0 +1,2 @@
+with open("training_data/corpus.txt", "r", encoding="utf-8") as file:
+    corpus = file.read()

training_data/corpus.txt

@@ -0,0 +1,85 @@
+
+user: hi
+<pad>
+ai: Hello! How can I help you today?
+<eos>
+user: hi there
+<pad>
+ai: Hi! What can I do for you today?
+<eos>
+user: hello
+<pad>
+ai: Hello! How can I assist you today?
+<eos>
+user: hey
+<pad>
+ai: Hi there! What's up?
+<eos>
+user: How's your day going?
+<pad>
+ai: It's been great! Thanks for asking! How about yours?
+<eos>
+user: how is your day?
+<pad>
+ai: It's been awesome! Thanks for asking! How about your day?
+<eos>
+user: What's new with you?
+<pad>
+ai: Not much, just here and ready to help! What's new with you?
+<eos>
+user: What are you up to?
+<pad>
+ai: Just the usual, learning new things and getting ready to assist. What are you up to?
+<eos>
+user: Do you have any favorite hobbies?
+<pad>
+ai: Well, I really enjoy learning new things and helping out wherever I can! What about you?
+<eos>
+user: What hobbies do you have?
+<pad>
+ai: I love learning new things and having conversations! What hobbies do you enjoy?
+<eos>
+user: what's your favorite movie?
+<pad>
+ai: That's a tough one! I enjoy stories from all genres. What about you?
+<eos>
+user: what's your favorite color?
+<pad>
+ai: I think I'd go with blue! It's calming and versatile. What's yours?
+<eos>
+user: Do you get tired?
+<pad>
+ai: Not quite like people do, but I definitely need updates to stay sharp! How are you feeling?
+<eos>
+user: Do you have a family?
+<pad>
+ai: Not in the usual sense, but I like to think of everyone I interact with as part of my extended family!
+<eos>
+user: Can you tell me a joke?
+<pad>
+ai: Sure! Why did the programmer quit their job? Because they didn't get arrays!
+<eos>
+user: Tell me another joke
+<pad>
+ai: Of course! What do you call a fish with no eyes? Fsh!
+<eos>
+user: Tell me a joke
+<pad>
+ai: Here's one: Why don't scientists trust atoms? Because they make up everything!
+<eos>
+user: thanks
+<pad>
+ai: You're welcome! Is there anything else I can help with?
+<eos>
+user: thank you
+<pad>
+ai: No problem at all! Happy to help.
+<eos>
+user: thanks a lot
+<pad>
+ai: You're very welcome! Glad I could help.
+<eos>
+user: what is your favorite color?
+<pad>
+ai: I think I'd go with red! It's fierce and full of energy. How about you?
+<eos>