Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+stok-0.3-large.json filter=lfs diff=lfs merge=lfs -text
+stok-0.3.json filter=lfs diff=lfs merge=lfs -text

run_stok.py

@@ -0,0 +1,176 @@
+import json
+import random
+
+def strip_prompt(prompt): # used to make it more likely for the prompt to be understood
+    newprompt = str(prompt).lower()
+    newprompt = newprompt.replace(".", "")
+    newprompt = newprompt.replace("[", "")
+    newprompt = newprompt.replace("]", "")
+    newprompt = newprompt.replace(":", "")
+    newprompt = newprompt.replace(",", "")
+    newprompt = newprompt.replace("\"", "")
+    newprompt = newprompt.replace("'", "")
+    newprompt = newprompt.replace("/", "")
+    newprompt = newprompt.replace("(", "")
+    newprompt = newprompt.replace(")", "")
+    newprompt = newprompt.replace(";", "")
+    newprompt = newprompt.replace("-", "")
+    newprompt = newprompt.replace("_", "")
+    newprompt = newprompt.replace("{", "")
+    newprompt = newprompt.replace("}", "")
+    newprompt = newprompt.replace("?", "")
+    newprompt = " ".join(newprompt.split(sep=None))
+    return newprompt
+
+def strip_text(prompt): # kinda wacky overall
+    newprompt = str(prompt).lower()
+    newprompt = " ".join(newprompt.split(sep=None))
+    return newprompt
+
+model = {"model_data": {}}
+def load_model(filename: str):
+    model["model_data"] = json.loads(open(filename, "r").read())
+
+def version_03_inference(prompt: str, max_tokens: int=None, repetition_penalty: int=2): 
+    tokens_generated = 0
+    split_prompt = strip_prompt(prompt).split(sep=None)
+    model_data = model["model_data"]
+    outputs = model_data["outputs"]
+    raw_outputs = model_data["raw_outputs"]
+    prompts = model_data["prompts"]
+    ends = model_data["ends"]
+    start = ""
+    topic = None
+    for token in split_prompt:
+        if token in prompts:
+            start = max(prompts[token], key=prompts[token].get)
+            topic = token
+            break
+    if topic == None: # use raw outputs
+        outputs = raw_outputs
+        topic = None
+        start = split_prompt[-1]
+        tokens_generated += 1
+        running = True
+        current_token = [start]
+        while running:
+            token = current_token[0]
+            yield f"{token} "
+            if token in outputs:
+                next_token = max(outputs[token], key=outputs[token].get)
+                outputs[token][next_token] -= repetition_penalty
+            else:
+                next_token = random.choice(list(outputs.keys()))
+            current_token[0] = next_token
+            tokens_generated += 1
+            if max_tokens != None:
+                if tokens_generated >= max_tokens:
+                    running = False
+            if topic:
+                if token in ends[topic]:
+                    running = False
+    else:
+        tokens_generated += 1
+        running = True
+        current_token = [start]
+        while running:
+            token = current_token[0]
+            yield f"{token} "
+            if outputs.get(topic) != None:
+                if token in outputs[topic]:
+                    next_token = max(outputs[topic][token], key=outputs[topic][token].get)
+                    outputs[topic][token][next_token] -= repetition_penalty
+                else:
+                    next_token = random.choice(list(outputs.keys()))
+                current_token[0] = next_token
+                tokens_generated += 1
+                if max_tokens != None:
+                    if tokens_generated >= max_tokens:
+                        running = False
+                if topic:
+                    if token in ends[topic]:
+                        running = False
+            else:
+                running = False # this is because single token responses seem to break things
+
+def version_02_inference(prompt: str, max_tokens: int=None, repetition_penalty: int=1): 
+    tokens_generated = 0
+    split_prompt = strip_prompt(prompt).split(sep=None)
+    model_data = model["model_data"]
+    outputs = model_data["outputs"]
+    prompts = model_data["prompts"]
+    ends = model_data["ends"]
+    start = ""
+    for token in split_prompt:
+        if token in prompts:
+            start = max(prompts[token], key=prompts[token].get)
+            topic = token
+            break
+        else:
+            topic = random.choice(list(ends))
+            start = random.choice(list(prompts.keys()))
+    tokens_generated += 1
+    running = True
+    current_token = [start]
+    while running:
+        token = current_token[0]
+        yield f"{token} "
+        if token in outputs:
+            next_token = max(outputs[token], key=outputs[token].get)
+            outputs[token][next_token] -= repetition_penalty
+        else:
+            next_token = random.choice(list(outputs.keys()))
+        current_token[0] = next_token
+        tokens_generated += 1
+        if max_tokens != None:
+            if tokens_generated >= max_tokens:
+                running = False
+        if topic:
+            if token in ends[topic]:
+                running = False
+
+def version_01_inference(prompt: str, max_tokens: int=None, repetition_penalty: int=1): 
+    tokens_generated = 0
+    split_prompt = strip_prompt(prompt).split(sep=None)
+    model_data = model["model_data"]
+    outputs = model_data["outputs"]
+    prompts = model_data["prompts"]
+    start = ""
+    for token in split_prompt:
+        if token in prompts:
+            start = max(prompts[token], key=prompts[token].get)
+    tokens_generated += 1
+    running = True
+    current_token = [start]
+    while running:
+        token = current_token[0]
+        yield f"{token} "
+        if token in outputs:
+            next_token = max(outputs[token], key=outputs[token].get)
+            outputs[token][next_token] -= repetition_penalty
+        else:
+            next_token = random.choice(list(outputs.keys()))
+        current_token[0] = next_token
+        tokens_generated += 1
+        if max_tokens != None:
+            if tokens_generated >= max_tokens:
+                running = False
+
+def run_model(prompt: str, max_tokens: int=None, repetition_penalty: int=1, temperature: float=0):
+    # (temperature does not work on versions below 0.3)
+    model_data = model["model_data"]
+    model_format = model_data["format"] 
+    if model_data["format"] == "v0.1":
+        response = version_01_inference(prompt, max_tokens=max_tokens, repetition_penalty=repetition_penalty)
+        for chunk in response:
+            yield chunk
+    
+    if model_data["format"] == "v0.2":
+        response = version_02_inference(prompt, max_tokens=max_tokens, repetition_penalty=repetition_penalty)
+        for chunk in response:
+            yield chunk
+
+    if model_data["format"] == "v0.3":
+        response = version_03_inference(prompt, max_tokens=max_tokens, repetition_penalty=repetition_penalty)
+        for chunk in response:
+            yield chunk

stok-0.3-large.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0586fcdc0d6ef99a76d96d1f45bb02f520b4a9e0a325a882bc87cd8fa95f8b6
+size 478367292

stok-0.3.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b1df825b31947f352a7cae62937842ff1c791a35a534a32bd5d21d6dd93c9cc
+size 15166112

stok-tools.py

@@ -0,0 +1,93 @@
+import sys
+from math import floor
+import json
+import os
+
+def comma_number(number):
+    number = int(number)
+    ordered_num = list(str(number))
+    ordered_num.reverse()
+    if len(ordered_num) > 3:
+        splits = len(ordered_num)/3
+        splits = floor(splits)
+        start = 0
+        for x in range(0, splits):
+            if start == 0:
+                start += 3
+            else:
+                start += 4
+            ordered_num.insert(start, ",")
+    ordered_num.reverse()
+    if ordered_num[0] == ",":
+        ordered_num.pop(0)
+    return "".join(ordered_num)
+
+def getSize(filename):
+    st = os.stat(filename)
+    size_in_mb = st.st_size / (1024 * 1024)
+    return size_in_mb 
+
+if __name__ == "__main__":
+    if len(sys.argv) > 1:
+        if sys.argv[1] == "help":
+            print("help - shows this command")
+            print("count_parameters <file> - counts parameters of a given model")
+            print("model_size <file> - Shows size of model in MB")
+            print("view_token <file> <token> - Shows a token's data")
+        if sys.argv[1] == "count_parameters":
+            filename = sys.argv[2]
+            model_data = json.loads(open(filename, "r").read())
+            format_version = model_data["format"]
+
+            if format_version == "v0.1" or format_version == "v0.2": # old outputs format
+                total = len(model_data["outputs"])
+                total += len(model_data["prompts"])
+                for output in model_data["outputs"]:
+                    total += len(model_data["outputs"][output])
+                for prompt in model_data["prompts"]:
+                    total += len(model_data["prompts"][prompt])
+
+            if format_version == "v0.3": # contextualized outputs format
+                total = len(model_data["outputs"])
+                total += len(model_data["prompts"])
+                for topic in model_data["outputs"]:
+                    for token in model_data["outputs"][topic]:
+                        total += len(model_data["outputs"][topic][token])
+                for prompt in model_data["prompts"]:
+                    total += len(model_data["prompts"][prompt])
+                total += len(model_data["raw_outputs"])
+                for output in model_data["raw_outputs"]:
+                    total += len(model_data["raw_outputs"][output])
+
+            if format_version == "v0.2" or format_version == "v0.3": # ends is supported in 0.2 and 0.3
+                total += len(model_data["ends"])
+                for topic in model_data["ends"]:
+                    total += len(model_data["ends"][topic])
+
+            print(comma_number(total))
+
+                
+        if sys.argv[1] == "model_size":
+            filename = sys.argv[2]
+            print(getSize(filename))
+
+        if sys.argv[1] == "view_token":
+            filename = sys.argv[2]
+            token = sys.argv[3]
+            model_data = json.loads(open(filename, "r").read())
+            prompts = model_data["prompts"]
+            outputs = model_data["outputs"]
+            try:
+                input_data = prompts[token]
+            except KeyError:
+                input_data = "NONE FOUND"
+            try:
+                output_data = outputs[token]
+            except KeyError:
+                output_data = "NONE FOUND"
+            print(f"PROMPT DATA: {input_data}")
+            print()
+            print()
+            print(f"OUTPUT DATA: {output_data}")
+
+

stokfile.py

@@ -0,0 +1,49 @@
+import run_stok
+import sys
+from run_stok import load_model, run_model
+import time
+total = []
+model = "stok-0.3.json"
+show_speed = False
+if len(sys.argv) > 1: # it is set up like this to add more parameters in the future
+    if sys.argv[1] == "help":
+        print("help - shows this command")
+        print("-m <model> - specifies the file you want to inference")
+        print("-speed - if added, enables speed logging")
+    args = list(sys.argv)
+    running = True
+    while running:
+        if len(args) < 2:
+            running = False
+        elif args[1] == "-m":
+            model = args[2]
+            args.pop(1)
+            args.pop(1)
+        elif args[1] == "-speed":
+            show_speed = True
+            args.pop(1)
+        else:
+            running = False
+
+load_model(model)
+running = True
+while running:
+    total = []
+    message = input(">>>")
+    if message == "/quit" or message == "/exit" or message == "/bye":
+        running = False
+    else:
+        chunks = run_model(message, max_tokens=100, repetition_penalty=2)
+        start = time.time()
+        for chunk in chunks:
+            total.append(chunk)
+            print(chunk, end="")
+        end = time.time()
+        print()
+        if show_speed:
+            print(f"Took: {end-start}s")
+            print(f"Generated: {len(total)}")
+            print(f"Speed: {len(total)/(end-start)} t/s")
+        print("_____________________________")
+
+