Upload tensor_type_testing.py

tensor_type_testing.py

@@ -0,0 +1,128 @@
+# tensor_type_testing.py
+# Python 3.11.2
+
+import os
+import sys
+
+import numpy      as np
+import easy_llama as ez
+
+from typing import Union
+
+INPUT_TEXTS_AS_TEXT: list[str] = []
+
+for i in range(10):
+    with open(f'./inputs/{i}.txt', 'r') as file:
+        INPUT_TEXTS_AS_TEXT.append(file.read())
+
+BASELINE_MODEL_PATH = '/opt/workspace/gguf/Qwen2.5-14B-BF16.gguf'
+BASELINE_MODEL_FILENAME = os.path.basename(BASELINE_MODEL_PATH)
+QUANT_MODEL_DIR = '/opt/workspace/gguf/'
+QUANT_MODEL_FILES = [
+    'Qwen2.5-14B-Q2_K.gguf',
+    'Qwen2.5-14B-EQ2_K-FQ8_0-AQ8_0-OQ8_0.gguf',
+    'Qwen2.5-14B-EQ8_0-FQ2_K-AQ8_0-OQ8_0.gguf',
+    'Qwen2.5-14B-EQ8_0-FQ8_0-AQ2_K-OQ8_0.gguf',
+    'Qwen2.5-14B-EQ8_0-FQ8_0-AQ8_0-OQ2_K.gguf',
+    'Qwen2.5-14B-Q8_0.gguf'
+]
+
+def msd(a: np.ndarray, b: np.ndarray) -> np.floating:
+    return np.mean((a - b) ** 2)
+
+def tokenize_prompt(llama: ez.Llama, prompt: str) -> list[int]:
+    return llama.tokenize(
+        text_bytes=prompt.encode('utf-8', 'strict'),
+        add_special=True,
+        parse_special=False
+    )
+
+def eval_text(llama: ez.Llama, text_toks: list[int]) -> np.ndarray:
+    llama.reset()
+    logits = llama.eval(input_tokens=text_toks, logits_all=True)
+    return logits
+
+def load_llama(model_file: str) -> ez.Llama:
+    return ez.Llama(
+        path_model=model_file,
+        n_gpu_layers=10,
+        use_mmap=False,
+        use_mlock=False,
+        n_ctx=5120,
+        offload_kqv=True,
+        warmup=False,
+        verbose=False
+    )
+
+def get_model_results(model_path: str) -> list[np.ndarray]:
+    print('Load model...')
+    Llama = load_llama(model_path)
+    print('Evaluate prompts...')
+    results = [eval_text(Llama, prompt) for prompt in input_texts_as_tokens]
+    print('Unload model...')
+    Llama.free()
+    return results
+
+def main() -> int:
+
+    global input_texts_as_tokens
+
+    results: dict[str, list[Union[list[np.floating], np.floating]]] = {}
+
+    baseline_llama = load_llama(BASELINE_MODEL_PATH)
+    input_texts_as_tokens = [
+        tokenize_prompt(baseline_llama, text) for text in INPUT_TEXTS_AS_TEXT
+    ]
+    n_inputs = len(input_texts_as_tokens)
+    max_len_input = max(len(toks) for toks in input_texts_as_tokens)
+    min_len_input = min(len(toks) for toks in input_texts_as_tokens)
+    avg_len_input = sum(len(toks) for toks in input_texts_as_tokens) / n_inputs
+    n_input_tokens = sum(len(toks) for toks in input_texts_as_tokens)
+    print(f'          Number of input texts: {len(input_texts_as_tokens)}')
+    print(f'Shortest input length in tokens: {min_len_input}')
+    print(f' Longest input length in tokens: {max_len_input}')
+    print(f' Average input length in tokens: {avg_len_input}')
+    print(f'   Total number of input tokens: {n_input_tokens}')
+    print('-' * 80)
+    baseline_llama.free()
+
+    print(f'Evaluating baseline model {BASELINE_MODEL_FILENAME}...')
+    baseline_results = get_model_results(BASELINE_MODEL_PATH)
+
+    for quant_file in QUANT_MODEL_FILES:
+        quant_path = os.path.join(QUANT_MODEL_DIR, quant_file)
+        if not os.path.exists(quant_path):
+            print(f"Error: {quant_path} not found. Skipping.")
+            continue
+        
+        print('-' * 80)
+        print(f'Now processing: {quant_file}')
+        quant_results = get_model_results(quant_path)
+        
+        print(f'Compute MSD...')
+        deviations = [
+            msd(baseline_results[i], quant_results[i]) for i in range(len(quant_results))
+        ]
+        avg = np.mean(deviations)
+
+        results[quant_file] = [deviations, avg]
+
+        print(
+            f'Mean-Squared Deviation - '
+            f'{BASELINE_MODEL_FILENAME} vs. {os.path.basename(quant_path)}:'
+        )
+        for i in range(len(input_texts_as_tokens)):
+            print(f'-- Prompt {i}: {deviations[i]}')
+        print(f'Average MSD: {avg}')
+    
+    print('-' * 80)
+    print(f'Average Mean-Squared Deviation compared to {BASELINE_MODEL_FILENAME}:')
+    print('-' * 80)
+    for k, v in results.items():
+        print(f'{k:>60} -- {v[1]}')
+    print('-' * 80)
+
+    return 0
+
+if __name__ == '__main__':
+    sys.exit(main())