add GGUF

README.md

@@ -78,6 +78,53 @@ generated_ids = model.generate(
 response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
 print(response)
 ```
+## **Implementation for GGUF**
+
+```python
+%pip install --quiet https://github.com/abetlen/llama-cpp-python/releases/download/v0.2.90-cu124/llama_cpp_python-0.2.90-cp310-cp310-linux_x86_64.whl
+import transformers
+import torch
+from llama_cpp import Llama
+import os
+import requests
+
+local_dir = "your local dir"
+directory_path = r'{local_dir}/Pathumma-llm-text-1.0.0'
+if not os.path.exists(directory_path):
+    os.mkdir(directory_path)
+
+if not os.path.exists(f'{local_dir}/Pathumma-llm-text-1.0.0/Pathumma-llm-it-7b-Q4_K_M.gguf'):
+    !wget -O f'{local_dir}/Pathumma-llm-text-1.0.0/Pathumma-llm-it-7b-Q4_K_M.gguf' "https://huggingface.co/nectec/Pathumma-llm-text-1.0.0/resolve/main/Pathumma-llm-it-7b-Q4_K_M.gguf?download=true"
+
+# Initialize the Llama model
+llm = Llama(model_path=f'{local_dir}/Pathumma-llm-text-1.0.0/Pathumma-llm-it-7b-Q4_K_M.gguf', n_gpu_layers=-1, n_ctx=8192,verbose=False)
+tokenizer = transformers.AutoTokenizer.from_pretrained("nectec/Pathumma-llm-text-1.0.0")
+
+memory = [{'content': 'You are Pathumma LLM, created by NECTEC (National Electronics and Computer Technology Center). Your are a helpful assistant.', 'role': 'system'},]
+
+def generate(instuction,memory=memory):
+    memory.append({'content': instuction, 'role': 'user'})
+    p = tokenizer.apply_chat_template(
+        memory,
+        tokenize=False,
+        add_generation_prompt=True
+    )
+    response = llm(
+        p,
+        max_tokens=2048,
+        temperature=0.2,
+        top_p=0.95,
+        repeat_penalty=1.1,
+        top_k=40,
+        min_p=0.05,
+        stop=["<|im_end|>"]
+    )
+    output = response['choices'][0]['text']
+    memory.append({'content': output, 'role': 'assistant'})
+    return output
+
+print(generate("คุณคือใคร"))
+```
 
 ## **Evaluation Performance**
 | Model | m3exam | thaiexam | xcopa | belebele | xnli | thaisentiment | XL sum | flores200 eng > th | flores200 th > eng | iapp | AVG(NLU) | AVG(MC) | AVG(NLG) |