Update README.md

README.md

@@ -13,36 +13,6 @@ datasets:
 * max. seq. length: 1024 tokens
 * code in code/
 
-## Inference
-```python
-from transformers import AutoModelForCausalLM, AutoTokenizer
-import torch
-
-modelpath="g-ronimo/phi-2-OpenHermes-2.5"
-
-model = AutoModelForCausalLM.from_pretrained(
-    modelpath,    
-    torch_dtype=torch.bfloat16,
-    device_map="auto",
-    # attn_implementation="flash_attention_2",
-)
-tokenizer = AutoTokenizer.from_pretrained(modelpath) 
-
-messages = [
-    {"role": "user", "content": "what does it mean to be successful?"},
-]
-        
-input_tokens = tokenizer.apply_chat_template(
-    messages, 
-    add_generation_prompt=True,
-    return_tensors="pt"
-).to("cuda")
-output_tokens = model.generate(input_tokens, max_new_tokens=500)
-output = tokenizer.decode(output_tokens[0])
-
-print(output)
-```
-
 ## Evals
 
 |                                   Model                                    |AGIEval|GPT4All|TruthfulQA|Bigbench|Average|
@@ -125,4 +95,32 @@ Average: 35.9%
 
 Average score: 45.3%
 
-Elapsed time: 01:24:18
+## Inference
+```python
+from transformers import AutoModelForCausalLM, AutoTokenizer
+import torch
+
+modelpath="g-ronimo/phi-2-OpenHermes-2.5"
+
+model = AutoModelForCausalLM.from_pretrained(
+    modelpath,    
+    torch_dtype=torch.bfloat16,
+    device_map="auto",
+    # attn_implementation="flash_attention_2",
+)
+tokenizer = AutoTokenizer.from_pretrained(modelpath) 
+
+messages = [
+    {"role": "user", "content": "what does it mean to be successful?"},
+]
+        
+input_tokens = tokenizer.apply_chat_template(
+    messages, 
+    add_generation_prompt=True,
+    return_tensors="pt"
+).to("cuda")
+output_tokens = model.generate(input_tokens, max_new_tokens=500)
+output = tokenizer.decode(output_tokens[0])
+
+print(output)
+```