Update README.md

README.md

@@ -15,6 +15,81 @@ This model is based on work reported in https://doi.org/10.1002/advs.202306724.
 
 This repository includes both, Hugging Face transformers and GGUF files (in different versions, the q5_K_M is recommended). 
 
+#### Hugging Face transformers files: Loading and inference
+
+```
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from accelerate import infer_auto_device_map
+
+model = AutoModelForCausalLM.from_pretrained(
+    model_name,
+    trust_remote_code=True,
+    device_map="auto", #device_map="cuda:0",
+    torch_dtype=  torch.bfloat16,
+    # use_flash_attention_2=True,
+)
+
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+```
+Chat template
+```
+messages = [
+    {"role": "system", "content": "You are a friendly materials scientist."},
+    {"role": "user", "content": "What is the strongest spider silk material?"},
+    {"role": "assistant", "content": "Sample response."},
+]
+prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+```
+
+'<|system|>\nYou are a friendly materials scientist.</s>\n<|user|>\nWhat is the strongest spider silk material?</s>\n<|assistant|>\nSample response.</s>\n<|assistant|>\n'
+
+```
+device='cuda'
+def generate_response (text_input="Biological materials offer amazing possibilities, such as",
+                      num_return_sequences=1,
+                      temperature=1.,  
+                      max_new_tokens=127,
+                      num_beams=1,
+                      top_k = 50,
+                      top_p =0.9,repetition_penalty=1.,eos_token_id=2,verbatim=False,
+                      exponential_decay_length_penalty_fac=None,
+                      ):
+
+    inputs = tokenizer.encode(text_input,  add_special_tokens  =False,  return_tensors ='pt')
+    if verbatim:
+        print ("Length of input, tokenized: ", inputs.shape, inputs)
+    with torch.no_grad():
+          outputs = model.generate(input_ids=inputs.to(device), 
+                                   max_new_tokens=max_new_tokens,
+                                   temperature=temperature, #value used to modulate the next token probabilities.
+                                   num_beams=num_beams,
+                                   top_k = top_k,
+                                   top_p =top_p,
+                                   num_return_sequences = num_return_sequences, eos_token_id=eos_token_id,
+                                   do_sample =True, 
+                                   repetition_penalty=repetition_penalty,
+                                  )
+    return tokenizer.batch_decode(outputs[:,inputs.shape[1]:].detach().cpu().numpy(), skip_special_tokens=True)
+
+```
+Then:
+```
+messages = [
+    {"role": "system", "content": "You are a friendly materials scientist."},
+    {"role": "user", "content": "What is the strongest spider silk material?"},
+]
+prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+
+output_text=generate_response (text_input=prompt, eos_token_id=eos_token,
+                                num_return_sequences=1, repetition_penalty=1.,
+                                top_p=0.9, top_k=512,  
+                                temperature=0.1,max_new_tokens=512, verbatim=False,
+                               )
+print (output_text)
+```
+
+#### GGUF files: Loading and inference
+
 ```
 from llama_cpp import Llama
 
@@ -80,4 +155,4 @@ deltat=time.time() - start_time
 print("--- %s seconds ---" % deltat)
 toked=tokenizer(res)
 print ("Tokens per second (generation): ", len (toked['input_ids'])/deltat)
-```
+```