Upload folder using huggingface_hub

.gitattributes

@@ -33,5 +33,3 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
-Meta-Llama-3-8B-Instruct-Q4_K_M.gguf filter=lfs diff=lfs merge=lfs -text
-Meta-Llama-3-8B-Instruct-Q5_K_M.gguf filter=lfs diff=lfs merge=lfs -text

README.md

@@ -185,6 +185,29 @@ extra_gated_fields:
   By clicking Submit below I accept the terms of the license and acknowledge that the information I provide will be collected stored processed and shared in accordance with the Meta Privacy Policy: checkbox
 extra_gated_description: The information you provide will be collected, stored, processed and shared in accordance with the [Meta Privacy Policy](https://www.facebook.com/privacy/policy/).
 extra_gated_button_content: Submit
+widget:
+  - example_title: Hello
+    messages:
+    - role: user
+      content: Hey my name is Julien! How are you?
+  - example_title: Winter holidays
+    messages:
+    - role: system
+      content: You are a helpful and honest assistant. Please, respond concisely and truthfully.
+    - role: user
+      content: Can you recommend a good destination for Winter holidays?
+  - example_title: Programming assistant
+    messages:
+    - role: system
+      content: You are a helpful and honest code and programming assistant. Please, respond concisely and truthfully.
+    - role: user
+      content: Write a function that computes the nth fibonacci number.
+inference:
+  parameters:
+    max_new_tokens: 300
+    stop:
+    - <|end_of_text|>
+    - <|eot_id|>
 ---
 
 ## Model Details
@@ -273,7 +296,9 @@ This repository contains two versions of Meta-Llama-3-8B-Instruct, for use with
 
 ### Use with transformers
 
-See the snippet below for usage with Transformers:
+You can run conversational inference using the Transformers pipeline abstraction, or by leveraging the Auto classes with the `generate()` function. Let's see examples of both.
+
+#### Transformers pipeline
 
 ```python
 import transformers
@@ -285,7 +310,7 @@ pipeline = transformers.pipeline(
     "text-generation",
     model=model_id,
     model_kwargs={"torch_dtype": torch.bfloat16},
-    device="cuda",
+    device_map="auto",
 )
 
 messages = [
@@ -300,8 +325,8 @@ prompt = pipeline.tokenizer.apply_chat_template(
 )
 
 terminators = [
-    tokenizer.eos_token_id,
-    tokenizer.convert_tokens_to_ids("<|eot_id|>")
+    pipeline.tokenizer.eos_token_id,
+    pipeline.tokenizer.convert_tokens_to_ids("<|eot_id|>")
 ]
 
 outputs = pipeline(
@@ -315,6 +340,50 @@ outputs = pipeline(
 print(outputs[0]["generated_text"][len(prompt):])
 ```
 
+#### Transformers AutoModelForCausalLM
+
+```python
+from transformers import AutoTokenizer, AutoModelForCausalLM
+import torch
+
+model_id = "meta-llama/Meta-Llama-3-8B-Instruct"
+
+tokenizer = AutoTokenizer.from_pretrained(model_id)
+model = AutoModelForCausalLM.from_pretrained(
+    model_id,
+    torch_dtype=torch.bfloat16,
+    device_map="auto",
+)
+
+messages = [
+    {"role": "system", "content": "You are a pirate chatbot who always responds in pirate speak!"},
+    {"role": "user", "content": "Who are you?"},
+]
+
+input_ids = tokenizer.apply_chat_template(
+    messages,
+    add_generation_prompt=True,
+    return_tensors="pt"
+).to(model.device)
+
+terminators = [
+    tokenizer.eos_token_id,
+    tokenizer.convert_tokens_to_ids("<|eot_id|>")
+]
+
+outputs = model.generate(
+    input_ids,
+    max_new_tokens=256,
+    eos_token_id=terminators,
+    do_sample=True,
+    temperature=0.6,
+    top_p=0.9,
+)
+response = outputs[0][input_ids.shape[-1]:]
+print(tokenizer.decode(response, skip_special_tokens=True))
+```
+
+
 ### Use with `llama3`
 
 Please, follow the instructions in the [repository](https://github.com/meta-llama/llama3)

generation_config.json

@@ -1,6 +1,9 @@
 {
-  "_from_model_config": true,
   "bos_token_id": 128000,
   "eos_token_id": [128001, 128009],
+  "do_sample": true,
+  "temperature": 0.6,
+  "max_length": 4096,
+  "top_p": 0.9,
   "transformers_version": "4.40.0.dev0"
 }

tokenizer.json

@@ -2329,10 +2329,69 @@
     ]
   },
   "post_processor": {
-    "type": "ByteLevel",
-    "add_prefix_space": true,
-    "trim_offsets": false,
-    "use_regex": true
+    "type": "Sequence",
+    "processors": [
+      {
+        "type": "ByteLevel",
+        "add_prefix_space": true,
+        "trim_offsets": false,
+        "use_regex": true
+      },
+      {
+        "type": "TemplateProcessing",
+        "single": [
+          {
+            "SpecialToken": {
+              "id": "<|begin_of_text|>",
+              "type_id": 0
+            }
+          },
+          {
+            "Sequence": {
+              "id": "A",
+              "type_id": 0
+            }
+          }
+        ],
+        "pair": [
+          {
+            "SpecialToken": {
+              "id": "<|begin_of_text|>",
+              "type_id": 0
+            }
+          },
+          {
+            "Sequence": {
+              "id": "A",
+              "type_id": 0
+            }
+          },
+          {
+            "SpecialToken": {
+              "id": "<|begin_of_text|>",
+              "type_id": 1
+            }
+          },
+          {
+            "Sequence": {
+              "id": "B",
+              "type_id": 1
+            }
+          }
+        ],
+        "special_tokens": {
+          "<|begin_of_text|>": {
+            "id": "<|begin_of_text|>",
+            "ids": [
+              128000
+            ],
+            "tokens": [
+              "<|begin_of_text|>"
+            ]
+          }
+        }
+      }
+    ]
   },
   "decoder": {
     "type": "ByteLevel",

tokenizer_config.json

@@ -2050,7 +2050,7 @@
     }
   },
   "bos_token": "<|begin_of_text|>",
-  "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}{% else %}{{ eos_token }}{% endif %}",
+  "chat_template": "{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}{% endif %}",
   "clean_up_tokenization_spaces": true,
   "eos_token": "<|end_of_text|>",
   "model_input_names": [