Update README.md

README.md

@@ -40,7 +40,16 @@ The model is trained for two epoches in the aforementioned data.
 
 ## Evaluation
 
-...
+Adapted models are evaluated on [ITA-Bench])(https://github.com/SapienzaNLP/ita-bench).
+
+| Model | MMLU (5-shots) | ARC-C (5-shots) | Hellaswag (0-shots) | IFEval (inst_level) |
+|------|-----|------|------|------|
+| Llama-3.1-SAVA | 56.9 | 42.3 | 58.1 | 62.3 |
+| Llama-3.1-LAPT | 58.5 | 47.9 | 62.4 | 67.3 |
+| Mistral-0.1-SAVA | 51.5 | 41.6 | 57.5 | 61.7 |
+| **Mistral-0.1-LAPT** | 52.9 | 39.9 | 58.4 | 60.0 |
+| Llama-3.1-Original | 47.4 | 43.1 | 57.9 | 66.8 |
+| Mistral-0.1-Original | 41.6 | 38.9 | 50.0 | 42.2 |
 
 ## Use with Transformers
 
@@ -52,17 +61,45 @@ Make sure to update your transformers installation via `pip install --upgrade tr
 import transformers
 import torch
 
-model_id = "SemanticAlignment/Mistral-v0.1-Italian-FVT-instruct"
+model_id = "SemanticAlignment/Mistral-v0.1-Italian-LAPT-instruct"
+
+tokenizer = AutoTokenizer.from_pretrained(model_name)
 
-pipeline = transformers.pipeline(
-    "text-generation", model=model_id, model_kwargs={"torch_dtype": torch.bfloat16}, device_map="auto"
+generator = pipeline(
+    "text-generation",
+    model=model_name,
+    device_map="auto",
+    dtype=torch.bfloat16
+)
+
+conversations.append([
+    {"role": "system", "content": "Sei un assistente utile, rispondi in modo conciso e coerente."},
+    {"role": "user", "content": "Cosa si può fare in una bella giornata di sole?"},
+])
+
+chat_samples = tokenizer.apply_chat_template(conversations, tokenize=False)
+
+# get number of prompt tokens
+prompt_tokens_number = len(tokenizer(chat_samples)["input_ids"])
+
+outputs = generator(
+    conversations,
+    max_new_tokens=2048,
+    eos_token_id=[
+        tokenizer.eos_token_id,
+        tokenizer.convert_tokens_to_ids("<|eot_id|>"),
+    ],
 )
 
-pipeline("Cosa si può fare in una bella giornata di sole?")
 ```
 
 Code: https://github.com/SapienzaNLP/sava
 
+## Aknowledgement
+Thanks to Leonardo Colosi (colosi@diag.uniroma1.it) for helping in instruction tuning phase.
+
+We acknowledge ISCRA for awarding this project access to the LEONARDO supercomputer, owned by the EuroHPC Joint Undertaking, hosted by CINECA (Italy).
+
 ## Citation
 
 If you use any part of this work, please consider citing the paper as follows: