Upload files

README.md

@@ -1,3 +1,106 @@
 ---
 license: apache-2.0
+base_model: togethercomputer/RedPajama-INCITE-Base-3B-v1
+datasets:
+  - https://huggingface.co/datasets/johnrobinsn/alpaca-cleaned
+tags:
+  - lora
+  - alpaca
+  - peft
+  - redpajama
 ---
+
+# RedPajama-3B-instruct-lora
+
+This is an instruction fine-tuned model of https://huggingface.co/togethercomputer/RedPajama-INCITE-Base-3B-v1, using `int8` mixed training.
+
+## Training dataset
+
+Cleaned version of alpaca from https://huggingface.co/datasets/johnrobinsn/alpaca-cleaned.
+
+## How to use
+
+```Python
+from huggingface_hub import model_info, hf_hub_download
+from peft import LoraConfig, get_peft_model, set_peft_model_state_dict, TaskType
+from textwrap import dedent
+from transformers import AutoModelForCausalLM, AutoTokenizer
+import torch
+
+model_id = "pcuenq/RedPajama-3B-instruct-lora"
+
+# Load base model
+
+info = model_info(model_id)
+base_model = info.cardData["base_model"]
+model = AutoModelForCausalLM.from_pretrained(
+    base_model,
+    load_in_8bit=True,
+    device_map="auto",
+)
+tokenizer = AutoTokenizer.from_pretrained(model_id)
+
+# Prepare for LoRA
+
+lora_config = LoraConfig(
+ r=8, 
+ lora_alpha=16,
+ target_modules=["query_key_value"],
+ lora_dropout=0.05,
+ bias="none",
+ task_type=TaskType.CAUSAL_LM
+)
+
+model = get_peft_model(model, lora_config)
+
+# Download and apply LoRA weights
+lora_filename = hf_hub_download(repo_id=model_id, filename="lora.bin")
+lora_dict = torch.load(lora_filename)
+set_peft_model_state_dict(model, lora_dict)
+
+# Run inference
+
+def generate_prompt(instruction, inputs=None):
+    if inputs is not None:
+        return dedent(
+            f"""\
+            Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.
+
+            ### Instruction:
+            {instruction}
+
+            ### Input:
+            {inputs}
+
+            ### Response:
+            """
+        )
+    else:
+        return dedent(
+            f"""\
+            Below is an instruction that describes a task. Write a response that appropriately completes the request.
+
+            ### Instruction:
+            {instruction}
+
+            ### Response:
+            """
+        )
+
+prompt = generate_prompt("Has humankind ever set foot on the Moon?")
+inputs = tokenizer(prompt, return_tensors='pt').to(model.device)
+input_length = inputs.input_ids.shape[1]
+outputs = model.generate(
+    **inputs, max_new_tokens=50, do_sample=True, temperature=1.0, top_p=0.7, top_k=50, return_dict_in_generate=True
+)
+tokens = outputs.sequences[0, input_length:]
+
+# Strip from first <eos>
+eos_pos = (tokens == tokenizer.eos_token_id).nonzero()
+if eos_pos.numel() > 0:
+    tokens = tokens[:eos_pos[0].item()]
+    
+output_str = tokenizer.decode(tokens)
+print(output_str)
+```
+

lora.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ba829adbc86d07bbfe8d76f88ea21e2092b69e926e327a4236638aa470b80a5
+size 10504543

special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<eos>",
+  "unk_token": "<|endoftext|>"
+}

tokenizer_config.json

@@ -0,0 +1,9 @@
+{
+  "add_prefix_space": false,
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endoftext|>",
+  "model_max_length": 2048,
+  "tokenizer_class": "GPTNeoXTokenizer",
+  "unk_token": "<|endoftext|>"
+}