2

app.py

@@ -1,11 +1,12 @@
-from datasets import load_dataset
-from transformers import AutoModelForCausalLM, AutoTokenizer, GenerationConfig, TrainingArguments, Trainer
+from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig, TrainingArguments, Trainer, pipeline
+from peft import PeftModel, PeftConfig
+from huggingface_hub import login
+import bitsandbytes as bnb
 import torch
 import time
-import evaluate
 import pandas as pd
 import numpy as np
-from huggingface_hub import login
+
 
 import streamlit as st
 
@@ -18,41 +19,17 @@ st.set_page_config(
 login(token='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', add_to_git_credential=True)
 
 st.title("Code Generation")
-huggingface_dataset_name = "red1xe/code_instructions"
-dataset = load_dataset(huggingface_dataset_name)
-model_name='bigcode/starcoder'
+st.write('MODEL: TinyPixel/Llama-2-7B-bf16-sharded')
+bnb_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+    bnb_4bit_use_double_quant=True,
+    bnb_4bit_quant_type="nf4",
+    bnb_4bit_compute_dtype=torch.bfloat16
+)
+model_name='TinyPixel/Llama-2-7B-bf16-sharded'
 tokenizer = AutoTokenizer.from_pretrained(model_name)
-original_model = AutoModelForCausalLM.from_pretrained(model_name)
-
-x = st.slider(label='Select a sample', min_value=0, max_value=1000, value=500, step=10)
-if st.button("Show Sample"):
-    index = x
-
-    input = dataset['test'][index]['input']
-    instruction = dataset['test'][index]['instruction']
-    output = dataset['test'][index]['output']
-
-    prompt = f"""
-    Answer the following question.
-
-    {input} {instruction}
-
-    Answer:
-    """
-
-    inputs = tokenizer(prompt, return_tensors='pt')
-    outputs = tokenizer.decode(
-        original_model.generate(
-            inputs["input_ids"],
-            max_new_tokens=200,
-        )[0],
-        skip_special_tokens=True
-    )
-
-    dash_line = '-'.join('' for x in range(100))
-    st.write(dash_line)
-    st.write(f'INPUT PROMPT:\n{prompt}')
-    st.write(dash_line)
-    st.write(f'BASELINE HUMAN SUMMARY:\n{output}\n')
-    st.write(dash_line)
-    st.write(f'MODEL GENERATION - ZERO SHOT:\n{outputs}')
+peft_model_base = AutoModelForCausalLM.from_pretrained(model_name, quantization_config=bnb_config)
+peft_model = PeftModel.from_pretrained(peft_model_base,
+                                       'red1xe/Llama-2-7B-codeGPT',
+                                       torch_dtype=torch.bfloat16,
+                                       is_trainable=False)

requirements.txt

@@ -5,4 +5,5 @@ datasets==2.11.0
 evaluate==0.4.0 
 rouge_score==0.1.2 
 loralib==0.1.1 
-peft==0.3.0 
+peft==0.3.0
+bitsandbytes>=0.41.1