Upload 8 files

app.py

@@ -0,0 +1,163 @@
+from PIL import Image
+import gradio as gr
+
+import torch
+import torch.nn as nn
+from transformers import AutoTokenizer, pipeline
+from transformers import AutoModelForCausalLM
+from torchvision import transforms
+from transformers import CLIPProcessor, CLIPModel
+
+from model import build_mlp_vector_projector
+
+
+device = "cpu"
+# Load the CLIP model and processor
+clip_model_name = "openai/clip-vit-base-patch16"
+clip_model = CLIPModel.from_pretrained(clip_model_name).to(device)
+clip_processor = CLIPProcessor.from_pretrained(clip_model_name)
+
+clip_transform = transforms.Compose(
+    [
+        transforms.Resize((224, 224)),
+        transforms.ToTensor()
+    ]
+)
+
+
+def process_image(img_path):
+    image = Image.open(img_path).convert("RGB")
+    image = clip_transform(image)
+    inputs = clip_processor(text=[""], images=image,
+                            return_tensors="pt", padding=True)
+    inputs = {k: v.to(device) for k, v in inputs.items()}
+    img_embedding = clip_model(**inputs).image_embeds
+    img_proj_head = build_mlp_vector_projector().to(device)
+    img_proj_head.load_state_dict(torch.load(
+        'stage_2_proj_head_v3.pth', map_location=torch.device(device)))
+    img_tokens = img_proj_head(img_embedding)
+    return img_tokens
+
+
+phi_model_name = "microsoft/phi-2"
+text_tokenizer = AutoTokenizer.from_pretrained(
+    phi_model_name, trust_remote_code=True)
+with torch.no_grad():
+    tuned_phi2 = AutoModelForCausalLM.from_pretrained(
+        "stage2_adaptor", trust_remote_code=True,
+        device=device, torch_dtype=torch.float16
+    )
+    base_phi2_text = AutoModelForCausalLM.from_pretrained(
+        phi_model_name, trust_remote_code=True,
+        device_map="auto", torch_dtype=torch.float16
+    )
+print("phi2 model loaded")
+
+audio_model_name = "openai/whisper-small"
+audio_pipe = pipeline(
+    task="automatic-speech-recognition",
+    model=audio_model_name,
+    chunk_length_s=30,
+    device=device)
+
+
+def process_text(text, count):
+    inputs = text_tokenizer(text, return_tensors="pt",
+                            return_attention_mask=False)
+    prediction = text_tokenizer.batch_decode(
+        base_phi2_text.generate(
+            **inputs,
+            max_new_tokens=count,
+            bos_token_id=text_tokenizer.bos_token_id,
+            eos_token_id=text_tokenizer.eos_token_id,
+            pad_token_id=text_tokenizer.pad_token_id
+        )
+    )
+    return prediction[0].rstrip('<|endoftext|>').rstrip("\n")
+
+
+def process_audio(audio):
+    if audio is None:
+        raise gr.Error(
+            "Please provide an audio file or record your input"
+        )
+
+    text = audio_pipe(
+        audio,
+        batch_size=8,
+        generate_kwargs={"task": "transcribe"},
+        return_timestamps=True
+    )["text"]
+    return text
+
+
+def generate_response(image, audio, text, count):
+    count = int(count)
+    if audio:
+        text_from_audio = process_audio(audio)
+    if text:
+        overall_input = text + text_from_audio
+    if image:
+        img_tokens = process_image(image)
+        q_tokens = text_tokenizer.encode(
+            overall_input,
+            return_tensors='pt').to(device)
+        question_token_embeddings = base_phi2_text.get_submodule(
+            'model.embed_tokens')(q_tokens).to(device)
+        inputs = torch.concat(
+            (img_tokens.unsqueeze(0), question_token_embeddings),
+            axis=-2).to(device)
+        prediction = text_tokenizer.batch_decode(
+            tuned_phi2.generate(
+                inputs_embeds=inputs,
+                max_new_tokens=30,
+                bos_token_id=text_tokenizer.bos_token_id,
+                eos_token_id=text_tokenizer.eos_token_id,
+                pad_token_id=text_tokenizer.pad_token_id
+            )
+        )
+        return prediction[0].rstrip('<|endoftext|>').rstrip("\n")
+    else:
+        return process_text(overall_input, count)
+
+    return prediction[0].strip('<|endoftext|>').rstrip("\n")
+
+
+with gr.Blocks() as demo:
+    gr.Markdown("# **AnyModeAssistant**")
+    gr.Markdown("Use any mode text/image/audio to interact with AI assistant")
+    with gr.Column():
+        with gr.Row("Text"):
+            text_input = gr.Textbox(placeholder="Enter your question here",
+                                    label="Input")
+        with gr.Row():
+            image_input = gr.Image(type="filepath")
+
+        with gr.Row("Audio mode"):
+            audio_input = gr.Audio(type="filepath")
+
+        with gr.Row("Image"):
+            response_count = gr.Textbox(
+                placeholder="Number of tokens to respond",
+                defualt=20,
+                label="Count")
+    with gr.Column():
+        response = gr.Textbox(label="AI Response")
+    with gr.Row():
+        submit_button = gr.Button("Submit")
+        submit_button.click(generate_response,
+                            inputs=[text_input, response_count,
+                                    image_input, audio_input],
+                            outputs=response)
+
+    # gr.Examples(
+    #     examples=[
+    #         ["What is a large language model?", "50"]
+    #     ],
+    #     # , image_input, image_text_input, audio_input],
+    #     inputs=[text_input, text_input_count],
+    #     outputs=[text_output],  # , image_text_output, audio_text_output],
+    #     fn=example_inference,
+    # )
+
+demo.launch()

dataset.py

@@ -0,0 +1,82 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from transformers import AutoTokenizer, AutoConfig
+import json 
+from torch.utils.data import Dataset, DataLoader
+
+instruct_dataset = f'./llava_instruct_150k.json'
+with open(instruct_dataset, 'r') as f:
+    instruct_data = json.load(f)
+
+class CustomTextDataset(Dataset):
+    def __init__(self, json_data, image_embedding_dict,  tokenizer, maxContext=512):
+        self.image_embedding_dict = image_embedding_dict
+        self.tokenizer = tokenizer
+        self.json_data = json_data
+        self.maxContext = maxContext
+        self.entries = []        
+        for entry in json_data:
+            image = entry['image']
+            image_embedding = self.getEmbeddingForImage(image)
+            if image_embedding is None:
+                continue
+            
+            conversations = entry['conversations']
+            for i in range(len(conversations)):
+                if conversations[i]['from'] == 'human':
+                    if len(conversations[i]['value'] + conversations[i + 1]['value']) > 512:
+                        continue
+                    question = 'Question: ' + conversations[i]['value'].lstrip('<image>\n')
+                    answer = 'Answer: ' + conversations[i + 1]['value']
+                    self.entries.append({
+                        'image_name': image,
+                        'image_embedding': image_embedding,
+                        'Question': question,
+                        'Answer': answer,
+                        'QnAText': question + answer
+                        }) 
+        print('------------- num entries = -----------------')
+        print(len(self.entries))
+
+    def getEmbeddingForImage(self, image):
+        if image in self.image_embedding_dict:
+            image_embedding = self.image_embedding_dict[image]
+            return image_embedding
+        else:
+            return None      
+
+    def __len__(self):
+        return len(self.entries)
+
+    def __getitem__(self, idx):
+        entry = self.entries[idx]
+        image_name = entry['image_name']
+        Q_caption_tokens = tokenizer.encode(entry['Question'], add_special_tokens=True)
+        QnA_captions_tokens = tokenizer.encode(entry['QnAText'], add_special_tokens=True)
+        QTokensLength = len(Q_caption_tokens)
+        QnA_length = len(QnA_captions_tokens)
+        
+
+        QnA_captions_tokens = QnA_captions_tokens + \
+            [tokenizer.pad_token_id] * (self.maxContext - len(QnA_captions_tokens))       
+
+        return {'image_name': entry['image_name'], 
+                'QText': entry['Question'], 
+                'AText': entry['Answer'], 
+                'image_embedding':  entry['image_embedding'].to("cuda"), 
+                'QnA_tokens': torch.tensor(QnA_captions_tokens),
+                'QTokensLength': QTokensLength,
+                'QnA_length': QnA_length
+               }
+
+
+
+imgEmbDict = torch.load('img_embeddings_dict.pth')
+
+model_name = "microsoft/phi-2"
+tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+tokenizer.pad_token = tokenizer.eos_token
+
+custom_dataset = CustomTextDataset(instruct_data, imgEmbDict,  tokenizer)
+custom_dataloader = DataLoader(custom_dataset, batch_size=10, shuffle=True)

model.py

@@ -0,0 +1,36 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from transformers import AutoTokenizer, AutoConfig
+
+class _MLPVectorProjector(nn.Module):
+    def __init__(
+        self,
+        input_hidden_size: int = 512,
+        lm_hidden_size: int = 2560,
+        num_layers: int = 1,
+        width: int = 4
+    ):
+        super(_MLPVectorProjector, self).__init__()
+        self.mlps = nn.ModuleList()
+        for _ in range(width):
+            mlp = [nn.Linear(input_hidden_size, lm_hidden_size, bias=False)]
+            for _ in range(1, num_layers):
+                mlp.append(nn.GELU())
+                mlp.append(nn.Linear(lm_hidden_size, lm_hidden_size, bias=False))
+            self.mlps.append(nn.Sequential(*mlp))
+
+    def forward(self, x):
+        return torch.cat([mlp(x) for mlp in self.mlps], dim=-2)
+
+
+def build_mlp_vector_projector(
+    input_hidden_size: int = 512, 
+    lm_hidden_size: int = 2560,
+    num_layers: int = 1,
+    num_tokens: int = 4
+):
+    return _MLPVectorProjector(
+        input_hidden_size, lm_hidden_size, num_layers, num_tokens
+    )
+

stage2_adaptor/README.md

@@ -0,0 +1,204 @@
+---
+library_name: peft
+base_model: microsoft/phi-2
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure 
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+
+
+### Framework versions
+
+- PEFT 0.7.1

stage2_adaptor/adapter_config.json

@@ -0,0 +1,31 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": {
+    "base_model_class": "PhiForCausalLM",
+    "parent_library": "transformers_modules.microsoft.phi-2.cb02e3efd822d6dbc1ca7e0dff31c29a11550411.modeling_phi"
+  },
+  "base_model_name_or_path": "microsoft/phi-2",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 64,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "fc2",
+    "fc1",
+    "out_proj",
+    "Wqkv"
+  ],
+  "task_type": null
+}

stage2_adaptor/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef13f8dac86c856f0ec2c6847f6a0b058b2387bba0b896b89b7cd2cc835b5965
+size 209731504

stage_2_proj_head_v3.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d7cb2049e9e62b633caacc5311380e55b842c4c51f23755546ec6490f93f119
+size 20973485