Upload modeling_vlm.py with huggingface_hub

modeling_vlm.py

@@ -330,6 +330,7 @@ class MultiModalityCausalLM(MultiModalityPreTrainedModel):
             hidden_states = outputs.last_hidden_state
             logits = self.gen_head(hidden_states)
 
+
             logits_cond = logits[0::2, :]
             logits_uncond = logits[1::2, :]
 
@@ -645,7 +646,7 @@ class MultiModalityCausalLM(MultiModalityPreTrainedModel):
                 # print('nmsl: ', ii, ind)
                 if ii % 4 == 0:
                     offset = ind[1] + 2
-                    inputs_embeds[ind[0], offset: offset + image_embeds_input.shape[1], :] = image_embeds_input[(ii // 2) % img_len]
+                    inputs_embeds[ind[0], offset: offset + image_embeds_input.shape[1], :] = image_embeds_input[ii // 4]
 
             generated_tokens = torch.zeros((3 * input_ids.size(0), image_token_num_per_image), dtype=torch.int).cuda()
 
@@ -657,6 +658,8 @@ class MultiModalityCausalLM(MultiModalityPreTrainedModel):
             # torch.save(inputs_embeds, '/data/bxh_data/unify_model/hidden_states.pt')
 
             logits = self.gen_head(hidden_states)
+            print('logits.shape', logits.shape) # [3, 1760, 16384])
+            print(labels.shape) # [3, 1760]
 
             # logits_cond = logits[0::2, :]
             # logits_uncond = logits[1::2, :]
@@ -679,6 +682,7 @@ class MultiModalityCausalLM(MultiModalityPreTrainedModel):
                 shift_labels = labels[..., 1:].contiguous()
                 shift_labels = shift_labels.view(-1)
                 shift_labels = shift_labels.to(shift_logits.device)
+                print(shift_logits.shape, shift_labels.shape)
                 loss = loss_fct(shift_logits, shift_labels)
             else:
                 loss = None