update

ixc_utils.py

@@ -66,7 +66,7 @@ def Video_transform(img, hd_num=25):
 
     return img
 
-def frame2img(imgs):
+def frame2img(imgs, font_path):
     new_imgs = []
     for img in imgs:
         w, h = img.size
@@ -83,7 +83,8 @@ def frame2img(imgs):
     new_w = 0
     new_h = 0
     pad = 40
-    font = ImageFont.truetype(os.path.join(config._name_or_path, "SimHei.ttf"), pad)
+    print (font_path)
+    font = ImageFont.truetype(os.path.join(font_path, "SimHei.ttf"), pad)
     if w > h:
         for im in imgs:
             w,h = im.size
@@ -135,6 +136,5 @@ def load_video(video_path, num_frm=32, start=None, end=None):
         indices = [int(i*step_size) for i in range(num_frm)]
         images = [images[i] for i in indices]
     images = [Image.fromarray(arr) for arr in images]
-    image = frame2img(images)
-    return image
+    return images
 

modeling_internlm_xcomposer2.py

@@ -45,7 +45,7 @@ import torchvision.transforms as transforms
 from torchvision.transforms.functional import InterpolationMode
 
 from .build_mlp import build_vision_projector, build_vision_tower
-from .ixc_utils import Image_transform, Video_transform, load_video
+from .ixc_utils import Image_transform, Video_transform, load_video, frame2img
 from .configuration_internlm_xcomposer2 import InternLMXcomposer2Config
 from .modeling_internlm2 import (InternLM2_INPUTS_DOCSTRING, InternLM2Model,
                                  InternLM2PreTrainedModel)
@@ -102,6 +102,7 @@ class InternLMXComposer2ForCausalLM(InternLM2PreTrainedModel):
             config.hidden_size, config.vocab_size, bias=False)
         self.tokenizer = None
         self.hd_num = 25
+        self._path = config._name_or_path
 
         self.max_length = config.max_length
         print(f'Set max length to {self.max_length}')
@@ -163,6 +164,7 @@ class InternLMXComposer2ForCausalLM(InternLM2PreTrainedModel):
                 image = Image_transform(image, hd_num = hd_num)
             elif ext.lower() in video_extensions:
                 image = load_video(image)
+                image = frame2img(image, self._path)
                 image = Video_transform(image, hd_num = hd_num)
             else:
                 print ('Unknow input format', image)