update

ixc_utils.py

@@ -124,7 +124,10 @@ def load_video(video_path, num_frm=32, start=None, end=None):
     start_idx = 0 if start is None else start
     end_idx = len(vid) if end is None else end
     all_pos = list(range(start_idx, end_idx, t_stride))
-    images = [vid[i].numpy() for i in all_pos]
+    try:
+        images = [vid[i].numpy() for i in all_pos]
+    except:
+        images = [vid[i].asnumpy() for i in all_pos]
     if len(images) > num_frm:
         num_frm = min(num_frm, len(images))
         step_size = len(images) / (num_frm + 1)

ixc_utils.py~

@@ -0,0 +1,139 @@
+import torch
+import numpy as np
+import torchvision
+from PIL import Image, ImageDraw, ImageFont
+from torchvision.transforms.functional import InterpolationMode
+import torchvision.transforms as transforms
+from decord import VideoReader
+
+def padding_336(b, pad=336):
+    width, height = b.size
+    tar = int(np.ceil(height / pad) * pad)
+    top_padding = 0 # int((tar - height)/2)
+    bottom_padding = tar - height - top_padding
+    left_padding = 0
+    right_padding = 0
+    b = transforms.functional.pad(b, [left_padding, top_padding, right_padding, bottom_padding], fill=[255,255,255])
+
+    return b
+
+def Image_transform(img, hd_num=25):
+    width, height = img.size
+    trans = False
+    if width < height:
+        img = img.transpose(Image.TRANSPOSE)
+        trans = True
+        width, height = img.size
+    ratio = (width/ height)
+    scale = 1
+    while scale*np.ceil(scale/ratio) <= hd_num:
+        scale += 1
+    scale -= 1
+    scale = min(np.ceil(width / 560), scale)
+    new_w = int(scale * 560)
+    new_h = int(new_w / ratio)
+    #print (scale, f'{height}/{new_h}, {width}/{new_w}')
+
+    img = transforms.functional.resize(img, [new_h, new_w],)
+    img = padding_336(img, 560)
+    width, height = img.size
+    if trans:
+        img = img.transpose(Image.TRANSPOSE)
+
+    return img
+
+
+def Video_transform(img, hd_num=25):
+    width, height = img.size
+    trans = False
+    if width < height:
+        img = img.transpose(Image.TRANSPOSE)
+        trans = True
+        width, height = img.size
+    ratio = (width/ height)
+    scale = 1
+    new_h = int(scale * 560)
+    new_w = int(new_h * ratio)
+    #print (new_h, new_w)
+
+    img = transforms.functional.resize(img, [new_h, new_w],)
+    img = img.transpose(Image.TRANSPOSE)
+    img = padding_336(img, 560)
+    width, height = img.size
+    if not trans:
+        img = img.transpose(Image.TRANSPOSE)
+
+    return img
+
+def frame2img(imgs):
+    new_imgs = []
+    for img in imgs:
+        w, h = img.size
+        scale = w/h
+        if w > h:
+            new_w = 560 * 2
+            new_h = int(560 * 2 / scale)
+        else:
+            new_w = int(560 * 2 * scale)
+            new_h = 560 * 2
+        img = transforms.functional.resize(img, [new_h, new_w],)
+        new_imgs.append(img)
+    imgs = new_imgs
+    new_w = 0
+    new_h = 0
+    pad = 40
+    font = ImageFont.truetype(os.path.join(config._name_or_path, "SimHei.ttf"), pad)
+    if w > h:
+        for im in imgs:
+            w,h = im.size
+            new_w = max(new_w, w)
+            new_h += h + 10 + pad
+        new_img = Image.new('RGB', (new_w, new_h), 'white')
+        draw = ImageDraw.Draw(new_img)
+        curr_h = 0
+        for idx, im in enumerate(imgs):
+            w,h = im.size
+            new_img.paste(im, (0, pad + curr_h))
+            draw.text((0, curr_h ), f'<IMAGE {idx}>', font=font, fill='black')
+            if idx + 1 < len(imgs):
+                draw.line([(0, pad +curr_h + h +5), (new_w, pad +curr_h + h +5)], fill = 'black', width=2)
+            curr_h += h + 10 + pad
+        #print (new_w, new_h)
+    else:
+        for im in imgs:
+            w,h = im.size
+            new_w += w + 10
+            new_h = max(new_h, h)
+        new_h += pad
+        new_img = Image.new('RGB', (new_w, new_h), 'white')
+        draw = ImageDraw.Draw(new_img)
+        curr_w = 0
+        for idx, im in enumerate(imgs):
+            w,h = im.size
+            new_img.paste(im, (curr_w, pad))
+            draw.text((curr_w, 0), f'<IMAGE {idx}>', font=font, fill='black')
+            if idx + 1 < len(imgs):
+                draw.line([(curr_w + w + 5, 0), (curr_w + w + 5, new_h)], fill = 'black', width=2)
+            curr_w += w + 10
+    return new_img
+
+def load_video(video_path, num_frm=32, start=None, end=None):
+    vid = VideoReader(video_path, num_threads=1)
+    fps = vid.get_avg_fps()
+    t_stride = int(round(float(fps) / int(1)))
+    start_idx = 0 if start is None else start
+    end_idx = len(vid) if end is None else end
+    all_pos = list(range(start_idx, end_idx, t_stride))
+    try:
+        images = [vid[i].numpy() for i in all_pos]
+    except:
+        images = [vid[i].asnumpy() for i in all_pos]
+    if len(images) > num_frm:
+        num_frm = min(num_frm, len(images))
+        step_size = len(images) / (num_frm + 1)
+        indices = [int(i*step_size) for i in range(num_frm)]
+        images = [images[i] for i in indices]
+    images = [Image.fromarray(arr) for arr in images]
+    image = frame2img(images)
+    return image
+