Upload folder using huggingface_hub

infer_video.py

@@ -11,7 +11,8 @@ from tqdm import tqdm
 import torchvision.transforms as T
 from PIL import Image
 import argparse
-from decord import VideoReader, cpu
+import cv2
+# from decord import VideoReader, cpu
 
 
 """
@@ -70,8 +71,8 @@ def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height, image_
                 best_ratio = ratio
     return best_ratio
 
-def dynamic_preprocess(image, min_num=1, max_num=12, image_size=448, use_thumbnail=False):
-    orig_width, orig_height = image.size
+def dynamic_preprocess(image:np.array, min_num=1, max_num=12, image_size=448, use_thumbnail=False):
+    orig_height, orig_width,  = image.shape[:2]
     aspect_ratio = orig_width / orig_height
 
     # calculate the existing image aspect ratio
@@ -90,7 +91,8 @@ def dynamic_preprocess(image, min_num=1, max_num=12, image_size=448, use_thumbna
     blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
 
     # resize the image
-    resized_img = image.resize((target_width, target_height))
+    # resized_img = image.resize((target_width, target_height))
+    resized_img = cv2.resize(image, (target_width, target_height))
     processed_images = []
     for i in range(blocks):
         box = (
@@ -100,11 +102,13 @@ def dynamic_preprocess(image, min_num=1, max_num=12, image_size=448, use_thumbna
             ((i // (target_width // image_size)) + 1) * image_size
         )
         # split the image
-        split_img = resized_img.crop(box)
+        # split_img = resized_img.crop(box)
+        split_img = resized_img[box[1]:box[3], box[0]:box[2]]
         processed_images.append(split_img)
     assert len(processed_images) == blocks
     if use_thumbnail and len(processed_images) != 1:
-        thumbnail_img = image.resize((image_size, image_size))
+        # thumbnail_img = image.resize((image_size, image_size))
+        thumbnail_img = cv2.resize(image, (image_size, image_size))
         processed_images.append(thumbnail_img)
     return processed_images
 
@@ -165,23 +169,63 @@ def get_index(bound, fps, max_frame, first_idx=0, num_segments=32):
     return frame_indices
 
 
-def load_video(video_path, bound=None, input_size=448, max_num=1, num_segments=32):
-    vr = VideoReader(video_path, ctx=cpu(0), num_threads=1)
-    max_frame = len(vr) - 1
-    fps = float(vr.get_avg_fps())
+# def load_video(video_path, bound=None, input_size=448, max_num=1, num_segments=32):
+#     vr = VideoReader(video_path, ctx=cpu(0), num_threads=1)
+#     max_frame = len(vr) - 1
+#     fps = float(vr.get_avg_fps())
+
+#     pixel_values_list, num_patches_list = [], []
+#     transform = build_transform(input_size=input_size)
+#     frame_indices = get_index(bound, fps, max_frame, first_idx=0, num_segments=num_segments)
+#     for frame_index in frame_indices:
+#         img = Image.fromarray(vr[frame_index].asnumpy()).convert('RGB')
+        # img = dynamic_preprocess(img, image_size=input_size, use_thumbnail=True, max_num=max_num)
+#         pixel_values = [transform(tile) for tile in img]
+#         pixel_values = torch.stack(pixel_values)
+#         num_patches_list.append(pixel_values.shape[0])
+#         pixel_values_list.append(pixel_values)
+#     pixel_values = torch.cat(pixel_values_list)
+#     return pixel_values, num_patches_list
+
+def pre_process(image, input_size=448, max_num=12):
+    images = dynamic_preprocess(image, image_size=input_size, use_thumbnail=True, max_num=max_num)
+    pixel_values = [img_preprocess(image, input_size) for image in images]
+    pixel_values = np.concatenate(pixel_values, axis=0)
+    return pixel_values
+
+def img_preprocess(img, input_size):
+    IMAGENET_MEAN = np.array([0.485, 0.456, 0.406], dtype=np.float32)
+    IMAGENET_STD = np.array((0.229, 0.224, 0.225), dtype=np.float32)
+    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
+    img = cv2.resize(img, (input_size, input_size))
+    img = img.astype(np.float32) / 255.0
+    img = (img - IMAGENET_MEAN) / IMAGENET_STD
+    img = img.transpose(2, 0, 1).reshape(1, 3, input_size, input_size)
+    return img
+def load_video_opencv(video_path, bound=None, num_segments=32):
+    cap = cv2.VideoCapture(video_path)
+    if not cap.isOpened():
+        raise IOError(f"Cannot open video: {video_path}")
+    
+    fps = cap.get(cv2.CAP_PROP_FPS)
+    max_frame = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) - 1
 
-    pixel_values_list, num_patches_list = [], []
-    transform = build_transform(input_size=input_size)
     frame_indices = get_index(bound, fps, max_frame, first_idx=0, num_segments=num_segments)
+
+    images_list = []
     for frame_index in frame_indices:
-        img = Image.fromarray(vr[frame_index].asnumpy()).convert('RGB')
-        img = dynamic_preprocess(img, image_size=input_size, use_thumbnail=True, max_num=max_num)
-        pixel_values = [transform(tile) for tile in img]
-        pixel_values = torch.stack(pixel_values)
-        num_patches_list.append(pixel_values.shape[0])
+        cap.set(cv2.CAP_PROP_POS_FRAMES, frame_index)
+        ret, frame = cap.read()
+        if not ret:
+            print(f"⚠ Failed to read frame {frame_index}")
+            continue
+        images_list.append(frame)
+    pixel_values_list = []
+    cap.release()
+    for img in images_list:
+        pixel_values = pre_process(img, input_size=448, max_num=1)
         pixel_values_list.append(pixel_values)
-    pixel_values = torch.cat(pixel_values_list)
-    return pixel_values, num_patches_list
+    return pixel_values_list
 
 
 if __name__ == "__main__":
@@ -200,7 +244,6 @@ if __name__ == "__main__":
                         help="Your question that you want to ask the model.")
     args = parser.parse_args()
 
-
     hf_model_path = args.hf_model
     axmodel_path = args.axmodel_path
     vit_axmodel_path = args.vit_model
@@ -209,10 +252,10 @@ if __name__ == "__main__":
     config = AutoConfig.from_pretrained(hf_model_path, trust_remote_code=True)
     tokenizer = AutoTokenizer.from_pretrained(hf_model_path, trust_remote_code=True, use_fast=False)
     # set the max number of tiles in `max_num`
-    pixel_values, num_patches_list = load_video(video_path, num_segments=8, max_num=1)
+    pixel_values_list = load_video_opencv(video_path, num_segments=8)
     # Frame1: <image>\nFrame2: <image>\n...\nFrame8: <image>\n{question}
 
-    pixel_values_list = [e[None, ...] for e in pixel_values]
+    # pixel_values_list = [e[None, ...] for e in pixel_values]
     if pixel_values_list is not None:
         print(f"输入帧数: {len(pixel_values_list)}")
         print("preprocess image done!")
@@ -221,7 +264,7 @@ if __name__ == "__main__":
         vit_session = InferenceSession(vit_axmodel_path)
         vit_output_list = []
         for idx, pixel_values in enumerate(pixel_values_list):
-            vit_output = vit_session.run(None, {"image": pixel_values.numpy()})[0]
+            vit_output = vit_session.run(None, {"image": pixel_values})[0]
             vit_output_list.append(vit_output.copy()) # 避免 vit 输出结果使用同一块内存
 
         print(f"vit_output.shape is {vit_output_list[0].shape}, vit feature extract done!")
@@ -288,7 +331,7 @@ if __name__ == "__main__":
     ]
     print(f"slice_indexs is {slice_indexs}")
     prefill_len = prefill_slice_len * slice_indexs[-1] if slice_indexs[-1] != 0 else prefill_slice_len # 这里的 128 就是 prefill_slice_len
-
+    # prefill_len = token_len if token_len < lastN else lastN
     if prefill_len > 0:
         for slice_index in slice_indexs:
             indices = np.array(
@@ -385,6 +428,7 @@ if __name__ == "__main__":
         next_token = token_ids[start_indice]
         indices = np.array([start_indice], np.uint32).reshape((1, 1))
         data = embeds[next_token, :].reshape((1, 1, cfg.hidden_size)).astype(bfloat16)
+        #print(data.shape)
         for i in range(cfg.num_hidden_layers):
             input_feed = {
                 "K_cache": k_caches[i],

internvl3_2b_axmodel/qwen2_p128_l0_together.axmodel

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:31115b997b4dcc41f4c6ba97658016940c3cee17882f55c3075ac22357f7e941
-size 63496708
+oid sha256:b1dd071450c814b4ff63d1c8625ea462a12e6c1f0e2d9450ab79cf93cdf70ae7
+size 81360616

internvl3_2b_axmodel/qwen2_p128_l10_together.axmodel

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bb2428ad66ec54ce9281e831abe25c3d9e83f89ff3b14066a490dfafe1eeb585
-size 63496708
+oid sha256:2fff56a436e3b92ff19e8558a4ef0505acf7166914edf359262f39200f24cecd
+size 81360616

internvl3_2b_axmodel/qwen2_p128_l11_together.axmodel

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:47d5e607133f7d6addeaffb37d76a56e5969c6d32069730bca151dc9a45f2ed8
-size 63496708
+oid sha256:146176cd6941ca5256c8982bcb3099f84c99c599474704589275df6998400549
+size 81360616

internvl3_2b_axmodel/qwen2_p128_l12_together.axmodel

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0f224442c6eb37a04b16d2a3c8cad051cc29653ca7c3f547359747fb42b50c09
-size 63496708
+oid sha256:137d2f2df726ebe1c7bc3a1c7ae7b3b5d2bc3a715537a95aa309b3412c4d4e71
+size 81360616

internvl3_2b_axmodel/qwen2_p128_l13_together.axmodel

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f9586839e6e53343d33c6897a7f7de2a291dca700524a13a34b902e86d987139
-size 63496708
+oid sha256:2592a19121c884a18e6b05a3b030b26e95f353121d32adffefa7faa5c486eca2
+size 81360616

internvl3_2b_axmodel/qwen2_p128_l14_together.axmodel

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:72046dd0e4b9ecba9a7f4091dae8a3f45b054e49dfd3ab609482a2edc019b8d2
-size 63496708
+oid sha256:e721a2515cc165434190010e03d14688100a9e9be933eff15f8dbe54efabc947
+size 81360616

internvl3_2b_axmodel/qwen2_p128_l15_together.axmodel

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c9fbec6c5c4f7c9b6b5b2631835bc08620b9763818a0954361ea4bca321941e2
-size 63496708
+oid sha256:24a626bbd133f62b3f3ce23c4aa4cd5277a57c9783129b31ea2c372494f2a9ed
+size 81360616

internvl3_2b_axmodel/qwen2_p128_l16_together.axmodel

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6e57ef1574952fc62a174d0ec45f0230646e816a8246bb50cc935735e1f1a93e
-size 63496708
+oid sha256:593e7816e379c5f8e07b16095aab81a10c7be5ee8314186e25d430b30e4da38b
+size 81360616

internvl3_2b_axmodel/qwen2_p128_l17_together.axmodel

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:31ee093b10e87b874a92a3a7d57c0e1aed8da199e3bc278e76ef8f6ee2ba1b57
-size 63496708
+oid sha256:a859330bdb75de08a56aabacf312aa97ac4abc3862103d3bde312b7eaee33123
+size 81360616

internvl3_2b_axmodel/qwen2_p128_l18_together.axmodel

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5dfd9149ee03bc07fc0bd0c9872126d47b341bc04b44f885ae2b8a268a600699
-size 63496708
+oid sha256:7d5b40d734848f644249a560a82655429cdbfcbed99cfe97d280aa41e27a39fb
+size 81360616

internvl3_2b_axmodel/qwen2_p128_l19_together.axmodel

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b2da9cd1dc89a16f42e1542891c063e9da117e876b96b5100fc52c030deef5b5
-size 63496708
+oid sha256:c185d30a7c2a824f06756bad0e70eb9f50f73bca0f4817a69e1a393928a80944
+size 81360616

internvl3_2b_axmodel/qwen2_p128_l1_together.axmodel

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5544c2d489c58cc0044af4690810a1ffc890e29fdab38c9a2c7bd73a089af149
-size 63496708
+oid sha256:f99d6fa520114d30900e7659e28d1c5dccd257faad27d439734e4be755fa3300
+size 81360616

internvl3_2b_axmodel/qwen2_p128_l20_together.axmodel

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5212839cc201e6ca5e90ff1ec1e9fbf253588dad5ad1c837bcc9053848f25af4
-size 63496708
+oid sha256:c763458a03e27b42b99aa785997191ece692c2e557b29481ca2b580cc6b8fab0
+size 81360616

internvl3_2b_axmodel/qwen2_p128_l21_together.axmodel

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:62c194ef1e7e79b69b83e1175cb2c2bd45cd508dcf9fd4659bd31ee04ef4965a
-size 63496708
+oid sha256:de52167d4fd5bc023884f76c3858acfb0062df830eb7e28f83871ae2c687026a
+size 81360616

internvl3_2b_axmodel/qwen2_p128_l22_together.axmodel

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2e435f31fe0aa2e1ac1bf8fa9aad70d264296d6753ee1b48a19f9f6c6021b135
-size 63496708
+oid sha256:4643590e0d72b5d039eeed0ec0ce6b855b002b5e43f9a8a6cbe6a6c2cd57159d
+size 81360616

internvl3_2b_axmodel/qwen2_p128_l23_together.axmodel

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d3f1f334bbc2dd2b1290ab726c66f3044bfa184398d6ef7fbe50f48f1398c368
-size 63496708
+oid sha256:3bcaf68c2cebbe8366868371dc0603bca4f9850bff5b897b3b9a712d391b775f
+size 81360616

internvl3_2b_axmodel/qwen2_p128_l24_together.axmodel

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a327bc245121dd324b3bef6e56398323cdf526f96af469210055e09a5c577336
-size 63496708
+oid sha256:0098dac4e276ef6490a883f6348fce60a7f190ce97a93c7831f411725cff04ab
+size 81360616

internvl3_2b_axmodel/qwen2_p128_l25_together.axmodel

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a1dfc35719298adbd618ca38c4142a42c19ffe580bd49eac1801c55c3ef1ab8b
-size 63496708
+oid sha256:ff65751a675eb14684357f327b11cccb331b3306edd0a2391f6d88a56cbe625c
+size 81360616

internvl3_2b_axmodel/qwen2_p128_l26_together.axmodel

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:122fb2c087434f4b9d38b8ee2d84c1f60a771d994a1e79ce87303ac0611258e8
-size 63496708
+oid sha256:15075f5c69f967e5adf64c795436967075feabc5b31ffd50f07dbaaff85f2262
+size 81360616

internvl3_2b_axmodel/qwen2_p128_l27_together.axmodel

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b965d9ce79f24ccf3bb690ab645a8e57acb660e416fb782f6b765103f577018a
-size 63496708
+oid sha256:943ebf93b6ef43f6884f1cfff8236e5f869bf0dae32fb1f34c4e7fab919bf734
+size 81360616

internvl3_2b_axmodel/qwen2_p128_l2_together.axmodel

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2cbc9b29068fc8e4b3488704ca9e7f85946b1aea5bbce8e9b6c15eabd801f944
-size 63498820
+oid sha256:df93cf776113bed20d6779dd895e2e61d13c9928aa073e0a595e65c2584bc7df
+size 81364200

internvl3_2b_axmodel/qwen2_p128_l3_together.axmodel

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d3ca02b44fda0690ecafa8a36d5a0f4a56ebe6e73702bca84c1089c3fbf548fe
-size 63496708
+oid sha256:58beca6260114f56c9aa5a6696725161588fa3d2c7ab6324d00715daaa6afbe1
+size 81360616

internvl3_2b_axmodel/qwen2_p128_l4_together.axmodel

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:daa424dcaa5e2c376a2485c4112f9be6b1c3a7f57dfc01ff25a4eb045ab72270
-size 63496708
+oid sha256:dc0da75d869839eeba66a3dd5863add115c2a97de83703ae807918feaec1ffa0
+size 81360616

internvl3_2b_axmodel/qwen2_p128_l5_together.axmodel

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c9a4c51af440e9a4f66f4287931779f3bff8efee4fc283368a5e42090d5bb0f7
-size 63496708
+oid sha256:d9c4a078e5288d4287607c0c9cccb7f1259a89d38de8b60118505dbd535af4d2
+size 81360616

internvl3_2b_axmodel/qwen2_p128_l6_together.axmodel

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e94535ca7805298f78f04275f27f44a20536c84985d78bb2faada5be18b72d96
-size 63496708
+oid sha256:3e74bdc2623b3eb8876c68c982d6e3d48ae804b79412dd7304994d65ae650a89
+size 81360616

internvl3_2b_axmodel/qwen2_p128_l7_together.axmodel

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:890b63d139eafbaf118ca3c203f7cafbac4aef02b2fffacbbe54b215d359d862
-size 63496708
+oid sha256:342fed4e97d800cc9824379bac2792c4a51f824a2b2236c50dfd476689e1b553
+size 81360616

internvl3_2b_axmodel/qwen2_p128_l8_together.axmodel

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1435d343d533f375ef4d34859dd8ad6d6885c6d80bebb7852b59ce783d005c31
-size 63496708
+oid sha256:ec4266b62c2040b7a0c689e3cc0bfec1b6634875b0b8681bd9f7e2ce3975e654
+size 81360616

internvl3_2b_axmodel/qwen2_p128_l9_together.axmodel

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b9d1d65a56b3727e8c224afae6446d536502e70ce31d003287d37ddba5886722
-size 63496708
+oid sha256:e6ee2e4269d546948822160db37943cc5f6d2d444823c502465c8ed445f06f81
+size 81360616

internvl3_2b_axmodel/qwen2_post.axmodel

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8246a085a945cb020db3aa7d4da5f94db41573298ef711360acb969a20a542d9
+oid sha256:6339b7f4ec86a8f2f1a496431f5027eaf30434b2026b3376ff34391d3c8d73b9
 size 254396183