Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,15 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+result/eval/msrvtt_qa/.ipynb_checkpoints/qwen2_vl_7b_msrvtt_prefix2-checkpoint.json filter=lfs diff=lfs merge=lfs -text
+result/eval/msrvtt_qa/.ipynb_checkpoints/qwen2_vl_7b_msrvtt_prefix3-checkpoint.json filter=lfs diff=lfs merge=lfs -text
+result/eval/msrvtt_qa/.ipynb_checkpoints/qwen2_vl_7b_msrvtt_prefix4-checkpoint.json filter=lfs diff=lfs merge=lfs -text
+result/eval/msrvtt_qa/llava_next_video_msrvtt_0.json filter=lfs diff=lfs merge=lfs -text
+result/eval/msrvtt_qa/llava_next_video_msrvtt_1.json filter=lfs diff=lfs merge=lfs -text
+result/eval/msrvtt_qa/llava_next_video_msrvtt_2.json filter=lfs diff=lfs merge=lfs -text
+result/eval/msrvtt_qa/qwen2_vl_7b_msrvtt.json filter=lfs diff=lfs merge=lfs -text
+result/eval/msrvtt_qa/qwen2_vl_7b_msrvtt_prefix1.json filter=lfs diff=lfs merge=lfs -text
+result/eval/msrvtt_qa/qwen2_vl_7b_msrvtt_prefix2.json filter=lfs diff=lfs merge=lfs -text
+result/eval/msrvtt_qa/qwen2_vl_7b_msrvtt_prefix3.json filter=lfs diff=lfs merge=lfs -text
+result/eval/msrvtt_qa/qwen2_vl_7b_msrvtt_prefix4.json filter=lfs diff=lfs merge=lfs -text
+result/eval/msrvtt_qa/qwen2_vl_7b_msrvtt_prefix5.json filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,2 @@
+**/__pycache__/
+./act_log.err

.ipynb_checkpoints/act_log-checkpoint.err

@@ -0,0 +1,107 @@
+[h264 @ 0xb32da00] mmco: unref short failure
+[h264 @ 0xb32da00] mmco: unref short failure
+[h264 @ 0xa5b8040] mmco: unref short failure
+[h264 @ 0xa5b8040] mmco: unref short failure
+[h264 @ 0xb32da00] mmco: unref short failure
+[h264 @ 0xb32da00] mmco: unref short failure
+[h264 @ 0xa2c0a80] mmco: unref short failure
+[h264 @ 0xa2c0a80] mmco: unref short failure
+[h264 @ 0xa31e800] mmco: unref short failure
+[h264 @ 0xa9dad00] mmco: unref short failure
+[h264 @ 0xa9dad00] mmco: unref short failure
+[h264 @ 0xa9dad00] mmco: unref short failure
+[h264 @ 0xa9dad00] mmco: unref short failure
+[h264 @ 0x92ff200] mmco: unref short failure
+[h264 @ 0xa6f26c0] mmco: unref short failure
+[h264 @ 0xa6f26c0] mmco: unref short failure
+[h264 @ 0xa239f80] mmco: unref short failure
+Unrecognized keys in `rope_scaling` for 'rope_type'='default': {'mrope_section'}
+You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`.
+You are attempting to use Flash Attention 2.0 without specifying a torch dtype. This might lead to unexpected behaviour
+`Qwen2VLRotaryEmbedding` can now be fully parameterized by passing the model config through the `config` argument. All other arguments will be removed in v4.46
+
+Unrecognized keys in `rope_scaling` for 'rope_type'='default': {'mrope_section'}
+You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`.
+You are attempting to use Flash Attention 2.0 without specifying a torch dtype. This might lead to unexpected behaviour
+
+Unrecognized keys in `rope_scaling` for 'rope_type'='default': {'mrope_section'}
+You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`.
+[h264 @ 0x9e195c0] mmco: unref short failure
+
+`Qwen2VLRotaryEmbedding` can now be fully parameterized by passing the model config through the `config` argument. All other arguments will be removed in v4.46
+Unrecognized keys in `rope_scaling` for 'rope_type'='default': {'mrope_section'}
+You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`.
+Unrecognized keys in `rope_scaling` for 'rope_type'='default': {'mrope_section'}
+You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`.
+You are attempting to use Flash Attention 2.0 without specifying a torch dtype. This might lead to unexpected behaviour
+You are attempting to use Flash Attention 2.0 without specifying a torch dtype. This might lead to unexpected behaviour
+`Qwen2VLRotaryEmbedding` can now be fully parameterized by passing the model config through the `config` argument. All other arguments will be removed in v4.46
+
+
+
+
+You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`.
+You are attempting to use Flash Attention 2.0 without specifying a torch dtype. This might lead to unexpected behaviour
+
+
+
+
+[h264 @ 0x92ff200] mmco: unref short failure
+[h264 @ 0x92ff200] mmco: unref short failure
+
+You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`.
+You are attempting to use Flash Attention 2.0 without specifying a torch dtype. This might lead to unexpected behaviour
+`Qwen2VLRotaryEmbedding` can now be fully parameterized by passing the model config through the `config` argument. All other arguments will be removed in v4.46
+
+
+[h264 @ 0x9e195c0] mmco: unref short failure
+[h264 @ 0x9e195c0] mmco: unref short failure
+Unrecognized keys in `rope_scaling` for 'rope_type'='default': {'mrope_section'}
+You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`.
+You are attempting to use Flash Attention 2.0 without specifying a torch dtype. This might lead to unexpected behaviour
+`Qwen2VLRotaryEmbedding` can now be fully parameterized by passing the model config through the `config` argument. All other arguments will be removed in v4.46
+
+multiprocessing.pool.RemoteTraceback: 
+"""
+Traceback (most recent call last):
+  File "/jizhicfs/hymiezhao/miniconda3_lh/envs/ml_hf/lib/python3.10/multiprocessing/pool.py", line 125, in worker
+    result = (True, func(*args, **kwds))
+  File "/jizhicfs/hymiezhao/miniconda3_lh/envs/ml_hf/lib/python3.10/multiprocessing/pool.py", line 48, in mapstar
+    return list(map(*args))
+  File "/jizhicfs/hymiezhao/ml/video_llm_template/task/eval/qwen2_vl_qa_eval.py", line 205, in batch_run
+    llm_messages = qwen2_vl_answer(model, processor, batch_samples, device=rank)
+  File "/jizhicfs/hymiezhao/ml/video_llm_template/task/eval/qwen2_vl_qa_eval.py", line 164, in qwen2_vl_answer
+    generated_ids = model.generate(**inputs, max_new_tokens=100, do_sample=False).cpu()
+  File "/jizhicfs/hymiezhao/miniconda3_lh/envs/ml_hf/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context
+    return func(*args, **kwargs)
+  File "/jizhicfs/hymiezhao/miniconda3_lh/envs/ml_hf/lib/python3.10/site-packages/transformers/generation/utils.py", line 2057, in generate
+    result = self._sample(
+  File "/jizhicfs/hymiezhao/miniconda3_lh/envs/ml_hf/lib/python3.10/site-packages/transformers/generation/utils.py", line 3021, in _sample
+    outputs = self(**model_inputs, return_dict=True)
+  File "/jizhicfs/hymiezhao/miniconda3_lh/envs/ml_hf/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1553, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/jizhicfs/hymiezhao/miniconda3_lh/envs/ml_hf/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1562, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/jizhicfs/hymiezhao/miniconda3_lh/envs/ml_hf/lib/python3.10/site-packages/transformers/models/qwen2_vl/modeling_qwen2_vl.py", line 1719, in forward
+    logits = logits.float()
+RuntimeError: r.nvmlDeviceGetNvLinkRemoteDeviceType_ INTERNAL ASSERT FAILED at "../c10/cuda/driver_api.cpp":27, please report a bug to PyTorch. Can't find nvmlDeviceGetNvLinkRemoteDeviceType: /lib64/libnvidia-ml.so.1: undefined symbol: nvmlDeviceGetNvLinkRemoteDeviceType
+"""
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/jizhicfs/hymiezhao/miniconda3_lh/envs/ml_hf/lib/python3.10/runpy.py", line 196, in _run_module_as_main
+    return _run_code(code, main_globals, None,
+  File "/jizhicfs/hymiezhao/miniconda3_lh/envs/ml_hf/lib/python3.10/runpy.py", line 86, in _run_code
+    exec(code, run_globals)
+  File "/jizhicfs/hymiezhao/ml/video_llm_template/task/eval/qwen2_vl_qa_eval.py", line 253, in <module>
+    main(args)
+  File "/jizhicfs/hymiezhao/ml/video_llm_template/task/eval/qwen2_vl_qa_eval.py", line 240, in main
+    result_lists = pool.map(func, range(world_size))
+  File "/jizhicfs/hymiezhao/miniconda3_lh/envs/ml_hf/lib/python3.10/multiprocessing/pool.py", line 367, in map
+    return self._map_async(func, iterable, mapstar, chunksize).get()
+  File "/jizhicfs/hymiezhao/miniconda3_lh/envs/ml_hf/lib/python3.10/multiprocessing/pool.py", line 774, in get
+    raise self._value
+RuntimeError: r.nvmlDeviceGetNvLinkRemoteDeviceType_ INTERNAL ASSERT FAILED at "../c10/cuda/driver_api.cpp":27, please report a bug to PyTorch. Can't find nvmlDeviceGetNvLinkRemoteDeviceType: /lib64/libnvidia-ml.so.1: undefined symbol: nvmlDeviceGetNvLinkRemoteDeviceType
+/jizhicfs/hymiezhao/miniconda3_lh/envs/ml_hf/lib/python3.10/multiprocessing/resource_tracker.py:224: UserWarning: resource_tracker: There appear to be 1 leaked semaphore objects to clean up at shutdown
+  warnings.warn('resource_tracker: There appear to be %d '

act_log.err

@@ -0,0 +1,264 @@
+[h264 @ 0x9b59c40] mmco: unref short failure
+[h264 @ 0x9b59c40] mmco: unref short failure
+[h264 @ 0x86a1a40] mmco: unref short failure
+[h264 @ 0x86a1a40] mmco: unref short failure
+[h264 @ 0x9eeff00] mmco: unref short failure
+[h264 @ 0x9eeff00] mmco: unref short failure
+[h264 @ 0x9741000] mmco: unref short failure
+[h264 @ 0x8a2a980] mmco: unref short failure
+[h264 @ 0x8a2a980] mmco: unref short failure
+[h264 @ 0x8241080] mmco: unref short failure
+[h264 @ 0x8241080] mmco: unref short failure
+[h264 @ 0x831f5c0] mmco: unref short failure
+[h264 @ 0x9064ac0] mmco: unref short failure
+[h264 @ 0x9064ac0] mmco: unref short failure
+[h264 @ 0x83a9cc0] mmco: unref short failure
+[h264 @ 0x83a9cc0] mmco: unref short failure
+[h264 @ 0xed7b140] mmco: unref short failure
+[h264 @ 0xed7b140] mmco: unref short failure
+[h264 @ 0x8892600] mmco: unref short failure
+[h264 @ 0x20ace480] mmco: unref short failure
+[h264 @ 0x20ace480] mmco: unref short failure
+[h264 @ 0x9aedd80] mmco: unref short failure
+[h264 @ 0x15769000] mmco: unref short failure
+[h264 @ 0x15769000] mmco: unref short failure
+[h264 @ 0x9821380] mmco: unref short failure
+[h264 @ 0x9821380] mmco: unref short failure
+[h264 @ 0x111680c0] mmco: unref short failure
+[h264 @ 0x83b4dc0] mmco: unref short failure
+[h264 @ 0x83b4dc0] mmco: unref short failure
+[h264 @ 0x9d5e500] mmco: unref short failure
+[h264 @ 0x9d5e500] mmco: unref short failure
+[h264 @ 0x12ff4840] mmco: unref short failure
+[h264 @ 0x12ff4840] mmco: unref short failure
+[h264 @ 0x976dfc0] mmco: unref short failure
+[h264 @ 0xd4e1180] mmco: unref short failure
+[h264 @ 0xd4e1180] mmco: unref short failure
+[h264 @ 0x976ab80] mmco: unref short failure
+[h264 @ 0x976ab80] mmco: unref short failure
+[h264 @ 0x9d6b5c0] mmco: unref short failure
+[h264 @ 0xfceee00] mmco: unref short failure
+[h264 @ 0xfceee00] mmco: unref short failure
+[h264 @ 0x838ff40] mmco: unref short failure
+[h264 @ 0x838ff40] mmco: unref short failure
+[h264 @ 0x1e7dba40] mmco: unref short failure
+[h264 @ 0x1e7dba40] mmco: unref short failure
+[h264 @ 0x953b3c0] mmco: unref short failure
+[h264 @ 0x953b3c0] mmco: unref short failure
+[h264 @ 0x9d5e500] mmco: unref short failure
+[h264 @ 0x9d5e500] mmco: unref short failure
+[h264 @ 0x11e6fcc0] mmco: unref short failure
+[h264 @ 0x93d7480] mmco: unref short failure
+[h264 @ 0x93d7480] mmco: unref short failure
+[h264 @ 0x12ff4840] mmco: unref short failure
+[h264 @ 0x12ff4840] mmco: unref short failure
+[h264 @ 0xfceee00] mmco: unref short failure
+[h264 @ 0xfceee00] mmco: unref short failure
+[h264 @ 0xfceee00] mmco: unref short failure
+[h264 @ 0x8bc3980] mmco: unref short failure
+[h264 @ 0x8bc3980] mmco: unref short failure
+[h264 @ 0x9c304c0] mmco: unref short failure
+[h264 @ 0x9c304c0] mmco: unref short failure
+[h264 @ 0xb6e6a40] mmco: unref short failure
+[h264 @ 0x8b27a00] mmco: unref short failure
+[h264 @ 0x8b27a00] mmco: unref short failure
+[h264 @ 0x1b4bdc40] mmco: unref short failure
+[h264 @ 0x1b4bdc40] mmco: unref short failure
+[h264 @ 0x175507c0] mmco: unref short failure
+[h264 @ 0x175507c0] mmco: unref short failure
+[h264 @ 0x95b2cc0] mmco: unref short failure
+[h264 @ 0x13094500] mmco: unref short failure
+[h264 @ 0x97cf300] mmco: unref short failure
+[h264 @ 0x97cf300] mmco: unref short failure
+[h264 @ 0x8338900] mmco: unref short failure
+[h264 @ 0x846fbc0] mmco: unref short failure
+[h264 @ 0x846fbc0] mmco: unref short failure
+[h264 @ 0x8654080] mmco: unref short failure
+[h264 @ 0x15e09fc0] mmco: unref short failure
+[h264 @ 0x15e09fc0] mmco: unref short failure
+[h264 @ 0x8654080] mmco: unref short failure
+[h264 @ 0x120ec300] mmco: unref short failure
+[h264 @ 0x120ec300] mmco: unref short failure
+[h264 @ 0x9518d00] mmco: unref short failure
+[h264 @ 0x9068a00] mmco: unref short failure
+[h264 @ 0x9068a00] mmco: unref short failure
+[h264 @ 0x9068a00] mmco: unref short failure
+[h264 @ 0x9068a00] mmco: unref short failure
+[h264 @ 0x9068a00] mmco: unref short failure
+[h264 @ 0x15769000] mmco: unref short failure
+[h264 @ 0x84bffc0] mmco: unref short failure
+[h264 @ 0x84bffc0] mmco: unref short failure
+[h264 @ 0x15769000] mmco: unref short failure
+[h264 @ 0x1aa11900] mmco: unref short failure
+[h264 @ 0x1aa11900] mmco: unref short failure
+[h264 @ 0x82bb480] mmco: unref short failure
+[h264 @ 0x82bb480] mmco: unref short failure
+[h264 @ 0x10d43940] mmco: unref short failure
+[h264 @ 0x9973540] mmco: unref short failure
+[h264 @ 0x9973540] mmco: unref short failure
+[h264 @ 0xa2da800] mmco: unref short failure
+[h264 @ 0xa2da800] mmco: unref short failure
+[h264 @ 0x9973540] mmco: unref short failure
+[h264 @ 0x9973540] mmco: unref short failure
+[h264 @ 0x10d43940] mmco: unref short failure
+[h264 @ 0x15e09fc0] mmco: unref short failure
+[h264 @ 0x15e09fc0] mmco: unref short failure
+[h264 @ 0x10d43940] mmco: unref short failure
+[h264 @ 0x8bc3980] mmco: unref short failure
+[h264 @ 0x8bc3980] mmco: unref short failure
+[h264 @ 0x90b2bc0] mmco: unref short failure
+[h264 @ 0x715d0d00] mmco: unref short failure
+[h264 @ 0x28108040] mmco: unref short failure
+[h264 @ 0x846fbc0] mmco: unref short failure
+[h264 @ 0x846fbc0] mmco: unref short failure
+[h264 @ 0x90b2bc0] mmco: unref short failure
+[h264 @ 0x90b2bc0] mmco: unref short failure
+[h264 @ 0x9973540] mmco: unref short failure
+[h264 @ 0x9973540] mmco: unref short failure
+[h264 @ 0x715d0d00] mmco: unref short failure
+[h264 @ 0x715d0d00] mmco: unref short failure
+[h264 @ 0x82bb480] mmco: unref short failure
+[h264 @ 0x82bb480] mmco: unref short failure
+[h264 @ 0x13c79580] mmco: unref short failure
+[h264 @ 0x13c79580] mmco: unref short failure
+[h264 @ 0x9518d00] mmco: unref short failure
+[h264 @ 0x85513c0] mmco: unref short failure
+[h264 @ 0x85513c0] mmco: unref short failure
+[h264 @ 0x1029a380] mmco: unref short failure
+[h264 @ 0x1029a380] mmco: unref short failure
+[h264 @ 0x20e53000] mmco: unref short failure
+[h264 @ 0x120ec300] mmco: unref short failure
+[h264 @ 0x93eb780] mmco: unref short failure
+[h264 @ 0x93eb780] mmco: unref short failure
+[h264 @ 0xd3c0080] mmco: unref short failure
+[h264 @ 0xd3c0080] mmco: unref short failure
+[h264 @ 0x1029a380] mmco: unref short failure
+[h264 @ 0x1b4bdc40] mmco: unref short failure
+[h264 @ 0x1b4bdc40] mmco: unref short failure
+[h264 @ 0x20e53000] mmco: unref short failure
+[h264 @ 0x20e53000] mmco: unref short failure
+[h264 @ 0x1aa11900] mmco: unref short failure
+[h264 @ 0x1aa11900] mmco: unref short failure
+[h264 @ 0x8b27a00] mmco: unref short failure
+[h264 @ 0x8b27a00] mmco: unref short failure
+[h264 @ 0x47b64100] mmco: unref short failure
+[h264 @ 0x47b64100] mmco: unref short failure
+[h264 @ 0x8b27a00] mmco: unref short failure
+[h264 @ 0x8b27a00] mmco: unref short failure
+[h264 @ 0xe857000] mmco: unref short failure
+[h264 @ 0xe857000] mmco: unref short failure
+[h264 @ 0x838ff40] mmco: unref short failure
+[h264 @ 0x838ff40] mmco: unref short failure
+[h264 @ 0x97b04c0] mmco: unref short failure
+[h264 @ 0x66aeab00] mmco: unref short failure
+[h264 @ 0x66aeab00] mmco: unref short failure
+[h264 @ 0xf7cf380] mmco: unref short failure
+[h264 @ 0xf7cf380] mmco: unref short failure
+[h264 @ 0xf616140] mmco: unref short failure
+[h264 @ 0xf616140] mmco: unref short failure
+[h264 @ 0x41642140] mmco: unref short failure
+[h264 @ 0x41642140] mmco: unref short failure
+[h264 @ 0x8654080] mmco: unref short failure
+[h264 @ 0x8654080] mmco: unref short failure
+[h264 @ 0x8414080] mmco: unref short failure
+[h264 @ 0xa2da800] mmco: unref short failure
+[h264 @ 0x47b64100] mmco: unref short failure
+[h264 @ 0x47b64100] mmco: unref short failure
+[h264 @ 0x8b27a00] mmco: unref short failure
+[h264 @ 0x8b27a00] mmco: unref short failure
+[h264 @ 0x1aa11900] mmco: unref short failure
+[h264 @ 0x1aa11900] mmco: unref short failure
+[h264 @ 0x18542300] mmco: unref short failure
+[h264 @ 0x18542300] mmco: unref short failure
+[h264 @ 0x1cac37c0] mmco: unref short failure
+[h264 @ 0x1cac37c0] mmco: unref short failure
+[h264 @ 0x551b7980] mmco: unref short failure
+[h264 @ 0x551b7980] mmco: unref short failure
+[h264 @ 0x130a5d00] mmco: unref short failure
+[h264 @ 0x130a5d00] mmco: unref short failure
+[h264 @ 0x47b64100] mmco: unref short failure
+[h264 @ 0x1029a380] mmco: unref short failure
+[h264 @ 0x1029a380] mmco: unref short failure
+[h264 @ 0x551b7980] mmco: unref short failure
+[h264 @ 0x9518d00] mmco: unref short failure
+[h264 @ 0x9518d00] mmco: unref short failure
+[h264 @ 0x12ff4840] mmco: unref short failure
+[h264 @ 0x12ff4840] mmco: unref short failure
+[h264 @ 0x37079f80] mmco: unref short failure
+[h264 @ 0x37079f80] mmco: unref short failure
+[h264 @ 0x130a5d00] mmco: unref short failure
+[h264 @ 0x37079f80] mmco: unref short failure
+[h264 @ 0x37079f80] mmco: unref short failure
+You are using a model of type llava_next to instantiate a model of type llava_next_video. This is not supported for all configurations of models and can yield errors.
+[h264 @ 0x838ff40] mmco: unref short failure
+[h264 @ 0xe857000] mmco: unref short failure
+[h264 @ 0x13002b80] mmco: unref short failure
+[h264 @ 0x13002b80] mmco: unref short failure
+[h264 @ 0x13c79580] mmco: unref short failure
+[h264 @ 0x13c79580] mmco: unref short failure
+
+[h264 @ 0x1cac37c0] mmco: unref short failure
+[h264 @ 0x1cac37c0] mmco: unref short failure
+[h264 @ 0x9518d00] mmco: unref short failure
+[h264 @ 0x9518d00] mmco: unref short failure
+[h264 @ 0x21946e00] mmco: unref short failure
+[h264 @ 0x21946e00] mmco: unref short failure
+[h264 @ 0x869f680] mmco: unref short failure
+[h264 @ 0x9518d00] mmco: unref short failure
+
+[h264 @ 0x20e53000] mmco: unref short failure
+[h264 @ 0x37f53c80] mmco: unref short failure
+[h264 @ 0x37f53c80] mmco: unref short failure
+
+[h264 @ 0x2bd85600] mmco: unref short failure
+[h264 @ 0x175507c0] mmco: unref short failure
+[h264 @ 0x175507c0] mmco: unref short failure
+You are using a model of type llava_next to instantiate a model of type llava_next_video. This is not supported for all configurations of models and can yield errors.
+You are using a model of type llava_next to instantiate a model of type llava_next_video. This is not supported for all configurations of models and can yield errors.
+[h264 @ 0x37f53c80] mmco: unref short failure
+You are using a model of type llava_next to instantiate a model of type llava_next_video. This is not supported for all configurations of models and can yield errors.
+You are using a model of type llava_next to instantiate a model of type llava_next_video. This is not supported for all configurations of models and can yield errors.
+[h264 @ 0x2034de00] mmco: unref short failure
+You are using a model of type llava_next to instantiate a model of type llava_next_video. This is not supported for all configurations of models and can yield errors.
+
+
+
+
+
+
+Starting from v4.46, the `logits` model output will have the same type as the model (except at train time, where it will always be FP32)
+
+
+
+
+
+
+
+Chat templates should be in a 'chat_template.json' file but found key='chat_template' in the processor's config. Make sure to move your template to its own file.
+Chat templates should be in a 'chat_template.json' file but found key='chat_template' in the processor's config. Make sure to move your template to its own file.
+Chat templates should be in a 'chat_template.json' file but found key='chat_template' in the processor's config. Make sure to move your template to its own file.
+Chat templates should be in a 'chat_template.json' file but found key='chat_template' in the processor's config. Make sure to move your template to its own file.
+Chat templates should be in a 'chat_template.json' file but found key='chat_template' in the processor's config. Make sure to move your template to its own file.
+Chat templates should be in a 'chat_template.json' file but found key='chat_template' in the processor's config. Make sure to move your template to its own file.
+Expanding inputs for image/video tokens in LLaVa-NeXT-Video should be done in processing. Please add `patch_size` and `vision_feature_select_strategy` to the model's processing config or set directly with `processor.patch_size = {{patch_size}}` and processor.vision_feature_select_strategy = {{vision_feature_select_strategy}}`. Using processors without these attributes in the config is deprecated and will throw an error in v4.47.
+Chat templates should be in a 'chat_template.json' file but found key='chat_template' in the processor's config. Make sure to move your template to its own file.
+Expanding inputs for image.video tokens in LLaVa-NeXT-Video should be done in processing. Please add `patch_size` and `vision_feature_select_strategy` to the model's processing config or set directly with `processor.patch_size = {{patch_size}}` and processor.vision_feature_select_strategy = {{vision_feature_select_strategy}}`. Using processors without these attributes in the config is deprecated and will throw an error in v4.47.
+Expanding inputs for image/video tokens in LLaVa-NeXT-Video should be done in processing. Please add `patch_size` and `vision_feature_select_strategy` to the model's processing config or set directly with `processor.patch_size = {{patch_size}}` and processor.vision_feature_select_strategy = {{vision_feature_select_strategy}}`. Using processors without these attributes in the config is deprecated and will throw an error in v4.47.
+Expanding inputs for image/video tokens in LLaVa-NeXT-Video should be done in processing. Please add `patch_size` and `vision_feature_select_strategy` to the model's processing config or set directly with `processor.patch_size = {{patch_size}}` and processor.vision_feature_select_strategy = {{vision_feature_select_strategy}}`. Using processors without these attributes in the config is deprecated and will throw an error in v4.47.
+Expanding inputs for image/video tokens in LLaVa-NeXT-Video should be done in processing. Please add `patch_size` and `vision_feature_select_strategy` to the model's processing config or set directly with `processor.patch_size = {{patch_size}}` and processor.vision_feature_select_strategy = {{vision_feature_select_strategy}}`. Using processors without these attributes in the config is deprecated and will throw an error in v4.47.
+Starting from v4.46, the `logits` model output will have the same type as the model (except at train time, where it will always be FP32)
+Expanding inputs for image/video tokens in LLaVa-NeXT-Video should be done in processing. Please add `patch_size` and `vision_feature_select_strategy` to the model's processing config or set directly with `processor.patch_size = {{patch_size}}` and processor.vision_feature_select_strategy = {{vision_feature_select_strategy}}`. Using processors without these attributes in the config is deprecated and will throw an error in v4.47.
+Expanding inputs for image/video tokens in LLaVa-NeXT-Video should be done in processing. Please add `patch_size` and `vision_feature_select_strategy` to the model's processing config or set directly with `processor.patch_size = {{patch_size}}` and processor.vision_feature_select_strategy = {{vision_feature_select_strategy}}`. Using processors without these attributes in the config is deprecated and will throw an error in v4.47.
+Expanding inputs for image.video tokens in LLaVa-NeXT-Video should be done in processing. Please add `patch_size` and `vision_feature_select_strategy` to the model's processing config or set directly with `processor.patch_size = {{patch_size}}` and processor.vision_feature_select_strategy = {{vision_feature_select_strategy}}`. Using processors without these attributes in the config is deprecated and will throw an error in v4.47.
+Expanding inputs for image/video tokens in LLaVa-NeXT-Video should be done in processing. Please add `patch_size` and `vision_feature_select_strategy` to the model's processing config or set directly with `processor.patch_size = {{patch_size}}` and processor.vision_feature_select_strategy = {{vision_feature_select_strategy}}`. Using processors without these attributes in the config is deprecated and will throw an error in v4.47.
+Expanding inputs for image.video tokens in LLaVa-NeXT-Video should be done in processing. Please add `patch_size` and `vision_feature_select_strategy` to the model's processing config or set directly with `processor.patch_size = {{patch_size}}` and processor.vision_feature_select_strategy = {{vision_feature_select_strategy}}`. Using processors without these attributes in the config is deprecated and will throw an error in v4.47.
+Expanding inputs for image.video tokens in LLaVa-NeXT-Video should be done in processing. Please add `patch_size` and `vision_feature_select_strategy` to the model's processing config or set directly with `processor.patch_size = {{patch_size}}` and processor.vision_feature_select_strategy = {{vision_feature_select_strategy}}`. Using processors without these attributes in the config is deprecated and will throw an error in v4.47.
+Expanding inputs for image.video tokens in LLaVa-NeXT-Video should be done in processing. Please add `patch_size` and `vision_feature_select_strategy` to the model's processing config or set directly with `processor.patch_size = {{patch_size}}` and processor.vision_feature_select_strategy = {{vision_feature_select_strategy}}`. Using processors without these attributes in the config is deprecated and will throw an error in v4.47.
+Expanding inputs for image.video tokens in LLaVa-NeXT-Video should be done in processing. Please add `patch_size` and `vision_feature_select_strategy` to the model's processing config or set directly with `processor.patch_size = {{patch_size}}` and processor.vision_feature_select_strategy = {{vision_feature_select_strategy}}`. Using processors without these attributes in the config is deprecated and will throw an error in v4.47.
+Starting from v4.46, the `logits` model output will have the same type as the model (except at train time, where it will always be FP32)
+Starting from v4.46, the `logits` model output will have the same type as the model (except at train time, where it will always be FP32)
+Expanding inputs for image.video tokens in LLaVa-NeXT-Video should be done in processing. Please add `patch_size` and `vision_feature_select_strategy` to the model's processing config or set directly with `processor.patch_size = {{patch_size}}` and processor.vision_feature_select_strategy = {{vision_feature_select_strategy}}`. Using processors without these attributes in the config is deprecated and will throw an error in v4.47.
+Starting from v4.46, the `logits` model output will have the same type as the model (except at train time, where it will always be FP32)
+Starting from v4.46, the `logits` model output will have the same type as the model (except at train time, where it will always be FP32)
+Starting from v4.46, the `logits` model output will have the same type as the model (except at train time, where it will always be FP32)
+Starting from v4.46, the `logits` model output will have the same type as the model (except at train time, where it will always be FP32)

config/eval/activitynet_qa/.ipynb_checkpoints/llava_next_video_eval0_config-checkpoint.json

@@ -0,0 +1,23 @@
+{
+    "dataset":{
+        "dataset_name":"Activitynet_QA",
+        "q_json_path":"/home/user/students/ml/dataset/Activitynet_Zero_Shot_QA/test_q.json",  
+        "a_json_path":"/home/user/students/ml/dataset/Activitynet_Zero_Shot_QA/test_a.json",  
+        "video_path":"/home/user/students/ml/dataset/Activitynet_Zero_Shot_QA/all_test",          
+        "data_type":"video",                           
+        "bound":"False",                                 
+        "question_key":"question",                    
+        "answer_key":"answer",                         
+        "name_key":"video_name",                      
+        "video_postfix":[".mp4",".mkv"],
+        "num_segments":8
+    },
+    "model":{
+        "model_path":"/home/user/students/ml/model/LLaVA-NeXT-Video-7B-DPO-hf"
+    },
+    "conv_mode":"llava_next_video_template",
+    "save_path":"./result/eval/activitynet_qa",
+    "experiment_name":"llava_next_video_activitynet_0",
+    "description":"simple inference using llava_next_video with llava_next_video_template in activitynet",
+    "batch_size":4
+}

config/eval/activitynet_qa/.ipynb_checkpoints/qwen2_vl_eval_config-checkpoint.json

@@ -0,0 +1,22 @@
+{
+    "dataset":{
+        "dataset_name": "Activitynet_QA",
+        "q_json_path":"/jizhicfs/hymiezhao/datasets/Activitynet_Zero_Shot_QA/test_q.json",  
+        "a_json_path":"/jizhicfs/hymiezhao/datasets/Activitynet_Zero_Shot_QA/test_a.json",  
+        "video_path":"/jizhicfs/hymiezhao/datasets/Activitynet_Zero_Shot_QA/all_test",          
+        "data_type":"video",                           
+        "bound":"False",                                 
+        "question_key":"question",                    
+        "answer_key":"answer",                         
+        "name_key":"video_name",                      
+        "video_postfix":[".mp4",".mkv"],
+        "num_segments":8
+    },
+    "model":{
+        "model_path":"/jizhicfs/hymiezhao/models/Qwen2-VL-7B-Instruct"
+    },
+    "save_path":"./result/eval/activitynet_qa",
+    "experiment_name":"qwen2_vl_7b_activitynet",
+    "description":"simple inference using qwen2 in activitynet",
+    "batch_size":4
+}

config/eval/activitynet_qa/.ipynb_checkpoints/qwen2_vl_eval_config_prefix1-checkpoint.json

@@ -0,0 +1,23 @@
+{
+    "dataset":{
+        "dataset_name": "Activitynet_QA",
+        "q_json_path":"/jizhicfs/hymiezhao/datasets/Activitynet_Zero_Shot_QA/test_q.json",  
+        "a_json_path":"/jizhicfs/hymiezhao/datasets/Activitynet_Zero_Shot_QA/test_a.json",  
+        "video_path":"/jizhicfs/hymiezhao/datasets/Activitynet_Zero_Shot_QA/all_test",          
+        "data_type":"video",                           
+        "bound":"False",                                 
+        "question_key":"question",                    
+        "answer_key":"answer",                         
+        "name_key":"video_name",                      
+        "video_postfix":[".mp4",".mkv"],
+        "num_segments":8
+    },
+    "model":{
+        "model_path":"/jizhicfs/hymiezhao/models/Qwen2-VL-7B-Instruct"
+    },
+    "prefix":"Watch this video carefully and answer the question: ",
+    "save_path":"./result/eval/activitynet_qa",
+    "experiment_name":"qwen2_vl_7b_activitynet_prefix1",
+    "description":"prefix1 inference using qwen2 in activitynet",
+    "batch_size":4
+}

config/eval/activitynet_qa/.ipynb_checkpoints/qwen2_vl_eval_config_prefix2-checkpoint.json

@@ -0,0 +1,23 @@
+{
+    "dataset":{
+        "dataset_name": "Activitynet_QA",
+        "q_json_path":"/jizhicfs/hymiezhao/datasets/Activitynet_Zero_Shot_QA/test_q.json",  
+        "a_json_path":"/jizhicfs/hymiezhao/datasets/Activitynet_Zero_Shot_QA/test_a.json",  
+        "video_path":"/jizhicfs/hymiezhao/datasets/Activitynet_Zero_Shot_QA/all_test",          
+        "data_type":"video",                           
+        "bound":"False",                                 
+        "question_key":"question",                    
+        "answer_key":"answer",                         
+        "name_key":"video_name",                      
+        "video_postfix":[".mp4",".mkv"],
+        "num_segments":8
+    },
+    "model":{
+        "model_path":"/jizhicfs/hymiezhao/models/Qwen2-VL-7B-Instruct"
+    },
+    "prefix":"You are an expert at video question answering. Please watch this video carefully and answer the question: ",
+    "save_path":"./result/eval/activitynet_qa",
+    "experiment_name":"qwen2_vl_7b_activitynet_prefix2",
+    "description":"prefix2 inference using qwen2 in activitynet",
+    "batch_size":4
+}

config/eval/activitynet_qa/.ipynb_checkpoints/qwen2_vl_eval_config_prefix3-checkpoint.json

@@ -0,0 +1,23 @@
+{
+    "dataset":{
+        "dataset_name": "Activitynet_QA",
+        "q_json_path":"/jizhicfs/hymiezhao/datasets/Activitynet_Zero_Shot_QA/test_q.json",  
+        "a_json_path":"/jizhicfs/hymiezhao/datasets/Activitynet_Zero_Shot_QA/test_a.json",  
+        "video_path":"/jizhicfs/hymiezhao/datasets/Activitynet_Zero_Shot_QA/all_test",          
+        "data_type":"video",                           
+        "bound":"False",                                 
+        "question_key":"question",                    
+        "answer_key":"answer",                         
+        "name_key":"video_name",                      
+        "video_postfix":[".mp4",".mkv"],
+        "num_segments":8
+    },
+    "model":{
+        "model_path":"/jizhicfs/hymiezhao/models/Qwen2-VL-7B-Instruct"
+    },
+    "prefix":"You are an expert at video question answering. Please watch this video carefully and pay close attention to the actions and states of the main characters. Then, answer the question: ",
+    "save_path":"./result/eval/activitynet_qa",
+    "experiment_name":"qwen2_vl_7b_activitynet_prefix3",
+    "description":"prefix3 inference using qwen2 in activitynet",
+    "batch_size":4
+}

config/eval/activitynet_qa/.ipynb_checkpoints/qwen2_vl_eval_config_prefix4-checkpoint.json

@@ -0,0 +1,23 @@
+{
+    "dataset":{
+        "dataset_name": "Activitynet_QA",
+        "q_json_path":"/jizhicfs/hymiezhao/datasets/Activitynet_Zero_Shot_QA/test_q.json",  
+        "a_json_path":"/jizhicfs/hymiezhao/datasets/Activitynet_Zero_Shot_QA/test_a.json",  
+        "video_path":"/jizhicfs/hymiezhao/datasets/Activitynet_Zero_Shot_QA/all_test",          
+        "data_type":"video",                           
+        "bound":"False",                                 
+        "question_key":"question",                    
+        "answer_key":"answer",                         
+        "name_key":"video_name",                      
+        "video_postfix":[".mp4",".mkv"],
+        "num_segments":8
+    },
+    "model":{
+        "model_path":"/jizhicfs/hymiezhao/models/Qwen2-VL-7B-Instruct"
+    },
+    "prefix":"You are an expert at video question answering. Please watch this video carefully, focus on the content described in the following question. Then, answer the question: ",
+    "save_path":"./result/eval/activitynet_qa",
+    "experiment_name":"qwen2_vl_7b_activitynet_prefix4",
+    "description":"prefix4 inference using qwen2 in activitynet",
+    "batch_size":4
+}

config/eval/activitynet_qa/.ipynb_checkpoints/qwen2_vl_eval_config_prefix5-checkpoint.json

@@ -0,0 +1,23 @@
+{
+    "dataset":{
+        "dataset_name": "Activitynet_QA",
+        "q_json_path":"/jizhicfs/hymiezhao/datasets/Activitynet_Zero_Shot_QA/test_q.json",  
+        "a_json_path":"/jizhicfs/hymiezhao/datasets/Activitynet_Zero_Shot_QA/test_a.json",  
+        "video_path":"/jizhicfs/hymiezhao/datasets/Activitynet_Zero_Shot_QA/all_test",          
+        "data_type":"video",                           
+        "bound":"False",                                 
+        "question_key":"question",                    
+        "answer_key":"answer",                         
+        "name_key":"video_name",                      
+        "video_postfix":[".mp4",".mkv"],
+        "num_segments":8
+    },
+    "model":{
+        "model_path":"/jizhicfs/hymiezhao/models/Qwen2-VL-7B-Instruct"
+    },
+    "prefix":"You are an expert at video question answering. Please watch this video carefully, focus on the frames and information which is highly related to the following question. Then, answer the question: ",
+    "save_path":"./result/eval/activitynet_qa",
+    "experiment_name":"qwen2_vl_7b_activitynet_prefix5",
+    "description":"prefix5 inference using qwen2 in activitynet",
+    "batch_size":4
+}

config/eval/activitynet_qa/llava_next_video_eval0_config.json

@@ -0,0 +1,23 @@
+{
+    "dataset":{
+        "dataset_name":"Activitynet_QA",
+        "q_json_path":"/home/user/students/ml/dataset/Activitynet_Zero_Shot_QA/test_q.json",  
+        "a_json_path":"/home/user/students/ml/dataset/Activitynet_Zero_Shot_QA/test_a.json",  
+        "video_path":"/home/user/students/ml/dataset/Activitynet_Zero_Shot_QA/all_test",          
+        "data_type":"video",                           
+        "bound":"False",                                 
+        "question_key":"question",                    
+        "answer_key":"answer",                         
+        "name_key":"video_name",                      
+        "video_postfix":[".mp4",".mkv"],
+        "num_segments":8
+    },
+    "model":{
+        "model_path":"/home/user/students/ml/model/LLaVA-NeXT-Video-7B-DPO-hf"
+    },
+    "conv_mode":"llava_next_video_template",
+    "save_path":"./result/eval/activitynet_qa",
+    "experiment_name":"llava_next_video_activitynet_0",
+    "description":"simple inference using llava_next_video with llava_next_video_template in activitynet",
+    "batch_size":4
+}

config/eval/activitynet_qa/llava_next_video_eval1_config.json

@@ -0,0 +1,23 @@
+{
+    "dataset":{
+        "dataset_name":"Activitynet_QA",
+        "q_json_path":"/home/user/students/ml/dataset/Activitynet_Zero_Shot_QA/test_q.json",  
+        "a_json_path":"/home/user/students/ml/dataset/Activitynet_Zero_Shot_QA/test_a.json",  
+        "video_path":"/home/user/students/ml/dataset/Activitynet_Zero_Shot_QA/all_test",          
+        "data_type":"video",                           
+        "bound":"False",                                 
+        "question_key":"question",                    
+        "answer_key":"answer",                         
+        "name_key":"video_name",                      
+        "video_postfix":[".mp4",".mkv"],
+        "num_segments":8
+    },
+    "model":{
+        "model_path":"/home/user/students/ml/model/LLaVA-NeXT-Video-7B-DPO-hf"
+    },
+    "conv_mode":"llava_next_video_template_with_space_after_assistant",
+    "save_path":"./result/eval/activitynet_qa",
+    "experiment_name":"llava_next_video_activitynet_1",
+    "description":"blank after assistant",
+    "batch_size":4
+}

config/eval/activitynet_qa/llava_next_video_eval2_config.json

@@ -0,0 +1,23 @@
+{
+    "dataset":{
+        "dataset_name":"Activitynet_QA",
+        "q_json_path":"/home/user/students/ml/dataset/Activitynet_Zero_Shot_QA/test_q.json",  
+        "a_json_path":"/home/user/students/ml/dataset/Activitynet_Zero_Shot_QA/test_a.json",  
+        "video_path":"/home/user/students/ml/dataset/Activitynet_Zero_Shot_QA/all_test",          
+        "data_type":"video",                           
+        "bound":"False",                                 
+        "question_key":"question",                    
+        "answer_key":"answer",                         
+        "name_key":"video_name",                      
+        "video_postfix":[".mp4",".mkv"],
+        "num_segments":8
+    },
+    "model":{
+        "model_path":"/home/user/students/ml/model/LLaVA-NeXT-Video-7B-DPO-hf"
+    },
+    "conv_mode":"llava_next_video_template_with_space_after_user",
+    "save_path":"./result/eval/activitynet_qa",
+    "experiment_name":"llava_next_video_activitynet_2",
+    "description":"llava_next_video_template_with_space_after_user in activitynet_qa",
+    "batch_size":4
+}

config/eval/activitynet_qa/qwen2_vl_eval_config.json

@@ -0,0 +1,22 @@
+{
+    "dataset":{
+        "dataset_name": "Activitynet_QA",
+        "q_json_path":"/jizhicfs/hymiezhao/datasets/Activitynet_Zero_Shot_QA/test_q.json",  
+        "a_json_path":"/jizhicfs/hymiezhao/datasets/Activitynet_Zero_Shot_QA/test_a.json",  
+        "video_path":"/jizhicfs/hymiezhao/datasets/Activitynet_Zero_Shot_QA/all_test",          
+        "data_type":"video",                           
+        "bound":"False",                                 
+        "question_key":"question",                    
+        "answer_key":"answer",                         
+        "name_key":"video_name",                      
+        "video_postfix":[".mp4",".mkv"],
+        "num_segments":8
+    },
+    "model":{
+        "model_path":"/jizhicfs/hymiezhao/models/Qwen2-VL-7B-Instruct"
+    },
+    "save_path":"./result/eval/activitynet_qa",
+    "experiment_name":"qwen2_vl_7b_activitynet",
+    "description":"simple inference using qwen2 in activitynet",
+    "batch_size":4
+}

config/eval/activitynet_qa/qwen2_vl_eval_config_prefix1.json

@@ -0,0 +1,23 @@
+{
+    "dataset":{
+        "dataset_name": "Activitynet_QA",
+        "q_json_path":"/jizhicfs/hymiezhao/datasets/Activitynet_Zero_Shot_QA/test_q.json",  
+        "a_json_path":"/jizhicfs/hymiezhao/datasets/Activitynet_Zero_Shot_QA/test_a.json",  
+        "video_path":"/jizhicfs/hymiezhao/datasets/Activitynet_Zero_Shot_QA/all_test",          
+        "data_type":"video",                           
+        "bound":"False",                                 
+        "question_key":"question",                    
+        "answer_key":"answer",                         
+        "name_key":"video_name",                      
+        "video_postfix":[".mp4",".mkv"],
+        "num_segments":8
+    },
+    "model":{
+        "model_path":"/jizhicfs/hymiezhao/models/Qwen2-VL-7B-Instruct"
+    },
+    "prefix":"Watch this video carefully and answer the question: ",
+    "save_path":"./result/eval/activitynet_qa",
+    "experiment_name":"qwen2_vl_7b_activitynet_prefix1",
+    "description":"prefix1 inference using qwen2 in activitynet",
+    "batch_size":4
+}

config/eval/activitynet_qa/qwen2_vl_eval_config_prefix2.json

@@ -0,0 +1,23 @@
+{
+    "dataset":{
+        "dataset_name": "Activitynet_QA",
+        "q_json_path":"/jizhicfs/hymiezhao/datasets/Activitynet_Zero_Shot_QA/test_q.json",  
+        "a_json_path":"/jizhicfs/hymiezhao/datasets/Activitynet_Zero_Shot_QA/test_a.json",  
+        "video_path":"/jizhicfs/hymiezhao/datasets/Activitynet_Zero_Shot_QA/all_test",          
+        "data_type":"video",                           
+        "bound":"False",                                 
+        "question_key":"question",                    
+        "answer_key":"answer",                         
+        "name_key":"video_name",                      
+        "video_postfix":[".mp4",".mkv"],
+        "num_segments":8
+    },
+    "model":{
+        "model_path":"/jizhicfs/hymiezhao/models/Qwen2-VL-7B-Instruct"
+    },
+    "prefix":"You are an expert at video question answering. Please watch this video carefully and answer the question: ",
+    "save_path":"./result/eval/activitynet_qa",
+    "experiment_name":"qwen2_vl_7b_activitynet_prefix2",
+    "description":"prefix2 inference using qwen2 in activitynet",
+    "batch_size":4
+}

config/eval/activitynet_qa/qwen2_vl_eval_config_prefix3.json

@@ -0,0 +1,23 @@
+{
+    "dataset":{
+        "dataset_name": "Activitynet_QA",
+        "q_json_path":"/jizhicfs/hymiezhao/datasets/Activitynet_Zero_Shot_QA/test_q.json",  
+        "a_json_path":"/jizhicfs/hymiezhao/datasets/Activitynet_Zero_Shot_QA/test_a.json",  
+        "video_path":"/jizhicfs/hymiezhao/datasets/Activitynet_Zero_Shot_QA/all_test",          
+        "data_type":"video",                           
+        "bound":"False",                                 
+        "question_key":"question",                    
+        "answer_key":"answer",                         
+        "name_key":"video_name",                      
+        "video_postfix":[".mp4",".mkv"],
+        "num_segments":8
+    },
+    "model":{
+        "model_path":"/jizhicfs/hymiezhao/models/Qwen2-VL-7B-Instruct"
+    },
+    "prefix":"You are an expert at video question answering. Please watch this video carefully and pay close attention to the actions and states of the main characters. Then, answer the question: ",
+    "save_path":"./result/eval/activitynet_qa",
+    "experiment_name":"qwen2_vl_7b_activitynet_prefix3",
+    "description":"prefix3 inference using qwen2 in activitynet",
+    "batch_size":4
+}

config/eval/activitynet_qa/qwen2_vl_eval_config_prefix4.json

@@ -0,0 +1,23 @@
+{
+    "dataset":{
+        "dataset_name": "Activitynet_QA",
+        "q_json_path":"/jizhicfs/hymiezhao/datasets/Activitynet_Zero_Shot_QA/test_q.json",  
+        "a_json_path":"/jizhicfs/hymiezhao/datasets/Activitynet_Zero_Shot_QA/test_a.json",  
+        "video_path":"/jizhicfs/hymiezhao/datasets/Activitynet_Zero_Shot_QA/all_test",          
+        "data_type":"video",                           
+        "bound":"False",                                 
+        "question_key":"question",                    
+        "answer_key":"answer",                         
+        "name_key":"video_name",                      
+        "video_postfix":[".mp4",".mkv"],
+        "num_segments":8
+    },
+    "model":{
+        "model_path":"/jizhicfs/hymiezhao/models/Qwen2-VL-7B-Instruct"
+    },
+    "prefix":"You are an expert at video question answering. Please watch this video carefully, focus on the content described in the following question. Then, answer the question: ",
+    "save_path":"./result/eval/activitynet_qa",
+    "experiment_name":"qwen2_vl_7b_activitynet_prefix4",
+    "description":"prefix4 inference using qwen2 in activitynet",
+    "batch_size":4
+}

config/eval/activitynet_qa/qwen2_vl_eval_config_prefix5.json

@@ -0,0 +1,23 @@
+{
+    "dataset":{
+        "dataset_name": "Activitynet_QA",
+        "q_json_path":"/jizhicfs/hymiezhao/datasets/Activitynet_Zero_Shot_QA/test_q.json",  
+        "a_json_path":"/jizhicfs/hymiezhao/datasets/Activitynet_Zero_Shot_QA/test_a.json",  
+        "video_path":"/jizhicfs/hymiezhao/datasets/Activitynet_Zero_Shot_QA/all_test",          
+        "data_type":"video",                           
+        "bound":"False",                                 
+        "question_key":"question",                    
+        "answer_key":"answer",                         
+        "name_key":"video_name",                      
+        "video_postfix":[".mp4",".mkv"],
+        "num_segments":8
+    },
+    "model":{
+        "model_path":"/jizhicfs/hymiezhao/models/Qwen2-VL-7B-Instruct"
+    },
+    "prefix":"You are an expert at video question answering. Please watch this video carefully, focus on the frames and information which is highly related to the following question. Then, answer the question: ",
+    "save_path":"./result/eval/activitynet_qa",
+    "experiment_name":"qwen2_vl_7b_activitynet_prefix5",
+    "description":"prefix5 inference using qwen2 in activitynet",
+    "batch_size":4
+}

config/eval/activitynet_qa/test_config.json

@@ -0,0 +1,23 @@
+{
+    "dataset":{
+        "dataset_name":"Activitynet_QA",
+        "q_json_path":"/home/user/students/ml/dataset/Activitynet_Zero_Shot_QA/test_q.json",  
+        "a_json_path":"/home/user/students/ml/dataset/Activitynet_Zero_Shot_QA/test_a.json",  
+        "video_path":"/home/user/students/ml/dataset/Activitynet_Zero_Shot_QA/all_test",          
+        "data_type":"video",                           
+        "bound":"False",                                 
+        "question_key":"question",                    
+        "answer_key":"answer",                         
+        "name_key":"video_name",                      
+        "video_postfix":[".mp4",".mkv"],
+        "num_segments":8
+    },
+    "model":{
+        "model_path":"/home/user/students/ml/model/LLaVA-NeXT-Video-7B-DPO-hf"
+    },
+    "conv_mode":"llava_next_video_template",
+    "save_path":"./result/eval/activitynet_qa",
+    "experiment_name":"test",
+    "description":"simple inference using llava_next_video with llava_next_video_template in activitynet_qa",
+    "batch_size":4
+}

config/eval/msrvtt_qa/.ipynb_checkpoints/llava_next_video_eval0_config-checkpoint.json

@@ -0,0 +1,23 @@
+{
+    "dataset":{
+        "dataset_name":"MSRVTT_QA",
+        "q_json_path":"/mnt/hwfile/opendatalab/chenjingzhou/ml/dataset/MSRVTT_Zero_Shot_QA/test_q.json",  
+        "a_json_path":"/mnt/hwfile/opendatalab/chenjingzhou/ml/dataset/MSRVTT_Zero_Shot_QA/test_a.json",  
+        "video_path":"/mnt/hwfile/opendatalab/chenjingzhou/ml/dataset/MSRVTT_Zero_Shot_QA/videos/all",          
+        "data_type":"video",                           
+        "bound":"False",                                 
+        "question_key":"question",                    
+        "answer_key":"answer",                         
+        "name_key":"video_name",                      
+        "video_postfix":[".mp4"],
+        "num_segments":8
+    },
+    "model":{
+        "model_path":"/mnt/hwfile/opendatalab/chenjingzhou/ml/model/llava-next-video-7b-DPO-hf"
+    },
+    "conv_mode":"llava_next_video_template",
+    "save_path":"./result/eval/msrvtt_qa",
+    "experiment_name":"llava_next_video_msrvtt_0",
+    "description":"simple inference using llava_next_video with llava_next_video_template in MSRVTT",
+    "batch_size":32
+}

config/eval/msrvtt_qa/.ipynb_checkpoints/qwen2_vl_eval_config-checkpoint.json

@@ -0,0 +1,22 @@
+{
+    "dataset":{
+        "dataset_name": "MSRVTT_QA",
+        "q_json_path":"/jizhicfs/hymiezhao/datasets/MSRVTT_Zero_Shot_QA/test_q.json",  
+        "a_json_path":"/jizhicfs/hymiezhao/datasets/MSRVTT_Zero_Shot_QA/test_a.json",  
+        "video_path":"/jizhicfs/hymiezhao/datasets/MSRVTT_Zero_Shot_QA/videos/all",          
+        "data_type":"video",                           
+        "bound":"False",                                 
+        "question_key":"question",                    
+        "answer_key":"answer",                         
+        "name_key":"video_name",                      
+        "video_postfix":[".mp4"],
+        "num_segments":8
+    },
+    "model":{
+        "model_path":"/jizhicfs/hymiezhao/models/Qwen2-VL-7B-Instruct"
+    },
+    "save_path":"./result/eval/msrvtt_qa",
+    "experiment_name":"qwen2_vl_7b_msrvtt",
+    "description":"simple inference using qwen2 in msrvtt",
+    "batch_size":8
+}

config/eval/msrvtt_qa/.ipynb_checkpoints/qwen2_vl_eval_config_prefix1-checkpoint.json

@@ -0,0 +1,23 @@
+{
+    "dataset":{
+        "dataset_name": "MSRVTT_QA",
+        "q_json_path":"/jizhicfs/hymiezhao/datasets/MSRVTT_Zero_Shot_QA/test_q.json",  
+        "a_json_path":"/jizhicfs/hymiezhao/datasets/MSRVTT_Zero_Shot_QA/test_a.json",  
+        "video_path":"/jizhicfs/hymiezhao/datasets/MSRVTT_Zero_Shot_QA/videos/all",          
+        "data_type":"video",                           
+        "bound":"False",                                 
+        "question_key":"question",                    
+        "answer_key":"answer",                         
+        "name_key":"video_name",                      
+        "video_postfix":[".mp4"],
+        "num_segments":8
+    },
+    "model":{
+        "model_path":"/jizhicfs/hymiezhao/models/Qwen2-VL-7B-Instruct"
+    },
+    "prefix": "Watch this video carefully and answer the question: ",
+    "save_path":"./result/eval/msrvtt_qa",
+    "experiment_name":"qwen2_vl_7b_msrvtt_prefix1",
+    "description":"prefix1 inference using qwen2 in msrvtt",
+    "batch_size":8
+}

config/eval/msrvtt_qa/.ipynb_checkpoints/qwen2_vl_eval_config_prefix2-checkpoint.json

@@ -0,0 +1,23 @@
+{
+    "dataset":{
+        "dataset_name": "MSRVTT_QA",
+        "q_json_path":"/jizhicfs/hymiezhao/datasets/MSRVTT_Zero_Shot_QA/test_q.json",  
+        "a_json_path":"/jizhicfs/hymiezhao/datasets/MSRVTT_Zero_Shot_QA/test_a.json",  
+        "video_path":"/jizhicfs/hymiezhao/datasets/MSRVTT_Zero_Shot_QA/videos/all",          
+        "data_type":"video",                           
+        "bound":"False",                                 
+        "question_key":"question",                    
+        "answer_key":"answer",                         
+        "name_key":"video_name",                      
+        "video_postfix":[".mp4"],
+        "num_segments":8
+    },
+    "model":{
+        "model_path":"/jizhicfs/hymiezhao/models/Qwen2-VL-7B-Instruct"
+    },
+    "prefix": "You are an expert at video question answering. Please watch this video carefully and answer the question: ",
+    "save_path":"./result/eval/msrvtt_qa",
+    "experiment_name":"qwen2_vl_7b_msrvtt_prefix2",
+    "description":"prefix2 inference using qwen2 in msrvtt",
+    "batch_size":8
+}

config/eval/msrvtt_qa/.ipynb_checkpoints/qwen2_vl_eval_config_prefix3-checkpoint.json

@@ -0,0 +1,23 @@
+{
+    "dataset":{
+        "dataset_name": "MSRVTT_QA",
+        "q_json_path":"/jizhicfs/hymiezhao/datasets/MSRVTT_Zero_Shot_QA/test_q.json",  
+        "a_json_path":"/jizhicfs/hymiezhao/datasets/MSRVTT_Zero_Shot_QA/test_a.json",  
+        "video_path":"/jizhicfs/hymiezhao/datasets/MSRVTT_Zero_Shot_QA/videos/all",          
+        "data_type":"video",                           
+        "bound":"False",                                 
+        "question_key":"question",                    
+        "answer_key":"answer",                         
+        "name_key":"video_name",                      
+        "video_postfix":[".mp4"],
+        "num_segments":8
+    },
+    "model":{
+        "model_path":"/jizhicfs/hymiezhao/models/Qwen2-VL-7B-Instruct"
+    },
+    "prefix": "You are an expert at video question answering. Please watch this video carefully and pay close attention to the actions and states of the main characters. Then, answer the question: ",
+    "save_path":"./result/eval/msrvtt_qa",
+    "experiment_name":"qwen2_vl_7b_msrvtt_prefix3",
+    "description":"prefix3 inference using qwen2 in msrvtt",
+    "batch_size":8
+}

config/eval/msrvtt_qa/.ipynb_checkpoints/qwen2_vl_eval_config_prefix4-checkpoint.json

@@ -0,0 +1,23 @@
+{
+    "dataset":{
+        "dataset_name": "MSRVTT_QA",
+        "q_json_path":"/jizhicfs/hymiezhao/datasets/MSRVTT_Zero_Shot_QA/test_q.json",  
+        "a_json_path":"/jizhicfs/hymiezhao/datasets/MSRVTT_Zero_Shot_QA/test_a.json",  
+        "video_path":"/jizhicfs/hymiezhao/datasets/MSRVTT_Zero_Shot_QA/videos/all",          
+        "data_type":"video",                           
+        "bound":"False",                                 
+        "question_key":"question",                    
+        "answer_key":"answer",                         
+        "name_key":"video_name",                      
+        "video_postfix":[".mp4"],
+        "num_segments":8
+    },
+    "model":{
+        "model_path":"/jizhicfs/hymiezhao/models/Qwen2-VL-7B-Instruct"
+    },
+    "prefix": "You are an expert at video question answering. Please watch this video carefully, focus on the content described in the following question. Then, answer the question: ",
+    "save_path":"./result/eval/msrvtt_qa",
+    "experiment_name":"qwen2_vl_7b_msrvtt_prefix4",
+    "description":"prefix4 inference using qwen2 in msrvtt",
+    "batch_size":8
+}

config/eval/msrvtt_qa/.ipynb_checkpoints/qwen2_vl_eval_config_prefix5-checkpoint.json

@@ -0,0 +1,23 @@
+{
+    "dataset":{
+        "dataset_name": "MSRVTT_QA",
+        "q_json_path":"/jizhicfs/hymiezhao/datasets/MSRVTT_Zero_Shot_QA/test_q.json",  
+        "a_json_path":"/jizhicfs/hymiezhao/datasets/MSRVTT_Zero_Shot_QA/test_a.json",  
+        "video_path":"/jizhicfs/hymiezhao/datasets/MSRVTT_Zero_Shot_QA/videos/all",          
+        "data_type":"video",                           
+        "bound":"False",                                 
+        "question_key":"question",                    
+        "answer_key":"answer",                         
+        "name_key":"video_name",                      
+        "video_postfix":[".mp4"],
+        "num_segments":8
+    },
+    "model":{
+        "model_path":"/jizhicfs/hymiezhao/models/Qwen2-VL-7B-Instruct"
+    },
+    "prefix": "You are an expert at video question answering. Please watch this video carefully, focus on the frames and information which is highly related to the following question. Then, answer the question: ",
+    "save_path":"./result/eval/msrvtt_qa",
+    "experiment_name":"qwen2_vl_7b_msrvtt_prefix5",
+    "description":"prefix5 inference using qwen2 in msrvtt",
+    "batch_size":8
+}

config/eval/msrvtt_qa/llava_next_video_eval0_config.json

@@ -0,0 +1,23 @@
+{
+    "dataset":{
+        "dataset_name":"MSRVTT_QA",
+        "q_json_path":"/mnt/hwfile/opendatalab/chenjingzhou/ml/dataset/MSRVTT_Zero_Shot_QA/test_q.json",  
+        "a_json_path":"/mnt/hwfile/opendatalab/chenjingzhou/ml/dataset/MSRVTT_Zero_Shot_QA/test_a.json",  
+        "video_path":"/mnt/hwfile/opendatalab/chenjingzhou/ml/dataset/MSRVTT_Zero_Shot_QA/videos/all",          
+        "data_type":"video",                           
+        "bound":"False",                                 
+        "question_key":"question",                    
+        "answer_key":"answer",                         
+        "name_key":"video_name",                      
+        "video_postfix":[".mp4"],
+        "num_segments":8
+    },
+    "model":{
+        "model_path":"/mnt/hwfile/opendatalab/chenjingzhou/ml/model/llava-next-video-7b-DPO-hf"
+    },
+    "conv_mode":"llava_next_video_template",
+    "save_path":"./result/eval/msrvtt_qa",
+    "experiment_name":"llava_next_video_msrvtt_0",
+    "description":"simple inference using llava_next_video with llava_next_video_template in MSRVTT",
+    "batch_size":32
+}

config/eval/msrvtt_qa/llava_next_video_eval1_config.json

@@ -0,0 +1,23 @@
+{
+    "dataset":{
+        "dataset_name":"MSRVTT_QA",
+        "q_json_path":"/mnt/hwfile/opendatalab/chenjingzhou/ml/dataset/MSRVTT_Zero_Shot_QA/test_q.json",  
+        "a_json_path":"/mnt/hwfile/opendatalab/chenjingzhou/ml/dataset/MSRVTT_Zero_Shot_QA/test_a.json",  
+        "video_path":"/mnt/hwfile/opendatalab/chenjingzhou/ml/dataset/MSRVTT_Zero_Shot_QA/videos/all",          
+        "data_type":"video",                           
+        "bound":"False",                                 
+        "question_key":"question",                    
+        "answer_key":"answer",                         
+        "name_key":"video_name",                      
+        "video_postfix":[".mp4"],
+        "num_segments":8
+    },
+    "model":{
+        "model_path":"/mnt/hwfile/opendatalab/chenjingzhou/ml/model/llava-next-video-7b-DPO-hf"
+    },
+    "conv_mode":"llava_next_video_template_with_space_after_assistant",
+    "save_path":"./result/eval/msrvtt_qa",
+    "experiment_name":"llava_next_video_msrvtt_1",
+    "description":"blank after assistant",
+    "batch_size":32
+}

config/eval/msrvtt_qa/llava_next_video_eval2_config.json

@@ -0,0 +1,23 @@
+{
+    "dataset":{
+        "dataset_name":"MSRVTT_QA",
+        "q_json_path":"/mnt/hwfile/opendatalab/chenjingzhou/ml/dataset/MSRVTT_Zero_Shot_QA/test_q.json",  
+        "a_json_path":"/mnt/hwfile/opendatalab/chenjingzhou/ml/dataset/MSRVTT_Zero_Shot_QA/test_a.json",  
+        "video_path":"/mnt/hwfile/opendatalab/chenjingzhou/ml/dataset/MSRVTT_Zero_Shot_QA/videos/all",          
+        "data_type":"video",                           
+        "bound":"False",                                 
+        "question_key":"question",                    
+        "answer_key":"answer",                         
+        "name_key":"video_name",                      
+        "video_postfix":[".mp4"],
+        "num_segments":8
+    },
+    "model":{
+        "model_path":"/mnt/hwfile/opendatalab/chenjingzhou/ml/model/llava-next-video-7b-DPO-hf"
+    },
+    "conv_mode":"llava_next_video_template_with_space_after_user",
+    "save_path":"./result/eval/msrvtt_qa",
+    "experiment_name":"llava_next_video_msrvtt_2",
+    "description":"llava_next_video_template_with_space_after_user in msrvtt_qa",
+    "batch_size":32
+}

config/eval/msrvtt_qa/qwen2_vl_eval_config.json

@@ -0,0 +1,22 @@
+{
+    "dataset":{
+        "dataset_name": "MSRVTT_QA",
+        "q_json_path":"/jizhicfs/hymiezhao/datasets/MSRVTT_Zero_Shot_QA/test_q.json",  
+        "a_json_path":"/jizhicfs/hymiezhao/datasets/MSRVTT_Zero_Shot_QA/test_a.json",  
+        "video_path":"/jizhicfs/hymiezhao/datasets/MSRVTT_Zero_Shot_QA/videos/all",          
+        "data_type":"video",                           
+        "bound":"False",                                 
+        "question_key":"question",                    
+        "answer_key":"answer",                         
+        "name_key":"video_name",                      
+        "video_postfix":[".mp4"],
+        "num_segments":8
+    },
+    "model":{
+        "model_path":"/jizhicfs/hymiezhao/models/Qwen2-VL-7B-Instruct"
+    },
+    "save_path":"./result/eval/msrvtt_qa",
+    "experiment_name":"qwen2_vl_7b_msrvtt",
+    "description":"simple inference using qwen2 in msrvtt",
+    "batch_size":8
+}

config/eval/msrvtt_qa/qwen2_vl_eval_config_prefix1.json

@@ -0,0 +1,23 @@
+{
+    "dataset":{
+        "dataset_name": "MSRVTT_QA",
+        "q_json_path":"/jizhicfs/hymiezhao/datasets/MSRVTT_Zero_Shot_QA/test_q.json",  
+        "a_json_path":"/jizhicfs/hymiezhao/datasets/MSRVTT_Zero_Shot_QA/test_a.json",  
+        "video_path":"/jizhicfs/hymiezhao/datasets/MSRVTT_Zero_Shot_QA/videos/all",          
+        "data_type":"video",                           
+        "bound":"False",                                 
+        "question_key":"question",                    
+        "answer_key":"answer",                         
+        "name_key":"video_name",                      
+        "video_postfix":[".mp4"],
+        "num_segments":8
+    },
+    "model":{
+        "model_path":"/jizhicfs/hymiezhao/models/Qwen2-VL-7B-Instruct"
+    },
+    "prefix": "Watch this video carefully and answer the question: ",
+    "save_path":"./result/eval/msrvtt_qa",
+    "experiment_name":"qwen2_vl_7b_msrvtt_prefix1",
+    "description":"prefix1 inference using qwen2 in msrvtt",
+    "batch_size":8
+}

config/eval/msrvtt_qa/qwen2_vl_eval_config_prefix2.json

@@ -0,0 +1,23 @@
+{
+    "dataset":{
+        "dataset_name": "MSRVTT_QA",
+        "q_json_path":"/jizhicfs/hymiezhao/datasets/MSRVTT_Zero_Shot_QA/test_q.json",  
+        "a_json_path":"/jizhicfs/hymiezhao/datasets/MSRVTT_Zero_Shot_QA/test_a.json",  
+        "video_path":"/jizhicfs/hymiezhao/datasets/MSRVTT_Zero_Shot_QA/videos/all",          
+        "data_type":"video",                           
+        "bound":"False",                                 
+        "question_key":"question",                    
+        "answer_key":"answer",                         
+        "name_key":"video_name",                      
+        "video_postfix":[".mp4"],
+        "num_segments":8
+    },
+    "model":{
+        "model_path":"/jizhicfs/hymiezhao/models/Qwen2-VL-7B-Instruct"
+    },
+    "prefix": "You are an expert at video question answering. Please watch this video carefully and answer the question: ",
+    "save_path":"./result/eval/msrvtt_qa",
+    "experiment_name":"qwen2_vl_7b_msrvtt_prefix2",
+    "description":"prefix2 inference using qwen2 in msrvtt",
+    "batch_size":8
+}

config/eval/msrvtt_qa/qwen2_vl_eval_config_prefix3.json

@@ -0,0 +1,23 @@
+{
+    "dataset":{
+        "dataset_name": "MSRVTT_QA",
+        "q_json_path":"/jizhicfs/hymiezhao/datasets/MSRVTT_Zero_Shot_QA/test_q.json",  
+        "a_json_path":"/jizhicfs/hymiezhao/datasets/MSRVTT_Zero_Shot_QA/test_a.json",  
+        "video_path":"/jizhicfs/hymiezhao/datasets/MSRVTT_Zero_Shot_QA/videos/all",          
+        "data_type":"video",                           
+        "bound":"False",                                 
+        "question_key":"question",                    
+        "answer_key":"answer",                         
+        "name_key":"video_name",                      
+        "video_postfix":[".mp4"],
+        "num_segments":8
+    },
+    "model":{
+        "model_path":"/jizhicfs/hymiezhao/models/Qwen2-VL-7B-Instruct"
+    },
+    "prefix": "You are an expert at video question answering. Please watch this video carefully and pay close attention to the actions and states of the main characters. Then, answer the question: ",
+    "save_path":"./result/eval/msrvtt_qa",
+    "experiment_name":"qwen2_vl_7b_msrvtt_prefix3",
+    "description":"prefix3 inference using qwen2 in msrvtt",
+    "batch_size":8
+}

config/eval/msrvtt_qa/qwen2_vl_eval_config_prefix4.json

@@ -0,0 +1,23 @@
+{
+    "dataset":{
+        "dataset_name": "MSRVTT_QA",
+        "q_json_path":"/jizhicfs/hymiezhao/datasets/MSRVTT_Zero_Shot_QA/test_q.json",  
+        "a_json_path":"/jizhicfs/hymiezhao/datasets/MSRVTT_Zero_Shot_QA/test_a.json",  
+        "video_path":"/jizhicfs/hymiezhao/datasets/MSRVTT_Zero_Shot_QA/videos/all",          
+        "data_type":"video",                           
+        "bound":"False",                                 
+        "question_key":"question",                    
+        "answer_key":"answer",                         
+        "name_key":"video_name",                      
+        "video_postfix":[".mp4"],
+        "num_segments":8
+    },
+    "model":{
+        "model_path":"/jizhicfs/hymiezhao/models/Qwen2-VL-7B-Instruct"
+    },
+    "prefix": "You are an expert at video question answering. Please watch this video carefully, focus on the content described in the following question. Then, answer the question: ",
+    "save_path":"./result/eval/msrvtt_qa",
+    "experiment_name":"qwen2_vl_7b_msrvtt_prefix4",
+    "description":"prefix4 inference using qwen2 in msrvtt",
+    "batch_size":8
+}

config/eval/msrvtt_qa/qwen2_vl_eval_config_prefix5.json

@@ -0,0 +1,23 @@
+{
+    "dataset":{
+        "dataset_name": "MSRVTT_QA",
+        "q_json_path":"/jizhicfs/hymiezhao/datasets/MSRVTT_Zero_Shot_QA/test_q.json",  
+        "a_json_path":"/jizhicfs/hymiezhao/datasets/MSRVTT_Zero_Shot_QA/test_a.json",  
+        "video_path":"/jizhicfs/hymiezhao/datasets/MSRVTT_Zero_Shot_QA/videos/all",          
+        "data_type":"video",                           
+        "bound":"False",                                 
+        "question_key":"question",                    
+        "answer_key":"answer",                         
+        "name_key":"video_name",                      
+        "video_postfix":[".mp4"],
+        "num_segments":8
+    },
+    "model":{
+        "model_path":"/jizhicfs/hymiezhao/models/Qwen2-VL-7B-Instruct"
+    },
+    "prefix": "You are an expert at video question answering. Please watch this video carefully, focus on the frames and information which is highly related to the following question. Then, answer the question: ",
+    "save_path":"./result/eval/msrvtt_qa",
+    "experiment_name":"qwen2_vl_7b_msrvtt_prefix5",
+    "description":"prefix5 inference using qwen2 in msrvtt",
+    "batch_size":8
+}

config/eval/msrvtt_qa/test_config.json

@@ -0,0 +1,23 @@
+{
+    "dataset":{
+        "dataset_name":"MSRVTT_QA",
+        "q_json_path":"/home/user/students/ml/dataset/MSRVTT_Zero_Shot_QA/test_q.json",  
+        "a_json_path":"/home/user/students/ml/dataset/MSRVTT_Zero_Shot_QA/test_a.json",  
+        "video_path":"/home/user/students/ml/dataset/MSRVTT_Zero_Shot_QA/videos/all",          
+        "data_type":"video",                           
+        "bound":"False",                                 
+        "question_key":"question",                    
+        "answer_key":"answer",                         
+        "name_key":"video_name",                      
+        "video_postfix":[".mp4"],
+        "num_segments":8
+    },
+    "model":{
+        "model_path":"/home/user/students/ml/model/LLaVA-NeXT-Video-7B-DPO-hf"
+    },
+    "conv_mode":"llava_next_video_template",
+    "save_path":"./result/eval/msrvtt_qa",
+    "experiment_name":"test",
+    "description":"simple inference using llava_next_video with llava_next_video_template in msrvtt_qa",
+    "batch_size":4
+}

config/eval/msvd_qa/.ipynb_checkpoints/qwen2_vl_eval_config-checkpoint.json

@@ -0,0 +1,22 @@
+{
+    "dataset":{
+        "dataset_name": "MSVD_QA",
+        "q_json_path":"/jizhicfs/hymiezhao/datasets/MSVD_Zero_Shot_QA/test_q.json",  
+        "a_json_path":"/jizhicfs/hymiezhao/datasets/MSVD_Zero_Shot_QA/test_a.json",  
+        "video_path":"/jizhicfs/hymiezhao/datasets/MSVD_Zero_Shot_QA/videos",          
+        "data_type":"video",                           
+        "bound":"False",                                 
+        "question_key":"question",                    
+        "answer_key":"answer",                         
+        "name_key":"video_name",                      
+        "video_postfix":[".avi"],
+        "num_segments":8
+    },
+    "model":{
+        "model_path":"/jizhicfs/hymiezhao/models/Qwen2-VL-7B-Instruct"
+    },
+    "save_path":"./result/eval/msvd_qa",
+    "experiment_name":"qwen2_vl_7b_msvd",
+    "description":"simple inference using qwen2 in msvd",
+    "batch_size":4
+}

config/eval/msvd_qa/.ipynb_checkpoints/qwen2_vl_eval_config_prefix1-checkpoint.json

@@ -0,0 +1,23 @@
+{
+    "dataset":{
+        "dataset_name": "MSVD_QA",
+        "q_json_path":"/jizhicfs/hymiezhao/datasets/MSVD_Zero_Shot_QA/test_q.json",  
+        "a_json_path":"/jizhicfs/hymiezhao/datasets/MSVD_Zero_Shot_QA/test_a.json",  
+        "video_path":"/jizhicfs/hymiezhao/datasets/MSVD_Zero_Shot_QA/videos",          
+        "data_type":"video",                           
+        "bound":"False",                                 
+        "question_key":"question",                    
+        "answer_key":"answer",                         
+        "name_key":"video_name",                      
+        "video_postfix":[".avi"],
+        "num_segments":8
+    },
+    "model":{
+        "model_path":"/jizhicfs/hymiezhao/models/Qwen2-VL-7B-Instruct"
+    },
+    "prefix":"Watch this video carefully and answer the question: ",
+    "save_path":"./result/eval/msvd_qa",
+    "experiment_name":"qwen2_vl_7b_msvd_prefix1",
+    "description":"prefix1 inference using qwen2 in msvd",
+    "batch_size":4
+}

config/eval/msvd_qa/.ipynb_checkpoints/qwen2_vl_eval_config_prefix2-checkpoint.json

@@ -0,0 +1,23 @@
+{
+    "dataset":{
+        "dataset_name": "MSVD_QA",
+        "q_json_path":"/jizhicfs/hymiezhao/datasets/MSVD_Zero_Shot_QA/test_q.json",  
+        "a_json_path":"/jizhicfs/hymiezhao/datasets/MSVD_Zero_Shot_QA/test_a.json",  
+        "video_path":"/jizhicfs/hymiezhao/datasets/MSVD_Zero_Shot_QA/videos",          
+        "data_type":"video",                           
+        "bound":"False",                                 
+        "question_key":"question",                    
+        "answer_key":"answer",                         
+        "name_key":"video_name",                      
+        "video_postfix":[".avi"],
+        "num_segments":8
+    },
+    "model":{
+        "model_path":"/jizhicfs/hymiezhao/models/Qwen2-VL-7B-Instruct"
+    },
+    "prefix":"You are an expert at video question answering. Please watch this video carefully and answer the question: ",
+    "save_path":"./result/eval/msvd_qa",
+    "experiment_name":"qwen2_vl_7b_msvd_prefix2",
+    "description":"prefix2 inference using qwen2 in msvd",
+    "batch_size":4
+}

config/eval/msvd_qa/.ipynb_checkpoints/qwen2_vl_eval_config_prefix3-checkpoint.json

@@ -0,0 +1,23 @@
+{
+    "dataset":{
+        "dataset_name": "MSVD_QA",
+        "q_json_path":"/jizhicfs/hymiezhao/datasets/MSVD_Zero_Shot_QA/test_q.json",  
+        "a_json_path":"/jizhicfs/hymiezhao/datasets/MSVD_Zero_Shot_QA/test_a.json",  
+        "video_path":"/jizhicfs/hymiezhao/datasets/MSVD_Zero_Shot_QA/videos",          
+        "data_type":"video",                           
+        "bound":"False",                                 
+        "question_key":"question",                    
+        "answer_key":"answer",                         
+        "name_key":"video_name",                      
+        "video_postfix":[".avi"],
+        "num_segments":8
+    },
+    "model":{
+        "model_path":"/jizhicfs/hymiezhao/models/Qwen2-VL-7B-Instruct"
+    },
+    "prefix":"You are an expert at video question answering. Please watch this video carefully and pay close attention to the actions and states of the main characters. Then, answer the question: ",
+    "save_path":"./result/eval/msvd_qa",
+    "experiment_name":"qwen2_vl_7b_msvd_prefix3",
+    "description":"prefix3 inference using qwen2 in msvd",
+    "batch_size":4
+}

config/eval/msvd_qa/.ipynb_checkpoints/qwen2_vl_eval_config_prefix4-checkpoint.json

@@ -0,0 +1,23 @@
+{
+    "dataset":{
+        "dataset_name": "MSVD_QA",
+        "q_json_path":"/jizhicfs/hymiezhao/datasets/MSVD_Zero_Shot_QA/test_q.json",  
+        "a_json_path":"/jizhicfs/hymiezhao/datasets/MSVD_Zero_Shot_QA/test_a.json",  
+        "video_path":"/jizhicfs/hymiezhao/datasets/MSVD_Zero_Shot_QA/videos",          
+        "data_type":"video",                           
+        "bound":"False",                                 
+        "question_key":"question",                    
+        "answer_key":"answer",                         
+        "name_key":"video_name",                      
+        "video_postfix":[".avi"],
+        "num_segments":8
+    },
+    "model":{
+        "model_path":"/jizhicfs/hymiezhao/models/Qwen2-VL-7B-Instruct"
+    },
+    "prefix":"You are an expert at video question answering. Please watch this video carefully, focus on the content described in the following question. Then, answer the question: ",
+    "save_path":"./result/eval/msvd_qa",
+    "experiment_name":"qwen2_vl_7b_msvd_prefix4",
+    "description":"prefix4 inference using qwen2 in msvd",
+    "batch_size":4
+}

config/eval/msvd_qa/.ipynb_checkpoints/qwen2_vl_eval_config_prefix5-checkpoint.json

@@ -0,0 +1,23 @@
+{
+    "dataset":{
+        "dataset_name": "MSVD_QA",
+        "q_json_path":"/jizhicfs/hymiezhao/datasets/MSVD_Zero_Shot_QA/test_q.json",  
+        "a_json_path":"/jizhicfs/hymiezhao/datasets/MSVD_Zero_Shot_QA/test_a.json",  
+        "video_path":"/jizhicfs/hymiezhao/datasets/MSVD_Zero_Shot_QA/videos",          
+        "data_type":"video",                           
+        "bound":"False",                                 
+        "question_key":"question",                    
+        "answer_key":"answer",                         
+        "name_key":"video_name",                      
+        "video_postfix":[".avi"],
+        "num_segments":8
+    },
+    "model":{
+        "model_path":"/jizhicfs/hymiezhao/models/Qwen2-VL-7B-Instruct"
+    },
+    "prefix":"You are an expert at video question answering. Please watch this video carefully, focus on the frames and information which is highly related to the following question. Then, answer the question: ",
+    "save_path":"./result/eval/msvd_qa",
+    "experiment_name":"qwen2_vl_7b_msvd_prefix5",
+    "description":"prefix5 inference using qwen2 in msvd",
+    "batch_size":4
+}

config/eval/msvd_qa/llava_next_video_eval0_config.json

@@ -0,0 +1,23 @@
+{
+    "dataset":{
+        "dataset_name": "MSVD_QA",
+        "q_json_path":"/mnt/hwfile/opendatalab/chenjingzhou/ml/dataset/MSVD_Zero_Shot_QA/test_q.json",  
+        "a_json_path":"/mnt/hwfile/opendatalab/chenjingzhou/ml/dataset/MSVD_Zero_Shot_QA/test_a.json",  
+        "video_path":"/mnt/hwfile/opendatalab/chenjingzhou/ml/dataset/MSVD_Zero_Shot_QA/videos",          
+        "data_type":"video",                           
+        "bound":"False",                                 
+        "question_key":"question",                    
+        "answer_key":"answer",                         
+        "name_key":"video_name",                      
+        "video_postfix":[".avi"],
+        "num_segments":8
+    },
+    "model":{
+        "model_path":"/mnt/hwfile/opendatalab/chenjingzhou/ml/model/llava-next-video-7b-DPO-hf"
+    },
+    "conv_mode":"llava_next_video_template",
+    "save_path":"./result/eval/msvd_qa",
+    "experiment_name":"llava_next_video_msvd_0",
+    "description":"simple inference using llava_next_video with llava_next_video_template in msvd",
+    "batch_size":32
+}

config/eval/msvd_qa/llava_next_video_eval1_config.json

@@ -0,0 +1,23 @@
+{
+    "dataset":{
+        "dataset_name": "MSVD_QA",
+        "q_json_path":"/mnt/hwfile/opendatalab/chenjingzhou/ml/dataset/MSVD_Zero_Shot_QA/test_q.json",  
+        "a_json_path":"/mnt/hwfile/opendatalab/chenjingzhou/ml/dataset/MSVD_Zero_Shot_QA/test_a.json",  
+        "video_path":"/mnt/hwfile/opendatalab/chenjingzhou/ml/dataset/MSVD_Zero_Shot_QA/videos",          
+        "data_type":"video",                           
+        "bound":"False",                                 
+        "question_key":"question",                    
+        "answer_key":"answer",                         
+        "name_key":"video_name",                      
+        "video_postfix":[".avi"],
+        "num_segments":8
+    },
+    "model":{
+        "model_path":"/mnt/hwfile/opendatalab/chenjingzhou/ml/model/llava-next-video-7b-DPO-hf"
+    },
+    "conv_mode":"llava_next_video_template_with_space_after_assistant",
+    "save_path":"./result/eval/msvd_qa",
+    "experiment_name":"llava_next_video_msvd_1",
+    "description":"blank after assistant",
+    "batch_size":32
+}

config/eval/msvd_qa/llava_next_video_eval2_config.json

@@ -0,0 +1,23 @@
+{
+    "dataset":{
+        "dataset_name": "MSVD_QA",
+        "q_json_path":"/mnt/hwfile/opendatalab/chenjingzhou/ml/dataset/MSVD_Zero_Shot_QA/test_q.json",  
+        "a_json_path":"/mnt/hwfile/opendatalab/chenjingzhou/ml/dataset/MSVD_Zero_Shot_QA/test_a.json",  
+        "video_path":"/mnt/hwfile/opendatalab/chenjingzhou/ml/dataset/MSVD_Zero_Shot_QA/videos",          
+        "data_type":"video",                           
+        "bound":"False",                                 
+        "question_key":"question",                    
+        "answer_key":"answer",                         
+        "name_key":"video_name",                      
+        "video_postfix":[".avi"],
+        "num_segments":8
+    },
+    "model":{
+        "model_path":"/mnt/hwfile/opendatalab/chenjingzhou/ml/model/llava-next-video-7b-DPO-hf"
+    },
+    "conv_mode":"llava_next_video_template_with_space_after_user",
+    "save_path":"./result/eval/msvd_qa",
+    "experiment_name":"llava_next_video_msvd_2",
+    "description":"llava_next_video_template_with_space_after_user in msvd",
+    "batch_size":32
+}

config/eval/msvd_qa/qwen2_vl_eval_config.json

@@ -0,0 +1,22 @@
+{
+    "dataset":{
+        "dataset_name": "MSVD_QA",
+        "q_json_path":"/jizhicfs/hymiezhao/datasets/MSVD_Zero_Shot_QA/test_q.json",  
+        "a_json_path":"/jizhicfs/hymiezhao/datasets/MSVD_Zero_Shot_QA/test_a.json",  
+        "video_path":"/jizhicfs/hymiezhao/datasets/MSVD_Zero_Shot_QA/videos",          
+        "data_type":"video",                           
+        "bound":"False",                                 
+        "question_key":"question",                    
+        "answer_key":"answer",                         
+        "name_key":"video_name",                      
+        "video_postfix":[".avi"],
+        "num_segments":8
+    },
+    "model":{
+        "model_path":"/jizhicfs/hymiezhao/models/Qwen2-VL-7B-Instruct"
+    },
+    "save_path":"./result/eval/msvd_qa",
+    "experiment_name":"qwen2_vl_7b_msvd",
+    "description":"simple inference using qwen2 in msvd",
+    "batch_size":4
+}

config/eval/msvd_qa/qwen2_vl_eval_config_prefix1.json

@@ -0,0 +1,23 @@
+{
+    "dataset":{
+        "dataset_name": "MSVD_QA",
+        "q_json_path":"/jizhicfs/hymiezhao/datasets/MSVD_Zero_Shot_QA/test_q.json",  
+        "a_json_path":"/jizhicfs/hymiezhao/datasets/MSVD_Zero_Shot_QA/test_a.json",  
+        "video_path":"/jizhicfs/hymiezhao/datasets/MSVD_Zero_Shot_QA/videos",          
+        "data_type":"video",                           
+        "bound":"False",                                 
+        "question_key":"question",                    
+        "answer_key":"answer",                         
+        "name_key":"video_name",                      
+        "video_postfix":[".avi"],
+        "num_segments":8
+    },
+    "model":{
+        "model_path":"/jizhicfs/hymiezhao/models/Qwen2-VL-7B-Instruct"
+    },
+    "prefix":"Watch this video carefully and answer the question: ",
+    "save_path":"./result/eval/msvd_qa",
+    "experiment_name":"qwen2_vl_7b_msvd_prefix1",
+    "description":"prefix1 inference using qwen2 in msvd",
+    "batch_size":4
+}

config/eval/msvd_qa/qwen2_vl_eval_config_prefix2.json

@@ -0,0 +1,23 @@
+{
+    "dataset":{
+        "dataset_name": "MSVD_QA",
+        "q_json_path":"/jizhicfs/hymiezhao/datasets/MSVD_Zero_Shot_QA/test_q.json",  
+        "a_json_path":"/jizhicfs/hymiezhao/datasets/MSVD_Zero_Shot_QA/test_a.json",  
+        "video_path":"/jizhicfs/hymiezhao/datasets/MSVD_Zero_Shot_QA/videos",          
+        "data_type":"video",                           
+        "bound":"False",                                 
+        "question_key":"question",                    
+        "answer_key":"answer",                         
+        "name_key":"video_name",                      
+        "video_postfix":[".avi"],
+        "num_segments":8
+    },
+    "model":{
+        "model_path":"/jizhicfs/hymiezhao/models/Qwen2-VL-7B-Instruct"
+    },
+    "prefix":"You are an expert at video question answering. Please watch this video carefully and answer the question: ",
+    "save_path":"./result/eval/msvd_qa",
+    "experiment_name":"qwen2_vl_7b_msvd_prefix2",
+    "description":"prefix2 inference using qwen2 in msvd",
+    "batch_size":4
+}