File size: 1,609 Bytes
1c980b1 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 |
import os
def split_jsonl_by_size(input_path, output_dir, target_size_mb):
target_size = target_size_mb * 1024 * 1024 # 转换为字节
os.makedirs(output_dir, exist_ok=True)
part_num = 1
current_size = 0
current_file = None
line_counter = 0 # 用于跟踪行号
with open(input_path, 'r', encoding='utf-8') as infile:
for line in infile:
line_counter += 1
line_bytes = line.encode('utf-8')
line_size = len(line_bytes)
# 新增：检查单行大小是否超过1MB
if line_size > 1 * 1024 * 1024:
raise ValueError(
f"JSON object at line {line_counter} exceeds 1MB limit "
f"(actual size: {line_size / 1024 / 1024:.2f}MB)"
)
# 原有分割逻辑
if current_file and (current_size + line_size > target_size):
current_file.close()
current_file = None
current_size = 0
part_num += 1
if not current_file:
output_path = os.path.join(output_dir, f'vlm_requests_{part_num}.jsonl')
current_file = open(output_path, 'wb')
current_size = 0
current_file.write(line_bytes)
current_size += line_size
if current_file:
current_file.close()
# 示例调用
split_jsonl_by_size('/mnt/data/users/zys/proj/vlm_reasoning/request/vlm_batch_requests.jsonl', '/mnt/data/users/zys/proj/vlm_reasoning/upload/vlm', 450) |