File size: 1,249 Bytes
1c980b1 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 |
def analyze_jsonl_size(input_path):
over_count = 0
total_over_size = 0 # 总大小，单位字节
max_size = 0 # 最大对象大小，单位字节
line_counter = 0 # 行号跟踪（可选）
with open(input_path, 'r', encoding='utf-8') as file:
for line in file:
line_counter += 1
# 计算当前行的字节大小
line_bytes = line.encode('utf-8')
current_size = len(line_bytes)
# 检查是否超过1MB
if current_size > 1 * 1024 * 1024:
over_count += 1
total_over_size += current_size
if current_size > max_size:
max_size = current_size
# 输出统计结果
print(f"Number of objects exceeding 1MB: {over_count}")
if over_count > 0:
avg_size_mb = (total_over_size / over_count) / (1024 * 1024)
max_size_mb = max_size / (1024 * 1024)
print(f"Average size of oversized objects: {avg_size_mb:.2f} MB")
print(f"Largest object size: {max_size_mb:.2f} MB")
else:
print("No objects exceed the 1MB limit.")
# 示例调用
analyze_jsonl_size('/mnt/data/users/zys/proj/vlm_reasoning/request/vlm_batch_requests.jsonl') |