File size: 4,640 Bytes
46b244e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 | #!/usr/bin/env python3
# -*- coding: utf-8 -*-
import json
from collections import Counter
def analyze_resource_names_distribution(file_path):
"""专门分析resource_names字段的分布情况，包括单个资源和资源组合"""
with open(file_path, 'r', encoding='utf-8') as f:
data = json.load(f)
print(f"总数据条数: {len(data)}")
print("=" * 60)
# 统计resource_names字段的分布
resource_names_counter = Counter() # 单个资源的频次
resource_combinations_counter = Counter() # 资源组合的频次
total_resource_names_entries = 0
empty_resource_names = 0
for item in data:
output = item.get('output', '')
try:
output_json = json.loads(output)
# 检查是否有resource_names字段
if 'resource_names' in output_json:
total_resource_names_entries += 1
resource_names_list = output_json['resource_names']
if not resource_names_list: # 空列表
empty_resource_names += 1
else:
# 统计每个资源名称
for resource in resource_names_list:
resource_names_counter[resource] += 1
# 统计资源组合（将列表转换为排序的元组作为key）
if len(resource_names_list) > 1:
# 对资源名称排序以确保组合的一致性
sorted_combination = tuple(sorted(resource_names_list))
resource_combinations_counter[sorted_combination] += 1
elif len(resource_names_list) == 1:
# 单个资源也算作一种"组合"
single_resource = tuple(resource_names_list)
resource_combinations_counter[single_resource] += 1
except json.JSONDecodeError:
continue
print(f"🎯 resource_names字段分布统计:")
print(f"包含resource_names字段的条目数: {total_resource_names_entries}")
print(f"resource_names为空列表的条目数: {empty_resource_names}")
print(f"不同资源名称的种类数: {len(resource_names_counter)}")
print(f"不同资源组合的种类数: {len(resource_combinations_counter)}")
print("=" * 60)
# 1. 单个资源名称频次统计
print("📊 各资源名称出现频次（按频次降序）:")
print("-" * 60)
sorted_resources = sorted(resource_names_counter.items(), key=lambda x: x[1], reverse=True)
for i, (resource, count) in enumerate(sorted_resources, 1):
print(f"{i:3d}. {resource}: {count}")
# 2. 资源组合频次统计
print(f"\n🔗 资源组合出现频次（按频次降序）:")
print("-" * 60)
sorted_combinations = sorted(resource_combinations_counter.items(), key=lambda x: x[1], reverse=True)
for i, (combination, count) in enumerate(sorted_combinations, 1):
if len(combination) == 1:
combo_str = f"[单个资源] {combination[0]}"
else:
combo_str = f"[{len(combination)}个资源] " + " + ".join(combination)
print(f"{i:3d}. {combo_str}: {count}")
# 按资源主体分类统计
print(f"\n📋 按资源主体分类统计:")
print("-" * 60)
resource_by_category = {}
for resource, count in resource_names_counter.items():
if '-' in resource:
category = resource.split('-')[0]
if category not in resource_by_category:
resource_by_category[category] = {}
resource_by_category[category][resource] = count
else:
# 没有'-'的资源名称
if '其他' not in resource_by_category:
resource_by_category['其他'] = {}
resource_by_category['其他'][resource] = count
# 按类别输出
for category, resources in sorted(resource_by_category.items()):
total_in_category = sum(resources.values())
print(f"\n{category} (总计: {total_in_category}):")
for resource, count in sorted(resources.items(), key=lambda x: x[1], reverse=True):
print(f" {resource}: {count}")
return resource_names_counter, resource_combinations_counter, total_resource_names_entries, empty_resource_names
if __name__ == "__main__":
file_path = "/home/ziqiang/LLaMA-Factory/data/ocr_text_orders_08_14_test_v4.json"
analyze_resource_names_distribution(file_path)
|