Spaces:
Paused
Paused
| import json | |
| import os | |
| from collections import defaultdict | |
| def find_duplicate_dat_keys(json_file_path, output_dir="./duplicate_reports"): | |
| """ | |
| 检测JSON中重复的.dat键名，记录重复次数和位置，并生成报告 | |
| :param json_file_path: JSON文件路径 | |
| :param output_dir: 报告输出目录 | |
| """ | |
| try: | |
| # 创建输出目录 | |
| os.makedirs(output_dir, exist_ok=True) | |
| report_path = os.path.join(output_dir, "duplicate_keys_report.txt") | |
| # 第一步：读取原始JSON内容，记录所有.dat键名及其出现位置（行号） | |
| key_positions = defaultdict(list) # {键名: [行号1, 行号2, ...]} | |
| all_keys = [] # 存储所有键名（保留顺序） | |
| line_num = 0 | |
| with open(json_file_path, 'r', encoding='utf-8') as f: | |
| for line in f: | |
| line_num += 1 | |
| stripped_line = line.strip() | |
| # 匹配格式："xxx.dat": ...（考虑可能的空格和引号） | |
| if '"' in stripped_line and '.dat"' in stripped_line and ':' in stripped_line: | |
| # 提取键名（处理可能的空格，如 " 3070b.dat " : 8 ） | |
| parts = stripped_line.split('"') | |
| if len(parts) >= 3: | |
| key = parts[1].strip() | |
| if key.endswith('.dat'): | |
| key_positions[key].append(line_num) | |
| all_keys.append(key) | |
| # 第二步：分析重复键 | |
| total_original = len(all_keys) | |
| unique_keys = len(key_positions) | |
| duplicate_keys = {k: v for k, v in key_positions.items() if len(v) > 1} | |
| total_duplicates = len(duplicate_keys) | |
| total_overwritten = total_original - unique_keys # 被覆盖的键数量 | |
| # 第三步：生成报告 | |
| with open(report_path, 'w', encoding='utf-8') as report: | |
| # 报告头部 | |
| report.write("=" * 80 + "\n") | |
| report.write(f"JSON重复.dat键名检测报告\n") | |
| report.write(f"检测文件: {json_file_path}\n") | |
| report.write(f"检测时间: {os.popen('date').read().strip()}\n") | |
| report.write("=" * 80 + "\n\n") | |
| # 统计信息 | |
| report.write(f"【基本统计】\n") | |
| report.write(f"1. 原始.dat键名总数量: {total_original} 个\n") | |
| report.write(f"2. 唯一.dat键名数量: {unique_keys} 个\n") | |
| report.write(f"3. 存在重复的.dat键名数量: {total_duplicates} 个\n") | |
| report.write(f"4. 被覆盖的键数量 (重复导致): {total_overwritten} 个\n\n") | |
| # 重复键详情 | |
| if duplicate_keys: | |
| report.write("=" * 80 + "\n") | |
| report.write(f"【重复键详情】(共{total_duplicates}个)\n") | |
| report.write("=" * 80 + "\n") | |
| # 按重复次数排序（从高到低） | |
| sorted_duplicates = sorted(duplicate_keys.items(), | |
| key=lambda x: len(x[1]), | |
| reverse=True) | |
| for i, (key, positions) in enumerate(sorted_duplicates, 1): | |
| repeat_count = len(positions) | |
| report.write(f"\n{i}. 键名: {key}\n") | |
| report.write(f" - 重复次数: {repeat_count} 次 (实际保留1次，被覆盖{repeat_count-1}次)\n") | |
| report.write(f" - 出现行号: {', '.join(map(str, positions))}\n") | |
| report.write(f" - 建议: 检查上述行号的键值对，保留需要的条目\n") | |
| else: | |
| report.write("【重复键详情】\n未发现任何重复的.dat键名\n") | |
| report.write("\n" + "=" * 80 + "\n") | |
| report.write("报告结束。重复键可能导致数据解析不完整，请根据实际需求处理。\n") | |
| # 控制台输出摘要 | |
| print("=" * 80) | |
| print(f"JSON重复键检测完成！") | |
| print(f"检测文件: {json_file_path}") | |
| print("=" * 80) | |
| print(f"基本统计:") | |
| print(f" - 原始键总数: {total_original}") | |
| print(f" - 唯一键数量: {unique_keys}") | |
| print(f" - 重复键数量: {total_duplicates}") | |
| print(f" - 被覆盖键数量: {total_overwritten}") | |
| print("=" * 80) | |
| print(f"详细报告已保存至: {report_path}") | |
| if duplicate_keys: | |
| print(f"前5个重复键示例:") | |
| for i, (key, positions) in enumerate(sorted(duplicate_keys.items())[:5], 1): | |
| print(f" {i}. {key} (重复{len(positions)}次，行号: {positions[:3]}...)") | |
| return duplicate_keys | |
| except FileNotFoundError: | |
| print(f"❌ 错误: 未找到JSON文件 -> {json_file_path}") | |
| return None | |
| except Exception as e: | |
| print(f"❌ 检测失败: {str(e)}") | |
| return None | |
| if __name__ == "__main__": | |
| # 替换为你的JSON文件路径 | |
| JSON_FILE_PATH = "/public/home/wangshuo/gap/assembly/data/car_1k/subset_self/label_mapping_freq.json" | |
| find_duplicate_dat_keys(JSON_FILE_PATH) | |