File size: 3,797 Bytes
d8bfe4a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 | import json
import os
import random
from collections import defaultdict
# ==========================================
# 配置部分
# ==========================================
INPUT_FILE_PATH = "/workspace/echoloc/datas/Emotion_Speech_Dataset/mixed_emotion_dataset.json"
# 定义需要的切分任务
# 格式: ("文件名后缀", 比例)
# 注意：所有比例之和不能超过 1.0
SPLIT_TASKS = [
("1000", 0.0001),
("10000", 0.001),
("100000", 0.01),
("1000000", 0.1),
("20pct", 0.2),
("50pct", 0.5)
]
# ==========================================
# 主逻辑
# ==========================================
def split_dataset_no_overlap():
if not os.path.exists(INPUT_FILE_PATH):
print(f"错误: 找不到文件 {INPUT_FILE_PATH}")
return
# 1. 读取数据
print(f"正在读取主文件...")
with open(INPUT_FILE_PATH, 'r', encoding='utf-8') as f:
all_data = json.load(f)
print(f"读取完成。总样本数: {len(all_data)}")
# 2. 分桶：按长度将数据归类
print("正在按长度分桶并打乱...")
buckets = defaultdict(list)
for item in all_data:
seq_len = len(item['segments'])
buckets[seq_len].append(item)
sorted_lengths = sorted(buckets.keys())
# 3. 桶内打乱 (Shuffle)
# 这是关键步骤：先彻底打乱，后面顺序切片也就等于随机切片了
for l in sorted_lengths:
print(f" Length {l}: {len(buckets[l])} 条 -> 打乱中...")
random.shuffle(buckets[l])
# 4. 依次切分 (Slicing)
dir_name = os.path.dirname(INPUT_FILE_PATH)
base_name = os.path.basename(INPUT_FILE_PATH)
file_name_no_ext = os.path.splitext(base_name)[0]
# 初始化每个桶的起始游标 (Offset)，初始都是 0
bucket_offsets = {l: 0 for l in sorted_lengths}
print("\n>>> 开始无重叠切分任务")
for suffix, ratio in SPLIT_TASKS:
subset_data = []
stats_info = []
print(f"\n正在生成 [{suffix}] (占比 {ratio*100}%)...")
for l in sorted_lengths:
total_in_bucket = len(buckets[l])
count_to_take = int(total_in_bucket * ratio)
start_idx = bucket_offsets[l]
end_idx = start_idx + count_to_take
# 边界检查
if end_idx > total_in_bucket:
print(f" [警告] 长度 {l} 的数据不够了！需要至 {end_idx}，但只有 {total_in_bucket}。")
end_idx = total_in_bucket
# 切片：取 [start : end] 这一段
chunk = buckets[l][start_idx:end_idx]
subset_data.extend(chunk)
stats_info.append(f"Len{l}:{len(chunk)}")
# 更新游标，供下一个任务使用
bucket_offsets[l] = end_idx
# 再次打乱 subset_data，避免数据是按长度排序的 (222...333...444...)
random.shuffle(subset_data)
# 保存文件
output_filename = f"{file_name_no_ext}_{suffix}.json"
output_path = os.path.join(dir_name, output_filename)
print(f" - 组成分布: {', '.join(stats_info)}")
print(f" - 总样本数: {len(subset_data)}")
print(f" - 保存路径: {output_path}")
with open(output_path, 'w', encoding='utf-8') as f:
json.dump(subset_data, f, ensure_ascii=False, indent=2)
# 检查剩余数据
print("\n所有任务完成。")
print("剩余未使用的数据量 (按长度):")
for l in sorted_lengths:
remaining = len(buckets[l]) - bucket_offsets[l]
print(f" Length {l}: {remaining} 条")
if __name__ == "__main__":
split_dataset_no_overlap()
|