#coding:utf-8 ''' write by ygq create on 2025-07-24 update MnMs data clean https://github.com/openmedlab/Awesome-Medical-Dataset/blob/main/resources/M&Ms.md https://zhuanlan.zhihu.com/p/694831343 来自 6 个国际医疗中心 的 340 名受试者 的 CMR 数据。 覆盖 4 个主流 MRI 设备厂商（Siemens, Philips, GE, Canon）。 数据集文件结构如下，数据集被组织成训练集、验证集和测试集三个主目录，其中训练集进一步分为有标注和无标注的子目录。每个有标注的子目录包含病人的成像文件以及相应的标注数据。 M&Ms ├── Training │ ├── Labeled │ │ ├── A0S9V9 │ │ │ ├── A0S9V9_sa.nii.gz │ │ │ └── A0S9V9_sa_gt.nii.gz │ │ ├── A1D0Q7 │ │ ├── A1D9Z7 │ │ └── ... │ └── Unlabeled ├── Validation ├── Testing └── 211230_M&Ms_Dataset_information_diagnosis_opendataset.csv 对训练集有标注的 150 例数据进行图像尺寸统计，size 的格式为 (x,y,z,frame) 经验丰富的临床医生对心脏磁共振（CMR）图像进行了分割，参考了 ACDC 的标注标准，标注了左心室（LV）、右心室（RV）血池以及左心室心肌（MYO）的轮廓，标签分别为：1（LV）、2（MYO）和3（RV）。 ''' import os import glob import pandas as pd import SimpleITK as sitk import argparse import json from tqdm import tqdm from util import meta_data import util import numpy as np # from bert_helper import * meta_id_name='External code' meta_vendor_name='VendorName' meta_centre_name='Centre' meta_pathology_name='Pathology' meta_ed_name='ED' meta_es_name='ES' meta_age_name='Age' meta_sex_name='Sex' meta_height_name='Height' meta_weight_name='Weight' TASK_VALUE="segmentation" CLAMP_RANGE_CT = [-300,300] CLAMP_RANGE_MRI = None # MRI images threshold placeholder TBC... TARGET_VOXEL_SPACING=None LABEL_DICT={ "0":"backgroud", "1":"LV",#左心室 Blood Pools "2":"MYO",#左心室心肌 "3":"RV"#右心室 Blood Pools } # def find_metadata_files(path): # # for Cancer Image Archive (TCIA) dataset # search_pattern = os.path.join(path, '**', 'metadata.csv') # return glob.glob(search_pattern, recursive=True) def find_metadata_files(path): # for Cancer Image Archive (TCIA) dataset search_pattern = os.path.join(path, '*.csv') return glob.glob(search_pattern, recursive=True) ##added by yanguoqing on 20250527 def find_image_dirs(path): return os.listdir(path) ##modify by yanguoqing on 20250527 def load_dicom_images(folder_path): reader = sitk.ImageSeriesReader() dicom_names = reader.GetGDCMSeriesFileNames(folder_path) reader.SetFileNames(dicom_names) image = reader.Execute() return dicom_names,image ##added by yanguoqing on 20250527 def load_dicom_tag(imgs): reader = sitk.ImageFileReader() # dicom_names = reader.GetGDCMSeriesFileNames(folder_path) reader.SetFileName(imgs) reader.ReadImageInformation() # 仅读取元信息，不加载像素数据 # metadata_keys = reader.GetMetaDataKeys() tag=reader.Execute() return tag def load_nrrd(fp): return sitk.ReadImage(fp) def save_nifti(image, output_path, folder_path): # Set metadata in the NIfTI file's header output_dirpath = os.path.dirname(output_path) if not os.path.exists(output_dirpath): print(f"Creating directory {output_dirpath}") os.makedirs(output_dirpath) # Set metadata in the NIfTI file's header image.SetMetaData("FolderPath", folder_path) sitk.WriteImage(image, output_path) ##modify by yanguoqing on 20250527 def convert_windows_to_linux_path(windows_path): # Replace backslashes with forward slashes and remove the drive letter # Some meta files have windows paths, but the data is stored on a linux server linux_path = windows_path.replace('\\', '/') if ':' in linux_path: linux_path = linux_path.split(':', 1)[1] return linux_path def main(target_path, output_dir): metadata_files = find_metadata_files(target_path) pid_dirs=find_image_dirs(target_path) pid_dirs=["Training","Testing","Validation"] failed_files = [] if not os.path.isdir(output_dir): os.makedirs(output_dir) json_output_path = os.path.join(output_dir, 'nifti_mappings.json') failed_files_path = os.path.join(output_dir, 'failed_files.json') meta = meta_data() # Initialize the JSON file if not os.path.exists(json_output_path): with open(json_output_path, 'w') as json_file: json.dump({}, json_file) meta_file=os.path.join(target_path,'211230_M&Ms_Dataset_information_diagnosis_opendataset.csv') if os.path.isfile(meta_file): mf_flag=True df_meta=pd.read_csv(meta_file,sep=',') else: mf_flag=False if pid_dirs: for pid_dir in tqdm(pid_dirs, desc="Processing pid dirs"): if not os.path.isdir(os.path.join(target_path,pid_dir)): continue if pid_dir =="Training": tr_flag=True else: tr_flag=False label_flag=False if not tr_flag: image_dirs=find_image_dirs(os.path.join(target_path,pid_dir)) unlabeled_list=image_dirs else: image_dir_1=find_image_dirs(os.path.join(target_path,pid_dir,'Labeled')) image_dir_2=find_image_dirs(os.path.join(target_path,pid_dir,'Unlabeled')) unlabeled_list=image_dir_2 image_dirs=image_dir_1+image_dir_2 for data_dir in tqdm(image_dirs, desc="Processing images files"): location=data_dir if not tr_flag: full_path=os.path.join(target_path,pid_dir,data_dir) else: if data_dir in unlabeled_list: full_path=os.path.join(target_path,pid_dir,"Unlabeled",data_dir) else: full_path=os.path.join(target_path,pid_dir,"Labeled",data_dir) label_flag=True data_info_row=df_meta[df_meta[meta_id_name]==data_dir] if data_info_row.shape[0]>0: data_info_row=data_info_row.reset_index() #print(data_info_row[meta_id_name]) meta_image_id=data_info_row[meta_id_name][0] meta_vendor=data_info_row[meta_vendor_name][0] meta_centre=data_info_row[meta_centre_name][0] meta_pathology=data_info_row[meta_pathology_name][0] meta_age=data_info_row[meta_age_name][0] meta_sex=data_info_row[meta_sex_name][0] meta_height=data_info_row[meta_height_name][0] meta_weigth=data_info_row[meta_weight_name][0] meta_ed=data_info_row[meta_ed_name][0] meta_es=data_info_row[meta_es_name][0] else: meta_image_id=data_dir meta_vendor='' meta_centre='' meta_pathology='' meta_age='' meta_sex='' meta_height='' meta_weigth='' meta_ed='' meta_es='' # full_path = convert_windows_to_linux_path(full_path) if not os.path.isdir(full_path): continue try: print(full_path) full_path_image=os.path.join(full_path,"%s_sa.nii.gz"%data_dir) if label_flag: full_path_label=os.path.join(full_path,"%s_sa_gt.nii.gz"%data_dir) if not os.path.isfile(full_path_label): full_path_label=None else: full_path_label=None sitk_img_original = util.load_nifti(full_path_image) if sitk_img_original is None: print(f" Failed to load image: {full_path_image}") continue modality="MRI" study='MnMs'##Dataset_name CIA_other_info = { 'metadata_file':'' # 'Series_Description':serise_desc } CIA_other_info['split'] = pid_dir if mf_flag: CIA_other_info['metadata_file']=meta_file original_spacing = list(sitk_img_original.GetSpacing()) original_size = list(sitk_img_original.GetSize()) sitk_img_processed = sitk_img_original # is_4d_image = msd_dataset_info.get("tensorImageSize", "3D").upper() == "4D" or sitk_img_original.GetDimension() == 4 is_4d_image = sitk_img_original.GetDimension() == 4 frame_flag=False # --- Resampling Logic (Revised for 4D) --- if is_4d_image: # Always process 4D images channel-wise for resampling # logging.info(f" Processing 4D image channel-wise: {original_img_full_path}") # Keep log for errors only channels = [] num_channels = original_size[3] if len(original_size) == 4 and sitk_img_original.GetDimension() == 4 else 1 channel_target_spacing = TARGET_VOXEL_SPACING if TARGET_VOXEL_SPACING else original_spacing[:3] # Use 3D spacing for i in range(num_channels): extractor = sitk.ExtractImageFilter() current_3d_channel_size = original_size[:3] if sitk_img_original.GetDimension() == 4: extractor.SetSize([current_3d_channel_size[0], current_3d_channel_size[1], current_3d_channel_size[2], 0]) extractor.SetIndex([0,0,0,i]) channel_3d_img = extractor.Execute(sitk_img_original) else: channel_3d_img = sitk_img_original if i > 0: break channel_resampler = util.get_unisize_resampler( channel_3d_img, 'linear', spacing=channel_target_spacing, size=current_3d_channel_size ) if channel_resampler: channels.append(channel_resampler.Execute(channel_3d_img)) else: channels.append(channel_3d_img) if channels: if len(channels) > 1: # Only join if there are multiple channels sitk_img_processed = sitk.JoinSeriesImageFilter().Execute(channels) ##aded by yanguoqing on 2025-08-11 frame_flag=True imgDict={} for kf_idx in range(num_channels): imgDict[str(kf_idx)]='none' if str(meta_ed):imgDict[str(meta_ed)]='ed' if str(meta_es):imgDict[str(meta_es)]='es' meta.add_keyvalue('ImgDict',imgDict) elif len(channels) == 1: # If only one channel resulted (e.g. original was 3D misidentified as 4D by tensorImageSize) sitk_img_processed = channels[0] elif TARGET_VOXEL_SPACING: # 3D image with target spacing img_resampler_obj = util.get_unisize_resampler(sitk_img_original, 'linear', spacing=TARGET_VOXEL_SPACING, size=original_size) if img_resampler_obj: sitk_img_processed = img_resampler_obj.Execute(sitk_img_original) else: # 3D image, no TARGET_VOXEL_SPACING img_resampler_obj = util.get_unisize_resampler(sitk_img_original, 'linear', spacing=original_spacing, size=original_size) if img_resampler_obj: sitk_img_processed = img_resampler_obj.Execute(sitk_img_original) ## CIA_other_info['Image_id']=meta_image_id CIA_other_info['Vendor']=meta_vendor CIA_other_info['Centre']=str(meta_centre) CIA_other_info['Pathology']=str(meta_pathology) CIA_other_info['Age']=str(meta_age) CIA_other_info['Sex']=meta_sex CIA_other_info['Height']=str(meta_height) CIA_other_info['Weight']=str(meta_weigth) CIA_other_info['ED']=str(meta_ed) CIA_other_info['ES']=str(meta_es) # --- End Resampling Logic --- is_processed_4d = sitk_img_processed.GetDimension() == 4 clamp_range_to_use=None if clamp_range_to_use and is_processed_4d: clamped_channels_final = [] num_channels_final = sitk_img_processed.GetSize()[3] if len(sitk_img_processed.GetSize()) == 4 else 1 for i in range(num_channels_final): extractor = sitk.ExtractImageFilter() proc_size_final = sitk_img_processed.GetSize() extractor.SetSize([proc_size_final[0], proc_size_final[1], proc_size_final[2], 0]) extractor.SetIndex([0,0,0,i]) channel_3d_img_to_clamp = extractor.Execute(sitk_img_processed) clamped_channels_final.append(util.clamp_image(channel_3d_img_to_clamp, clamp_range_to_use)) if clamped_channels_final: if len(clamped_channels_final) > 1: sitk_img_processed = sitk.JoinSeriesImageFilter().Execute(clamped_channels_final) elif len(clamped_channels_final) == 1: sitk_img_processed = clamped_channels_final[0] elif clamp_range_to_use: # 3D image sitk_img_processed = util.clamp_image(sitk_img_processed, clamp_range_to_use) output_path = os.path.join(output_dir,data_dir, f"{data_dir}.nii.gz") # output_path=convert_windows_to_linux_path(output_path) save_nifti(sitk_img_processed, output_path, full_path_image) print(f"Saved NIfTI file to {output_path}") label_path_dict = {} processed_lbl_full_path = os.path.join(output_dir, data_dir, TASK_VALUE, f"{data_dir}.nii.gz") print(processed_lbl_full_path,full_path_label,tr_flag,label_flag) if tr_flag and label_flag and os.path.exists(full_path_label): sitk_lbl_original = util.load_nifti(full_path_label) if not sitk_lbl_original: print(f" Failed to load label: {full_path_label}") processed_lbl_full_path = None continue if sitk_lbl_original: label_resampler = sitk.ResampleImageFilter() reference_for_label = sitk_img_processed # Default to processed image if sitk_img_processed.GetDimension() == 4: num_comp_proc = sitk_img_processed.GetSize()[3] if len(sitk_img_processed.GetSize()) == 4 else 1 if num_comp_proc > 0: extractor = sitk.ExtractImageFilter() proc_img_size_for_lbl_ref = sitk_img_processed.GetSize() extractor.SetSize([proc_img_size_for_lbl_ref[0], proc_img_size_for_lbl_ref[1], proc_img_size_for_lbl_ref[2], 0]) extractor.SetIndex([0,0,0,0]) try: reference_for_label = extractor.Execute(sitk_img_processed) except Exception as ref_err: print(f" Failed to extract 3D reference from 4D image: {output_path} for label alignment.") # print(traceback.format_exc()) reference_for_label = None else: # Fallback if extraction fails print(f" Could not extract 3D reference for label from 4D image {output_path}. Label may not be correctly resampled.") reference_for_label = None # This will cause an issue below if not handled sitk_lbl_processed = None if reference_for_label and reference_for_label.GetDimension() > 0: label_resampler.SetInterpolator(sitk.sitkNearestNeighbor) label_resampler.SetOutputPixelType(sitk_lbl_original.GetPixelID()) if sitk_lbl_original.GetDimension() == 4: lbl_channels = [] lbl_size = list(sitk_lbl_original.GetSize()) for i in range(lbl_size[3]): extractor = sitk.ExtractImageFilter() extractor.SetSize([lbl_size[0], lbl_size[1], lbl_size[2], 0]) extractor.SetIndex([0, 0, 0, i]) single_channel = extractor.Execute(sitk_lbl_original) label_resampler.SetReferenceImage(reference_for_label) resampled_channel = label_resampler.Execute(single_channel) lbl_channels.append(resampled_channel) if len(lbl_channels) > 1: sitk_lbl_processed = sitk.JoinSeriesImageFilter().Execute(lbl_channels) elif len(lbl_channels) == 1: sitk_lbl_processed = lbl_channels[0] else: label_resampler.SetReferenceImage(reference_for_label) sitk_lbl_processed = label_resampler.Execute(sitk_lbl_original) if processed_lbl_full_path: if sitk_img_processed.GetSize()[:3] != sitk_lbl_processed.GetSize()[:3]: print(f" Mismatch between image and label size (ignoring channels):") print(f" Image size: {sitk_img_processed.GetSize()}") print(f" Label size: {sitk_lbl_processed.GetSize()}") util.save_nifti(sitk_lbl_processed, processed_lbl_full_path, full_path_label) else: print(f" Failed to set reference image for label resampling for {full_path_label}. Saving original label.") util.save_nifti(sitk_lbl_original, processed_lbl_full_path, full_path_label) # Save original # processed_lbl_full_path should still point to this saved original label else: processed_lbl_full_path = None else: processed_lbl_full_path = None if processed_lbl_full_path: label_path_dict['heart'] = processed_lbl_full_path print('compare image and label size',sitk_img_original.GetSize(),sitk_lbl_original.GetSize()) print('compare image and label size',sitk_img_processed.GetSize(),sitk_lbl_processed.GetSize()) try: assert sitk_img_processed.GetSize() == sitk_lbl_processed.GetSize() except Exception as e: failed_files.append(full_path_label) continue except RuntimeError: failed_files.append(full_path_image) print(f"Failed to load MnMs images from {full_path_image}") continue size_processed = list(sitk_img_processed.GetSize()) print('size_processed',size_processed) # meta.add_keyvalue('Image_id',meta_image_id) meta.add_keyvalue('Spacing_mm',min(original_spacing[:3]))##保留前三个x,y,z的最小spacing meta.add_keyvalue('OriImg_path',full_path_image) meta.add_keyvalue('Size',size_processed) # 这里用处理后的size -- YH Jachin meta.add_keyvalue('Modality',modality) meta.add_keyvalue('Dataset_name',study) meta.add_keyvalue('ROI','chest') if processed_lbl_full_path: print(label_path_dict.keys()) meta.add_keyvalue('Task',TASK_VALUE) # meta.add_keyvalue('Label_tissue',list(label_path_dict.keys())) meta.add_keyvalue('Label_path',{TASK_VALUE:label_path_dict}) meta.add_keyvalue('Label_Dict',LABEL_DICT) meta.add_extra_keyvalue('Metadata',CIA_other_info) # Write the mapping to the JSON file on the fly with open(json_output_path, 'r+') as json_file: existing_mappings = json.load(json_file) existing_mappings[output_path] = meta.get_meta_data() json_file.seek(0) print(existing_mappings) json.dump(existing_mappings, json_file, indent=4) json_file.truncate() # else: # print("No metadata.csv files found.") with open(failed_files_path, "w") as json_file: json.dump(failed_files, json_file) print(f"The list has been written to {failed_files_path}") print(f"Saved NIfTI mappings to {json_output_path}") if __name__ == "__main__": parser = argparse.ArgumentParser(description="Process DICOM files and save as NIfTI.") parser.add_argument("--target_path", type=str, help="Path to the target directory containing metadata files.", default="/home/data/Github/data/data_gen_def/DATASETS/MnMs/OpenDataset/") parser.add_argument("--output_dir", type=str, help="Directory to save the NIfTI files.", default="/home/data/Github/data/data_gen_def/DATASETS_processed/MnMs/") args = parser.parse_args() print(args.target_path, args.output_dir) main(args.target_path, args.output_dir)