add minor improvements on loading and saving

cytof/classes.py

@@ -37,17 +37,6 @@ from sklearn.cluster import KMeans
 from itertools import product
 
 
-# SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
-# sys.path.append(os.path.dirname(SCRIPT_DIR))
-
-# # from hyperion_segmentation import cytof_nuclei_segmentation, cytof_cell_segmentation, visualize_segmentation
-# # from utils import (save_multi_channel_img, generate_color_dict, show_color_table, 
-# # visualize_scatter, visualize_expression, _get_thresholds, _generate_summary)
-
-# from cytof.hyperion_segmentation import cytof_nuclei_segmentation, cytof_cell_segmentation, visualize_segmentation
-# from cytof.utils import (save_multi_channel_img, generate_color_dict, show_color_table, 
-# visualize_scatter, visualize_expression, _get_thresholds, _generate_summary)
-
 ## added for test
 import platform
 from pathlib import Path
@@ -61,18 +50,6 @@ from hyperion_segmentation import cytof_nuclei_segmentation, cytof_cell_segmenta
 from cytof.utils import (save_multi_channel_img, generate_color_dict, show_color_table, 
 visualize_scatter, visualize_expression, _get_thresholds, _generate_summary)
 
-# def _get_colors(n):
-#     base_colors = np.array([(1, 0, 0), (0, 1, 0), (0, 0, 1),
-#                             (0, 1, 1), (1, 0, 1), (1, 1, 0),
-#                             (1, 1, 1)])
-
-#     n0 = len(base_colors)
-#     if n <= n0:
-#         colours = base_colors[:n]
-#     else:
-#         colours = np.vstack((base_colors, cm.rainbow(np.linspace(0, 1, n-n0))[:,:-1]))
-#     return colours
-
 def get_name(dfrow):
     return os.path.join(dfrow['path'], dfrow['ROI'])
 
@@ -118,6 +95,9 @@ class CytofImage():
         return f"CytofImage(slide={self.slide}, roi={self.roi})"
 
     def save_cytof(self, savename: str):
+        directory = os.path.dirname(savename)
+        if not os.path.exists(directory):
+            os.makedirs(directory)
         pkl.dump(self, open(savename, "wb"))
 
     def get_markers(self, imarker0: Optional[str] = None):
@@ -599,11 +579,6 @@ class CytofImage():
         else:
             channels = self.channels
         '''assert all([x.lower() in channels_temp for x in channels]), "Not all provided channels are available!"'''
-        # for (i, chn) in enumerate(channels):
-        #     savename = os.path.join(savedir, f"{chn}.tiff")
-        #     im_temp = self.image[..., i]
-        #     im_temp_ = np.clip(im_temp / np.quantile(im_temp, 0.99), 0, 1)
-        #     save_multi_channel_img((im_temp_ * 255).astype(np.uint8), savename)
         for chn in channels:
             savename = os.path.join(savedir, f"{chn}{ext}")
             #         i = channels_temp.index(chn.lower())
@@ -1190,7 +1165,6 @@ class CytofCohort():
         """
         self.cytof_images = cytof_images or {}
         self.df_cohort    = df_cohort# or None# pd.read_csv(file_cohort) # the slide-ROI
-#         self.df_io  = None pd.read_csv(file_io) # the input-output correspondence file
         self.feat_sets = {
             "all": ["cell_sum", "cell_ave", "cell_morphology"],
             "cell_sum": ["cell_sum", "cell_morphology"],
@@ -1214,6 +1188,9 @@ class CytofCohort():
         return f"CytofCohort(name={self.name})"
     
     def save_cytof_cohort(self, savename):
+        directory = os.path.dirname(savename)
+        if not os.path.exists(directory):
+            os.makedirs(directory)
         pkl.dump(self, open(savename, "wb"))
     
     def batch_process_feature(self):
@@ -1247,8 +1224,8 @@ class CytofCohort():
 
 
     def batch_process(self, params: Dict):
-        sys.path.append("../../CLIscripts")
-        from single_roi import process_single, SetParameters
+        sys.path.append("../CLIscripts")
+        from process_single_roi import process_single, SetParameters
         for i, (slide, roi, fname) in self.df_cohort.iterrows():
             paramsi = SetParameters(filename=fname,
                         outdir=self.dir_out,
@@ -1395,9 +1372,8 @@ class CytofCohort():
             kmeans = KMeans(n_clusters=k, random_state=42).fit(df_feature)
             communities = kmeans.labels_
         else: 
-            communities, graph, Q = phenograph.cluster(df_feature, k=k, seed=42, n_jobs=10)   # run PhenoGraph
+            communities, graph, Q = phenograph.cluster(df_feature, k=k, n_jobs=-1)   # run PhenoGraph
 
-        print('Performing dimensionality reduction with UMAP now...')
         # project to 2D using UMAP
         umap_2d = umap.UMAP(n_components=2, init='random', random_state=0)
         proj_2d = umap_2d.fit_transform(df_feature)
@@ -1699,14 +1675,12 @@ class CytofCohort():
         thres = kwars.get(_, default_thres[_])
         """print("{}: {}".format(_, thres))"""
         networks = self._gather_roi_kneighbor_graphs(key_pheno, method=method, **{_: thres})
-        # networks = _gather_roi_kneighbor_graphs(self, key_pheno, method=method, **{_: thres})
 
         if level == "slide":
             keys = ['edge_nums', 'expected_percentage', 'num_cell']
             for slide in self.df_cohort['Slide'].unique():
                 cond = self.df_cohort['Slide'] == slide
                 df_slide = self.df_cohort.loc[cond, :]
-#                 rois = df_slide.apply(lambda row: get_name(row), axis=1).values
                 rois = df_slide['ROI'].values
                 '''keys = list(networks.values())[0].keys()'''
                 networks[slide] = {}
@@ -1756,6 +1730,7 @@ class CytofCohort():
                            figsize=(6, 6), row_cluster=False, col_cluster=False)
             plt.title(f_key)
             plt.show()
+
         # IMPORTANT: attch to individual ROIs
         self.attach_individual_roi_pheno(key_pheno, override=True)
         return interacts, clustergrid
@@ -1792,7 +1767,6 @@ class CytofCohort():
                                                                         visualize=vis_thres,
                                                                         verbose=verbose)
         setattr(self, "marker_thresholds", thres)
-        print('done getting threshold for summary')  
 
         # split to each ROI
         _attr_marker_pos, seen = [], 0
@@ -1818,29 +1792,6 @@ class CytofCohort():
             seen += 1
         return _attr_marker_pos
 
-    def get_roi_co_exp_compoent(self, feature_name, accumul_type):
-        """
-        Creates the components for ROI level co-expression. This is expected to be used in cohort analysis
-        """
-
-        # initialize dictionaries to record componenets of the log odds-ratio
-        co_positive_counts_dict = dict() # theta in manuscript
-        expected_counts_dict = dict() # E in manuscript
-        num_cells_dict = dict()
-
-        for i, cytof_img in enumerate(self.cytof_images.values()):
-            slide, roi = cytof_img.slide, cytof_img.roi
-
-            # compute the co-expression at the ROI level, but does not compute the final probability
-            df_co_pos_prob, df_expected_prob, n_cell = cytof_img.roi_co_expression(feature_name=feature_name, accumul_type=accumul_type, return_components=True)
-            
-            # store the counts and number of cells separately
-            co_positive_counts_dict[roi] = df_co_pos_prob
-            expected_counts_dict[roi] = df_expected_prob
-            num_cells_dict[roi] = n_cell
-
-        return co_positive_counts_dict, expected_counts_dict, num_cells_dict
-
     def co_expression_analysis(self, 
                                 normq: int = 75,
                                 feat_type: str = "normed", 
@@ -1941,175 +1892,3 @@ class CytofCohort():
             slide_co_expression_dict[slide_key] = (edge_percentage_norm, df_expected.columns)
 
         return slide_co_expression_dict
-
-
-    def _gather_roi_co_exp(self, 
-                        feat_name: str, 
-                        accumul_type: str = "sum"):
-        """roi level co-expression analysis"""
-        n_attr = f"df_feature_{feat_name}"
-        expected_percentages = {}
-        edge_percentages     = {}
-        num_cells            = {}
-        df_slide_roi         = self.df_cohort
-
-        for i, cytof_img in enumerate(self.cytof_images.values()):
-            slide, roi = cytof_img.slide, cytof_img.roi
-            df_feat = getattr(cytof_img , n_attr)
-
-            if i == 0:
-                # all gene (marker) columns
-                marker_col_all = [x for x in df_feat.columns if f"cell_{accumul_type}" in x]
-                ids = [cytof_img.channels.index(x.split(f"_cell_{accumul_type}")[0]) for x in marker_col_all]
-                marker_all = list(np.array(cytof_img.markers, dtype=object)[np.array(ids)])
-                n_marker = len(marker_col_all)  
-            n_cell   = len(df_feat)
-            
-            # corresponding marker positive info file
-            if not hasattr(cytof_img, f"cell_count_{feat_name}_{accumul_type}"):
-                print('no marker positive analysis found. Generating analysis...')
-                self.generate_summary()
-                
-            df_info_cell = getattr(cytof_img, f"cell_count_{feat_name}_{accumul_type}")
-            pos_nums   = df_info_cell["positive counts"].values
-            pos_ratios = df_info_cell["positive ratio"].values
-            thresholds = df_info_cell["threshold"].values
-
-            # create new expected_percentage matrix for each ROI
-            expected_percentage = np.zeros((n_marker, n_marker))
-            edge_percentage     = np.zeros_like(expected_percentage)
-
-            """expected_percentage
-            an N by N matrix, where N represent for the number of total gene (marker)
-            each ij-th element represents for the percentage that both the i-th and the j-th gene is "positive"
-            based on the threshold defined previously"""
-            for ii in range(n_marker):
-                for jj in range(n_marker):
-                    expected_percentage[ii, jj] = pos_nums[ii] * pos_nums[jj]
-            expected_percentages[roi] = expected_percentage    
-            
-            """edge_percentage
-            an N by N matrix, where N represent for the number of gene (marker)
-            each ij-th element represents for the percentage of cells that show positive in both i-th and j-th gene
-            """
-            edge_nums = np.zeros_like(expected_percentage)
-            for ii in range(n_marker):
-                _x = df_feat[marker_col_all[ii]].values > thresholds[ii] 
-                # _x = df_feat[marker_col_all[ii]].values > thresholds[marker_idx[ii]]
-                for jj in range(n_marker):
-                    _y = df_feat[marker_col_all[jj]].values > thresholds[jj] 
-                    # _y = df_feat[marker_col_all[jj]].values > thresholds[marker_idx[jj]]
-                    edge_nums[ii, jj] = np.sum(np.all([_x, _y], axis=0)) # / n_cell
-            edge_percentages[roi] = edge_nums
-            num_cells[roi]        = n_cell
-
-        return expected_percentages, edge_percentages, num_cells, marker_all, marker_col_all    
-
-    def _co_expression_analysis_(self, 
-                            normq: int = 75,
-                            feat_type: str = "normed", 
-                            co_exp_markers: Union[str, List] = "all", 
-                            accumul_type: Union[str, List[str]] = "sum", 
-                            level: str = "slide", 
-                            verbose: bool = False,
-                            clustergrid=None):
-        
-        assert level in ["slide", "roi"], "Only slide or roi levels are accepted!"
-        assert feat_type in ["original", "normed", "scaled"]
-        if feat_type == "original":
-            feat_name = ""
-        elif feat_type == "normed":
-            feat_name = f"{normq}normed"
-        else:
-            feat_name = f"{normq}normed_scaled"
-        if verbose:
-            print(feat_name)
-
-        expected_percentages, edge_percentages, num_cells, marker_all, marker_col_all = \
-        self._gather_roi_co_exp(feat_name)
-        
-
-        if co_exp_markers != "all":
-            # assert (isinstance(co_exp_markers, list) and all([x in cytof_img.markers for x in co_exp_markers]))
-            assert (isinstance(co_exp_markers, list) and all([x in marker_all for x in co_exp_markers]))
-            marker_idx = np.array([marker_all.index(x) for x in co_exp_markers])
-            marker_all = [marker_all[x] for x in marker_idx]
-            marker_col_all = [marker_col_all[x] for x in marker_idx]
-        else:
-            marker_idx = np.arange(len(marker_all))
-
-        df_slide_roi = self.df_cohort
-        if level == "slide":
-            # expected_percentages, edge_percentages = {}, {}
-            for slide in df_slide_roi["Slide"].unique():  ## for each slide
-                print('co-exp slide:', slide)
-                print('exp-perc keys before:', expected_percentages.keys())
-                for seen_roi, f_roi in enumerate(df_slide_roi.loc[df_slide_roi["Slide"] == slide, "ROI"]):  ## for each ROI
-                    roi = f_roi.replace(".txt", "")
-                    print('roi finding:', roi)
-                    if roi not in expected_percentages:
-                        continue
-                    if seen_roi == 0:
-                        expected_percentages[slide] = expected_percentages[roi]
-                        edge_percentages[slide] = edge_percentages[roi]
-                        num_cells[slide] = num_cells[roi]
-                    else:
-                        expected_percentages[slide] += expected_percentages[roi]
-                        edge_percentages[slide] += edge_percentages[roi]
-                        num_cells[slide] += num_cells[roi]
-                    expected_percentages.pop(roi)
-                    edge_percentages.pop(roi)
-                    num_cells.pop(roi)
-        
-        print('exp-perc keys after:', expected_percentages.keys())
-        # print('exp_after:', expected_percentages)
-
-        co_exps = {}
-        for key, expected_percentage in expected_percentages.items():
-            print('key for co-exp:', key)
-            # print(type(expected_percentage), type(num_cells[key]))
-            # print('exp_perc:', expected_percentage)
-            expected_percentage = expected_percentage / num_cells[key] ** 2
-            edge_percentage = edge_percentages[key] / num_cells[key]
-
-            # Normalize
-            edge_percentage_norm = np.log10(edge_percentage / expected_percentage + 0.1)
-
-            # Fix Nan
-            edge_percentage_norm[np.isnan(edge_percentage_norm)] = np.log10(1 + 0.1)
-
-            co_exps[key] = edge_percentage_norm
-
-        # plot
-        for f_key, edge_percentage_norm in co_exps.items():
-            # fig, ax = plt.subplots(1,1, figsize=(6,6))
-            # # ax = sns.heatmap(edge_percentage_norm, center=np.log10(1 + 0.1),
-            # sns.heatmap(edge_percentage_norm[marker_idx, :][:, marker_idx], center=np.log10(1 + 0.1),
-            #             cmap='RdBu_r', vmin=-1, vmax=3, xticklabels=marker_all, yticklabels=marker_all, ax=ax)
-            # ax.set_aspect('equal')
-            # plt.title(f_key)
-            # plt.show()
-
-            if clustergrid is None:
-                plt.figure()
-                clustergrid = sns.clustermap(edge_percentage_norm[marker_idx, :][:, marker_idx],
-                                            # clustergrid = sns.clustermap(edge_percentage_norm,
-                                            center=np.log10(1 + 0.1), cmap='RdBu_r', vmin=-1, vmax=3,
-                                            xticklabels=marker_all, yticklabels=marker_all)
-                if verbose: 
-                    plt.title(f_key)
-                    plt.show()
-
-            else:
-                if verbose:
-                    plt.figure()
-                    sns.clustermap(edge_percentage_norm[marker_idx, :][:, marker_idx] \
-                                [clustergrid.dendrogram_row.reordered_ind, :]\
-                                [:, clustergrid.dendrogram_row.reordered_ind],
-                                center=np.log10(1 + 0.1), cmap='RdBu_r', vmin=-1, vmax=3,
-                                xticklabels=np.array(marker_all)[clustergrid.dendrogram_row.reordered_ind],
-                                yticklabels=np.array(marker_all)[clustergrid.dendrogram_row.reordered_ind],
-                                figsize=(6, 6), row_cluster=False, col_cluster=False)
-                    plt.title(f_key)
-                    plt.show()
-        return co_exps, marker_all, marker_idx, clustergrid