From e97143acd661f3f4629bd90af67467a3b66f6b67 Mon Sep 17 00:00:00 2001 From: simone Date: Sun, 22 May 2022 11:42:22 +0200 Subject: [PATCH 01/10] adding segmentation --- pysmFISH/pipeline.py | 124 +++++++++++++++++++++---------------------- 1 file changed, 62 insertions(+), 62 deletions(-) diff --git a/pysmFISH/pipeline.py b/pysmFISH/pipeline.py index f0bc37b..b0353e9 100644 --- a/pysmFISH/pipeline.py +++ b/pysmFISH/pipeline.py @@ -1285,7 +1285,7 @@ def processing_fresh_tissue_step( f"cannot process fresh tissue because missing running_functions attr" ) - '''( + ( self.ds_beads, self.ds_nuclei, self.nuclei_metadata, @@ -1302,69 +1302,69 @@ def processing_fresh_tissue_step( diameter_size=self.diameter_size, parsing=parsing, save_steps_output=self.save_intermediate_steps, + ) + + pickle.dump( + [ + self.ds_beads, + self.ds_nuclei, + self.metadata, + ], + open( + Path(self.experiment_fpath) + / "fresh_tissue" + / "segmentation" + / "ds_tmp_data.pkl", + "wb", + ), + ) + + (self.ds_beads, self.ds_nuclei, self.nuclei_metadata) = pickle.load( + open( + Path(self.experiment_fpath) + / "fresh_tissue" + / "segmentation" + / "ds_tmp_data.pkl", + "rb", + ), + ) + + # Segmentation + '''fov_processing.segmentation_graph( + self.ds_nuclei, + self.chunk_size, + self.experiment_fpath, + self.fresh_tissue_segmentation_engine, + self.diameter_size, )''' - # pickle.dump( - # [ - # self.ds_beads, - # self.ds_nuclei, - # self.metadata, - # ], - # open( - # Path(self.experiment_fpath) - # / "fresh_tissue" - # / "segmentation" - # / "ds_tmp_data.pkl", - # "wb", - # ), - # ) - - # (self.ds_beads, self.ds_nuclei, self.nuclei_metadata) = pickle.load( - # open( - # Path(self.experiment_fpath) - # / "fresh_tissue" - # / "segmentation" - # / "ds_tmp_data.pkl", - # "rb", - # ), - # ) - - # # Segmentation - # fov_processing.segmentation_graph( - # self.ds_nuclei, - # self.chunk_size, - # self.experiment_fpath, - # self.fresh_tissue_segmentation_engine, - # self.diameter_size, - # ) - - # ( - # self.nuclei_org_tiles, - # self.nuclei_adjusted_coords, - # ) = stitching.stitched_beads_on_nuclei_fresh_tissue( - # self.experiment_fpath, - # self.client, - # self.ds_nuclei, - # self.ds_beads, - # round_num=1, - # ) - - # pickle.dump( - # [ - # self.ds_beads, - # self.ds_nuclei, - # self.nuclei_metadata, - # self.nuclei_org_tiles, - # self.nuclei_adjusted_coords, - # ], - # open( - # Path(self.experiment_fpath) - # / "fresh_tissue" - # / "segmentation" - # / "tmp_data.pkl", - # "wb", - # ), - # ) + ( + self.nuclei_org_tiles, + self.nuclei_adjusted_coords, + ) = stitching.stitched_beads_on_nuclei_fresh_tissue( + self.experiment_fpath, + self.client, + self.ds_nuclei, + self.ds_beads, + round_num=1, + ) + + pickle.dump( + [ + self.ds_beads, + self.ds_nuclei, + self.nuclei_metadata, + self.nuclei_org_tiles, + self.nuclei_adjusted_coords, + ], + open( + Path(self.experiment_fpath) + / "fresh_tissue" + / "segmentation" + / "tmp_data.pkl", + "wb", + ), + ) ( self.ds_beads, From a357f2452479d065cfc3c101b1abfd31372714ca Mon Sep 17 00:00:00 2001 From: simone Date: Sun, 22 May 2022 13:04:32 +0200 Subject: [PATCH 02/10] segmentation --- pysmFISH/pipeline.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pysmFISH/pipeline.py b/pysmFISH/pipeline.py index b0353e9..8f4c1fd 100644 --- a/pysmFISH/pipeline.py +++ b/pysmFISH/pipeline.py @@ -1285,7 +1285,7 @@ def processing_fresh_tissue_step( f"cannot process fresh tissue because missing running_functions attr" ) - ( + '''( self.ds_beads, self.ds_nuclei, self.nuclei_metadata, @@ -1317,7 +1317,7 @@ def processing_fresh_tissue_step( / "ds_tmp_data.pkl", "wb", ), - ) + )''' (self.ds_beads, self.ds_nuclei, self.nuclei_metadata) = pickle.load( open( From c2a7ca1c9d2507e5b17cdefe4be0ab2702d37d31 Mon Sep 17 00:00:00 2001 From: simone Date: Sun, 22 May 2022 15:34:05 +0200 Subject: [PATCH 03/10] segmentation --- pysmFISH/pipeline.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/pysmFISH/pipeline.py b/pysmFISH/pipeline.py index 8f4c1fd..c7ac880 100644 --- a/pysmFISH/pipeline.py +++ b/pysmFISH/pipeline.py @@ -1285,7 +1285,7 @@ def processing_fresh_tissue_step( f"cannot process fresh tissue because missing running_functions attr" ) - '''( + ( self.ds_beads, self.ds_nuclei, self.nuclei_metadata, @@ -1317,7 +1317,7 @@ def processing_fresh_tissue_step( / "ds_tmp_data.pkl", "wb", ), - )''' + ) (self.ds_beads, self.ds_nuclei, self.nuclei_metadata) = pickle.load( open( @@ -1330,14 +1330,13 @@ def processing_fresh_tissue_step( ) # Segmentation - '''fov_processing.segmentation_graph( + fov_processing.segmentation_graph( self.ds_nuclei, self.chunk_size, self.experiment_fpath, self.fresh_tissue_segmentation_engine, self.diameter_size, - )''' - + ) ( self.nuclei_org_tiles, self.nuclei_adjusted_coords, From a49dc3172b1c506b507fb8e87b8474de1e0cfc5e Mon Sep 17 00:00:00 2001 From: simone Date: Sun, 22 May 2022 21:56:57 +0200 Subject: [PATCH 04/10] segmentation --- pysmFISH/pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysmFISH/pipeline.py b/pysmFISH/pipeline.py index c7ac880..1faa9f2 100644 --- a/pysmFISH/pipeline.py +++ b/pysmFISH/pipeline.py @@ -1380,7 +1380,7 @@ def processing_fresh_tissue_step( "rb", ), ) - + segmentation_output_path = ( Path(self.experiment_fpath) / "fresh_tissue" / "segmentation" ) From 949940e70f38030a9b786f6bf9372ba5041082be Mon Sep 17 00:00:00 2001 From: simone Date: Tue, 24 May 2022 14:50:21 +0200 Subject: [PATCH 05/10] gc.collect() --- pysmFISH/pipeline.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pysmFISH/pipeline.py b/pysmFISH/pipeline.py index 1faa9f2..f7f4bc6 100644 --- a/pysmFISH/pipeline.py +++ b/pysmFISH/pipeline.py @@ -1394,6 +1394,7 @@ def processing_fresh_tissue_step( self.client, self.min_overlapping_pixels_segmentation, ) + gc.collect() segmentation.register_assign( self.experiment_fpath, From b501ed0b8a633b1a542fb1e728366d8b4f489e97 Mon Sep 17 00:00:00 2001 From: simone Date: Wed, 25 May 2022 14:43:18 +0200 Subject: [PATCH 06/10] all_registrations_removed_large_shift -> new --- pysmFISH/pipeline.py | 30 +++++++++++++++++++++++++++++- pysmFISH/segmentation.py | 13 ++++++------- pysmFISH/stitching.py | 11 +++++++++-- 3 files changed, 44 insertions(+), 10 deletions(-) diff --git a/pysmFISH/pipeline.py b/pysmFISH/pipeline.py index f7f4bc6..b88b092 100644 --- a/pysmFISH/pipeline.py +++ b/pysmFISH/pipeline.py @@ -1395,7 +1395,35 @@ def processing_fresh_tissue_step( self.min_overlapping_pixels_segmentation, ) gc.collect() + + def processing_assign_dots(self): + segmentation_output_path = ( + Path(self.experiment_fpath) / "fresh_tissue" / "segmentation" + ) + + ( + self.ds_beads, + self.ds_nuclei, + self.nuclei_metadata, + self.nuclei_org_tiles, + self.nuclei_adjusted_coords, + ) = pickle.load( + open( + Path(self.experiment_fpath) + / "fresh_tissue" + / "segmentation" + / "tmp_data.pkl", + "rb", + ), + ) + + segmented_object_dict_recalculated = pickle.load( + open( + segmentation_output_path / ("segmented_objects_dict_recalculated_ids.pkl"), + "rb", + ), + ) segmentation.register_assign( self.experiment_fpath, segmented_object_dict_recalculated, @@ -1407,7 +1435,6 @@ def processing_fresh_tissue_step( segmentation_output_path, self.max_expansion_radius, self.hamming_distance, - centering_mode=centering_mode, ) # -------------------------------- @@ -1663,6 +1690,7 @@ def run_full(self): step_start = datetime.now() self.processing_fresh_tissue_step() + self.processing_assign_dots() self.logger.info( f"{self.experiment_fpath.stem} timing: \ Processing fresh tissue completed in {utils.nice_deltastring(datetime.now() - step_start)}." diff --git a/pysmFISH/segmentation.py b/pysmFISH/segmentation.py index ca71c21..8f46d5b 100644 --- a/pysmFISH/segmentation.py +++ b/pysmFISH/segmentation.py @@ -484,12 +484,11 @@ def create_label_image( ), ) - '''zarr_fpath = segmentation_output_path / "image_segmented_labels.zarr" + zarr_fpath = segmentation_output_path / "image_segmented_labels.zarr" store = zarr.DirectoryStore(zarr_fpath, "w") grp = zarr.group(store=store, overwrite=True) - grp.create_dataset(name="segmented_labels_image", data=img)''' - - np.save(os.path.join(segmentation_output_path,'segmented_labels_image.npy'),img) + grp.create_dataset(name="segmented_labels_image", data=img) + #np.save(os.path.join(segmentation_output_path,'segmented_labels_image.npy'),img) return segmented_object_dict_recalculated @@ -708,11 +707,11 @@ def register_assign( source_RNA_df.loc[:, ["r_transformed", "c_transformed"]] = transformed_points # Replace this with chunk loading in the expanding function - '''zarr_fpath = segmentation_output_path / "image_segmented_labels.zarr" + zarr_fpath = segmentation_output_path / "image_segmented_labels.zarr" store = zarr.DirectoryStore(zarr_fpath, "r") grp = zarr.group(store=store, overwrite=False) - segmented_img = grp["segmented_labels_image"][...]''' - segmented_img = np.load(os.path.join(segmentation_output_path,'segmented_labels_image.npy')) + segmented_img = grp["segmented_labels_image"][...] + #segmented_img = np.load(os.path.join(segmentation_output_path,'segmented_labels_image.npy')) # instantiate model CA = Cell_Assignment() diff --git a/pysmFISH/stitching.py b/pysmFISH/stitching.py index a4f6dc4..8b58564 100644 --- a/pysmFISH/stitching.py +++ b/pysmFISH/stitching.py @@ -1204,9 +1204,16 @@ def stitching_graph( for idx, cpl_dict in tiles_org.overlapping_regions.items(): overlapping_coords_reorganized.update(cpl_dict) - all_registrations_removed_large_shift = { + '''all_registrations_removed_large_shift = { k: v for (k, v) in all_registrations_dict.items() if np.all(np.abs(v[0]) < 20) - } + }''' + # Alejandro version + all_registrations_removed_large_shift = {} + for (k, v) in all_registrations_dict.items(): + if np.all(np.abs(v[0]) < 20): + all_registrations_removed_large_shift[k] = v + else: + all_registrations_removed_large_shift[k] = [np.array([0,0]), 1.0] cpls = all_registrations_removed_large_shift.keys() # cpls = list(unfolded_overlapping_regions_dict.keys()) From d5791e8b4ba288923cf582241a5b43896187f2f0 Mon Sep 17 00:00:00 2001 From: simone Date: Wed, 25 May 2022 14:56:22 +0200 Subject: [PATCH 07/10] all_registrations_removed_large_shift -> new --- pysmFISH/stitching.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/pysmFISH/stitching.py b/pysmFISH/stitching.py index 8b58564..65c1b19 100644 --- a/pysmFISH/stitching.py +++ b/pysmFISH/stitching.py @@ -1204,16 +1204,9 @@ def stitching_graph( for idx, cpl_dict in tiles_org.overlapping_regions.items(): overlapping_coords_reorganized.update(cpl_dict) - '''all_registrations_removed_large_shift = { + all_registrations_removed_large_shift = { k: v for (k, v) in all_registrations_dict.items() if np.all(np.abs(v[0]) < 20) - }''' - # Alejandro version - all_registrations_removed_large_shift = {} - for (k, v) in all_registrations_dict.items(): - if np.all(np.abs(v[0]) < 20): - all_registrations_removed_large_shift[k] = v - else: - all_registrations_removed_large_shift[k] = [np.array([0,0]), 1.0] + } cpls = all_registrations_removed_large_shift.keys() # cpls = list(unfolded_overlapping_regions_dict.keys()) @@ -1349,6 +1342,14 @@ def stitching_graph_fresh_nuclei( k: v for (k, v) in all_registrations_dict.items() if np.all(np.abs(v[0]) < 20) } + # Alejandro version + all_registrations_removed_large_shift = {} + for (k, v) in all_registrations_dict.items(): + if np.all(np.abs(v[0]) < 20): + all_registrations_removed_large_shift[k] = v + else: + all_registrations_removed_large_shift[k] = [np.array([0,0]), 1.0] + cpls = all_registrations_removed_large_shift.keys() # cpls = list(unfolded_overlapping_regions_dict.keys()) total_cpls = len(cpls) From 8d5ad8c2f31ab88fe9113450a14c83a4fbe5dec1 Mon Sep 17 00:00:00 2001 From: simone Date: Thu, 26 May 2022 09:51:18 +0200 Subject: [PATCH 08/10] gc_collect() and new expansion with fixed chunking issue --- pysmFISH/cell_assignment.py | 20 +++++++++-- pysmFISH/pipeline.py | 66 +++++++++++++++++++------------------ 2 files changed, 51 insertions(+), 35 deletions(-) diff --git a/pysmFISH/cell_assignment.py b/pysmFISH/cell_assignment.py index bf945a4..a105718 100644 --- a/pysmFISH/cell_assignment.py +++ b/pysmFISH/cell_assignment.py @@ -59,15 +59,22 @@ def expand_labels_mask(self, mask, distance, out_file_name='expanded', overlap_percentage = overlap_percentage / 100 overlap = int(chunk_size * overlap_percentage) - #Iterate over chunks and asign dots to cells + #Iterate over chunks for i, (view_min, view_max) in enumerate(sliding_window_view(chunks, 2)): print(i, view_min, view_max) if i == 0: view_min_extra = view_min view_max_extra = view_max + overlap + overlap_right = overlap + elif (view_max + overlap) > mask.shape[0]: + print('got here') + view_min_extra = view_min - overlap + view_max_extra = mask.shape[0] + overlap_right = mask.shape[0] - view_max else: view_min_extra = view_min - overlap - view_max_extra = view_max + overlap + view_max_extra = view_max + overlap + overlap_right = overlap #Expand labels img = mask[view_min_extra:view_max_extra, :] @@ -81,7 +88,14 @@ def expand_labels_mask(self, mask, distance, out_file_name='expanded', exp[view_min:, :] = img[overlap:] else: #Middle chunks - exp[view_min:view_max, :] = img[overlap:-overlap,:] + print(f' {overlap}, {overlap_right}') + print(f' {view_min}, {view_max}') + print(f' {view_min_extra}, {view_max_extra}') + print(f' {img.shape}') + print(f' {overlap}, {chunk_size+overlap}') + print(f' {img[overlap:chunk_size+overlap,:].shape}') + #exp[view_min:view_max, :] = img[overlap:-overlap_right,:] + exp[view_min:view_max, :] = img[overlap:chunk_size+overlap,:] return out_file_name diff --git a/pysmFISH/pipeline.py b/pysmFISH/pipeline.py index b88b092..29580af 100644 --- a/pysmFISH/pipeline.py +++ b/pysmFISH/pipeline.py @@ -1260,6 +1260,7 @@ def stitch_and_remove_dots_eel_graph_step(self, def processing_fresh_tissue_step( self, parsing=True, + reprocessing=True, tag_ref_beads="_ChannelEuropium_Cy3_", tag_nuclei="_ChannelCy3_", centering_mode='middle', @@ -1285,39 +1286,40 @@ def processing_fresh_tissue_step( f"cannot process fresh tissue because missing running_functions attr" ) - ( - self.ds_beads, - self.ds_nuclei, - self.nuclei_metadata, - ) = fov_processing.process_fresh_sample_graph( - self.experiment_fpath, - self.running_functions, - self.analysis_parameters, - self.client, - self.chunk_size, - tag_ref_beads=tag_ref_beads, - tag_nuclei=tag_nuclei, - eel_metadata=self.metadata, - fresh_tissue_segmentation_engine=self.fresh_tissue_segmentation_engine, - diameter_size=self.diameter_size, - parsing=parsing, - save_steps_output=self.save_intermediate_steps, - ) - - pickle.dump( - [ + if reprocessing: + ( self.ds_beads, self.ds_nuclei, - self.metadata, - ], - open( - Path(self.experiment_fpath) - / "fresh_tissue" - / "segmentation" - / "ds_tmp_data.pkl", - "wb", - ), - ) + self.nuclei_metadata, + ) = fov_processing.process_fresh_sample_graph( + self.experiment_fpath, + self.running_functions, + self.analysis_parameters, + self.client, + self.chunk_size, + tag_ref_beads=tag_ref_beads, + tag_nuclei=tag_nuclei, + eel_metadata=self.metadata, + fresh_tissue_segmentation_engine=self.fresh_tissue_segmentation_engine, + diameter_size=self.diameter_size, + parsing=parsing, + save_steps_output=self.save_intermediate_steps, + ) + + pickle.dump( + [ + self.ds_beads, + self.ds_nuclei, + self.metadata, + ], + open( + Path(self.experiment_fpath) + / "fresh_tissue" + / "segmentation" + / "ds_tmp_data.pkl", + "wb", + ), + ) (self.ds_beads, self.ds_nuclei, self.nuclei_metadata) = pickle.load( open( @@ -1397,7 +1399,7 @@ def processing_fresh_tissue_step( gc.collect() def processing_assign_dots(self): - + gc.collect() segmentation_output_path = ( Path(self.experiment_fpath) / "fresh_tissue" / "segmentation" ) From 9fbd3f1580d8146eb0eade3437462aec6723062c Mon Sep 17 00:00:00 2001 From: simone Date: Thu, 26 May 2022 10:09:44 +0200 Subject: [PATCH 09/10] removed centering mode, not needed with marcos pipeline --- pysmFISH/pipeline.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pysmFISH/pipeline.py b/pysmFISH/pipeline.py index 29580af..06d5d98 100644 --- a/pysmFISH/pipeline.py +++ b/pysmFISH/pipeline.py @@ -1263,7 +1263,6 @@ def processing_fresh_tissue_step( reprocessing=True, tag_ref_beads="_ChannelEuropium_Cy3_", tag_nuclei="_ChannelCy3_", - centering_mode='middle', ): """ This function create and run a processing graph that parse and filter the nuclei staining in fresh tissue From 72faf888b72c176224ca2ee77f8d0b856b5619c2 Mon Sep 17 00:00:00 2001 From: simone Date: Thu, 26 May 2022 16:46:36 +0200 Subject: [PATCH 10/10] m --- pysmFISH/pipeline.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/pysmFISH/pipeline.py b/pysmFISH/pipeline.py index 06d5d98..abdca0d 100644 --- a/pysmFISH/pipeline.py +++ b/pysmFISH/pipeline.py @@ -131,6 +131,9 @@ class Pipeline: diameter_size (int): Size of the diameter of the cells to segment using cellpose min_overlapping_pixels_segmentation (int): Size of the overlapping label objects fov_alignement_mode (str): clip or merged (default clipped). + clip_size (str): only if fov_alignment_mode is merge it will clip the clip_size length of borders. + remove_distinct_genes (bool): when stitching it will also remove overlapping dots of different genes if set to + true. Defaults to true Attributes: @@ -257,6 +260,9 @@ def __init__( self.max_expansion_radius = kwarg.pop("max_expansion_radius", 18) self.fov_alignement_mode = kwarg.pop("fov_alignement_mode", "clip") + self.clip_size = kwarg.pop("clip_size", 0) + self.remove_distinct_genes = kwarg.pop("remove_distinct_genes", False) + # ----------------------------------- # PROCESSING STEPS @@ -1164,8 +1170,8 @@ def stitch_and_remove_dots_eel_graph_old_room_step(self): # # ---------------------------------------------------------------- def stitch_and_remove_dots_eel_graph_step(self, - remove_distinct_genes=False, - clip_size=0,): + remove_distinct_genes=self.remove_distinct_genes, + clip_size=self.clip_size,): """ Function to stitch the different fovs and remove the duplicated @@ -1249,8 +1255,8 @@ def stitch_and_remove_dots_eel_graph_step(self, matching_dot_radius=self.same_dot_radius_duplicate_dots, out_folder=folder, exp_name=self.metadata["experiment_name"], - remove_distinct_genes=remove_distinct_genes, - clip_size=clip_size, + remove_distinct_genes=self.remove_distinct_genes, + clip_size=self.clip_size, verbose=False, ) # Set to False in pipeline