From 6638defbd968b5a68b8499df53f0d1e0f2d3421c Mon Sep 17 00:00:00 2001 From: David Young Date: Tue, 10 Jan 2023 13:56:30 +0800 Subject: [PATCH 01/12] Make blob verification matches debugging more readable - Group matches using original blob coordinates for easier comparison - Show scaled coordinates and distances separately with corresponding indices - Add type hints --- magmap/cv/verifier.py | 60 ++++++++++++++++++++++++++++--------------- 1 file changed, 40 insertions(+), 20 deletions(-) diff --git a/magmap/cv/verifier.py b/magmap/cv/verifier.py index a3f254f2d..7b990c6c2 100644 --- a/magmap/cv/verifier.py +++ b/magmap/cv/verifier.py @@ -44,26 +44,32 @@ def _match_blobs(blobs, blobs_master, close, close_master, dists): return matches -def find_closest_blobs_cdist(blobs, blobs_master, thresh=None, scaling=None): - """Find the closest blobs within a given tolerance using the +def find_closest_blobs_cdist( + blobs: np.ndarray, blobs_master: np.ndarray, + thresh: Optional[float] = None, scaling: + Optional[Sequence[float]] = None +) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: + """Find the closest blobs within a given tolerance using the Hungarian algorithm to find blob matches. Args: - blobs: Blobs as a 2D array of [n, [z, row, column, ...]]. + blobs: Blobs as a 2D array of ``[n, [z, row, column, ...]]``. blobs_master: Array in same format as ``blobs``. - thresh: Threshold distance beyond which blob pairings are excluded; + thresh: Threshold distance beyond which blob pairings are excluded; defaults to None to include all matches. - scaling: Sequence of scaling factors by which to multiply the - blob coordinates before computing distances, used to - scale coordinates from an anisotropic to isotropic - ROI before computing distances, which assumes isotropy. + scaling: Sequence of scaling factors by which to multiply the + blob coordinates before computing distances, used to + scale coordinates from an anisotropic to isotropic + ROI before computing distances, which assumes isotropy. Defaults to None. Returns: - Tuple of ``rowis`` and ``colis``, arrays of row and corresponding - column indices of the closest matches; and ``dists_closest``, an - array of corresponding distances for these matches. Only matches - within the given tolerance will be included. + Tuple of: + - ``rowis`` and ``colis``, arrays of row and corresponding + column indices of the closest matches + - ``dists_closest``, an array of corresponding distances for these + matches. Only matches within the given tolerance will be included. + """ blobs_scaled = blobs blobs_master_scaled = blobs_master @@ -73,7 +79,7 @@ def find_closest_blobs_cdist(blobs, blobs_master, thresh=None, scaling=None): blobs_scaled = np.multiply(blobs[:, :len_scaling], scaling) blobs_master_scaled = np.multiply(blobs_master[:, :len_scaling], scaling) - # find Euclidean distances between each pair of points and determine + # find Euclidean distances between each pair of points and determine # the optimal assignments using the Hungarian algorithm dists = distance.cdist(blobs_scaled, blobs_master_scaled) rowis, colis = optimize.linear_sum_assignment(dists) @@ -82,14 +88,28 @@ def find_closest_blobs_cdist(blobs, blobs_master, thresh=None, scaling=None): if thresh is not None: # filter out matches beyond the given threshold distance dists_in = dists_closest < thresh + if config.verbose: - for blob, blob_sc, blob_base, blob_base_sc, dist, dist_in in zip( - blobs[rowis], blobs_scaled[rowis], blobs_master[colis], - blobs_master_scaled[colis], dists_closest, - dists_in): - print("blob: {} (scaled {}), base: {} ({}), dist: {}, in? {}" - .format(blob[:3], blob_sc[:3], blob_base[:3], - blob_base_sc[:3], dist, dist_in)) + # show matches using original blob coordinates + for i, (blob, blob_base, dist_in) in enumerate(zip( + blobs[rowis], blobs_master[colis], dists_in)): + _logger.debug( + "%s: Detected blob: %s, truth blob: %s, in? %s", + i, blob[:3], blob_base[:3], dist_in) + _logger.debug("") + + # show corresponding scaled coordinates and distances + for i, (blob_sc, blob_base_sc, dist, dist_in) in enumerate(zip( + blobs_scaled[rowis], blobs_master_scaled[colis], + dists_closest, dists_in)): + _logger.debug( + "%s: Scaled dist: %s", i, dist) + _logger.debug( + " %s (detected)", blob_sc[:3]) + _logger.debug( + " %s (truth)", blob_base_sc[:3]) + _logger.debug("") + rowis = rowis[dists_in] colis = colis[dists_in] dists_closest = dists_closest[dists_in] From 14d50edfb3e0302315cfcf87fe0cae84f3f9a65a Mon Sep 17 00:00:00 2001 From: David Young Date: Thu, 2 Feb 2023 20:00:33 -0800 Subject: [PATCH 02/12] Make `openpyxl` package optional during region ID export Exporting region IDs has output both a CSV and Excel files. Make the Excel file output optional to avoid an error when the export package is not available since this package is optional. --- magmap/io/export_regions.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/magmap/io/export_regions.py b/magmap/io/export_regions.py index 8cc6734b4..584d3f1af 100644 --- a/magmap/io/export_regions.py +++ b/magmap/io/export_regions.py @@ -96,8 +96,12 @@ def color_cells(s): if rgbs is not None: df = df.style.apply(color_cells, subset="RGB") path_xlsx = "{}.xlsx".format(os.path.splitext(path)[0]) - df.to_excel(path_xlsx) - print("exported regions to styled spreadsheet: \"{}\"".format(path_xlsx)) + try: + df.to_excel(path_xlsx) + _logger.info("Exported regions to styled spreadsheet: %s", path_xlsx) + except ModuleNotFoundError: + raise ModuleNotFoundError( + config.format_import_err("openpyxl", task="formatting Excel files")) return df From 294d6b6282d7a97abd1da3be0778725c0d8a11e6 Mon Sep 17 00:00:00 2001 From: David Young Date: Thu, 2 Feb 2023 20:03:09 -0800 Subject: [PATCH 03/12] Add a plot labels option to customize rotation degrees Add `--plot_labels rotation=` CLI argument to set the amount of rotation. Apply this rotation to swarm and bar plot tasks. --- magmap/plot/plot_2d.py | 16 ++++++++++++---- magmap/settings/config.py | 2 ++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/magmap/plot/plot_2d.py b/magmap/plot/plot_2d.py index 02330c3c7..96106b8d1 100644 --- a/magmap/plot/plot_2d.py +++ b/magmap/plot/plot_2d.py @@ -1007,7 +1007,9 @@ def plot_swarm( x_unit: Optional[str] = None, y_unit: Optional[str] = None, legend_names: Optional[Sequence[str]] = None, col_vspan: Optional[str] = None, vspan_fmt: Optional[str] = None, - size=None, ax=None, **kwargs) -> "axes.Axes": + size: Optional[Sequence[float]] = None, + ax: Optional["axes.Axes"] = None, rotation: Optional[float] = None, + **kwargs) -> "axes.Axes": """Generate a swarm/jitter plot in Seaborn. Supports x-axis scaling and vertical spans. @@ -1032,6 +1034,8 @@ def plot_swarm( vspan_fmt: Vertical span label string format; defaulst to None. size: Figure size in ``width, height`` as inches; defaults to None. ax: Matplotlib axes; defaults to None. + rotation: x-axis ttext angle rotation in degrees. Defaults to None, + which will rotate by 45 degrees. **kwargs: Additional arguments, passed to :meth:`decorate_plot`. Returns: @@ -1071,7 +1075,9 @@ def plot_swarm( order=x_order, data=df, ax=ax) # scale x-axis ticks and rotate labels - plot_support.scale_xticks(ax, 45) + if rotation is None: + rotation = 45 + plot_support.scale_xticks(ax, rotation) legend = ax.get_legend() if legend: @@ -1417,6 +1423,7 @@ def main( hline = config.plot_labels[config.PlotLabels.HLINE] col_vspan = config.plot_labels[config.PlotLabels.VSPAN_COL] vspan_fmt = config.plot_labels[config.PlotLabels.VSPAN_FORMAT] + rotation = config.plot_labels[config.PlotLabels.ROTATION] # base output path for tasks that defer saving to post_plot base_out_path = None @@ -1433,7 +1440,8 @@ def main( size=size, show=False, groups=config.groups, col_vspan=col_vspan, vspan_fmt=vspan_fmt, prefix=config.prefix, save=False, - col_wt=col_wt, x_tick_labels=x_tick_lbls, rotation=45, + col_wt=col_wt, x_tick_labels=x_tick_lbls, + rotation=45 if rotation is None else rotation, err_cols_abs=err_col_abs) elif plot_2d_type is config.Plot2DTypes.BAR_PLOT_VOLS_STATS_EFFECTS: @@ -1519,7 +1527,7 @@ def main( ax = plot_swarm( pd.read_csv(config.filename), x_cols, data_cols, config.groups, group_col, x_lbl, y_lbl, x_unit, y_unit, legend_names, col_vspan, - vspan_fmt, size, title=title) + vspan_fmt, size, title=title, rotation=rotation) base_out_path = "swarm" elif plot_2d_type is config.Plot2DTypes.CAT_PLOT: diff --git a/magmap/settings/config.py b/magmap/settings/config.py index 3561b20f6..61a8ad138 100644 --- a/magmap/settings/config.py +++ b/magmap/settings/config.py @@ -369,6 +369,8 @@ class PlotLabels(Enum): VSPAN_FORMAT = auto() #: Background color as a Matplotlib or RGBA string. BACKGROUND = auto() + #: Rotation angle in degrees. + ROTATION = auto() #: dict[Any]: Plot labels set from command-line. From 9fa0055935b9d3804c20c1a379bc968a14f86b2b Mon Sep 17 00:00:00 2001 From: David Young Date: Thu, 2 Feb 2023 20:10:44 -0800 Subject: [PATCH 04/12] Support RGB in stack export - Support RGB set on the CLI when exporting a stack of images - Refactor plane processing to use the rescale/resize wrapper - Add multichannel support to this wrapper --- magmap/io/export_stack.py | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/magmap/io/export_stack.py b/magmap/io/export_stack.py index 24aad1803..b3f736002 100644 --- a/magmap/io/export_stack.py +++ b/magmap/io/export_stack.py @@ -109,7 +109,7 @@ def import_img(cls, i, rescale, multichannel): return i, img @classmethod - def process_plane(cls, i, target_size, rotate=None): + def process_plane(cls, i, target_size, multichannel, rotate=None): """Process corresponding planes from related images. Assumes that :attr:``imgs`` is a list of nested 2D image lists, @@ -120,6 +120,7 @@ def process_plane(cls, i, target_size, rotate=None): Args: i: Index within nested lists of :attr:``imgs`` to plot. target_size: Resize to this shape. + multichannel: True for multichannel images. rotate: Degrees by which to rotate; defaults to None. Returns: @@ -131,17 +132,9 @@ def process_plane(cls, i, target_size, rotate=None): cls.convert_imgs() imgs_proc = [] for j, img_stack in enumerate(cls.imgs): - if j == 0: - # atlas image - img = transform.resize( - img_stack[i], target_size, mode="reflect", - preserve_range=True, anti_aliasing=True, - order=cls.interp_order) - else: - # labels-based image, using nearest-neighbor interpolation - img = transform.resize( - img_stack[i], target_size, mode="reflect", - preserve_range=True, anti_aliasing=False, order=0) + order = cls.interp_order if j == 0 else 0 + img = cv_nd.rescale_resize( + img_stack[i], target_size, multichannel, True, order=order) imgs_proc.append(img) if rotate: # TODO: consider removing since not saved; should instead rotate @@ -182,12 +175,12 @@ def handle_extracted_plane(): ax = axs[imgi] plot_support.hide_axes(ax) - # multiple artists can be shown at each frame by collecting - # each group of artists in a list; overlay_images returns - # a nested list containing a list for each image, which in turn + # multiple artists can be shown at each frame by collecting + # each group of artists in a list; overlay_images returns + # a nested list containing a list for each image, which in turn # contains a list of artists for each channel overlaid = plot_support.ImageOverlayer( - ax, self.aspect, self.origin, ignore_invis=True) + ax, self.aspect, self.origin, ignore_invis=True, rgb=config.rgb) ax_imgs = overlaid.overlay_images( imgs, None, cmaps_all, check_single=True) if (colorbar is not None and len(ax_imgs) > 0 @@ -248,7 +241,7 @@ def handle_extracted_plane(): for i in range(num_images): # add rotation argument if necessary - args = (i, target_size) + args = (i, target_size, multichannel) if pool is None: # extract and handle without multiprocessing imgi, imgs = self.fn_process(*args) From fd49aaf9b5ac28d348d944354c1126cf4e064643 Mon Sep 17 00:00:00 2001 From: David Young Date: Thu, 2 Feb 2023 21:38:43 -0800 Subject: [PATCH 05/12] Remove TIF file extension from output file when exporting stack Files loaded directly as TIFs require the extension during loading, but the output file should not retain this extension. --- magmap/io/export_stack.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/magmap/io/export_stack.py b/magmap/io/export_stack.py index b3f736002..03ae027c1 100644 --- a/magmap/io/export_stack.py +++ b/magmap/io/export_stack.py @@ -531,6 +531,7 @@ def stack_to_img(paths, roi_offset, roi_size, series=None, subimg_offset=None, collage = num_paths > 1 figs = {} + path_base = paths[0] for i in range(nrows): for j in range(ncols): n = i * ncols + j @@ -542,6 +543,10 @@ def stack_to_img(paths, roi_offset, roi_size, series=None, subimg_offset=None, # TODO: test directory of images # TODO: consider not reloading first image np_io.setup_images(path_sub, series, subimg_offset, subimg_size) + if config.img5d.img_io is config.LoadIO.TIFFFILE: + # remove extension from TIF files, which needed ext to load + path_base = libmag.get_filename_without_ext(path_base) + stacker = setup_stack( config.image5d, path_sub, offset=roi_offset, roi_size=roi_size, slice_vals=config.slice_vals, @@ -590,7 +595,6 @@ def stack_to_img(paths, roi_offset, roi_size, series=None, subimg_offset=None, for fig_dict, img in zip(figs.values(), plotted_imgs): fig_dict["imgs"].append(img) - path_base = paths[0] for planei, fig_dict in figs.items(): if animated: # generate animated image (eg animated GIF or movie file) From 5fc8af91ab71a5e686afd0b1b80bea71838950bb Mon Sep 17 00:00:00 2001 From: David Young Date: Fri, 3 Feb 2023 16:31:56 -0800 Subject: [PATCH 06/12] Format import error for category plots Follow the same error formatting when Seaborn is missing in category plotting as for swarm plots. --- magmap/plot/plot_2d.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/magmap/plot/plot_2d.py b/magmap/plot/plot_2d.py index 96106b8d1..940fd1f80 100644 --- a/magmap/plot/plot_2d.py +++ b/magmap/plot/plot_2d.py @@ -1047,7 +1047,8 @@ def plot_swarm( """ if sns is None: - raise ImportError(config.format_import_err("seaborn", task="swarm plots")) + raise ImportError( + config.format_import_err("seaborn", task="swarm plots")) df_vspan = df if x_order is not None: @@ -1143,7 +1144,8 @@ def plot_catplot( """ if sns is None: - raise ImportError("Seaborn is required for swarm plots, please install") + raise ImportError( + config.format_import_err("seaborn", task="category plots")) if kwargs_plot is None: kwargs_plot = {} From 7a3b1b9c0f5b7daaa8d43cc1b427d090a03a7a29 Mon Sep 17 00:00:00 2001 From: David Young Date: Thu, 9 Feb 2023 14:01:58 +0800 Subject: [PATCH 07/12] Configure intercept term for linear regression using environment variable Add an `Intercept` environment field to configure whether to include the intercept term in linear regressions. --- clrstats/R/clrstats.R | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/clrstats/R/clrstats.R b/clrstats/R/clrstats.R index 6abb14e56..0ea59ee42 100644 --- a/clrstats/R/clrstats.R +++ b/clrstats/R/clrstats.R @@ -112,18 +112,29 @@ fitModel <- function(model, vals, genos, sides, ids=NULL) { # linear regression # TODO: see whether need to factorize genos if (length(unique(genos)) < 2) { - lm.formula <- as.formula(vals ~ sides) + form <- "vals ~ sides" } else if (length(unique(sides)) < 2) { - lm.formula <- as.formula(vals ~ genos) + form <- "vals ~ genos" } else { - lm.formula <- as.formula(vals ~ genos * sides) + form <- "vals ~ genos * sides" } + + intercept <- config.env$Intercept + if (!intercept) { + # remove the implied intercept term + form <- paste0(form, " + 0") + } + + # perform linear regression + lm.formula <- as.formula(form) fit <- lm(lm.formula) result.summary <- summary.lm(fit) result <- result.summary$coefficients - # remove first ("non-intercept") row - result <- result[-(1:1), ] + if (intercept) { + # remove first ("non-intercept") row + result <- result[-(1:1), ] + } } else if (model == kModel[3]) { # generalized estimating equations @@ -891,6 +902,8 @@ setupConfig <- function(name=NULL) { # region-ID map from MagellanMapper, which should contain all regions # including hierarchical/ontological ones config.env$Labels.Path <- "../region_ids.csv" + # include linear regression intercept term + config.env$Intercept <- TRUE } else if (endsWith(name, ".R")) { # load a profile file From 4ddb60eb020b41cc11921a9c9dd92fff1079e9f5 Mon Sep 17 00:00:00 2001 From: David Young Date: Thu, 9 Feb 2023 15:55:21 +0800 Subject: [PATCH 08/12] Generalize extracting stats into filtered stats table - Transfer all columns from the stat coefficients output table rather than hard-coding column names - Clean up unused variable initializations and for-loop sequence --- clrstats/R/clrstats.R | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/clrstats/R/clrstats.R b/clrstats/R/clrstats.R index 0ea59ee42..76a2a7883 100644 --- a/clrstats/R/clrstats.R +++ b/clrstats/R/clrstats.R @@ -663,10 +663,6 @@ filterStats <- function(stats, corr=NULL) { stats.filt <- stats[non.na, ] if (length(stats.filt$Stats) < 1) return(NULL) - filtered <- NULL - interactions <- NULL - offset <- 0 # number of columns ahead of coefficients - # get names and mean and CI columns cols.names <- names(stats.filt) cols.means.cis <- c( @@ -676,21 +672,26 @@ filterStats <- function(stats, corr=NULL) { cols.names[grepl(".sd", cols.names)], cols.names[grepl(".ci", cols.names)]) - # build data frame for pertinent coefficients from each type of main - # effect or interaction + # set up common columns + cols <- list("Region", "Volume", "Nuclei") + cols.orig <- cols + offset <- length(cols) # number of columns ahead of coefficients + + # get stats coefficients and main effects or interactions stats.coef <- stats.filt$Stats[1][[1]] interactions <- gsub(":", ".", rownames(stats.coef)) - cols <- list("Region", "Volume", "Nuclei") - cols.orig <- cols # points to original vector if it is mutated - offset <- length(cols) - cols.suffixes <- c( - ".n", ".effect", ".ci.low", ".ci.hi", ".effect.raw", ".ci.low.raw", - ".ci.hi.raw", ".p", ".pcorr", ".logp") + + # make a set of stat coefficient columns for each main effect/interaction + cols.suffixes <- paste0(".", tolower(colnames(stats.coef))) + cols.suffixes <- gsub("value", "effect", cols.suffixes) + cols.suffixes <- c(cols.suffixes, ".pcorr", ".logp") for (interact in interactions) { for (suf in cols.suffixes) { cols <- append(cols, paste0(interact, suf)) } } + + # build output data frame filtered <- data.frame(matrix(nrow=nrow(stats.filt), ncol=length(cols))) names(filtered) <- cols for (col in cols.orig) { @@ -699,7 +700,7 @@ filterStats <- function(stats, corr=NULL) { } num.stat.cols <- length(names(stats.coef)) - for (i in 1:nrow(stats.filt)) { + for (i in seq_len(nrow(stats.filt))) { if (is.na(stats.filt$Stats[i])) next # get coefficients, stored in one-element list stats.coef <- stats.filt$Stats[i][[1]] From 9ceed987af3ae49f22c6b64033a200c0f2cbd495 Mon Sep 17 00:00:00 2001 From: David Young Date: Thu, 9 Feb 2023 15:56:48 +0800 Subject: [PATCH 09/12] Store r-squared and intercept from linear regression - Extract additional linear regression values - Store extra columns in results data frame to output stats table --- clrstats/R/clrstats.R | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/clrstats/R/clrstats.R b/clrstats/R/clrstats.R index 76a2a7883..a84eb0be2 100644 --- a/clrstats/R/clrstats.R +++ b/clrstats/R/clrstats.R @@ -129,12 +129,18 @@ fitModel <- function(model, vals, genos, sides, ids=NULL) { lm.formula <- as.formula(form) fit <- lm(lm.formula) result.summary <- summary.lm(fit) - result <- result.summary$coefficients + result <- as.data.frame(result.summary$coefficients) + result$r.squared <- result.summary$r.squared + # store intercept coefficient + intercept.coef <- 0 if (intercept) { - # remove first ("non-intercept") row + # extract intercept coefficient and remove its row so result only + # has main effect and interactions + intercept.coef <- fit$coefficients[1] result <- result[-(1:1), ] } + result$intercept <- intercept.coef } else if (model == kModel[3]) { # generalized estimating equations @@ -209,6 +215,15 @@ fitModel <- function(model, vals, genos, sides, ids=NULL) { } coef.tab$P <- result[rows, 4] coef.tab$N <- length(vals) + + # transfer any additional stats to output table + extra.cols <- c("intercept", "r.squared") + for (col in extra.cols) { + if (col %in% colnames(result)) { + coef.tab[[col]] <- result[rows, col] + } + } + print(coef.tab) return(coef.tab) } From 83696912da1162fc944a9f916199fbf92fbd69d3 Mon Sep 17 00:00:00 2001 From: David Young Date: Fri, 10 Feb 2023 11:35:19 +0800 Subject: [PATCH 10/12] Support legend title and swarm plot customization - Add parameters to customize the legend title and pass arguments to the underlying Seaborn swarm plot, similar to the category plot wrapper - Use the data frame passed to `main` if given --- magmap/plot/plot_2d.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/magmap/plot/plot_2d.py b/magmap/plot/plot_2d.py index 940fd1f80..1c8c19d32 100644 --- a/magmap/plot/plot_2d.py +++ b/magmap/plot/plot_2d.py @@ -1009,7 +1009,8 @@ def plot_swarm( col_vspan: Optional[str] = None, vspan_fmt: Optional[str] = None, size: Optional[Sequence[float]] = None, ax: Optional["axes.Axes"] = None, rotation: Optional[float] = None, - **kwargs) -> "axes.Axes": + legend_title: Optional[str] = None, + kwargs_plot: Optional[Dict[str, Any]] = None, **kwargs) -> "axes.Axes": """Generate a swarm/jitter plot in Seaborn. Supports x-axis scaling and vertical spans. @@ -1036,6 +1037,9 @@ def plot_swarm( ax: Matplotlib axes; defaults to None. rotation: x-axis ttext angle rotation in degrees. Defaults to None, which will rotate by 45 degrees. + legend_title: Legened title; defaults to None. + kwargs_plot: Dictionary of arguments to :meth:`sns.swarmplot`; defaults + to None. **kwargs: Additional arguments, passed to :meth:`decorate_plot`. Returns: @@ -1050,6 +1054,9 @@ def plot_swarm( raise ImportError( config.format_import_err("seaborn", task="swarm plots")) + if kwargs_plot is None: + kwargs_plot = {} + df_vspan = df if x_order is not None: # reorder so vals in x_cols match the order of vals in x_order; @@ -1073,7 +1080,7 @@ def plot_swarm( # plot in seaborn ax = sns.swarmplot( x=x_cols, y=y_cols, hue=group_col, hue_order=legend_names, - order=x_order, data=df, ax=ax) + order=x_order, data=df, ax=ax, **kwargs_plot) # scale x-axis ticks and rotate labels if rotation is None: @@ -1085,7 +1092,7 @@ def plot_swarm( # make legend translucent in case it overlaps points and remove # legend title legend.get_frame().set(alpha=0.5) - legend.set_title(None) + legend.set_title(legend_title if legend_title else None) if col_vspan is not None: # add vertical spans @@ -1526,10 +1533,12 @@ def main( elif plot_2d_type is config.Plot2DTypes.SWARM_PLOT: # swarm/jitter plot + df_plot = pd.read_csv(config.filename) if df is None else df ax = plot_swarm( - pd.read_csv(config.filename), x_cols, data_cols, config.groups, + df_plot, x_cols, data_cols, config.groups, group_col, x_lbl, y_lbl, x_unit, y_unit, legend_names, col_vspan, - vspan_fmt, size, title=title, rotation=rotation) + vspan_fmt, size, ax=ax, title=title, rotation=rotation, + kwargs_plot=kwargs_plot, **kwargs) base_out_path = "swarm" elif plot_2d_type is config.Plot2DTypes.CAT_PLOT: From b6877c09e4a73874fd10fb7219bbb7ccfdcef4fb Mon Sep 17 00:00:00 2001 From: David Young Date: Fri, 10 Feb 2023 11:36:51 +0800 Subject: [PATCH 11/12] Option to not save 2D plot Add a parameter to `main` to specify whether to save the plot. --- magmap/plot/plot_2d.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/magmap/plot/plot_2d.py b/magmap/plot/plot_2d.py index 1c8c19d32..ed020978d 100644 --- a/magmap/plot/plot_2d.py +++ b/magmap/plot/plot_2d.py @@ -1394,7 +1394,8 @@ def post_plot(ax, out_path=None, save_ext=None, show=False): def main( ax: Optional["axes.Axes"] = None, df: Optional[pd.DataFrame] = None, - kwargs_plot: Optional[Dict[str, Any]] = None, **kwargs): + kwargs_plot: Optional[Dict[str, Any]] = None, save: bool = True, + **kwargs): """Perform 2D plot tasks. Args: @@ -1402,6 +1403,7 @@ def main( df: Data frame; defaults to None. kwargs_plot: Dictionary of args to the underlying plot function; defaults to None. + save: True (default) to save plot. kwargs: Additional args to :meth:`decorate_plot`. Returns: @@ -1553,9 +1555,8 @@ def main( if ax is not None: # perform plot post-processing tasks, including file save unless # savefig is None - post_plot( - ax, libmag.make_out_path(base_out_path), config.savefig, - config.show) + out_path = libmag.make_out_path(base_out_path) if save else None + post_plot(ax, out_path, config.savefig, config.show) return ax From 87f799fb9e1ab8274c5ff8632f99e2bdda3c037b Mon Sep 17 00:00:00 2001 From: David Young Date: Thu, 23 Feb 2023 12:13:45 +0800 Subject: [PATCH 12/12] Release notes for customizing exports round-up (#445) Also, update CLI docs for `--plot_labels`. --- docs/cli.md | 14 +++++++++----- docs/release/release_v1.6.md | 8 ++++++-- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index 9e30eee30..75d31946c 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -34,7 +34,7 @@ Argument | Sub-argument | Function | Ver Added | Last updated `--labels` | `[path_ref="""] [level=0] [ID=0] [orig_colors=1] [symmetric_colors=1] [binary=,] [translate_labels="] [translate_children=0]` | Atlas label settings; see `config.AtlasLabels`.
  • `path_ref`: Path to the labels reference file. Should have at least these columns: `Region` or `id` for region IDs, and `RegionName` or `name` for corresponding names. Atlases generated in >=v1.5.0 with this flag will copy this file into the atlas directory so that this argument is not needed when loading the atlas and images registered to it.
  • `level`: Ontology level. Structures in sub-levels will be grouped at this level for volume stats. | v1.0.0 | v1.0.0 `--transform` | `[rotate=0] [flip_vert=0] [flip_horiz=0] [flip=axis] [rescale=0] [interpolation=1]` | Image transformations; see `config.Transforms`.
    • `interpolation`: Interpolation order `n` for the main (intensity) image, where `n` is passed to the `order` argument in Scikit-image's [`transform.resize`](https://scikit-image.org/docs/dev/api/skimage.transform.html#skimage.transform.resize) function
    • *Since [v1.6.0](#changes-in-magellanmapper-v16):* `flip`: Flip the selected axis, where `axis` = 0 for the z-axis, 1 for the y-axis, and 2 for the x-axis
    | v1.0.0 | v1.6.0 `--reg_suffixes` | `[atlas=atlasVolume.mhd,...] [annotation=annotation.mhd,...] [borders=]` | Suffixes of registered images to load; see `config.RegSuffixes` for suffixes used throughout the package. Atlases and regenerated and registered with output paths based on the path given by `--img` or `--prefix`. For example:
    • "Base" path (from `--img` or `--prefix`): `/home/me/my_image`
    • Intensity/histology/"atlas" image: `/home/me/my_image_atlasVolume.nii.gz`
    • Annotation/labels image: `/home/me/my_image_annotation.nii.gz`
    • Other registered images, eg atlas edges: `/home/me/my_image_atlasEdge.nii.gz`
    To load intensity and annotations image: `./run.py --img /home/me/my_image --reg_suffixes atlasVolume.nii.gz annotation.nii.gz`.
    • *Since [v1.5.0](#changes-in-magellanmapper-v15):* Suffixes can be also given as an absolute path, such as a labels image not registered to the main image.
    • *Since [v1.6.0](#changes-in-magellanmapper-v16):* Multiple labels suffixes can be given, separated by commas.
    | v1.0.0 | [v1.5.0](#changes-in-magellanmapper-v15) -`--plot_labels` | `[title=] ...` | Plot labels; see `config.PlotLabels` for available parameters. | v1.0.0 | v1.0.0 +`--plot_labels` | `[title=<title>] [err_col_abs=<col>] [background=<color>] [vspan_col=<col>] [vspan_format=<str>] [rotation=<deg>] ...` | Plot labels; see `config.PlotLabels` for available parameters.<ul><li>*Since [v1.6.0](#changes-in-magellanmapper-v16):* Added several new sub-arguments.</li></ul> | v1.0.0 | [v1.6.0](#changes-in-magellanmapper-v16) `--set_meta` | `[resolutions=<x,y,z>] [magnification=<n>] [zoom=<n>] [shape=<c,x,y,z,...>] [dtype=<data-type>]` | Metadata to set when importing an image; see `config.MetaKeys`. | v1.0.0 | [v1.3.0](#changes-in-magellanmapper-v13) `--plane` | `<xy\|xz\|yz>` | Transpose to the given planar orientation | v1.0.0 | v1.0.0 `--show` | `<0\|1>` | Show images after generating them in atlas pipelines. `0` to not show, `1` to show. | v1.0.0 | [v1.3.0](#changes-in-magellanmapper-v13) @@ -53,10 +53,14 @@ Argument | Sub-argument | Function | Ver Added | Last updated Old | New | Version | Purpose of Change | --- | --- | --- | --- -None | `--rgb` | v1.6.0 | Open images in RGB(a) mode. -`--ref_suffixes annotation=` | `--transform [flip=axis] ...` | v1.6.0 | Flip the specified axis. -`--transform ...` | `--transform [interpolation=n] ...` | v1.6.0 | Interpolation order can be specified when exporting the main image. -`--transform ...` | `--transform [flip=axis] ...` | v1.6.0 | Flip the specified axis. +None | `--rgb` | v1.6a1 | Open images in RGB(a) mode. +`--plot_labels ...` | `--plot_labels err_col_abs=<col> ...` | v1.6a1 | Plot error bars with a column of absolute rather than relative values, now that Clrstats gives absolute values for effect sizes. +` ` | `--plot_labels background=<color>` | v1.6a1 | Change plot background color with a Matplotlib color string. +` ` | `--plot_labels vspan_col=<col> vspan_format=<str>` | v1.6a1 | Column denoting vertical span groups and string format for them, respectively. +` ` | `--plot_labels rotation=<deg>` | v1.6a2 | Change rotation in degrees. +`--ref_suffixes annotation=` | `--transform [flip=axis] ...` | v1.6a1 | Flip the specified axis. +`--transform ...` | `--transform [interpolation=n] ...` | v1.6a1 | Interpolation order can be specified when exporting the main image. +`--transform ...` | `--transform [flip=axis] ...` | v1.6a1 | Flip the specified axis. ## Changes in MagellanMapper v1.5 diff --git a/docs/release/release_v1.6.md b/docs/release/release_v1.6.md index e06c8cdb3..10bb50ef9 100644 --- a/docs/release/release_v1.6.md +++ b/docs/release/release_v1.6.md @@ -110,8 +110,8 @@ #### I/O -- Images can be viewed as RGB(A) using the `RGB` button or the `--rgb` CLI argument (#142) -- EXPERIMENTAL: Some TIF files can be loaded directly, without importing the file first (#90, #213, #242) +- Images can be viewed and exported as RGB(A) using the `RGB` button or the `--rgb` CLI argument (#142, #445) +- EXPERIMENTAL: Some TIF files can be loaded directly, without importing the file first (#90, #213, #242, #445) - The `--proc export_planes` task can export a subset of image planes specified by `--slice`, or an ROI specified by `--offset` and `--size` - Image metadata is stored in the `Image5d` image object (#115) - Better 2D image support @@ -119,6 +119,7 @@ - Unit factor conversions adapts to image dimensions (eg 2D vs 3D) (#132) - Fixed ROI padding during blob verification and match-based colocalization for 2D images (#380) - Multiple multiplane image files can be selected directly instead of relying on file auto-detection (#201) +- `openpyxl` package is now optional during region export (#445) - Fixed re-importing an image after loading it (#117) - Fixed to store the image path when loading a registered image as the main image, which fixes saving the experiment name used when saving blobs (#139) @@ -132,7 +133,9 @@ - `--plot labels err_col_abs=<col>`: plot error bars with a column of absolute rather than relative values, now that Clrstats gives absolute values for effect sizes - `--plot_labels background=<color>`: change plot background color with a Matplotlib color string - `--plot_labels vspan_col=<col> vspan_format=<str>`: column denoting vertical span groups and string format for them, respectively (#135, 137) + - `--plot_labels rotation=<deg>`: change rotation in degrees (#445) - The figure save wrapper (`plot_support.save_fig`) is more flexible (#215) +- 2D plots can be set not to save (#445) - Discrete colormaps can use [Matplotlib named colors](https://matplotlib.org/stable/gallery/color/named_colors.html) and use them for symmetric colors (#226) - Fixed errors when generating labels difference heat maps, and conditions can be set through `--plot_labels condition=cond1,cond2,...` (#132) - Fixed alignment of headers and columns in data frames printed to console (#109) @@ -148,6 +151,7 @@ - The labels reference path has been moved to an environment variable, which can be configured through `--labels <path>` (#147) - The Shapiro-Wilks test has been implemented in `meansModel` for consistent table output (#164) - A basic `NAMESPACE` file is provided to fix installation and exporting functions (#303) +- Linear regression intercept term can be toggled using the `Intercept` environment field, and r<sup>2</sup> and intercept are exported (#445) - Fixed t-test, which also provides Cohen's d as a standardized effect size through the `effectsize` package (#135) - Fixed jitter plot box plots to avoid covering labels (#147) - Fixed model fitting (#240, #304)