Upgrade to TensorFlow 2.8 (#25)

* Update to TF 2.8 and deepcell 0.12.0

* Update supported python versions

* Loosen scikit-image requirement

* Fix deprecated skimage.draw.circle

* Refactor EM functions to use DataFrames (#24)

* Fix keras imports

* Raise error for image dim

* Change cache pip

* Update application model

* Update requirements and Python versions in setup.py

* Bump release version

* Update deepcell version in README

* Update deepcell version for published Docker image

* Update badges

.github/workflows/docker-image.yaml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
 
     env:
-      DEEPCELL_VERSION: 0.11.0
+      DEEPCELL_VERSION: 0.12.0
 
     steps:
     - uses: actions/checkout@v2

.github/workflows/tests.yaml

@@ -12,7 +12,7 @@ jobs:
 
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8]
+        python-version: [3.7, 3.8, 3.9]
 
     steps:
     - uses: actions/checkout@v2
@@ -25,10 +25,17 @@ jobs:
     - name: Cache pip
       uses: actions/cache@v2
       with:
-        path: ${{ env.pythonLocation }}
-        key: ${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}
+        # path: ${{ env.pythonLocation }}
+        # key: ${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}
+        # restore-keys: |
+        #   ${{ env.pythonLocation }}-
+        # This path is specific to Ubuntu
+        path: ~/.cache/pip
+        # Look to see if there is a cache hit for the corresponding requirements file
+        key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
         restore-keys: |
-          ${{ env.pythonLocation }}-
+          ${{ runner.os }}-pip-
+          ${{ runner.os }}-
 
     - name: Install Dependencies
       run: |

Dockerfile

@@ -1,7 +1,7 @@
 # Use vanvalenlab/deepcell-tf as the base image
-# Change the build arg to edit the tensorflow version.
+# Change the build arg to edit the deepcell version.
 # Only supporting python3.
-ARG DEEPCELL_VERSION=0.11.0-gpu
+ARG DEEPCELL_VERSION=0.12.0-gpu
 
 FROM vanvalenlab/deepcell-tf:${DEEPCELL_VERSION}
 

README.md

@@ -2,7 +2,10 @@
 
 [![Build Status](https://github.com/vanvalenlab/deepcell-spots/workflows/build/badge.svg)](https://github.com/vanvalenlab/deepcell-spots/actions)
 [![Coverage Status](https://coveralls.io/repos/github/vanvalenlab/deepcell-spots/badge.svg)](https://coveralls.io/github/vanvalenlab/deepcell-spots)
-[![Apache 2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/vanvalenlab/deepcell-spots/blob/master/LICENSE)
+[![Modified Apache 2.0](https://img.shields.io/badge/license-Modified%20Apache%202-blue)](https://github.com/vanvalenlab/deepcell-spots/blob/master/LICENSE)
+[![PyPI version](https://badge.fury.io/py/DeepCell-Spots.svg)](https://badge.fury.io/py/DeepCell-Spots)
+[![PyPi Monthly Downloads](https://img.shields.io/pypi/dm/deepcell-spots)](https://pypistats.org/packages/deepcell-spots)
+[![Python Versions](https://img.shields.io/pypi/pyversions/deepcell-spots.svg)](https://pypi.org/project/deepcell-spots/)
 
 `deepcell-spots` is a deep learning library for fluorescent spot detection image analysis. It allows you to apply pre-existing models and train new deep learning models for spot detection. It is written in Python and built using [TensorFlow](https://github.com/tensorflow/tensorflow), [Keras](https://www.tensorflow.org/guide/keras) and [DeepCell](https://github.com/vanvalenlab/deepcell-tf).
 
@@ -15,7 +18,7 @@ Build and run a local docker container, similarly to the instructions for deepce
 ```bash
 git clone https://github.com/vanvalenlab/deepcell-spots.git
 cd deepcell-spots
-docker build --build-arg DEEPCELL_VERSION=0.11.0-gpu -t $USER/deepcell-spots . 
+docker build --build-arg DEEPCELL_VERSION=0.12.0-gpu -t $USER/deepcell-spots . 
 ```
 
 ### Run the new docker image

deepcell_spots/_version.py

@@ -27,7 +27,7 @@
 __title__ = 'DeepCell-Spots'
 __description__ = 'Deep learning for fluorescent spot detection'
 __url__ = 'https://github.com/vanvalenlab/deepcell-spots'
-__version__ = '0.2.1'
+__version__ = '0.3.0'
 __download_url__ = '{}/tarball/{}'.format(__url__, __version__)
 __author__ = 'The Van Valen Lab'
 __author_email__ = '[email protected]'

deepcell_spots/applications/spot_detection.py

@@ -39,7 +39,7 @@
 
 
 MODEL_PATH = ('https://deepcell-data.s3-us-west-1.amazonaws.com/'
-              'saved-models/SpotDetection-3.tar.gz')
+              'saved-models/SpotDetection-6.tar.gz')
 
 
 def output_to_dictionary(output_images, output_names):
@@ -92,7 +92,7 @@ def __init__(self, model=None):
         if model is None:
             archive_path = tf.keras.utils.get_file(
                 'SpotDetection.tgz', MODEL_PATH,
-                file_hash='2b9a46087b25e9aab20a2c9f67f4f559',
+                file_hash='a1f655f7daab87c17fffa5561c9f1439',
                 extract=True, cache_subdir='models'
             )
             model_path = os.path.splitext(archive_path)[0]

deepcell_spots/cluster_vis.py

@@ -74,44 +74,44 @@ def ca_to_adjacency_matrix(ca_matrix):
     return A
 
 
-def label_graph_ann(G, coords, exclude_last=False):
+def label_graph_ann(G, coords_df, exclude_last=False):
     """Labels the annotator associated with each node in the graph
 
     Args:
         G (networkx.Graph): Graph with edges indicating clusters of points
-        assumed to be derived from the same ground truth detection
-    coords (numpy.array): 2d-array of detected point locations for each
-        classical algorithm used
-    exclude_last (bool): Only set as True to exclude a point that has been
-        included for the purpose of normalization
+            assumed to be derived from the same ground truth detection
+        coords_df (DataFrame): Data frame with columns 'x' and 'y' which encode the
+            spot locations and 'Algorithm' which encodes the algorithm that
+            corresponds with that spot
+        exclude_last (bool): Only set as True to exclude a point that has been
+            included for the purpose of normalization
 
     Returns:
         networkx.Graph: Labeled graph
     """
 
     G_new = G.copy()
-    num_spots = [len(x) for x in coords]
+    algs = coords_df.Algorithm.unique()
+    num_spots = [len(coords_df.loc[coords_df['Algorithm'] == alg]) for alg in algs]
 
     # Create list of annotator labels
-    ann_labels = np.array([0] * num_spots[0])
-    for i in range(1, len(num_spots)):
-        temp_labels = np.array([i] * num_spots[i])
-        ann_labels = np.hstack((ann_labels, temp_labels))
+    labels = []
+    for i in range(len(num_spots)):
+        labels.extend([i] * num_spots[i])
 
     nodes = list(G_new.nodes)
 
     if exclude_last:
         for i in range(len(nodes) - 1):
-            G_new.nodes[i]['name'] = ann_labels[i]
+            G_new.nodes[i]['name'] = labels[i]
     else:
         for i in range(len(nodes)):
-            G_new.nodes[i]['name'] = ann_labels[i]
+            G_new.nodes[i]['name'] = labels[i]
 
     return G_new
 
 
 def label_graph_gt(G, detection_data, gt):
-
     """Labels the ground truth identity of each node in the graph.
 
     Intended for simulated data.
@@ -153,7 +153,6 @@ def label_graph_gt(G, detection_data, gt):
 
 
 def label_graph_prob(G, detection_data, p_matrix):
-
     """Labels the EM output probability of being a ground truth true detection
     for each cluster in the graph.
 

deepcell_spots/data_utils.py

@@ -28,7 +28,7 @@
 
 import numpy as np
 from sklearn.model_selection import train_test_split
-from tensorflow.python.keras import backend as K
+from tensorflow.keras import backend as K
 
 
 def slice_image(X, reshape_size, overlap=0):

deepcell_spots/data_utils_test.py

@@ -30,7 +30,7 @@
 
 import numpy as np
 from sklearn.model_selection import train_test_split
-from tensorflow.python.keras import backend as K
+from tensorflow.keras import backend as K
 from tensorflow.python.platform import test
 
 from deepcell_spots.data_utils import (get_data, slice_annotated_image,

deepcell_spots/dotnet.py

@@ -28,13 +28,13 @@
 
 from deepcell.layers import TensorProduct
 from deepcell.model_zoo import bn_feature_net_skip_2D
-from tensorflow.python.keras import backend as K
-from tensorflow.python.keras.initializers import RandomNormal
-from tensorflow.python.keras.layers import (Activation, BatchNormalization,
-                                            Conv2D, Input, Lambda, Permute,
-                                            Reshape, Softmax)
-from tensorflow.python.keras.models import Model
-from tensorflow.python.keras.regularizers import l2
+from tensorflow.keras import backend as K
+from tensorflow.keras.initializers import RandomNormal
+from tensorflow.keras.layers import (Activation, BatchNormalization,
+                                     Conv2D, Input, Lambda, Permute,
+                                     Reshape, Softmax)
+from tensorflow.keras.models import Model
+from tensorflow.keras.regularizers import l2
 
 
 def default_heads(input_shape, num_classes):
@@ -64,7 +64,6 @@ def classification_head(input_shape,
                         reg=1e-5,
                         init='he_normal',
                         name='classification_head'):
-
     """Creates a classification head.
 
     Args:
@@ -116,7 +115,7 @@ def offset_regression_head(input_shape,
             **options
         )(outputs)
 
-    outputs = Conv2D(filters=2, name='offset_regression', **options)(outputs)
+    outputs = Conv2D(filters=2, name='offset_regression_output', **options)(outputs)
 
     return Model(inputs=inputs, outputs=outputs, name=name)
 

deepcell_spots/dotnet_losses.py

@@ -32,7 +32,7 @@
 
 import tensorflow as tf
 from deepcell import losses
-from tensorflow.python.keras import backend as K
+from tensorflow.keras import backend as K
 
 
 # DIFFERENCE FROM DEEPCELL.losses: doesn't sum over channel axis.

deepcell_spots/image_generators.py

@@ -32,10 +32,10 @@
 
 import numpy as np
 from tensorflow.keras.utils import to_categorical
-from tensorflow.python.keras import backend as K
-from tensorflow.python.keras.preprocessing.image import (ImageDataGenerator,
-                                                         Iterator,
-                                                         array_to_img)
+from tensorflow.keras import backend as K
+from tensorflow.keras.preprocessing.image import (ImageDataGenerator,
+                                                  Iterator,
+                                                  array_to_img)
 
 from deepcell_spots.utils import (affine_transform_points,
                                   subpixel_distance_transform)

deepcell_spots/preprocessing_utils.py

@@ -35,7 +35,7 @@ def mean_std_normalize(image, epsilon=1e-07):
     """Normalize image data by subtracting standard deviation pixel value
     and dividing by mean pixel value
     Args:
-        image (numpy.array): numpy array of image data
+        image (numpy.array): 4D numpy array of image data
         epsilon (float): fuzz factor used in numeric expressions.
     Returns:
         numpy.array: normalized image data
@@ -44,6 +44,10 @@ def mean_std_normalize(image, epsilon=1e-07):
         logging.info('Converting image dtype to float')
     image = image.astype('float32')
 
+    if not len(np.shape(image)) == 4:
+        raise ValueError('Image must be 4D, input image shape was'
+                         ' {}.'.format(np.shape(image)))
+
     for batch in range(image.shape[0]):
         for channel in range(image.shape[-1]):
             img = image[batch, ..., channel]
@@ -56,7 +60,7 @@ def min_max_normalize(image, clip=False):
     """Normalize image data by subtracting minimum pixel value and
      dividing by the maximum pixel value
     Args:
-        image (numpy.array): numpy array of image data
+        image (numpy.array): 4D numpy array of image data
         clip (boolean): Defaults to false. Determines if pixel
             values are clipped by percentile.
     Returns:
@@ -66,6 +70,10 @@ def min_max_normalize(image, clip=False):
         logging.info('Converting image dtype to float')
     image = image.astype('float32')
 
+    if not len(np.shape(image)) == 4:
+        raise ValueError('Image must be 4D, input image shape was'
+                         ' {}.'.format(np.shape(image)))
+
     for batch in range(image.shape[0]):
         for channel in range(image.shape[-1]):
             img = image[batch, ..., channel]

deepcell_spots/simulate_data.py

@@ -64,7 +64,6 @@ def sim_gt_clusters(num_clusters, tp_ratio):
 
 
 def sim_detections(gt, tpr, fpr):
-
     """Simulates detection data for a set of ground truth cluster labels and an
     annotator with a specified TPR and FPR.
 
@@ -111,7 +110,6 @@ def sim_detections(gt, tpr, fpr):
 
 
 def sim_annotators(gt, tpr_list, fpr_list):
-
     """Simulate the detections of multiple annotators with different TPRs and
     FPRs on the same ground truth data.
 
@@ -154,7 +152,6 @@ def sim_annotators(gt, tpr_list, fpr_list):
 
 
 def percent_correct(gt, data_array):
-
     """Calculates the percent of detections correctly labeled.
 
     Returns a value from 0 to 1 indicating the fraction of detections correctly
@@ -193,7 +190,6 @@ def percent_correct(gt, data_array):
 
 
 def is_in_image(x, y, a, L):
-
     """Determines if a square with defined vertices is contained in an image
     with larger dimensions
 
@@ -222,7 +218,6 @@ def is_in_image(x, y, a, L):
 def is_overlapping(x_list, y_list, a_list, x, y, a):
     # check if a square with left corner at x,y,a
     # overlaps with other squares with corner coordinates and side length in the list
-
     """Determines if a square overlaps with a list of other squares.
 
     Returns boolean, true if square overlaps with any of squares in list,
@@ -378,8 +373,8 @@ def gaussian_spot_image_generator(L,
             # create segmentation mask
             label = np.zeros((L, L))     # create the image
             for spot_ind in range(len(x_list)):
-                rr, cc = skimage.draw.circle(
-                    x_list[spot_ind], y_list[spot_ind], sigma_list[spot_ind], shape=label.shape)
+                rr, cc = skimage.draw.disk(
+                    (x_list[spot_ind], y_list[spot_ind]), sigma_list[spot_ind], shape=label.shape)
                 label[cc, rr] = spot_ind + 1
         else:
             # create image with background 0, spot centers labeled with 1

deepcell_spots/singleplex.py

@@ -32,7 +32,6 @@
 
 
 def match_spots_to_cells(labeled_im, coords):
-
     """Assigns detected spots to regions of a labeled image.
 
     Returns a dictionary where keys are labeled regions of input image and
@@ -64,7 +63,6 @@ def match_spots_to_cells(labeled_im, coords):
 
 
 def process_spot_dict(spot_dict):
-
     """Processes spot dictionary into an array of coordinates and list of
     region labels for spots.
 
@@ -92,7 +90,6 @@ def process_spot_dict(spot_dict):
 
 
 def remove_nuc_spots_from_cyto(labeled_im_nuc, labeled_im_cyto, coords):
-
     """Removes spots in nuclear regions from spots assigned to cytoplasmic
     regions.