From 4af8364ff5735f29a1ff81cd30710464fe96325a Mon Sep 17 00:00:00 2001 From: Joost van Griethuysen Date: Mon, 19 Feb 2018 13:09:12 +0100 Subject: [PATCH 1/2] BUG: Fix formula error in NGTDM's Complexity and Contrast Erroneously divided by the square of number of voxels, instead of just the number of voxels. Additionally, use the number of voxels with at least 1 neighbor, which is lesser equal to the total number of voxels in the ROI (exclude voxels without neigbors). Update documentation accordingly --- radiomics/ngtdm.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/radiomics/ngtdm.py b/radiomics/ngtdm.py index 61f79e7d..b90fb7a7 100644 --- a/radiomics/ngtdm.py +++ b/radiomics/ngtdm.py @@ -58,7 +58,8 @@ class RadiomicsNGTDM(base.RadiomicsFeaturesBase): :math:`n_i` be the number of voxels in :math:`X_{gl}` with gray level :math:`i` - :math:`N_v` be the total number of voxels in :math:`X_{gl}` and equal to :math:`\sum{n_i}` + :math:`N_{v,p}` be the total number of voxels in :math:`X_{gl}` and equal to :math:`\sum{n_i}` (i.e. the number of voxels + with a valid region; at least 1 neighbor). :math:`N_{v,p} \leq N_p`, where :math:`N_p` is the total number of voxels in the ROI. :math:`p_i` be the gray level probability and equal to :math:`n_i/N_v` @@ -168,9 +169,6 @@ def _calculateMatrix(self): # Delete empty grey levels P_ngtdm = numpy.delete(P_ngtdm, numpy.where(P_ngtdm[:, 0] == 0), 0) - # Normalize P_ngtdm[:, 0] (= p_i) - P_ngtdm[:, 0] = P_ngtdm[:, 0] / self.coefficients['Np'] - return P_ngtdm def _calculateCMatrix(self): @@ -182,13 +180,16 @@ def _calculateCMatrix(self): # Delete empty grey levels P_ngtdm = numpy.delete(P_ngtdm, numpy.where(P_ngtdm[:, 0] == 0), 0) - # Normalize P_ngtdm[:, 0] (= p_i) - P_ngtdm[:, 0] = P_ngtdm[:, 0] / self.coefficients['Np'] - return P_ngtdm def _calculateCoefficients(self): - self.coefficients['p_i'] = self.P_ngtdm[:, 0] + Nvp = numpy.sum(self.P_ngtdm[:, 0]) # = No of voxels that have a valid region, lesser equal to Np + self.coefficients['Nvp'] = Nvp + if Nvp < self.coefficients['Np']: + self.logger.debug('Detected %d voxels without valid neighbors', self.coefficients['Np'] - Nvp) + + # Normalize P_ngtdm[:, 0] (= n_i) to obtain p_i + self.coefficients['p_i'] = self.P_ngtdm[:, 0] / Nvp self.coefficients['s_i'] = self.P_ngtdm[:, 1] self.coefficients['ivector'] = self.P_ngtdm[:, 2] @@ -221,19 +222,19 @@ def getContrastFeatureValue(self): Calculate and return the contrast. :math:`Contrast = \left(\frac{1}{N_{g,p}(N_{g,p}-1)}\displaystyle\sum^{N_g}_{i=1}\displaystyle\sum^{N_g}_{j=1}{p_{i}p_{j}(i-j)^2}\right) - \left(\frac{1}{N_p^2}\displaystyle\sum^{N_g}_{i=1}{s_i}\right)\text{, where }p_i \neq 0, p_j \neq 0` + \left(\frac{1}{N_{v,p}}\displaystyle\sum^{N_g}_{i=1}{s_i}\right)\text{, where }p_i \neq 0, p_j \neq 0` Contrast is a measure of the spatial intensity change, but is also dependent on the overall gray level dynamic range. Contrast is high when both the dynamic range and the spatial change rate are high, i.e. an image with a large range of gray levels, with large changes between voxels and their neighbourhood. """ Ngp = self.coefficients['Ngp'] - Np = self.coefficients['Np'] + Nvp = self.coefficients['Nvp'] p_i = self.coefficients['p_i'] s_i = self.coefficients['s_i'] i = self.coefficients['ivector'] contrast = (numpy.sum(p_i[:, None] * p_i[None, :] * (i[:, None] - i[None, :]) ** 2) / (Ngp * (Ngp - 1))) * \ - ((numpy.sum(s_i)) / (Np ** 2)) + ((numpy.sum(s_i)) / Nvp) return contrast @@ -263,18 +264,18 @@ def getComplexityFeatureValue(self): r""" Calculate and return the complexity. - :math:`Complexity = \frac{1}{N_p^2}\displaystyle\sum^{N_g}_{i = 1}\displaystyle\sum^{N_g}_{j = 1}{|i - j| + :math:`Complexity = \frac{1}{N_{v,p}}\displaystyle\sum^{N_g}_{i = 1}\displaystyle\sum^{N_g}_{j = 1}{|i - j| \frac{p_{i}s_{i} + p_{j}s_{j}}{p_i + p_j}}\text{, where }p_i \neq 0, p_j \neq 0` An image is considered complex when there are many primitive components in the image, i.e. the image is non-uniform and there are many rapid changes in gray level intensity. """ - Np = self.coefficients['Np'] + Nvp = self.coefficients['Nvp'] p_i = self.coefficients['p_i'] s_i = self.coefficients['s_i'] i = self.coefficients['ivector'] complexity = numpy.sum(numpy.abs(i[:, None] - i[None, :]) * ( - ((p_i * s_i)[:, None] + (p_i * s_i)[None, :]) / (p_i[:, None] + p_i[None, :]))) / Np ** 2 + ((p_i * s_i)[:, None] + (p_i * s_i)[None, :]) / (p_i[:, None] + p_i[None, :]))) / Nvp return complexity def getStrengthFeatureValue(self): From 940ea9460521af7c2db1915da37b56e83b24d251 Mon Sep 17 00:00:00 2001 From: Joost van Griethuysen Date: Tue, 6 Mar 2018 13:38:41 +0100 Subject: [PATCH 2/2] DOCS: Update release notes and baseline --- CHANGES.rst | 2 ++ data/baseline/baseline_ngtdm.csv | 12 ++++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index ec7cd1d8..5efaefc3 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -87,6 +87,8 @@ Feature Calculation Changes - Remove feature *Sum Variance*, as this is mathematically equal to *Cluster Tendency*. (`#300 `_) +- Fix feature formula error in NGTDM (incorrect use of square in *Complexity* and *Contrast*). + (`#351 `_) New Features ############ diff --git a/data/baseline/baseline_ngtdm.csv b/data/baseline/baseline_ngtdm.csv index 303c9dfb..9d5cf648 100644 --- a/data/baseline/baseline_ngtdm.csv +++ b/data/baseline/baseline_ngtdm.csv @@ -1,6 +1,6 @@ -Patient ID,general_info_BoundingBox,general_info_GeneralSettings,general_info_ImageHash,general_info_ImageSpacing,general_info_InputImages,general_info_MaskHash,general_info_Version,general_info_VolumeNum,general_info_VoxelNum,Coarseness,Complexity,Strength,Busyness,Contrast -brain1,"(162, 84, 11, 47, 70, 7)","{'distances': [1], 'voxelArrayShift': 2000, 'additionalInfo': True, 'enableCExtensions': True, 'weightingNorm': None, 'force2D': False, 'interpolator': None, 'resampledPixelSpacing': None, 'gldm_a': 0, 'label': 1, 'normalizeScale': 1, 'normalize': False, 'force2Ddimension': 0, 'removeOutliers': None, 'minimumROISize': None, 'binWidth': 25, 'minimumROIDimensions': 1, 'symmetricalGLCM': False, 'padDistance': 5}",5c9ce3ca174f0f8324aa4d277e0fef82dc5ac566,"(0.7812499999999999, 0.7812499999999999, 6.499999999999998)",{},9dc2c3137b31fd872997d92c9a92d5178126d9d3,1.2.0.post17.dev0+g46c7fb8,2,4137,0.0017120352713566309,0.34796054586526076,0.63073148273486113,1.4467306835644076,5.9914259435373462e-05 -brain2,"(205, 155, 8, 20, 15, 3)","{'distances': [1], 'voxelArrayShift': 2000, 'additionalInfo': True, 'enableCExtensions': True, 'weightingNorm': None, 'force2D': False, 'interpolator': None, 'resampledPixelSpacing': None, 'gldm_a': 0, 'label': 1, 'normalizeScale': 1, 'normalize': False, 'force2Ddimension': 0, 'removeOutliers': None, 'minimumROISize': None, 'binWidth': 25, 'minimumROIDimensions': 1, 'symmetricalGLCM': False, 'padDistance': 5}",f2b8fbc4d5d1da08a1a70e79a301f8a830139438,"(0.7812499999999999, 0.7812499999999999, 6.499999999999998)",{},b41049c71633e194bee4891750392b72eabd8800,1.2.0.post17.dev0+g46c7fb8,1,453,0.01286160865655888,1.6825211314863389,3.9663129914847497,0.15164513751048295,0.0002689553152317465 -breast1,"(21, 64, 8, 9, 12, 3)","{'distances': [1], 'voxelArrayShift': 2000, 'additionalInfo': True, 'enableCExtensions': True, 'weightingNorm': None, 'force2D': False, 'interpolator': None, 'resampledPixelSpacing': None, 'gldm_a': 0, 'label': 1, 'normalizeScale': 1, 'normalize': False, 'force2Ddimension': 0, 'removeOutliers': None, 'minimumROISize': None, 'binWidth': 25, 'minimumROIDimensions': 1, 'symmetricalGLCM': False, 'padDistance': 5}",016951a8f9e8e5de21092d9d62b77262f92e04a5,"(0.664062, 0.664062, 2.1)",{},5aa7d57fd57e83125b605c036c40f4a0d0cfd3e4,1.2.0.post17.dev0+g46c7fb8,1,143,0.050821154965410821,0.0082145909265791753,0.12459116813839596,8.9043951456360233,0.00044669705438426832 -lung1,"(206, 347, 32, 24, 26, 3)","{'distances': [1], 'voxelArrayShift': 2000, 'additionalInfo': True, 'enableCExtensions': True, 'weightingNorm': None, 'force2D': False, 'interpolator': None, 'resampledPixelSpacing': None, 'gldm_a': 0, 'label': 1, 'normalizeScale': 1, 'normalize': False, 'force2Ddimension': 0, 'removeOutliers': None, 'minimumROISize': None, 'binWidth': 25, 'minimumROIDimensions': 1, 'symmetricalGLCM': False, 'padDistance': 5}",34dca4200809a5e76c702d6b9503d958093057a3,"(0.5703125, 0.5703125, 5.0)",{},054d887740012177bd1f9031ddac2b67170af0f3,1.2.0.post17.dev0+g46c7fb8,1,837,0.0089851482891101057,0.73822878390479196,2.7864289040763568,0.19930533973333536,0.00021955754133650324 -lung2,"(318, 333, 15, 87, 66, 11)","{'distances': [1], 'voxelArrayShift': 2000, 'additionalInfo': True, 'enableCExtensions': True, 'weightingNorm': None, 'force2D': False, 'interpolator': None, 'resampledPixelSpacing': None, 'gldm_a': 0, 'label': 1, 'normalizeScale': 1, 'normalize': False, 'force2Ddimension': 0, 'removeOutliers': None, 'minimumROISize': None, 'binWidth': 25, 'minimumROIDimensions': 1, 'symmetricalGLCM': False, 'padDistance': 5}",14f57fd04838eb8c9cca2a0dd871d29971585975,"(0.6269531, 0.6269531, 5.0)",{},e284ff05593bc6cb2747261882e452d4efbccb3a,1.2.0.post17.dev0+g46c7fb8,1,24644,0.00020294841093398597,0.046405126305385076,0.61472046408820225,2.0631991387093871,1.1469959784621812e-06 +Patient ID,general_info_BoundingBox,general_info_EnabledImageTypes,general_info_GeneralSettings,general_info_ImageHash,general_info_ImageSpacing,general_info_MaskHash,general_info_NumpyVersion,general_info_PyWaveletVersion,general_info_SimpleITKVersion,general_info_Version,general_info_VolumeNum,general_info_VoxelNum,Coarseness,Complexity,Strength,Contrast,Busyness +brain1,"(162, 84, 11, 47, 70, 7)",{'Original': {}},"{'distances': [1], 'voxelArrayShift': 2000, 'additionalInfo': True, 'enableCExtensions': True, 'force2D': False, 'interpolator': None, 'resampledPixelSpacing': None, 'gldm_a': 0, 'weightingNorm': None, 'normalizeScale': 1, 'normalize': False, 'force2Ddimension': 0, 'removeOutliers': None, 'minimumROISize': None, 'binWidth': 25, 'label': 1, 'minimumROIDimensions': 1, 'preCrop': False, 'symmetricalGLCM': False, 'resegmentRange': None, 'padDistance': 5}",5c9ce3ca174f0f8324aa4d277e0fef82dc5ac566,"(0.7812499999999999, 0.7812499999999999, 6.499999999999998)",9dc2c3137b31fd872997d92c9a92d5178126d9d3,1.14.0,0.5.2,0.9.1,1.3.0.post61.dev0+g4af8364,2,4137,0.001712035271356631,1439.5127782445836,0.6307314827348611,0.24786529128414,1.4467306835644076 +brain2,"(205, 155, 8, 20, 15, 3)",{'Original': {}},"{'distances': [1], 'voxelArrayShift': 2000, 'additionalInfo': True, 'enableCExtensions': True, 'force2D': False, 'interpolator': None, 'resampledPixelSpacing': None, 'gldm_a': 0, 'weightingNorm': None, 'normalizeScale': 1, 'normalize': False, 'force2Ddimension': 0, 'removeOutliers': None, 'minimumROISize': None, 'binWidth': 25, 'label': 1, 'minimumROIDimensions': 1, 'preCrop': False, 'symmetricalGLCM': False, 'resegmentRange': None, 'padDistance': 5}",f2b8fbc4d5d1da08a1a70e79a301f8a830139438,"(0.7812499999999999, 0.7812499999999999, 6.499999999999998)",b41049c71633e194bee4891750392b72eabd8800,1.14.0,0.5.2,0.9.1,1.3.0.post61.dev0+g4af8364,1,453,0.01286160865655888,762.1820725633115,3.9663129914847497,0.12183675779998118,0.15164513751048295 +breast1,"(21, 64, 8, 9, 12, 3)",{'Original': {}},"{'distances': [1], 'voxelArrayShift': 2000, 'additionalInfo': True, 'enableCExtensions': True, 'force2D': False, 'interpolator': None, 'resampledPixelSpacing': None, 'gldm_a': 0, 'weightingNorm': None, 'normalizeScale': 1, 'normalize': False, 'force2Ddimension': 0, 'removeOutliers': None, 'minimumROISize': None, 'binWidth': 25, 'label': 1, 'minimumROIDimensions': 1, 'preCrop': False, 'symmetricalGLCM': False, 'resegmentRange': None, 'padDistance': 5}",016951a8f9e8e5de21092d9d62b77262f92e04a5,"(0.664062, 0.664062, 2.1)",5aa7d57fd57e83125b605c036c40f4a0d0cfd3e4,1.14.0,0.5.2,0.9.1,1.3.0.post61.dev0+g4af8364,1,143,0.05082115496541082,1.1746865025008222,0.12459116813839596,0.06387767877695037,8.904395145636023 +lung1,"(206, 347, 32, 24, 26, 3)",{'Original': {}},"{'distances': [1], 'voxelArrayShift': 2000, 'additionalInfo': True, 'enableCExtensions': True, 'force2D': False, 'interpolator': None, 'resampledPixelSpacing': None, 'gldm_a': 0, 'weightingNorm': None, 'normalizeScale': 1, 'normalize': False, 'force2Ddimension': 0, 'removeOutliers': None, 'minimumROISize': None, 'binWidth': 25, 'label': 1, 'minimumROIDimensions': 1, 'preCrop': False, 'symmetricalGLCM': False, 'resegmentRange': None, 'padDistance': 5}",34dca4200809a5e76c702d6b9503d958093057a3,"(0.5703125, 0.5703125, 5.0)",054d887740012177bd1f9031ddac2b67170af0f3,1.14.0,0.5.2,0.9.1,1.3.0.post61.dev0+g4af8364,1,837,0.008985148289110106,617.8974921283109,2.786428904076357,0.1837696620986532,0.19930533973333536 +lung2,"(318, 333, 15, 87, 66, 11)",{'Original': {}},"{'distances': [1], 'voxelArrayShift': 2000, 'additionalInfo': True, 'enableCExtensions': True, 'force2D': False, 'interpolator': None, 'resampledPixelSpacing': None, 'gldm_a': 0, 'weightingNorm': None, 'normalizeScale': 1, 'normalize': False, 'force2Ddimension': 0, 'removeOutliers': None, 'minimumROISize': None, 'binWidth': 25, 'label': 1, 'minimumROIDimensions': 1, 'preCrop': False, 'symmetricalGLCM': False, 'resegmentRange': None, 'padDistance': 5}",14f57fd04838eb8c9cca2a0dd871d29971585975,"(0.6269531, 0.6269531, 5.0)",e284ff05593bc6cb2747261882e452d4efbccb3a,1.14.0,0.5.2,0.9.1,1.3.0.post61.dev0+g4af8364,1,24644,0.00020294841093398597,1143.6079326699098,0.6147204640882022,0.028266568893221995,2.063199138709387