From dd7c445d850b7634bf0c39efd13c1cf0a4eebf2a Mon Sep 17 00:00:00 2001 From: Joost van Griethuysen Date: Wed, 21 Feb 2018 18:23:29 +0100 Subject: [PATCH 1/2] ENH: Refactor CLI scripts Combine the `pyradiomics` and `pyradiomicsbatch` entry point into 1 joint entry point `pyradiomics`. This new entry point operates in batch mode when input to `Image|Batch` has `.csv` extension. In batch mode, `Mask` argument is ignored. In single mode `Mask` inputs is required. Additionally, enables easy multi-threaded extraction by specifying `--jobs` / `-j` argument (with integer indicating number of parallel threads to use. Will only work in batch mode, as extraction is multi-threaded at the case level (1 thread per case). Removes argument `--label` / `-l`, specifying an override label to use is now achieved through specifying `-s "label:N"`, with N being the label value. Also includes a small update to initialization of the feature extractor: - In addition to a string pointing to a parameter file, feature extractor now also accepts a dictionary (top level specifies customization types 'setting', 'imageType' and 'featureClass') as 1st positional argument - Regardless of initialization with or without a customization file/dictionary, `kwargs` passed to the constructor are used to override settings. --- examples/batchprocessing_parallel.py | 2 +- radiomics/featureextractor.py | 44 +++-- radiomics/scripts/__init__.py | 266 +++++++++++++++++++++++++++ radiomics/scripts/segment.py | 194 +++++++++++++++++++ setup.py | 3 +- 5 files changed, 486 insertions(+), 23 deletions(-) create mode 100644 radiomics/scripts/segment.py diff --git a/examples/batchprocessing_parallel.py b/examples/batchprocessing_parallel.py index 0278e948..76f5f349 100644 --- a/examples/batchprocessing_parallel.py +++ b/examples/batchprocessing_parallel.py @@ -104,7 +104,7 @@ def run(case): imageFilepath = case['Image'] # Required maskFilepath = case['Mask'] # Required - label = case.get('Label', 1) # Optional + label = case.get('Label', None) # Optional # Instantiate Radiomics Feature extractor diff --git a/radiomics/featureextractor.py b/radiomics/featureextractor.py index 52e593c4..7bb02e6c 100644 --- a/radiomics/featureextractor.py +++ b/radiomics/featureextractor.py @@ -26,12 +26,14 @@ class RadiomicsFeaturesExtractor: signature specified by these settings for the passed image and labelmap combination. This function can be called repeatedly in a batch process to calculate the radiomics signature for all image and labelmap combinations. - At initialization, a parameters file can be provided containing all necessary settings. This is done by passing the - location of the file as the single argument in the initialization call, without specifying it as a keyword argument. - If such a file location is provided, any additional kwargs are ignored. - Alternatively, at initialisation, custom settings (*NOT enabled image types and/or feature classes*) can be provided - as keyword arguments, with the setting name as key and its value as the argument value (e.g. ``binWidth=25``). For - more information on possible settings and customization, see + At initialization, a parameters file (string pointing to yaml or json structured file) or dictionary can be provided + containing all necessary settings (top level containing keys "setting", "imageType" and/or "featureClass). This is + done by passing it as the first positional argument. If no positional argument is supplied, or the argument is not + either a dictionary or a string pointing to a valid file, defaults will be applied. + Moreover, at initialisation, custom settings (*NOT enabled image types and/or feature classes*) can be provided + as keyword arguments, with the setting name as key and its value as the argument value (e.g. ``binWidth=25``). + Settings specified here will override those in the parameter file/dict/default settings. + For more information on possible settings and customization, see :ref:`Customizing the Extraction `. By default, all features in all feature classes are enabled. @@ -47,28 +49,30 @@ def __init__(self, *args, **kwargs): self._enabledImagetypes = {} self._enabledFeatures = {} - if len(args) == 1 and isinstance(args[0], six.string_types): + if len(args) == 1 and isinstance(args[0], six.string_types) and os.path.isfile(args[0]): self.logger.info("Loading parameter file") - self.loadParams(args[0]) + self._applyParams(paramsFile=args[0]) + elif len(args) == 1 and isinstance(args[0], dict): + self.logger.info("Loading parameter dictionary") + self._applyParams(paramsDict=args[0]) else: # Set default settings and update with and changed settings contained in kwargs self.settings = self._getDefaultSettings() - if len(kwargs) > 0: - self.logger.info('Applying custom settings') - self.settings.update(kwargs) - else: - self.logger.info('No customized settings, applying defaults') - - self.logger.debug("Settings: %s", self.settings) + self.logger.info('No valid config parameter, applying defaults: %s', self.settings) self._enabledImagetypes = {'Original': {}} self.logger.info('Enabled image types: %s', self._enabledImagetypes) - self._enabledFeatures = {} + for featureClassName in self.getFeatureClassNames(): self._enabledFeatures[featureClassName] = [] self.logger.info('Enabled features: %s', self._enabledFeatures) + if len(kwargs) > 0: + self.logger.info('Applying custom setting overrides') + self.settings.update(kwargs) + self.logger.debug("Settings: %s", self.settings) + self._setTolerance() @classmethod @@ -270,8 +274,8 @@ def enableAllFeatures(self): Enable all classes and all features. .. note:: - Individual features that have been marked "deprecated" are not enabled by this function. They can still be enabled manually by - a call to :py:func:`~radiomics.base.RadiomicsBase.enableFeatureByName()`, + Individual features that have been marked "deprecated" are not enabled by this function. They can still be enabled + manually by a call to :py:func:`~radiomics.base.RadiomicsBase.enableFeatureByName()`, :py:func:`~radiomics.featureextractor.RadiomicsFeaturesExtractor.enableFeaturesByName()` or in the parameter file (by specifying the feature by name, not when enabling all features). However, in most cases this will still result only in a deprecation warning. @@ -293,8 +297,8 @@ def enableFeatureClassByName(self, featureClass, enabled=True): Enable or disable all features in given class. .. note:: - Individual features that have been marked "deprecated" are not enabled by this function. They can still be enabled manually by - a call to :py:func:`~radiomics.base.RadiomicsBase.enableFeatureByName()`, + Individual features that have been marked "deprecated" are not enabled by this function. They can still be enabled + manually by a call to :py:func:`~radiomics.base.RadiomicsBase.enableFeatureByName()`, :py:func:`~radiomics.featureextractor.RadiomicsFeaturesExtractor.enableFeaturesByName()` or in the parameter file (by specifying the feature by name, not when enabling all features). However, in most cases this will still result only in a deprecation warning. diff --git a/radiomics/scripts/__init__.py b/radiomics/scripts/__init__.py index e69de29b..0a7bf0e9 100644 --- a/radiomics/scripts/__init__.py +++ b/radiomics/scripts/__init__.py @@ -0,0 +1,266 @@ +#!/usr/bin/env python +import argparse +import csv +from functools import partial +import logging +from multiprocessing import cpu_count, Pool +import os +import sys + +from pykwalify.compat import yaml +import six.moves + +import radiomics +from . import segment + + +scriptlogger = logging.getLogger('radiomics.script') # holds logger for script events +logging_config = {} +relative_path_start = os.getcwd() + + +def parse_args(custom_arguments=None): + global relative_path_start + parser = argparse.ArgumentParser(usage='%(prog)s image|batch [mask] [Options]', + formatter_class=argparse.RawTextHelpFormatter) + + inputGroup = parser.add_argument_group(title='Input', + description='Input files and arguments defining the extraction:\n' + '- image and mask files (single mode) ' + 'or CSV-file specifying them (batch mode)\n' + '- Parameter file (.yml/.yaml or .json)\n' + '- Overrides for customization type 3 ("settings")\n' + '- Multi-threaded batch processing') + inputGroup.add_argument('input', metavar='{Image,Batch}FILE', + help='Image file (single mode) or CSV batch file (batch mode)') + inputGroup.add_argument('mask', nargs='?', metavar='MaskFILE', default=None, + help='Mask file identifying the ROI in the Image. \n' + 'Only required when in single mode, ignored otherwise.') + inputGroup.add_argument('--param', '-p', metavar='FILE', default=None, + help='Parameter file containing the settings to be used in extraction') + inputGroup.add_argument('--setting', '-s', metavar='"SETTING_NAME:VALUE"', action='append', default=[], type=str, + help='Additional parameters which will override those in the\n' + 'parameter file and/or the default settings. Multiple\n' + 'settings possible. N.B. Only works for customization\n' + 'type 3 ("setting").') + inputGroup.add_argument('--jobs', '-j', metavar='N', type=int, default=1, choices=six.moves.range(1, cpu_count() + 1), + help='(Batch mode only) Specifies the number of threads to use for\n' + 'parallel processing. This is applied at the case level;\n' + 'i.e. 1 thread per case. Actual number of workers used is\n' + 'min(cases, jobs).') + + outputGroup = parser.add_argument_group(title='Output', description='Arguments controlling output redirection and ' + 'the formatting of calculated results.') + outputGroup.add_argument('--out', '-o', metavar='FILE', type=argparse.FileType('a'), default=sys.stdout, + help='File to append output to') + outputGroup.add_argument('--skip-nans', action='store_true', + help='Add this argument to skip returning features that have an\n' + 'invalid result (NaN)') + outputGroup.add_argument('--format', '-f', choices=['csv', 'json', 'txt'], default='txt', + help='Format for the output.\n' + '"csv" (Default): one row of feature names, followed by one row of\n' + 'feature values per case.\n' + '"json": Features are written in a JSON format dictionary\n' + '(1 dictionary per case, 1 case per line) "{name:value}"\n' + '"txt": one feature per line in format "case-N_name:value"') + outputGroup.add_argument('--format-path', choices=['absolute', 'relative', 'basename'], default='absolute', + help='Controls input image and mask path formatting in the output.\n' + '"absolute" (Default): Absolute file paths.\n' + '"relative": File paths relative to current working directory.\n' + '"basename": Only stores filename.') + + loggingGroup = parser.add_argument_group(title='Logging', + description='Controls the (amount of) logging output to the ' + 'console and the (optional) log-file.') + loggingGroup.add_argument('--logging-level', metavar='LEVEL', + choices=['NOTSET', 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], + default='WARNING', help='Set capture level for logging') + loggingGroup.add_argument('--log-file', metavar='FILE', default=None, help='File to append logger output to') + loggingGroup.add_argument('--verbosity', '-v', action='store', nargs='?', default=3, const=4, type=int, + choices=[1, 2, 3, 4, 5], + help='Regulate output to stderr. By default [3], level\n' + 'WARNING and up are printed. By specifying this\n' + 'argument without a value, level INFO [4] is assumed.\n' + 'A higher value results in more verbose output.') + + parser.add_argument('--version', action='version', help='Print version and exit', + version='%(prog)s ' + radiomics.__version__) + + args = parser.parse_args(args=custom_arguments) # Exits with code 2 if parsing fails + + # Run the extraction + try: + _configureLogging(args) + scriptlogger.info('Starting PyRadiomics (version: %s)', radiomics.__version__) + results = _processInput(args) + if results is not None: + segment.processOutput(results, args.out, args.skip_nans, args.format, args.format_path, relative_path_start) + scriptlogger.info('Finished extraction successfully...') + else: + return 1 # Feature extraction error + except Exception: + scriptlogger.error('Error extracting features!', exc_info=True) + return 3 # Unknown error + return 0 # success + + +def _processInput(args): + global logging_config, relative_path_start, scriptlogger + scriptlogger.info('Processing input...') + + caseCount = 1 + num_workers = 1 + + # Check if input represents a batch file + if args.input.endswith('.csv'): + scriptlogger.debug('Loading batch file "%s"', args.input) + relative_path_start = os.path.dirname(args.input) + with open(args.input, mode='r') as batchFile: + cr = csv.DictReader(batchFile, lineterminator='\n') + + # Check if required Image and Mask columns are present + if 'Image' not in cr.fieldnames: + scriptlogger.error('Required column "Image" not present in input, unable to extract features...') + return None + if 'Mask' not in cr.fieldnames: + scriptlogger.error('Required column "Mask" not present in input, unable to extract features...') + return None + + cases = [] + for row_idx, row in enumerate(cr, start=2): + if row['Image'] is None or row['Mask'] is None: + scriptlogger.warning('Batch L%d: Missing required Image or Mask, skipping this case...', row_idx) + continue + imPath = row['Image'] + maPath = row['Mask'] + if not os.path.isabs(imPath): + imPath = os.path.abspath(os.path.join(relative_path_start, imPath)) + scriptlogger.debug('Updated relative image filepath to be relative to input CSV: %s', imPath) + if not os.path.isabs(maPath): + maPath = os.path.abspath(os.path.join(relative_path_start, maPath)) + scriptlogger.debug('Updated relative mask filepath to be relative to input CSV: %s', maPath) + cases.append(row) + cases[-1]['Image'] = imPath + cases[-1]['Mask'] = maPath + + caseCount = len(cases) + caseGenerator = _buildGenerator(args, cases) + num_workers = min(caseCount, args.jobs) + elif args.mask is not None: + caseGenerator = _buildGenerator(args, [{'Image': args.input, 'Mask': args.mask}]) + else: + scriptlogger.error('Input is not recognized as batch, no mask specified, cannot compute result!') + return None + + from radiomics.scripts import segment + + if num_workers > 1: # multiple cases, parallel processing enabled + scriptlogger.info('Input valid, starting parallel extraction from %d cases with %d workers...', + caseCount, num_workers) + pool = Pool(num_workers) + results = pool.map(partial(segment.extractSegment_parallel, parallel_config=logging_config), caseGenerator) + elif num_workers == 1: # single case or sequential batch processing + scriptlogger.info('Input valid, starting sequential extraction from %d case(s)...', + caseCount) + results = [] + for case in caseGenerator: + results.append(segment.extractSegment(*case)) + else: + # No cases defined in the batch + scriptlogger.error('No cases to process...') + return None + return results + + +def _buildGenerator(args, cases): + global scriptlogger + setting_overrides = _parseOverrides(args.setting) + + for case_idx, case in enumerate(cases, start=1): + yield case_idx, case, args.param, setting_overrides + + +def _parseOverrides(overrides): + global scriptlogger + setting_overrides = {} + + # parse overrides + if len(overrides) == 0: + scriptlogger.debug('No overrides found') + return setting_overrides + + scriptlogger.debug('Reading parameter schema') + schemaFile, schemaFuncs = radiomics.getParameterValidationFiles() + with open(schemaFile) as schema: + settingsSchema = yaml.load(schema)['mapping']['setting']['mapping'] + + # parse single value function + def parse_value(value, value_type): + if value_type == 'str': + return value # no conversion + elif value_type == 'int': + return int(value) + elif value_type == 'float': + return float(value) + elif value_type == 'bool': + return value == '1' or value.lower() == 'true' + else: + raise ValueError('Cannot understand value_type %s' % value_type) + + for setting in overrides: # setting = "setting_key:setting_value" + if ':' not in setting: + scriptlogger.warning('Incorrect format for override setting "%s", missing ":"', setting) + # split into key and value + setting_key, setting_value = setting.split(':', 2) + + # Check if it is a valid PyRadiomics Setting + if setting_key not in settingsSchema: + scriptlogger.warning('Did not recognize override %s, skipping...', setting_key) + continue + + # Try to parse the value by looking up its type in the settingsSchema + try: + setting_def = settingsSchema[setting_key] + setting_type = 'str' # If type is omitted in the schema, treat it as string (no conversion) + if 'seq' in setting_def: + # Multivalued setting + if len(setting_def['seq']) > 0 and 'type' in setting_def['seq'][0]: + setting_type = setting_def['seq'][0]['type'] + + setting_overrides[setting_key] = [parse_value(val, setting_type) for val in setting_value.split(',')] + scriptlogger.debug('Parsed "%s" as list (element type "%s"); value: %s', + setting_key, setting_type, setting_overrides[setting_key]) + else: + if 'type' in setting_def: + setting_type = setting_def['type'] + setting_overrides[setting_key] = parse_value(setting_value, setting_type) + scriptlogger.debug('Parsed "%s" as type "%s"; value: %s', setting_key, setting_type, setting_overrides[setting_key]) + + except Exception: + scriptlogger.warning('Could not parse value %s for setting %s, skipping...', setting_value, setting_key) + + return setting_overrides + + +def _configureLogging(args): + global scriptlogger, logging_config + + # Initialize Logging + logLevel = getattr(logging, args.logging_level) + rLogger = radiomics.logger + logging_config['logLevel'] = logLevel + + # Set up optional logging to file + if args.log_file is not None: + rLogger.setLevel(logLevel) + handler = logging.FileHandler(filename=args.log_file, mode='a') + handler.setFormatter(logging.Formatter("%(levelname)s:%(name)s: %(message)s")) + rLogger.addHandler(handler) + logging_config['logFile'] = args.log_file + + # Set verbosity of output (stderr) + verboseLevel = (6 - args.verbosity) * 10 # convert to python logging level + radiomics.setVerbosity(verboseLevel) + logging_config['verbosity'] = verboseLevel + + scriptlogger.debug('Logging initialized') diff --git a/radiomics/scripts/segment.py b/radiomics/scripts/segment.py new file mode 100644 index 00000000..bac2cc83 --- /dev/null +++ b/radiomics/scripts/segment.py @@ -0,0 +1,194 @@ +from collections import OrderedDict +import csv +from datetime import datetime +from functools import partial +import json +import logging +import os +import threading + +import numpy +import SimpleITK as sitk +import six + +from radiomics import featureextractor, setVerbosity + +caseLogger = logging.getLogger('radiomics.script') + + +def extractSegment(case_idx, case, config, config_override): + global caseLogger + + # Instantiate the output + feature_vector = OrderedDict(case) + + try: + t = datetime.now() + + imageFilepath = case['Image'] # Required + maskFilepath = case['Mask'] # Required + label = case.get('Label', None) # Optional + + # Instantiate Radiomics Feature extractor + extractor = featureextractor.RadiomicsFeaturesExtractor(config, **config_override) + + # Extract features + feature_vector.update(extractor.execute(imageFilepath, maskFilepath, label)) + + # Display message + delta_t = datetime.now() - t + caseLogger.info('Patient %s processed in %s', case_idx, delta_t) + + except Exception: + caseLogger.error('Feature extraction failed!', exc_info=True) + + return feature_vector + + +def extractSegment_parallel(args, parallel_config=None): + if parallel_config is not None: + _configurParallelExtraction(parallel_config) + # set thread name to patient name + threading.current_thread().name = 'case %s' % args[0] # args[0] = case_idx + return extractSegment(*args) + + +def extractSegmentWithTempFiles(case_idx, case, config, config_override, temp_dir): + global caseLogger + + filename = os.path.join(temp_dir, 'features_%s.csv' % case_idx) + if os.path.isfile(filename): + # Output already generated, load result (prevents re-extraction in case of interrupted process) + with open(filename, 'w') as outputFile: + reader = csv.reader(outputFile) + headers = reader.rows[0] + values = reader.rows[1] + feature_vector = OrderedDict(zip(headers, values)) + + caseLogger.info('Patient %s already processed, reading results...', case_idx) + else: + # Extract the set of features. Set parallel_config flag to None, as any logging initialization is already handled. + feature_vector = extractSegment(case_idx, case, config, config_override) + + # Store results in temporary separate files to prevent write conflicts + # This allows for the extraction to be interrupted. Upon restarting, already processed cases are found in the + # TEMP_DIR directory and loaded instead of re-extracted + with open(filename, 'w') as outputFile: + writer = csv.DictWriter(outputFile, fieldnames=list(feature_vector.keys()), lineterminator='\n') + writer.writeheader() + writer.writerow(feature_vector) + + return feature_vector + + +def extractSegmentWithTempFiles_parallel(args, parallel_config=None): + if parallel_config is not None: + _configurParallelExtraction(parallel_config) + # set thread name to patient name + threading.current_thread().name = 'case %s' % args[0] # args[0] = case_idx + return extractSegmentWithTempFiles(*args) + + +def processOutput(results, + outStream, + skip_nans=False, + format_output='csv', + format_path='absolute', + relative_path_start=''): + global caseLogger + caseLogger.info('Processing results...') + + # Store the header of all calculated features + headers = results[0].keys() + + # Set the formatting rule for image and mask paths + if format_path == 'absolute': + pathFormatter = os.path.abspath + elif format_path == 'relative': + pathFormatter = partial(os.path.relpath, start=relative_path_start) + elif format_path == 'basename': + pathFormatter = os.path.basename + else: + caseLogger.warning('Unrecognized format for paths (%s), reverting to default ("absolute")', format_path) + pathFormatter = os.path.abspath + + for case_idx, case in enumerate(results, start=1): + # if specified, skip NaN values + if skip_nans: + for key in list(case.keys()): + if isinstance(case[key], float) and numpy.isnan(case[key]): + caseLogger.debug('Case %d, feature %s computed NaN, removing from results', case_idx, key) + del case[key] + + # Format paths of image and mask files + case['Image'] = pathFormatter(case['Image']) + case['Mask'] = pathFormatter(case['Mask']) + + # Write out results + if format_output not in ('csv', 'json', 'txt'): + caseLogger.warning('Unrecognized format for output (%s), reverting to default ("csv")', format_output) + format_output = 'csv' + + if format_output == 'csv': + writer = csv.DictWriter(outStream, headers, lineterminator='\n') + if case_idx == 1: + writer.writeheader() + writer.writerow(case) # if skip_nans is enabled, nan-values are written as empty strings + elif format_output == 'json': + json.dump(case, outStream) + outStream.write('\n') + else: # txt + for k, v in six.iteritems(case): + outStream.write('Case-%d_%s: %s\n' % (case_idx, k, v)) + + +def _configurParallelExtraction(parallel_config): + """ + Initialize logging for parallel extraction. This needs to be done here, as it needs to be done for each thread that is + created. + """ + # Configure logging + ################### + + rLogger = logging.getLogger('radiomics') + + # Add logging to file is specified + logFile = parallel_config.get('logFile', None) + if logFile is not None: + logHandler = logging.FileHandler(filename=logFile, mode='a') + logHandler.setLevel(parallel_config.get('logLevel', logging.INFO)) + rLogger.addHandler(logHandler) + + # Include thread name in Log-message output for all handlers. + parallelFormatter = logging.Formatter('[%(asctime)-.19s] %(levelname)-.1s: (%(threadName)s) %(name)s: %(message)s') + for h in rLogger.handlers: + h.setFormatter(parallelFormatter) + + if parallel_config.get('addFilter', True): + # Define filter that allows messages from specified filter and level INFO and up, and level WARNING and up from + # other loggers. + class info_filter(logging.Filter): + def __init__(self, name): + super(info_filter, self).__init__(name) + self.level = logging.WARNING + + def filter(self, record): + if record.levelno >= self.level: + return True + if record.name == self.name and record.levelno >= logging.INFO: + return True + return False + + # Adding the filter to the first handler of the radiomics logger limits the info messages on the output to just + # those from radiomics.script, but warnings and errors from the entire library are also printed to the output. + # This does not affect the amount of logging stored in the log file. + outputhandler = rLogger.handlers[0] # Handler printing to the output + outputhandler.addFilter(info_filter('radiomics.batch')) + + # Ensures that log messages are being passed to the filter with the specified level + setVerbosity(parallel_config.get('verbosity', logging.INFO)) + + # Ensure the entire extraction for each cases is handled on 1 thread + #################################################################### + + sitk.ProcessObject_SetGlobalDefaultNumberOfThreads(1) diff --git a/setup.py b/setup.py index 4ef4cff8..5cdb653b 100644 --- a/setup.py +++ b/setup.py @@ -77,8 +77,7 @@ def run_tests(self): entry_points={ 'console_scripts': [ - 'pyradiomics=radiomics.scripts.commandline:main', - 'pyradiomicsbatch=radiomics.scripts.commandlinebatch:main' + 'pyradiomics=radiomics.scripts.__init__:parse_args', ]}, description='Radiomics features library for python', From 0f16cd2217018b62ccfe1e3ef992ed2114eaa2ab Mon Sep 17 00:00:00 2001 From: Joost van Griethuysen Date: Fri, 2 Mar 2018 10:05:32 +0100 Subject: [PATCH 2/2] DOCS: Update documentation for new entry point style. Additionally, re-add the `pyradiomicsbatch` as a deprecated entry point to preserve legacy functionality. Also re-add the `--label` argument as a deprecated argument in the `pyradiomics` entry point. --- docs/usage.rst | 27 ++++--- radiomics/scripts/__init__.py | 10 +++ radiomics/scripts/commandline.py | 112 -------------------------- radiomics/scripts/commandlinebatch.py | 4 + setup.py | 1 + 5 files changed, 30 insertions(+), 124 deletions(-) delete mode 100644 radiomics/scripts/commandline.py diff --git a/docs/usage.rst b/docs/usage.rst index a14ff3e6..118edd46 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -36,10 +36,8 @@ Example Command Line Use ---------------- -* PyRadiomics has 2 commandline scripts, ``pyradiomics`` is for single image feature extraction and ``pyradiomicsbatch`` - is for feature extraction from a batch of images and segmentations. - -* Both scripts can be run directly from a command line window, anywhere in your system. +* PyRadiomics can be used directly from the commandline via the entry point ``pyradiomics``. Depending on the input + provided, PyRadiomics is run in either single-extraction or batch-extraction mode. * To extract features from a single image and segmentation run:: @@ -47,7 +45,7 @@ Command Line Use * To extract features from a batch run:: - pyradiomicsbatch + pyradiomics * The input file for batch processing is a CSV file where the first row is contains headers and each subsequent row represents one combination of an image and a segmentation and contains at least 2 elements: 1) path/to/image, @@ -62,17 +60,21 @@ Command Line Use All headers should be unique and different from headers provided by PyRadiomics (``__``). +* Extraction can be customized by specifying a `parameter file ` in the ``--param`` + argument and/or by specifying override settings (only `type 3 customization `) in the + ``--setting`` argument. Multiple overrides can be used by specifying ``--setting`` multiple times. + * For more information on the possible command line arguments, run:: pyradiomics -h - pyradiomicsbatch -h --------------- Interactive Use --------------- -* (LINUX) Add pyradiomics to the environment variable PYTHONPATH: +* (LINUX) To run from source code, add pyradiomics to the environment variable PYTHONPATH (Not necessary when + PyRadiomics is installed): * ``setenv PYTHONPATH /path/to/pyradiomics/radiomics`` @@ -126,7 +128,8 @@ Using feature classes directly * This represents an example where feature classes are used directly, circumventing checks and preprocessing done by the radiomics feature extractor class, and is not intended as standard use example. -* (LINUX) Add pyradiomics to the environment variable PYTHONPATH: +* (LINUX) To run from source code, add pyradiomics to the environment variable PYTHONPATH (Not necessary when + PyRadiomics is installed): * ``setenv PYTHONPATH /path/to/pyradiomics/radiomics`` @@ -170,9 +173,8 @@ Setting Up Logging ------------------ PyRadiomics features extensive logging to help track down any issues with the extraction of features. -By default PyRadiomics logging reports messages of level INFO and up (giving some information on progress during -extraction and any warnings or errors that occur), and prints this to the output (stderr). By default, PyRadiomics does -not create a log file. +By default PyRadiomics logging reports messages of level WARNING and up (reporting any warnings or errors that occur), +and prints this to the output (stderr). By default, PyRadiomics does not create a log file. To change the amount of information that is printed to the output, use :py:func:`~radiomics.setVerbosity` in interactive use and the optional ``--verbosity`` argument in commandline use. @@ -193,4 +195,5 @@ handler to the pyradiomics logger:: radiomics.logger.setLevel(logging.DEBUG) To store a log file when running pyradiomics from the commandline, specify a file location in the optional -``--log-file`` argument. The amount of logging that is stored is controlled by the ``--log-level`` argument. +``--log-file`` argument. The amount of logging that is stored is controlled by the ``--log-level`` argument +(default level INFO and up). diff --git a/radiomics/scripts/__init__.py b/radiomics/scripts/__init__.py index 0a7bf0e9..9f4b21cd 100644 --- a/radiomics/scripts/__init__.py +++ b/radiomics/scripts/__init__.py @@ -82,6 +82,9 @@ def parse_args(custom_arguments=None): 'WARNING and up are printed. By specifying this\n' 'argument without a value, level INFO [4] is assumed.\n' 'A higher value results in more verbose output.') + parser.add_argument('--label', '-l', metavar='N', default=None, type=int, + help='(DEPRECATED) Value of label in mask to use for\n' + 'feature extraction.') parser.add_argument('--version', action='version', help='Print version and exit', version='%(prog)s ' + radiomics.__version__) @@ -176,6 +179,13 @@ def _buildGenerator(args, cases): global scriptlogger setting_overrides = _parseOverrides(args.setting) + # Section for deprecated argument label + if args.label is not None: + scriptlogger.warning('Argument "label" is deprecated. To specify a custom label, use argument "setting" as follows:' + '"--setting=label:N", where N is the a label value.') + setting_overrides['label'] = args.label + # End deprecated section + for case_idx, case in enumerate(cases, start=1): yield case_idx, case, args.param, setting_overrides diff --git a/radiomics/scripts/commandline.py b/radiomics/scripts/commandline.py deleted file mode 100644 index f9b67003..00000000 --- a/radiomics/scripts/commandline.py +++ /dev/null @@ -1,112 +0,0 @@ -#!/usr/bin/env python - -import argparse -import collections -import csv -import json -import logging -import os.path -import sys - -import numpy -import six - -import radiomics -from radiomics import featureextractor - -parser = argparse.ArgumentParser(usage='%(prog)s image mask [Options]') -parser.add_argument('image', metavar='Image', - help='Features are extracted from the Region Of Interest (ROI) in the image') -parser.add_argument('mask', metavar='Mask', help='Mask identifying the ROI in the Image') - -parser.add_argument('--out', '-o', metavar='FILE', nargs='?', type=argparse.FileType('w'), default=sys.stdout, - help='File to append output to') -parser.add_argument('--format', '-f', choices=['txt', 'csv', 'json'], default='txt', - help='Format for the output. ' - 'Default is "txt": one feature per line in format "name:value". For "csv": one row of feature ' - 'names, followed by one row of feature values. For "json": Features are written in a JSON ' - 'format dictionary "{name:value}"') -parser.add_argument('--skip-nans', action='store_true', - help='Add this argument to skip returning features that have an invalid result (NaN)') -parser.add_argument('--param', '-p', metavar='FILE', type=str, default=None, - help='Parameter file containing the settings to be used in extraction') -parser.add_argument('--label', '-l', metavar='N', default=None, type=int, - help='Value of label in mask to use for feature extraction') -parser.add_argument('--logging-level', metavar='LEVEL', - choices=['NOTSET', 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], - default='WARNING', help='Set capture level for logging') -parser.add_argument('--log-file', metavar='FILE', type=argparse.FileType('w'), default=None, - help='File to append logger output to') -parser.add_argument('--verbosity', '-v', action='store', nargs='?', default=3, const=4, type=int, choices=range(0, 6), - help='Regulate output to stderr. By default [3], level WARNING and up are printed. By specifying ' - 'this argument without a value, level INFO [4] is assumed. A higher value results in more verbose ' - 'output.') -parser.add_argument('--shorten-path', dest='shorten', action='store_true', - help='specify this argument to image and mask path to just the file names') -parser.add_argument('--version', action='version', help='Print version and exit', - version='%(prog)s ' + radiomics.__version__) - - -def main(): - args = parser.parse_args() - - # Initialize Logging - logLevel = getattr(logging, args.logging_level) - rLogger = radiomics.logger - - # Set up optional logging to file - if args.log_file is not None: - rLogger.setLevel(logLevel) - handler = logging.StreamHandler(args.log_file) - handler.setFormatter(logging.Formatter("%(levelname)s:%(name)s: %(message)s")) - rLogger.addHandler(handler) - - # Set verbosity of output (stderr) - verboseLevel = (6 - args.verbosity) * 10 # convert to python logging level - radiomics.setVerbosity(verboseLevel) - - # Initialize logging for script log messages - logger = rLogger.getChild('script') - - # Initialize extractor - try: - if args.param is not None: - extractor = featureextractor.RadiomicsFeaturesExtractor(args.param) - else: - extractor = featureextractor.RadiomicsFeaturesExtractor() - logger.info('Extracting features with kwarg settings: %s\n\tImage: %s\n\tMask: %s', - str(extractor.settings), os.path.abspath(args.image), os.path.abspath(args.mask)) - featureVector = collections.OrderedDict() - if args.shorten: - featureVector['Image'] = os.path.basename(args.image) - featureVector['Mask'] = os.path.basename(args.mask) - - featureVector.update(extractor.execute(args.image, args.mask, args.label)) - - # if specified, skip NaN values - if args.skip_nans: - for key in list(featureVector.keys()): - if isinstance(featureVector[key], float) and numpy.isnan(featureVector[key]): - logger.debug('Feature %s computed NaN, removing from results', key) - del featureVector[key] - - if args.format == 'csv': - writer = csv.writer(args.out, lineterminator='\n') - writer.writerow(list(featureVector.keys())) - writer.writerow(list(featureVector.values())) - elif args.format == 'json': - json.dump(featureVector, args.out) - args.out.write('\n') - else: - for k, v in six.iteritems(featureVector): - args.out.write('%s: %s\n' % (k, v)) - except Exception: - logger.error('FEATURE EXTRACTION FAILED', exc_info=True) - - args.out.close() - if args.log_file is not None: - args.log_file.close() - - -if __name__ == "__main__": - main() diff --git a/radiomics/scripts/commandlinebatch.py b/radiomics/scripts/commandlinebatch.py index b04686f6..a1b0748e 100644 --- a/radiomics/scripts/commandlinebatch.py +++ b/radiomics/scripts/commandlinebatch.py @@ -66,6 +66,10 @@ def main(): # Initialize logging for script log messages logger = rLogger.getChild('batch') + logger.warning("This entry point is deprecated. It's enhanced functionality for batch extraction is now available " + 'in the "pyradiomics" entry point. See "http://pyradiomics.readthedocs.io/en/latest/usage.html" ' + 'for more details.') + # Load patient list flists = [] try: diff --git a/setup.py b/setup.py index 5cdb653b..f9b27664 100644 --- a/setup.py +++ b/setup.py @@ -78,6 +78,7 @@ def run_tests(self): entry_points={ 'console_scripts': [ 'pyradiomics=radiomics.scripts.__init__:parse_args', + 'pyradiomicsbatch=radiomics.scripts.commandlinebatch:main' # Deprecated entry point ]}, description='Radiomics features library for python',