Parallel tools and compiling (#701)

* Parallel tools and compiling * add blank line * Add parallel compile test - Also try to work around lint and optional imports * disable import-error for optional import * fix indent * Major update wiht magics * fix compile lint * fix style * disable import errors * white space * Add compile with TextProgressBar test * Remove jupyter components * shorten parallel test time * Minimize the parallel routines * cleanup merge * updates * more updates * fix comtypes issue * add comtypes * make comtypes requirement on windows * update changelog * review updates * more updates * remove callback function * fix conflict * Add progressbars back * fix lint errors * better grabbing of progress bars * grab bars in order created * Make jupyter tools availabel by default * fix circular import * switch used order * fix move lint disable * fix signature * fix changelog and releases * revert erroneous changes from git history * revert one more * add back psutil requirement * remove high-level imports * refine changelog * move everything to transpiler folder
Qiskit · Sep 18, 2018 · c9c4ed5 · c9c4ed5
1 parent 1a60b23
commit c9c4ed5
Show file tree

Hide file tree

Showing 16 changed files with 691 additions and 50 deletions.
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -25,9 +25,11 @@ Added
 - Introduced new options for handling credentials (qiskitrc file, environment
   variables) and automatic registration. (#547)
 - Add OpenMP parallelization for Apple builds of the cpp simulator (#698).
+- Add parallelization utilities (#701)
+- Parallelize transpilation (#701)
 - New interactive visualizations (#765).
 - Added option to reverse the qubit order when plotting a circuit. (#762, #786)
-- Jupyter notebook magic function qiskit_job_status (#734).
+- Jupyter notebook magic function qiskit_job_status, qiskit_progress_bar (#701, #734)
 - Add a new function ``qobj_to_circuits`` to convert a Qobj object to
   a list of QuantumCircuit objects (#877)
 

diff --git a/qiskit/__init__.py b/qiskit/__init__.py
@@ -11,6 +11,7 @@
 """Main QISKit public functionality."""
 
 import os
+import sys
 import pkgutil
 
 # First, check for required Python and API version
@@ -37,6 +38,10 @@
 # to be placed *before* the wrapper imports or any non-import code.
 __path__ = pkgutil.extend_path(__path__, __name__)
 
+# Allow extending this namespace. Please note that currently this line needs
+# to be placed *before* the wrapper imports.
+__path__ = pkgutil.extend_path(__path__, __name__)
+
 from .wrapper._wrapper import (
     available_backends, local_backends, remote_backends,
     get_backend, compile, execute, register, unregister,

diff --git a/qiskit/_util.py b/qiskit/_util.py
@@ -1,5 +1,4 @@
 # -*- coding: utf-8 -*-
-
 # Copyright 2017, IBM.
 #
 # This source code is licensed under the Apache License, Version 2.0 found in
@@ -12,9 +11,11 @@
 import logging
 import re
 import sys
+import platform
 import warnings
 import socket
 from collections import UserDict
+import psutil
 
 API_NAME = 'IBMQuantumExperience'
 logger = logging.getLogger(__name__)
@@ -160,6 +161,22 @@ def _parse_ibmq_credentials(url, hub=None, group=None, project=None):
     return url
 
 
+def local_hardware_info():
+    """Basic hardware information about the local machine.
+
+    Gives actual number of CPU's in the machine, even when hyperthreading is
+    turned on.
+
+    Returns:
+        dict: The hardware information.
+
+    """
+    results = {'os': platform.system()}
+    results['memory'] = psutil.virtual_memory().total / (1024**3)
+    results['cpus'] = psutil.cpu_count(logical=False)
+    return results
+
+
 def _has_connection(hostname, port):
     """Checks to see if internet connection exists to host
     via specified port

diff --git a/qiskit/transpiler/__init__.py b/qiskit/transpiler/__init__.py
@@ -12,3 +12,6 @@
 
 # pylint: disable=redefined-builtin
 from ._transpiler import compile, transpile
+
+from ._parallel import parallel_map
+from ._progressbar import TextProgressBar
diff --git a/qiskit/transpiler/_parallel.py b/qiskit/transpiler/_parallel.py
@@ -0,0 +1,144 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2018, IBM.
+#
+# This source code is licensed under the Apache License, Version 2.0 found in
+# the LICENSE.txt file in the root directory of this source tree.
+
+# This file is part of QuTiP: Quantum Toolbox in Python.
+#
+#    Copyright (c) 2011 and later, Paul D. Nation and Robert J. Johansson.
+#    All rights reserved.
+#
+#    Redistribution and use in source and binary forms, with or without
+#    modification, are permitted provided that the following conditions are
+#    met:
+#
+#    1. Redistributions of source code must retain the above copyright notice,
+#       this list of conditions and the following disclaimer.
+#
+#    2. Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#
+#    3. Neither the name of the QuTiP: Quantum Toolbox in Python nor the names
+#       of its contributors may be used to endorse or promote products derived
+#       from this software without specific prior written permission.
+#
+#    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+#    PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#    HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+###############################################################################
+
+"""Routines for running Python functions in parallel using process pools
+from the multiprocessing library.
+"""
+
+import os
+import platform
+from multiprocessing import Pool
+from qiskit._qiskiterror import QISKitError
+from qiskit._util import local_hardware_info
+from ._receiver import receiver as rec
+from ._progressbar import BaseProgressBar
+
+# Number of local physical cpus
+CPU_COUNT = local_hardware_info()['cpus']
+
+# Set parallel ennvironmental variable
+os.environ['QISKIT_IN_PARALLEL'] = 'FALSE'
+
+
+def parallel_map(task, values, task_args=tuple(), task_kwargs={},  # pylint: disable=W0102
+                 num_processes=CPU_COUNT):
+    """
+    Parallel execution of a mapping of `values` to the function `task`. This
+    is functionally equivalent to::
+        result = [task(value, *task_args, **task_kwargs) for value in values]
+
+    On Windows this function defaults to a serial implimentation to avoid the
+    overhead from spawning processes in Windows.
+
+    Parameters:
+        task (func): Function that is to be called for each value in ``task_vec``.
+        values (array_like): List or array of values for which the ``task``
+                            function is to be evaluated.
+        task_args (list): Optional additional arguments to the ``task`` function.
+        task_kwargs (dict): Optional additional keyword argument to the ``task`` function.
+        num_processes (int): Number of processes to spawn.
+
+    Returns:
+        result: The result list contains the value of
+                ``task(value, *task_args, **task_kwargs)`` for
+                    each value in ``values``.
+
+    Raises:
+        QISKitError: If user interupts via keyboard.
+    """
+    # len(values) == 1
+    if len(values) == 1:
+        return [task(values[0], *task_args, **task_kwargs)]
+
+    # Get last element of the receiver channels
+    if any(rec.channels):
+        progress_bar = None
+        for idx in rec.channels:
+            if rec.channels[idx].type == 'progressbar' and not rec.channels[idx].touched:
+                progress_bar = rec.channels[idx]
+                break
+        if progress_bar is None:
+            progress_bar = BaseProgressBar()
+    else:
+        progress_bar = BaseProgressBar()
+
+    progress_bar.start(len(values))
+    nfinished = [0]
+
+    def _callback(x):  # pylint: disable=W0613
+        nfinished[0] += 1
+        progress_bar.update(nfinished[0])
+
+    # Run in parallel if not Win and not in parallel already
+    if platform.system() != 'Windows' and num_processes > 1 \
+       and os.getenv('QISKIT_IN_PARALLEL') == 'FALSE':
+        os.environ['QISKIT_IN_PARALLEL'] = 'TRUE'
+        try:
+            pool = Pool(processes=num_processes)
+
+            async_res = [pool.apply_async(task, (value,) + task_args, task_kwargs,
+                                          _callback) for value in values]
+
+            while not all([item.ready() for item in async_res]):
+                for item in async_res:
+                    item.wait(timeout=0.1)
+
+            pool.terminate()
+            pool.join()
+
+        except KeyboardInterrupt:
+            pool.terminate()
+            pool.join()
+            progress_bar.finished()
+            raise QISKitError('Keyboard interrupt in parallel_map.')
+
+        progress_bar.finished()
+        os.environ['QISKIT_IN_PARALLEL'] = 'FALSE'
+        return [ar.get() for ar in async_res]
+
+    # Cannot do parallel on Windows , if another parallel_map is running in parallel,
+    # or len(values) == 1.
+    results = []
+    for _, value in enumerate(values):
+        result = task(value, *task_args, **task_kwargs)
+        results.append(result)
+        _callback(0)
+    progress_bar.finished()
+    return results
diff --git a/qiskit/transpiler/_progressbar.py b/qiskit/transpiler/_progressbar.py
@@ -0,0 +1,129 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2018, IBM.
+#
+# This source code is licensed under the Apache License, Version 2.0 found in
+# the LICENSE.txt file in the root directory of this source tree.
+
+# This file is part of QuTiP: Quantum Toolbox in Python.
+#
+#    Copyright (c) 2011 and later, Paul D. Nation and Robert J. Johansson.
+#    All rights reserved.
+#
+#    Redistribution and use in source and binary forms, with or without
+#    modification, are permitted provided that the following conditions are
+#    met:
+#
+#    1. Redistributions of source code must retain the above copyright notice,
+#       this list of conditions and the following disclaimer.
+#
+#    2. Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#
+#    3. Neither the name of the QuTiP: Quantum Toolbox in Python nor the names
+#       of its contributors may be used to endorse or promote products derived
+#       from this software without specific prior written permission.
+#
+#    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+#    PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#    HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+###############################################################################
+
+"""Progress bars module"""
+
+import time
+import datetime
+import sys
+from ._receiver import receiver as rec
+
+
+class BaseProgressBar(object):
+    """An abstract progress bar with some shared functionality.
+    """
+
+    def __init__(self):
+        self.type = 'progressbar'
+        self.touched = False
+        self.channel_id = rec.add_channel(self)
+        self.iter = None
+        self.t_start = None
+        self.t_done = None
+
+    def start(self, iterations):
+        """Start the progress bar.
+
+        Parameters:
+            iterations (int): Number of iterations.
+        """
+        self.touched = True
+        self.iter = int(iterations)
+        self.t_start = time.time()
+
+    def update(self, n):
+        """Update status of progress bar.
+        """
+        pass
+
+    def time_elapsed(self):
+        """Return the time elapsed since start.
+
+        Returns:
+            elapsed_time: Time since progress bar started.
+        """
+        return "%6.2fs" % (time.time() - self.t_start)
+
+    def time_remaining_est(self, completed_iter):
+        """Estimate the remaining time left.
+
+        Parameters:
+            completed_iter (int): Number of iterations completed.
+
+        Returns:
+            est_time: Estimated time remaining.
+        """
+        if completed_iter:
+            t_r_est = (time.time() - self.t_start) / \
+                completed_iter*(self.iter-completed_iter)
+        else:
+            t_r_est = 0
+        date_time = datetime.datetime(1, 1, 1) + datetime.timedelta(seconds=t_r_est)
+        time_string = "%02d:%02d:%02d:%02d" % \
+            (date_time.day - 1, date_time.hour, date_time.minute, date_time.second)
+
+        return time_string
+
+    def finished(self):
+        """Run when progress bar has completed.
+        """
+        rec.remove_channel(self.channel_id)
+
+
+class TextProgressBar(BaseProgressBar):
+    """
+    A simple text-based progress bar.
+    """
+    def start(self, iterations):
+        self.touched = True
+        self.iter = int(iterations)
+        self.t_start = time.time()
+        pbar = '-' * 50
+        sys.stdout.write('\r|%s| %s%s%s [%s]' %
+                         (pbar, 0, '/', self.iter, ''))
+
+    def update(self, n):
+        filled_length = int(round(50 * n / self.iter))
+        pbar = u'█' * filled_length + '-' * (50 - filled_length)
+        time_left = self.time_remaining_est(n)
+        sys.stdout.write('\r|%s| %s%s%s [%s]' % (pbar, n, '/', self.iter, time_left))
+        if n == self.iter:
+            sys.stdout.write('\n')
+        sys.stdout.flush()