From f1a00ff3b9b51bdca90794b91d24ece92e9b1c31 Mon Sep 17 00:00:00 2001 From: Giulio Ungaretti Date: Tue, 7 Mar 2017 11:38:52 +0100 Subject: [PATCH] chore: Remove benchmarking and reloading Benchmarking should go if anything to its own repo reloading is dangerous, and it can trick many new users. --- benchmarking/mptest.py | 156 ------------------------ benchmarking/mptest_results_mac.txt | 117 ------------------ benchmarking/mptest_results_windows.txt | 88 ------------- benchmarking/thread_test.py | 55 --------- qcodes/utils/reload_code.py | 98 --------------- 5 files changed, 514 deletions(-) delete mode 100644 benchmarking/mptest.py delete mode 100644 benchmarking/mptest_results_mac.txt delete mode 100644 benchmarking/mptest_results_windows.txt delete mode 100644 benchmarking/thread_test.py delete mode 100644 qcodes/utils/reload_code.py diff --git a/benchmarking/mptest.py b/benchmarking/mptest.py deleted file mode 100644 index 4072ef921694..000000000000 --- a/benchmarking/mptest.py +++ /dev/null @@ -1,156 +0,0 @@ -# stress test multiprocessing -# run with: python mptest.py -# proc_count: the number of processes to spin up and load qcodes -# period: milliseconds to wait between calling each process -# repetitions: how many times to call each one -# start_method: multiprocessing method to use (fork, forkserver, spawn) - -import multiprocessing as mp -import sys -import os -import psutil -import time - -import qcodes as qc - - -timer = time.perf_counter - - -def print_perf(): - print(time.perf_counter()) - - -def mp_test(name, qin, qout, qglobal): - ''' - simple test that keeps a process running until asked to stop, - and looks for an attribute within qcodes just to ensure it has loaded it - ''' - delays = [] - first = True - while True: - item, qtime = qin.get() - if first: # ignore the first one... process is still starting - first = False - else: - delays.append(timer() - qtime) - if item == 'break': - qglobal.put({ - 'name': name, - 'avg': sum(delays) / len(delays), - 'max': max(delays) - }) - break - qout.put(repr(getattr(qc, item))) - - -def get_memory(pid): - mi = psutil.Process(pid).memory_info() - return mi.rss, mi.vms - - -def get_all_memory(processes, title): - main_memory = get_memory(os.getpid()) - proc_memory = [0, 0] - for proc in processes: - for i, v in enumerate(get_memory(proc.pid)): - proc_memory[i] += v - # print(v) - - return { - 'main_physical': main_memory[0]/1e6, - 'main_virtual': main_memory[1]/1e6, - 'proc_physical': proc_memory[0]/1e6, - 'proc_virtual': proc_memory[1]/1e6, - 'title': title - } - - -def format_memory(mem): - return ('{title}\n' - ' main: {main_physical:.0f} MB phys, ' - '{main_virtual:.0f} MB virt\n' - ' procs: {proc_physical:.0f} MB phys, ' - '{proc_virtual:.0f} MB virt\n' - '').format(**mem) - - -if __name__ == '__main__': - proc_count = int(sys.argv[-4]) - period = float(sys.argv[-3]) - reps = int(sys.argv[-2]) - method = sys.argv[-1] - mp.set_start_method(method) - - qglobal = mp.Queue() - - mem = [get_all_memory([], 'on startup')] - - queues = [] - processes = [] - resp_delays = [] - - t_before_start = timer() - - for proc_num in range(proc_count): - qin = mp.Queue() - qout = mp.Queue() - queues.append((qin, qout)) - p = mp.Process(target=mp_test, - args=('p{}'.format(proc_num), qin, qout, qglobal)) - processes.append(p) - p.start() - - start_delay = (timer() - t_before_start) * 1000 - - mem.append(get_all_memory(processes, 'procs started')) - - for i in range(reps): - for qin, qout in queues: - t1 = timer() - qin.put(('Loop', timer())) - loop_repr = qout.get() - if i: - # ignore the first one, process is still starting - resp_delays.append((timer() - t1) * 1000) - if(loop_repr != repr(qc.Loop)): - raise RuntimeError('{} != {}'.format(loop_repr, repr(qc.Loop))) - print('.', end='', flush=True) - time.sleep(period / 1000) - print('') - - mem.append(get_all_memory(processes, 'procs done working')) - - for qin, qout in queues: - qin.put(('break', timer())) - - t_before_join = timer() - for proc in processes: - proc.join() - join_delay = (timer() - t_before_join) * 1000 - - delays = [qglobal.get() for proc in processes] - avg_delay = sum([d['avg'] for d in delays]) * 1000 / len(delays) - max_delay = max([d['max'] for d in delays]) * 1000 - - avg_resp_delay = sum(resp_delays) / len(resp_delays) - max_resp_delay = max(resp_delays) - - print(('Ran {} procs using "{}" method\n' - 'sent messages every {} milliseconds, {} times\n' - ).format(proc_count, method, period, reps)) - - print('Milliseconds to start all processes: {:.3f}'.format(start_delay)) - print('Final join delay: {:.3f}\n'.format(join_delay)) - - print('Milliseconds to receive to queue request') - print(' avg: {:.6f}'.format(avg_delay)) - print(' max: {:.6f}\n'.format(max_delay)) - - print('Milliseconds to respond to queue request') - print(' avg: {:.6f}'.format(avg_resp_delay)) - print(' max: {:.6f}\n'.format(max_resp_delay)) - - # report on the memory results - for m in mem: - print(format_memory(m)) diff --git a/benchmarking/mptest_results_mac.txt b/benchmarking/mptest_results_mac.txt deleted file mode 100644 index ec8e2a7765d6..000000000000 --- a/benchmarking/mptest_results_mac.txt +++ /dev/null @@ -1,117 +0,0 @@ -Mac OS X -------------------- - -Alexs-Air-2:Qcodes alex$ p3 mptest.py 5 50 50 spawn -.................................................. -Ran 5 procs using "spawn" method -sent messages every 50.0 milliseconds, 50 times - -Milliseconds to start all processes: 30.601 -Final join delay: 130.124 - -Milliseconds to receive to queue request - avg: 0.321102 - max: 1.061535 - -Milliseconds to respond to queue request - avg: 0.607558 - max: 1.465819 - -on startup - main: 29 MB phys, 2498 MB virt - procs: 0 MB phys, 0 MB virt - -procs started - main: 29 MB phys, 2498 MB virt - procs: 8 MB phys, 12327 MB virt - -procs done working - main: 29 MB phys, 2524 MB virt - procs: 142 MB phys, 12509 MB virt - - -Alexs-Air-2:Qcodes alex$ p3 mptest.py 20 50 50 spawn -.................................................. -Ran 20 procs using "spawn" method -sent messages every 50.0 milliseconds, 50 times - -Milliseconds to start all processes: 512.364 -Final join delay: 456.155 - -Milliseconds to receive to queue request - avg: 0.302726 - max: 2.754109 - -Milliseconds to respond to queue request - avg: 0.555251 - max: 1.123662 - -on startup - main: 28 MB phys, 2498 MB virt - procs: 0 MB phys, 0 MB virt - -procs started - main: 29 MB phys, 2498 MB virt - procs: 150 MB phys, 49561 MB virt - -procs done working - main: 29 MB phys, 2603 MB virt - procs: 571 MB phys, 50029 MB virt - - -Alexs-Air-2:Qcodes alex$ p3 mptest.py 100 50 50 spawn -.................................................. -Ran 100 procs using "spawn" method -sent messages every 50.0 milliseconds, 50 times - -Milliseconds to start all processes: 5242.341 -Final join delay: 3806.779 - -Milliseconds to receive to queue request - avg: 0.527770 - max: 59.421732 - -Milliseconds to respond to queue request - avg: 0.416881 - max: 4.560305 - -on startup - main: 28 MB phys, 2497 MB virt - procs: 0 MB phys, 0 MB virt - -procs started - main: 30 MB phys, 2499 MB virt - procs: 1234 MB phys, 248003 MB virt - -procs done working - main: 11 MB phys, 3026 MB virt - procs: 2144 MB phys, 250162 MB virt - - -Alexs-Air-2:Qcodes alex$ p3 mptest.py 100 50 50 fork -.................................................. -Ran 100 procs using "fork" method -sent messages every 50.0 milliseconds, 50 times - -Milliseconds to start all processes: 447.375 -Final join delay: 113.426 - -Milliseconds to receive to queue request - avg: 0.351694 - max: 11.889809 - -Milliseconds to respond to queue request - avg: 0.494252 - max: 2.525026 - -on startup - main: 28 MB phys, 2498 MB virt - procs: 0 MB phys, 0 MB virt - -procs started - main: 29 MB phys, 2499 MB virt - procs: 444 MB phys, 249616 MB virt - -procs done working - main: 32 MB phys, 3025 MB virt - procs: 525 MB phys, 250142 MB virt diff --git a/benchmarking/mptest_results_windows.txt b/benchmarking/mptest_results_windows.txt deleted file mode 100644 index e267edeb280b..000000000000 --- a/benchmarking/mptest_results_windows.txt +++ /dev/null @@ -1,88 +0,0 @@ -Windows -------------------- - -c:\Qcodes>python mptest.py 5 50 50 spawn -.................................................. -Ran 5 procs using "spawn" method -sent messages every 50.0 milliseconds, 50 times - -Milliseconds to start all processes: 15.406 -Final join delay: 92.359 - -Milliseconds to receive to queue request - avg: -510.603209 - max: -509.684951 - -Milliseconds to respond to queue request - avg: 0.360488 - max: 0.659211 - -on startup - main: 36 MB phys, 31 MB virt - procs: 0 MB phys, 0 MB virt - -procs started - main: 36 MB phys, 31 MB virt - procs: 14 MB phys, 9 MB virt - -procs done working - main: 36 MB phys, 32 MB virt - procs: 179 MB phys, 155 MB virt - - -c:\Qcodes>python mptest.py 20 50 50 spawn -.................................................. -Ran 20 procs using "spawn" method -sent messages every 50.0 milliseconds, 50 times - -Milliseconds to start all processes: 126.134 -Final join delay: 228.260 - -Milliseconds to receive to queue request - avg: -1591.011659 - max: -1563.003352 - -Milliseconds to respond to queue request - avg: 0.365974 - max: 0.794690 - -on startup - main: 36 MB phys, 31 MB virt - procs: 0 MB phys, 0 MB virt - -procs started - main: 36 MB phys, 31 MB virt - procs: 98 MB phys, 60 MB virt - -procs done working - main: 37 MB phys, 32 MB virt - procs: 716 MB phys, 620 MB virt - - -c:\Qcodes>python mptest.py 100 50 50 spawn -.................................................. -Ran 100 procs using "spawn" method -sent messages every 50.0 milliseconds, 50 times - -Milliseconds to start all processes: 761.556 -Final join delay: 927.847 - -Milliseconds to receive to queue request - avg: -8560.453780 - max: -8470.848083 - -Milliseconds to respond to queue request - avg: 0.423033 - max: 1.299948 - -on startup - main: 35 MB phys, 31 MB virt - procs: 0 MB phys, 0 MB virt - -procs started - main: 37 MB phys, 32 MB virt - procs: 872 MB phys, 572 MB virt - -procs done working - main: 40 MB phys, 36 MB virt - procs: 3575 MB phys, 3094 MB virt diff --git a/benchmarking/thread_test.py b/benchmarking/thread_test.py deleted file mode 100644 index 40dac28bc692..000000000000 --- a/benchmarking/thread_test.py +++ /dev/null @@ -1,55 +0,0 @@ -import threading -import time - - -def sleeper(t, n, out, make_error): - time.sleep(t) - out[n] = n - if make_error: - raise RuntimeError('hello from # {}!'.format(n)) - - -def runmany(n, t, error_nums): - out = [None] * n - - t0 = time.perf_counter() - - threads = [ - CatchingThread(target=sleeper, args=(t, i, out, i in error_nums)) - for i in range(n)] - - # start threads backward - [t.start() for t in reversed(threads)] - [t.join() for t in threads] - - t1 = time.perf_counter() - - out_ok = [] - for i in range(n): - if out[i] != i: - out_ok += ['ERROR! out[{}] = {}'.format(i, out[i])] - - if not out_ok: - out_ok += ['all output correct'] - - print('{} parallel threads sleeping\n'.format(n) + - 'given time: {}\n'.format(t) + - 'resulting time: {}\n'.format(t1 - t0) + - '\n'.join(out_ok)) - - -class CatchingThread(threading.Thread): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.exception = None - - def run(self): - try: - super().run() - except Exception as e: - self.exception = e - - def join(self): - super().join() - if self.exception: - raise self.exception diff --git a/qcodes/utils/reload_code.py b/qcodes/utils/reload_code.py deleted file mode 100644 index 08cedf5d1dd6..000000000000 --- a/qcodes/utils/reload_code.py +++ /dev/null @@ -1,98 +0,0 @@ -import os -import sys -import imp -from traceback import format_exc - -# Routine to reload modules during execution, for development only. -# This is finicky and SHOULD NOT be used in regular experiments, -# as they can cause non-intuitive errors later on. It is not included in -# the base qcodes import, nor tested; Use at your own risk. - - -# see http://stackoverflow.com/questions/22195382/ -# how-to-check-if-a-module-library-package-is-part-of-the-python-standard-library -syspaths = [os.path.abspath(p) for p in sys.path] -stdlib = tuple(p for p in syspaths - if p.startswith((sys.prefix, sys.base_prefix)) - and 'site-packages' not in p) -# a few things in site-packages we will consider part of the standard lib -# it causes problems if we reload some of these, others are just stable -# dependencies - this is mainly for reloading our own code. -# could even whitelist site-packages items to allow, rather than to ignore? -otherlib = ('jupyter', 'ipy', 'IPy', 'matplotlib', 'numpy', 'scipy', 'pyvisa', - 'traitlets', 'zmq', 'tornado', 'dateutil', 'six', 'pexpect') -otherpattern = tuple('site-packages/' + n for n in otherlib) - - -def reload_code(pattern=None, lib=False, site=False): - ''' - reload all modules matching a given pattern - or all (non-built-in) modules if pattern is omitted - if lib is False (default), ignore the standard library and major packages - if site is False (default), ignore everything in site-packages, only reload - files in nonstandard paths - ''' - reloaded_files = [] - - for i in range(2): - # sometimes we need to reload twice to propagate all links, - # even though we reload the deepest modules first. Not sure if - # twice is always sufficient, but we'll try it. - for module in sys.modules.values(): - if (pattern is None or pattern in module.__name__): - reload_recurse(module, reloaded_files, lib, site) - - return reloaded_files - - -def is_good_module(module, lib=False, site=False): - ''' - is an object (module) a module we can reload? - if lib is False (default), ignore the standard library and major packages - ''' - # take out non-modules and underscore modules - name = getattr(module, '__name__', '_') - if name[0] == '_' or not isinstance(module, type(sys)): - return False - - # take out modules we can't find and built-ins - if name in sys.builtin_module_names or not hasattr(module, '__file__'): - return False - - path = os.path.abspath(module.__file__) - - if 'site-packages' in path and not site: - return False - - if not lib: - if path.startswith(stdlib) and 'site-packages' not in path: - return False - - for pattern in otherpattern: - if pattern in path: - return False - - return True - - -def reload_recurse(module, reloaded_files, lib, site): - ''' - recursively search module for its own dependencies to reload, - ignoring those already in reloaded_files - if lib is False (default), ignore the standard library and major packages - ''' - if (not is_good_module(module, lib, site) or - module.__file__ in reloaded_files): - return - - reloaded_files.append(module.__file__) - - try: - for name in dir(module): - module2 = getattr(module, name) - reload_recurse(module2, reloaded_files, lib, site) - imp.reload(module) - - except: - print('error reloading "{}"'.format(getattr(module, '__name__', '?'))) - print(format_exc())