Merge pull request #570 from DanielTimLee/test/parallel

Parallel tests implementation with multiprocessing from Daniel. It should speed up tests and improve the development. Good job, Daniel! Fixed: #569 Signed-off-by: Namhyung Kim <[email protected]>
namhyung · Nov 28, 2018 · b8c3c96 · b8c3c96
2 parents aa51e1f + dd4bf93
commit b8c3c96
Show file tree

Hide file tree

Showing 10 changed files with 128 additions and 54 deletions.
diff --git a/tests/runtest.py b/tests/runtest.py
@@ -1,8 +1,12 @@
 #!/usr/bin/env python
 
+import random
 import os, sys
+import tempfile
 import glob, re
 import subprocess as sp
+import multiprocessing
+import time
 
 class TestBase:
     supported_lang = {
@@ -20,13 +24,17 @@ class TestBase:
     TEST_SKIP = -7
     TEST_SUCCESS_FIXED = -8
 
-    objdir = 'objdir' in os.environ and os.environ['objdir'] or '..'
+    basedir = os.path.dirname(os.getcwd())
+    objdir = 'objdir' in os.environ and os.environ['objdir'] or basedir
     uftrace_cmd = objdir + '/uftrace --no-pager --no-event -L' + objdir
 
     default_cflags = ['-fno-inline', '-fno-builtin', '-fno-ipa-cp',
                       '-fno-omit-frame-pointer', '-D_FORTIFY_SOURCE=0']
 
     def __init__(self, name, result, lang='C', cflags='', ldflags='', sort='task'):
+        _tmp = tempfile.mkdtemp(prefix='test_%s_' % name)
+        os.chdir(_tmp)
+        self.test_dir = _tmp
         self.name = name
         self.result = result
         self.cflags = cflags
@@ -41,7 +49,19 @@ def pr_debug(self, msg):
         if self.debug:
             print(msg)
 
+    def gen_port(self):
+        self.port = random.randint(40000, 50000)
+
+    def convert_abs_path(self, build_cmd):
+        cmd = build_cmd.split()
+        src_idx = [i for i, _cmd in enumerate(cmd) if _cmd.startswith('s-')][0]
+        abs_src = os.path.join(self.basedir, 'tests', cmd[src_idx])
+        cmd[src_idx] = abs_src
+        return " ".join(cmd)
+
     def build_it(self, build_cmd):
+        build_cmd = self.convert_abs_path(build_cmd)
+
         try:
             p = sp.Popen(build_cmd.split(), stderr=sp.PIPE)
             if p.wait() != 0:
@@ -316,7 +336,7 @@ def fixup(self, cflags, result):
 
     def check_dependency(self, item):
         import os.path
-        return os.path.exists('../check-deps/' + item)
+        return os.path.exists('%s/check-deps/' % self.basedir + item)
 
     def check_perf_paranoid(self):
         try:
@@ -405,6 +425,8 @@ def timeout_handler(sig, frame):
 
         return ret
 
+    def __del__(self):
+        sp.call(['rm', '-rf', self.test_dir])
 
 RED     = '\033[1;31m'
 GREEN   = '\033[1;32m'
@@ -447,6 +469,7 @@ def timeout_handler(sig, frame):
     TestBase.TEST_SUCCESS_FIXED:  'Test succeeded (with some fixup)',
 }
 
+
 def run_single_case(case, flags, opts, arg):
     result = []
 
@@ -469,6 +492,15 @@ def run_single_case(case, flags, opts, arg):
 
     return result
 
+
+def save_test_result(result, case, shared):
+    shared.results[case] = result
+    shared.progress += 1
+    for r in result:
+        shared.stats[r] += 1
+        shared.total += 1
+
+
 def print_test_result(case, result, color):
     if sys.stdout.isatty() and color:
         result_list = [colored_result[r] for r in result]
@@ -480,6 +512,40 @@ def print_test_result(case, result, color):
     sys.stdout.write(output)
 
 
+def print_test_header(opts, flags):
+    optslen = len(opts)
+
+    header1 = '%-24s ' % 'Test case'
+    header2 = '-' * 24 + ':'
+    empty = '                      '
+
+    for flag in flags:
+        # align with optimization flags
+        header1 += ' ' + flag[:optslen] + empty[len(flag):optslen]
+        header2 += ' ' + opts
+
+    print(header1)
+    print(header2)
+
+
+
+
+def print_test_report(arg, shared):
+    success = shared.stats[TestBase.TEST_SUCCESS] + shared.stats[TestBase.TEST_SUCCESS_FIXED]
+    percent = 100.0 * success / shared.total
+
+    print("")
+    print("runtime test stats")
+    print("====================")
+    print("total %5d  Tests executed (success: %.2f%%)" % (shared.total, percent))
+    for r in res:
+        if sys.stdout.isatty() and arg.color:
+            result = colored_result[r]
+        else:
+            result = text_result[r]
+        print("  %s: %5d  %s" % (result, shared.stats[r], result_string[r]))
+
+
 def parse_argument():
     import argparse
 
@@ -503,9 +569,12 @@ def parse_argument():
                         help="suppress color in the output")
     parser.add_argument("-t", "--timeout", dest='timeout', default=5,
                         help="fail test if it runs more than TIMEOUT seconds")
+    parser.add_argument("-j", "--worker", dest='worker', type=int, default=multiprocessing.cpu_count(),
+                        help="Parallel worker count; using all core for default")
 
     return parser.parse_args()
 
+
 if __name__ == "__main__":
     # prevent to create .pyc files (it makes some tests failed)
     os.environ["PYTHONDONTWRITEBYTECODE"] = "1"
@@ -522,28 +591,24 @@ def parse_argument():
                 print("cannot find testcase for : %s" % arg.case)
                 sys.exit(0)
 
-    opts = ' '.join(sorted(['O'+o for o in arg.opts]))
-    optslen = len(opts)
-
-    header1 = '%-24s ' % 'Test case'
-    header2 = '-' * 24 + ':'
-    empty = '                      '
+    opts = ' '.join(sorted(['O' + o for o in arg.opts]))
 
     if arg.pg_flag:
         flags = ['pg']
     elif arg.if_flag:
         flags = ['finstrument-functions']
     else:
         flags = arg.flags.split()
-    for flag in flags:
-        # align with optimization flags
-        header1 += ' ' + flag[:optslen] + empty[len(flag):optslen]
-        header2 += ' ' + opts
 
-    print(header1)
-    print(header2)
+    from functools import partial
 
-    total = 0
+    manager = multiprocessing.Manager()
+    shared = manager.dict()
+
+    shared.tests_count = len(testcases)
+    shared.progress = 0
+    shared.results = dict()
+    shared.total = 0
     res = []
     res.append(TestBase.TEST_SUCCESS)
     res.append(TestBase.TEST_SUCCESS_FIXED)
@@ -555,26 +620,30 @@ def parse_argument():
     res.append(TestBase.TEST_UNSUPP_LANG)
     res.append(TestBase.TEST_SKIP)
 
-    stats = dict.fromkeys(res, 0)
+    shared.stats = dict.fromkeys(res, 0)
+    pool = multiprocessing.Pool(arg.worker)
 
     for tc in sorted(testcases):
-        name = tc[:-3]  # remove '.py'
-        result = run_single_case(name, flags, opts.split(), arg)
-        print_test_result(name, result, arg.color)
-        for r in result:
-            stats[r] += 1
-            total += 1
+        name = tc.split('.')[0]  # remove '.py'
+        clbk = partial(save_test_result, case=name, shared=shared)
+        pool.apply_async(run_single_case, callback=clbk,
+                         args=[name, flags, opts.split(), arg])
 
-    success = stats[TestBase.TEST_SUCCESS] + stats[TestBase.TEST_SUCCESS_FIXED]
-    percent = 100.0 * success / total
+    print("Start %s tests with %d worker" % (shared.tests_count, arg.worker))
+    print_test_header(opts, flags)
 
-    print("")
-    print("runtime test stats")
-    print("====================")
-    print("total %5d  Tests executed (success: %.2f%%)" % (total, percent))
-    for r in res:
-        if sys.stdout.isatty() and arg.color:
-            result = colored_result[r]
-        else:
-            result = text_result[r]
-        print("  %s: %5d  %s" % (result, stats[r], result_string[r]))
+    for tc in sorted(testcases):
+        name = tc.split('.')[0]  # remove '.py'
+
+        while name not in shared.results:
+            time.sleep(1)
+
+        print_test_result(name, shared.results[name], arg.color)
+
+    pool.close()
+    pool.join()
+
+    sys.stdout.write("\n")
+    sys.stdout.flush()
+
+    print_test_report(arg, shared)
diff --git a/tests/t141_recv_basic.py b/tests/t141_recv_basic.py
@@ -21,14 +21,15 @@ def __init__(self):
    2.405 us [28141] |   } /* a */
    3.005 us [28141] | } /* main */
 """)
+        self.gen_port()
 
     recv_p = None
 
     def pre(self):
-        recv_cmd = '%s recv -d %s' % (TestBase.uftrace_cmd, TDIR)
+        recv_cmd = '%s recv -d %s --port %s' % (TestBase.uftrace_cmd, TDIR, self.port)
         self.recv_p = sp.Popen(recv_cmd.split())
 
-        record_cmd = '%s record -H %s %s' % (TestBase.uftrace_cmd, 'localhost', 't-abc')
+        record_cmd = '%s record -H %s --port %s %s' % (TestBase.uftrace_cmd, 'localhost', self.port, 't-abc')
         sp.call(record_cmd.split())
         return TestBase.TEST_SUCCESS
 

diff --git a/tests/t142_recv_multi.py b/tests/t142_recv_multi.py
@@ -21,19 +21,20 @@ def __init__(self):
    2.405 us [28141] |   } /* a */
    3.005 us [28141] | } /* main */
 """)
+        self.gen_port()
 
     recv_p = None
 
     def pre(self):
-        recv_cmd = '%s recv -d %s' % (TestBase.uftrace_cmd, TDIR)
+        recv_cmd = '%s recv -d %s --port %s' % (TestBase.uftrace_cmd, TDIR, self.port)
         self.recv_p = sp.Popen(recv_cmd.split())
 
         # recorded but not used
-        record_cmd = '%s record -H %s %s' % (TestBase.uftrace_cmd, 'localhost', 't-abc')
+        record_cmd = '%s record -H %s --port %s %s' % (TestBase.uftrace_cmd, 'localhost', self.port, 't-abc')
         sp.call(record_cmd.split())
 
         # use this
-        record_cmd = '%s record -H %s -d %s %s' % (TestBase.uftrace_cmd, 'localhost', TDIR2, 't-abc')
+        record_cmd = '%s record -H %s --port %s -d %s %s' % (TestBase.uftrace_cmd, 'localhost', self.port, TDIR2, 't-abc')
         sp.call(record_cmd.split())
 
         return TestBase.TEST_SUCCESS

diff --git a/tests/t143_recv_kernel.py b/tests/t143_recv_kernel.py
@@ -22,6 +22,7 @@ def __init__(self):
   37.325 us [18343] |   } /* fclose */
  128.387 us [18343] | } /* main */
 """)
+        self.gen_port()
 
     recv_p = None
 
@@ -34,10 +35,10 @@ def pre(self):
         uftrace = TestBase.uftrace_cmd
         program = 't-' + self.name
 
-        recv_cmd = '%s recv -d %s' % (uftrace, TDIR)
+        recv_cmd = '%s recv -d %s --port %s' % (uftrace, TDIR, self.port)
         self.recv_p = sp.Popen(recv_cmd.split())
 
-        argument  = '-H %s -k -d %s' % ('localhost', TDIR2)
+        argument  = '-H %s -k -d %s --port %s' % ('localhost', TDIR2, self.port)
         argument += ' -N %s@kernel' % '_*do_page_fault'
 
         record_cmd = '%s record %s %s' % (uftrace, argument, program)

diff --git a/tests/t150_recv_event.py b/tests/t150_recv_event.py
@@ -18,17 +18,19 @@ def __init__(self):
    2.896 us [28141] |   } /* foo */
    3.017 us [28141] | } /* main */
 """)
+        self.gen_port()
 
     recv_p = None
 
     def pre(self):
-        recv_cmd = '%s recv -d %s' % (TestBase.uftrace_cmd, TDIR)
+        recv_cmd = '%s recv -d %s --port %s' % (TestBase.uftrace_cmd, TDIR, self.port)
         self.recv_p = sp.Popen(recv_cmd.split())
 
         server = '-H 127.0.0.1'
+        port   = '--port %s' % self.port
         option = '-E uftrace:event'
         prog   = 't-' + self.name
-        record_cmd = '%s record %s %s %s' % (TestBase.uftrace_cmd, server, option, prog)
+        record_cmd = '%s record %s %s %s %s' % (TestBase.uftrace_cmd, server, port, option, prog)
         sp.call(record_cmd.split())
         return TestBase.TEST_SUCCESS
 

diff --git a/tests/t151_recv_runcmd.py b/tests/t151_recv_runcmd.py
@@ -21,16 +21,18 @@ def __init__(self):
    2.405 us [28141] |   } /* a */
    3.005 us [28141] | } /* main */
 """)
+        self.gen_port()
 
     recv_p = None
     file_p = None
 
     def pre(self):
         self.file_p = open(TMPF, 'w+')
-        recv_cmd = TestBase.uftrace_cmd.split() + ['recv', '-d', TDIR, '--run-cmd', TestBase.uftrace_cmd + ' replay']
+        recv_cmd = TestBase.uftrace_cmd.split() + \
+                   ['recv', '-d', TDIR, '--port', str(self.port), '--run-cmd', TestBase.uftrace_cmd + ' replay']
         self.recv_p = sp.Popen(recv_cmd, stdout=self.file_p, stderr=self.file_p)
 
-        record_cmd = '%s record -H %s %s' % (TestBase.uftrace_cmd, 'localhost', 't-' + self.name)
+        record_cmd = '%s record -H %s --port %s %s' % (TestBase.uftrace_cmd, 'localhost', self.port, 't-' + self.name)
         sp.call(record_cmd.split())
         return TestBase.TEST_SUCCESS
 

diff --git a/tests/t157_script_python.py b/tests/t157_script_python.py
@@ -21,7 +21,7 @@ def pre(self):
 
     def runcmd(self):
         uftrace = TestBase.uftrace_cmd
-        options = '-F main -S ../scripts/count.py'
+        options = '-F main -S %s/scripts/count.py' % self.basedir
         return '%s script -d %s %s' % (uftrace, TDIR, options)
 
     def sort(self, output):

diff --git a/tests/t167_recv_sched.py b/tests/t167_recv_sched.py
@@ -26,6 +26,7 @@ def __init__(self):
    2.120 ms [  395] |   } /* foo */
    2.121 ms [  395] | } /* main */
 """)
+        self.gen_port()
 
     recv_p = None
 
@@ -35,10 +36,10 @@ def pre(self):
         if not TestBase.check_perf_paranoid(self):
             return TestBase.TEST_SKIP
 
-        recv_cmd = '%s recv -d %s' % (TestBase.uftrace_cmd, TDIR)
+        recv_cmd = '%s recv -d %s --port %s' % (TestBase.uftrace_cmd, TDIR, self.port)
         self.recv_p = sp.Popen(recv_cmd.split())
 
-        options = '-H %s -E %s' % ('localhost', 'linux:schedule')
+        options = '-H %s --port %s -E %s' % ('localhost', self.port, 'linux:schedule')
         record_cmd = '%s record %s %s' % (TestBase.uftrace_cmd, options, 't-' + self.name)
         sp.call(record_cmd.split())
         return TestBase.TEST_SUCCESS

diff --git a/tests/t199_script_info.py b/tests/t199_script_info.py
@@ -25,7 +25,7 @@ def pre(self):
 
     def runcmd(self):
         uftrace = TestBase.uftrace_cmd
-        options = '-F main -S ../scripts/info.py foo bar'
+        options = '-F main -S %s/scripts/info.py foo bar' % self.basedir
         return '%s script -d %s %s' % (uftrace, TDIR, options)
 
     def sort(self, output):