Run JUnit CI Tests with Compiled Query Execution

Jenkinsfile

@@ -140,8 +140,11 @@ pipeline {
                         sh script: 'sudo lsof -i -P -n | grep LISTEN || true', label: 'Check ports.'
                         sh script: 'cd build/bin && PYTHONPATH=../.. timeout 20m python3 -m script.testing.replication.tests_simple --build-type=debug', label: 'Replication (Simple)'
                         sh script: 'cd build && PYTHONPATH=.. timeout 20m python3 -m script.testing.junit --build-type=debug --query-mode=simple', label: 'UnitTest (Simple)'
+                        sh script: 'cd build && PYTHONPATH=.. timeout 60m python3 -m script.testing.junit --build-type=debug --query-mode=simple -a "compiled_query_execution=True" -a "bytecode_handlers_path=./bytecode_handlers_ir.bc"', label: 'UnitTest (Simple, Compiled Execution)'
                         sh script: 'cd build && PYTHONPATH=.. timeout 20m python3 -m script.testing.junit --build-type=debug --query-mode=extended', label: 'UnitTest (Extended)'
+                        sh script: 'cd build && PYTHONPATH=.. timeout 60m python3 -m script.testing.junit --build-type=debug --query-mode=extended -a "compiled_query_execution=True" -a "bytecode_handlers_path=./bytecode_handlers_ir.bc"', label: 'UnitTest (Extended, Compiled Execution)'
                         sh script: 'cd build && PYTHONPATH=.. timeout 20m python3 -m script.testing.junit --build-type=debug --query-mode=extended -a "pipeline_metrics_enable=True" -a "pipeline_metrics_sample_rate=100" -a "counters_enable=True" -a "query_trace_metrics_enable=True"', label: 'UnitTest (Extended with pipeline metrics, counters, and query trace metrics)'
+                        sh script: 'cd build && PYTHONPATH=.. timeout 60m python3 -m script.testing.junit --build-type=debug --query-mode=extended -a "pipeline_metrics_enable=True" -a "pipeline_metrics_sample_rate=100" -a "counters_enable=True" -a "query_trace_metrics_enable=True" -a "compiled_query_execution=True" -a "bytecode_handlers_path=./bytecode_handlers_ir.bc"', label: 'UnitTest (Extended, Compiled Execution with pipeline metrics, counters, and query trace metrics)'
                         sh script: 'sudo lsof -i -P -n | grep LISTEN || true', label: 'Check ports.'
                         sh 'cd build && timeout 1h ninja check-tpl'
                         sh 'cd build && timeout 1h ninja jumbotests'
@@ -183,7 +186,9 @@ pipeline {
                         sh script: 'cd build && timeout 10s sudo python3 -B ../script/testing/kill_server.py 15723', label: 'Kill PID(15723)'
                         sh script: 'sudo lsof -i -P -n | grep LISTEN || true', label: 'Check ports.'
                         sh script: 'cd build && PYTHONPATH=.. timeout 20m python3 -m script.testing.junit --build-type=debug --query-mode=simple', label: 'UnitTest (Simple)'
+                        sh script: 'cd build && PYTHONPATH=.. timeout 60m python3 -m script.testing.junit --build-type=debug --query-mode=simple -a "compiled_query_execution=True" -a "bytecode_handlers_path=./bytecode_handlers_ir.bc"', label: 'UnitTest (Simple, Compiled Execution)'
                         sh script: 'cd build && PYTHONPATH=.. timeout 20m python3 -m script.testing.junit --build-type=debug --query-mode=extended', label: 'UnitTest (Extended)'
+                        sh script: 'cd build && PYTHONPATH=.. timeout 60m python3 -m script.testing.junit --build-type=debug --query-mode=extended -a "compiled_query_execution=True" -a "bytecode_handlers_path=./bytecode_handlers_ir.bc"', label: 'UnitTest (Extended, Compiled Execution)'
                         sh script: 'sudo lsof -i -P -n | grep LISTEN || true', label: 'Check ports.'
                         sh 'cd build && timeout 1h ninja check-tpl'
                         sh 'cd build && timeout 1h ninja unittest'
@@ -232,7 +237,9 @@ pipeline {
                         sh script: 'sudo lsof -i -P -n | grep LISTEN || true', label: 'Check ports.'
                         sh script: 'cd build/bin && PYTHONPATH=../.. timeout 20m python3 -m script.testing.replication.tests_simple --build-type=debug', label: 'Replication (Simple)'
                         sh script: 'cd build && PYTHONPATH=.. timeout 20m python3 -m script.testing.junit --build-type=debug --query-mode=simple', label: 'UnitTest (Simple)'
+                        sh script: 'cd build && PYTHONPATH=.. timeout 60m python3 -m script.testing.junit --build-type=debug --query-mode=simple -a "compiled_query_execution=True" -a "bytecode_handlers_path=./bytecode_handlers_ir.bc"', label: 'UnitTest (Simple, Compiled Execution)'
                         sh script: 'cd build && PYTHONPATH=.. timeout 20m python3 -m script.testing.junit --build-type=debug --query-mode=extended', label: 'UnitTest (Extended)'
+                        sh script: 'cd build && PYTHONPATH=.. timeout 60m python3 -m script.testing.junit --build-type=debug --query-mode=extended -a "compiled_query_execution=True" -a "bytecode_handlers_path=./bytecode_handlers_ir.bc"', label: 'UnitTest (Extended, Compiled Execution)'
                         sh script: 'sudo lsof -i -P -n | grep LISTEN || true', label: 'Check ports.'
                         sh 'cd build && timeout 1h ninja check-tpl'
                         sh 'cd build && timeout 1h ninja jumbotests'
@@ -271,7 +278,9 @@ pipeline {
                         sh script: 'sudo lsof -i -P -n | grep LISTEN || true', label: 'Check ports.'
                         sh script: 'cd build/bin && PYTHONPATH=../.. timeout 20m python3 -m script.testing.replication.tests_simple --build-type=release', label: 'Replication (Simple)'
                         sh script: 'cd build && PYTHONPATH=.. timeout 20m python3 -m script.testing.junit --build-type=release --query-mode=simple', label: 'UnitTest (Simple)'
+                        sh script: 'cd build && PYTHONPATH=.. timeout 60m python3 -m script.testing.junit --build-type=release --query-mode=simple -a "compiled_query_execution=True" -a "bytecode_handlers_path=./bytecode_handlers_ir.bc"', label: 'UnitTest (Simple, Compiled Execution)'
                         sh script: 'cd build && PYTHONPATH=.. timeout 20m python3 -m script.testing.junit --build-type=release --query-mode=extended', label: 'UnitTest (Extended)'
+                        sh script: 'cd build && PYTHONPATH=.. timeout 60m python3 -m script.testing.junit --build-type=release --query-mode=extended -a "compiled_query_execution=True" -a "bytecode_handlers_path=./bytecode_handlers_ir.bc"', label: 'UnitTest (Extended, Compiled Execution)'
                         sh script: 'sudo lsof -i -P -n | grep LISTEN || true', label: 'Check ports.'
                         sh 'cd build && timeout 1h ninja check-tpl'
                         sh 'cd build && timeout 1h ninja jumbotests'
@@ -314,7 +323,9 @@ pipeline {
                         sh script: 'sudo lsof -i -P -n | grep LISTEN || true', label: 'Check ports.'
                         sh script: 'cd build/bin && PYTHONPATH=../.. timeout 20m python3 -m script.testing.replication.tests_simple --build-type=release', label: 'Replication (Simple)'
                         sh script: 'cd build && PYTHONPATH=.. timeout 20m python3 -m script.testing.junit --build-type=release --query-mode=simple', label: 'UnitTest (Simple)'
+                        sh script: 'cd build && PYTHONPATH=.. timeout 60m python3 -m script.testing.junit --build-type=release --query-mode=simple -a "compiled_query_execution=True" -a "bytecode_handlers_path=./bytecode_handlers_ir.bc"', label: 'UnitTest (Simple, Compiled Execution)'
                         sh script: 'cd build && PYTHONPATH=.. timeout 20m python3 -m script.testing.junit --build-type=release --query-mode=extended', label: 'UnitTest (Extended)'
+                        sh script: 'cd build && PYTHONPATH=.. timeout 60m python3 -m script.testing.junit --build-type=release --query-mode=extended -a "compiled_query_execution=True" -a "bytecode_handlers_path=./bytecode_handlers_ir.bc"', label: 'UnitTest (Extended, Compiled Execution)'
                         sh script: 'sudo lsof -i -P -n | grep LISTEN || true', label: 'Check ports.'
                         sh 'cd build && timeout 1h ninja check-tpl'
                         sh 'cd build && timeout 1h ninja jumbotests'

build-support/tpl_bytecode_handlers_ir_compiler.py

@@ -1,67 +1,115 @@
 #!/usr/bin/env python3
-"""
-Usage: tpl_bytecode_handlers_ir_compiler.py LLVM_COMPILER CMAKE_BINARY_DIR BCH_CPP BCH_OUT
-    where
-        LLVM_COMPILER       =   The path to clang++ on the system.
-        CMAKE_BINARY_DIR    =   The build directory, which must contain compile_commands.json.
-        BCH_CPP             =   The path to the bytecode_handlers_ip.cpp file to be compiled.
-        BCH_OUT             =   The output path for the compiled file.
-
-Compiles bytecode_handlers_ir.cpp with clang++.
-This script is necessary because it is not possible to mix compilers in a single CMake project
-without going through some superproject / externalproject shenanigans. Moreover, with "modern"
-target-based CMake, you do not have access to a convenient list of all the CMAKE_CXX_FLAGS
-that another compiler should compile with, and you do not have an easy way of extracting a
-full set of target properties for compilation.
-
-So instead, we parse compile_commands.json to get the flags that bytecode_handlers_ir.cpp
-should be compiled with. Note that we depend on the command being
-    /usr/bin/c++ (WE EXTRACT THIS) -o blah -c blah
-which is the case for at least CMake 3.16 on Ubuntu when generating for both Make and Ninja.
-
-Fortunately, we are only compiling with common flags which are shared by both gcc and clang.
-If this changes, we may need the above superproject / externalproject solutions.
-"""
+#
+# tpl_bytecode_handlers_ir_compiler.py
+# Compiles bytecode_handlers_ir.cpp with clang++.
+#
+# Usage: 
+#
+#   tpl_bytecode_handlers_ir_compiler.py LLVM_COMPILER CMAKE_BINARY_DIR BCH_CPP BCH_OUT
+#    
+#   where
+#         LLVM_COMPILER       =   The path to clang++ on the system.
+#         CMAKE_BINARY_DIR    =   The build directory, which must contain compile_commands.json.
+#         BCH_CPP             =   The path to the bytecode_handlers_ip.cpp file to be compiled.
+#         BCH_OUT             =   The output path for the compiled file.
+#
+#
+# This script is necessary because it is not possible to mix compilers in a single CMake project
+# without going through some superproject / externalproject shenanigans. Moreover, with "modern"
+# target-based CMake, you do not have access to a convenient list of all the CMAKE_CXX_FLAGS
+# that another compiler should compile with, and you do not have an easy way of extracting a
+# full set of target properties for compilation.
+#
+# So instead, we parse compile_commands.json to get the flags that bytecode_handlers_ir.cpp
+# should be compiled with. Note that we depend on the command being
+#     /usr/bin/c++ (WE EXTRACT THIS) -o blah -c blah
+# which is the case for at least CMake 3.16 on Ubuntu when generating for both Make and Ninja.
+#
+# Fortunately, we are only compiling with common flags which are shared by both gcc and clang.
+# If this changes, we may need the above superproject / externalproject solutions.
 
 import os
-import subprocess
 import sys
+import subprocess
+from typing import List
+
+# Script exit codes
+EXIT_SUCCESS =  0
+EXIT_FAILURE = -1
 
 PROGRAM_NAME = sys.argv[0]
+
 # The Clang compiler that will emit LLVM.
 PATH_TO_LLVM_COMPILER = sys.argv[1]
+
 # cd to the build directory, which should have a compile_commands.json file.
 PATH_TO_CMAKE_BINARY_DIR = sys.argv[2]
+
 # The bytecode_handlers_ir.cpp file to be compiled.
 PATH_TO_BCH_CPP = sys.argv[3]
+
 # The output path and filename.
 PATH_TO_BCH_OUT = sys.argv[4]
 
+# Those flags that we do not want passed through to
+# clang++ for compilation of the bytecode handlers
+FLAG_BLACKLIST = [
+    "",
+    "--coverage",          # Relevant?
+    "-fPIC",               # Relevant?
+    "-ggdb",               # No need for debug symbols
+    "-fsanitize=address"   # Don't want ASAN instrumentation
+]
+
+# Those flags that we want to transform in some way
+# as they are 
+FLAG_TRANSFORMS = {
+    "-O0": "-O3",  # Always optimize
+}
+
+def apply_transform(flag: str) -> str:
+    """
+    Apply any transformations defined for the flag if they
+    are present; otherwise return the flag unmodified.
+    :param flag The input flag
+    :return The flag with transformation applied
+    """
+    return FLAG_TRANSFORMS[flag] if flag in FLAG_TRANSFORMS else flag
 
-def GetClangFlags():
-    prev = ''
-    with open('compile_commands.json') as f:
+def get_clang_flags() -> List[str]:
+    """
+    Compute the flags passed to clang++ to compile the bytecodes.
+    :return A list of the flags to pass to clang++ (strings)
+    """
+    prev = ""
+    with open("compile_commands.json") as f:
         for line in f:
-            # Look for the line that ends with tpl.cpp". The preceding line should be the compilation command.
+            # Look for the line that ends with bytecode_handlers_ir.cpp".
+            # The preceding line should be the compilation command.
             if line.endswith('bytecode_handlers_ir.cpp"\n'):
                 command = prev
                 # Some magic parsing logic. I hate this.
                 _, _, _, command, _ = command.split('"')
                 # Remove the compiler (idx 0) and executable (-o blah -c blahblah).
                 command = command.split(' ')[1:-4]
                 # Return the compile command.
-                return [x for x in command if x != '' and x != '--coverage' and x != '-fPIC']
+                return [apply_transform(c) for c in filter(lambda x: x not in FLAG_BLACKLIST, command)]
+
             # Record the line for the next iteration.
             prev = line
     raise Exception("Could not find bytecode_handlers_ir.cpp in compile_commands.json.")
 
-
-if __name__ == '__main__':
+def main() -> int:
     os.chdir(PATH_TO_CMAKE_BINARY_DIR)
-    call = [PATH_TO_LLVM_COMPILER] + GetClangFlags() + ["-emit-llvm", "-o", PATH_TO_BCH_OUT, "-c", PATH_TO_BCH_CPP]
-    call = ' '.join(call)
+    call = [PATH_TO_LLVM_COMPILER] + get_clang_flags() + ["-emit-llvm", "-o", PATH_TO_BCH_OUT, "-c", PATH_TO_BCH_CPP]
+    call = " ".join(call)
     try:
         subprocess.check_call(call, shell=True)
-        print('{} invoked: {}'.format(PROGRAM_NAME, call))
+        print("{} invoked: {}".format(PROGRAM_NAME, call))
     except:
-        sys.exit(-1)
+        return EXIT_FAILURE
+
+    return EXIT_SUCCESS
+
+if __name__ == "__main__":
+    sys.exit(main())