rebase, fix compile issue

CMakeLists.txt

@@ -13,7 +13,7 @@
 # limitations under the License.
 cmake_minimum_required(VERSION 3.10)
 set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMake" ${CMAKE_MODULE_PATH})
-
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 # set the project name
 project(velox)
 

scripts/setup-helper-functions.sh

@@ -100,7 +100,7 @@ function get_cxx_flags {
     ;;
 
     "avx")
-      echo -n "-mavx2 -mfma -mavx -mf16c -mlzcnt -std=c++17"
+      echo -n "-mavx2 -mfma -mavx -mf16c -mlzcnt -std=c++17 -mbmi2 -mbmi"
     ;;
 
     "sse")

scripts/setup-ubuntu.sh

@@ -47,13 +47,13 @@ sudo --preserve-env apt install -y \
   liblz4-dev \
   libzstd-dev \
   libre2-dev \
-  libsnappy-dev \
-  liblzo2-dev \
-  protobuf-compiler \
   bison \
   flex \
-  tzdata
-
+  tzdata \
+  libsnappy-dev \
+  liblzo2-dev \
+  protobuf-compiler
+
 function run_and_time {
   time "$@"
   { echo "+ Finished running $*"; } 2> /dev/null

velox/connectors/hive/CMakeLists.txt

@@ -14,7 +14,7 @@
 
 add_library(velox_hive_connector HiveConnector.cpp FileHandle.cpp)
 
-target_link_libraries(velox_hive_connector velox_connector
+target_link_libraries(velox_hive_connector velox_connector velox_dwio_common_exception
                       velox_dwio_dwrf_reader velox_dwio_dwrf_writer velox_file)
 
 add_library(velox_hive_partition_function HivePartitionFunction.cpp)

velox/substrait/CMakeLists.txt

@@ -41,8 +41,18 @@ add_custom_command(
 add_custom_target(substrait_proto ALL DEPENDS ${PROTO_OUTPUT_FILES})
 add_dependencies(substrait_proto protobuf::libprotobuf)
 
-set(SRCS ${PROTO_SRCS} SubstraitUtils.cpp SubstraitToVeloxPlanValidator.cpp
-         SubstraitToVeloxExpr.cpp SubstraitToVeloxPlan.cpp TypeUtils.cpp VectorCreater.cpp)
+set(SRCS
+    ${PROTO_SRCS}
+    SubstraitParser.cpp
+    SubstraitToVeloxExpr.cpp
+    SubstraitToVeloxPlan.cpp
+    TypeUtils.cpp
+    VeloxToSubstraitExpr.cpp
+    VeloxToSubstraitPlan.cpp
+    VeloxToSubstraitType.cpp
+    SubstraitToVeloxPlanValidator.cpp
+    VectorCreater.cpp)
+
 add_library(velox_substrait_plan_converter ${SRCS})
 target_include_directories(velox_substrait_plan_converter
                            PUBLIC ${PROTO_OUTPUT_DIR})

velox/substrait/SubstraitParser.cpp

@@ -27,7 +27,7 @@ std::shared_ptr<SubstraitParser::SubstraitType> SubstraitParser::parseType(
   switch (substraitType.kind_case()) {
     case ::substrait::Type::KindCase::kBool: {
       typeName = "BOOLEAN";
-      nullability = sType.bool_().nullability();
+      nullability = substraitType.bool_().nullability();
       break;
     }
     case ::substrait::Type::KindCase::kI8: {
@@ -96,7 +96,7 @@ std::shared_ptr<SubstraitParser::SubstraitType> SubstraitParser::parseType(
     }
     case ::substrait::Type::KindCase::kDate: {
       typeName = "DATE";
-      nullability = sType.date().nullability();
+      nullability = substraitType.date().nullability();
       break;
     }
     default:
@@ -198,39 +198,14 @@ std::string SubstraitParser::findSubstraitFuncSpec(
   return map[id];
 }
 
-std::string SubstraitParser::getFunctionName(
-    const std::string& functionSpec) const {
+std::string SubstraitParser::getSubFunctionName(
+    const std::string& subFuncSpec) const {
   // Get the position of ":" in the function name.
-  std::size_t pos = functionSpec.find(":");
+  std::size_t pos = subFuncSpec.find(":");
   if (pos == std::string::npos) {
-    return functionSpec;
+    return subFuncSpec;
   }
-  return functionSpec.substr(0, pos);
-}
-
-void SubstraitParser::getFunctionTypes(
-    const std::string& functionSpec,
-    std::vector<std::string>& types) const {
-  types.clear();
-  // Get the position of ":" in the function name.
-  std::size_t pos = functionSpec.find(":");
-  // Get the parameter types.
-  std::string funcTypes;
-  if (pos == std::string::npos) {
-    return;
-  } else {
-    if (pos == functionSpec.size() - 1) {
-      return;
-    }
-    funcTypes = functionSpec.substr(pos + 1);
-  }
-  // Split the types with delimiter.
-  std::string delimiter = "_";
-  while ((pos = funcTypes.find(delimiter)) != std::string::npos) {
-    types.emplace_back(funcTypes.substr(0, pos));
-    funcTypes.erase(0, pos + delimiter.length());
-  }
-  types.emplace_back(funcTypes);
+  return subFuncSpec.substr(0, pos);
 }
 
 void SubstraitParser::getSubFunctionTypes(
@@ -260,8 +235,8 @@ void SubstraitParser::getSubFunctionTypes(
 std::string SubstraitParser::findVeloxFunction(
     const std::unordered_map<uint64_t, std::string>& functionMap,
     uint64_t id) const {
-  std::string funcSpec = findFunctionSpec(functionMap, id);
-  std::string funcName = getFunctionName(funcSpec);
+  std::string funcSpec = findSubstraitFuncSpec(functionMap, id);
+  std::string funcName = getSubFunctionName(funcSpec);
   return mapToVeloxFunction(funcName);
 }
 
@@ -274,7 +249,7 @@ std::string SubstraitParser::mapToVeloxFunction(
 
   // If not finding the mapping from Substrait function name to Velox function
   // name, the original Substrait function name will be used.
-  return substraitFunction;
+  return subFunc;
 }
 
 } // namespace facebook::velox::substrait

velox/substrait/SubstraitParser.h

@@ -65,18 +65,13 @@ class SubstraitParser {
   /// Currently, the input types in the function specification are not used. But
   /// in the future, they should be used for the validation according the
   /// specifications in Substrait yaml files.
-  const std::string& findFunctionSpec(
+  std::string findSubstraitFuncSpec(
       const std::unordered_map<uint64_t, std::string>& functionMap,
       uint64_t id) const;
 
   /// Extracts the function name for a function from specified compound name.
   /// When the input is a simple name, it will be returned.
-  std::string getFunctionName(const std::string& functionSpec) const;
-
-  /// Extracts argument types for a function from specified compound name.
-  void getFunctionTypes(
-      const std::string& functionSpec,
-      std::vector<std::string>& types) const;
+  std::string getSubFunctionName(const std::string& functionSpec) const;
 
   /// This function is used get the types from the compound name.
   void getSubFunctionTypes(