adding hdfs support

Signed-off-by: Yuan Zhou <[email protected]>
apache · May 12, 2022 · cec6b77 · cec6b77
1 parent 3bbbf86
commit cec6b77
Show file tree

Hide file tree

Showing 3 changed files with 38 additions and 1 deletion.
diff --git a/cpp/velox/CMakeLists.txt b/cpp/velox/CMakeLists.txt
@@ -41,6 +41,7 @@ set(VELOX_EXEC_LIB_PATH "${VELOX_BUILD_PATH}/exec/libvelox_exec.a")
 set(VELOX_CONNECTOR_LIB_PATH "${VELOX_BUILD_PATH}/connectors/libvelox_connector.a")
 set(VELOX_HIVE_PART_LIB_PATH "${VELOX_BUILD_PATH}/connectors/hive/libvelox_hive_partition_function.a")
 set(VELOX_HIVE_CONNECTOR_LIB_PATH "${VELOX_BUILD_PATH}/connectors/hive/libvelox_hive_connector.a")
+set(VELOX_FILESYSTEM_HDFS_LIB_PATH "${VELOX_BUILD_PATH}/connectors/hive/storage_adapters/hdfs/libvelox_hdfs.a")
 set(VELOX_DWIO_COMMON_LIB_PATH "${VELOX_BUILD_PATH}/dwio/common/libvelox_dwio_common.a")
 set(VELOX_DWIO_DWRF_PROTO_LIB_PATH "${VELOX_BUILD_PATH}/dwio/dwrf/proto/libvelox_dwio_dwrf_proto.a")
 set(VELOX_CORE_LIB_PATH "${VELOX_BUILD_PATH}/core/libvelox_core.a")
@@ -170,6 +171,19 @@ macro(find_snappy)
   set(SNAPPY_LIB_PATH "${SNAPPY_LIBRARY}")
 endmacro()
 
+# Find Snappy
+macro(find_libhdfs3)
+  # Find the snappy library
+  set(CMAKE_FIND_LIBRARY_SUFFIXES ".so")
+  find_library(LIBHDFS3_LIBRARY NAMES hdfs3 PATHS ${SYSTEM_LIB_PATH} ${SYSTEM_LIB64_PATH} ${SYSTEM_LIB_MULTIARCH_PATH} ${SYSTEM_LOCAL_LIB_PATH} ${SYSTEM_LOCAL_LIB64_PATH} NO_DEFAULT_PATH)
+  if (NOT LIBHDFS3_LIBRARY)
+     message(FATAL_ERROR "LIBHDFS3 Library Not Found")
+  else()
+    message(STATUS "LIBHDFS3 Library Can Be Found in ${LIBHDFS3_LIBRARY}")
+  endif()
+  set(LIBHDFS3_LIB_PATH "${LIBHDFS3_LIBRARY}")
+endmacro()
+
 find_package(JNI REQUIRED)
 
 macro(build_velox_exec)
@@ -178,6 +192,7 @@ macro(build_velox_exec)
   add_library(facebook::velox::exec STATIC IMPORTED)
   add_library(facebook::velox::connector STATIC IMPORTED)
   add_library(facebook::velox::connector::hive::part STATIC IMPORTED)
+  add_library(facebook::velox::connector::hive::storage_adapters::hdfs STATIC IMPORTED)
   add_library(facebook::velox::dwio::common STATIC IMPORTED)
   add_library(facebook::velox::dwio::dwrf::proto STATIC IMPORTED)
   add_library(facebook::velox::core::config STATIC IMPORTED)
@@ -220,6 +235,7 @@ macro(build_velox_exec)
   add_library(iberty STATIC IMPORTED)
   add_library(doubleconversion SHARED IMPORTED)
   add_library(snappy SHARED IMPORTED)
+  add_library(libhdfs3 SHARED IMPORTED)
   add_library(glog SHARED IMPORTED)
   add_library(fmt STATIC IMPORTED)
   add_library(gtest SHARED IMPORTED)
@@ -300,6 +316,10 @@ macro(build_velox_exec)
                         PROPERTIES IMPORTED_LOCATION "${SNAPPY_LIB_PATH}"
                                    INTERFACE_INCLUDE_DIRECTORIES
                                    "${BINARY_RELEASE_DIR}/include")
+  set_target_properties(libhdfs3
+	                PROPERTIES IMPORTED_LOCATION "${LIBHDFS3_LIB_PATH}"
+                                   INTERFACE_INCLUDE_DIRECTORIES
+                                   "${BINARY_RELEASE_DIR}/include")
   set_target_properties(facebook::velox::common::memory
                         PROPERTIES IMPORTED_LOCATION "${VELOX_COMMON_MEM_LIB_PATH}"
                                    INTERFACE_INCLUDE_DIRECTORIES
@@ -416,6 +436,11 @@ macro(build_velox_exec)
                         PROPERTIES IMPORTED_LOCATION "${VELOX_SUBSTRAIT_LIB_PATH}"
                                    INTERFACE_INCLUDE_DIRECTORIES
                                    "${BINARY_RELEASE_DIR}/include")
+  set_target_properties(facebook::velox::connector::hive::storage_adapters::hdfs
+	                PROPERTIES IMPORTED_LOCATION "${VELOX_FILESYSTEM_HDFS_LIB_PATH}"
+                                   INTERFACE_INCLUDE_DIRECTORIES
+                                   "${BINARY_RELEASE_DIR}/include")
+
 
   target_link_libraries(velox
                         LINK_PUBLIC spark_columnar_jni
@@ -424,6 +449,7 @@ macro(build_velox_exec)
                         LINK_PUBLIC facebook::velox::substrait
                         LINK_PUBLIC facebook::velox::connector
                         LINK_PUBLIC facebook::velox::connector::hive::part
+                        LINK_PUBLIC facebook::velox::connector::hive::storage_adapters::hdfs
                         LINK_PUBLIC facebook::velox::dwio::dwrf::proto
                         LINK_PUBLIC facebook::velox::functions::prestosql
                         LINK_PUBLIC facebook::velox::functions::prestosql::impl
@@ -465,6 +491,7 @@ macro(build_velox_exec)
                         LINK_PUBLIC iberty
                         LINK_PUBLIC doubleconversion
                         LINK_PUBLIC snappy
+                        LINK_PUBLIC libhdfs3
                         LINK_PUBLIC glog
                         LINK_PUBLIC fmt)
 endmacro()
@@ -491,4 +518,5 @@ find_double_conversion()
 find_fmt()
 find_gtest()
 find_snappy()
+find_libhdfs3()
 build_velox_exec()
diff --git a/cpp/velox/compute/VeloxPlanConverter.cc b/cpp/velox/compute/VeloxPlanConverter.cc
@@ -39,12 +39,20 @@ namespace compute {
 void VeloxInitializer::Init() {
   // Setup and register.
   filesystems::registerLocalFileSystem();
+
+  // register HDFS
+  filesystems::registerHdfsFileSystem();
+  std::unordered_map<std::string, std::string> configurationValues(
+      {{"hdfs_host", "sr595"}, {"hdfs_port", "9000"}});
+  auto memConfig =
+      std::make_shared<const core::MemConfig>(std::move(configurationValues));
+
   std::unique_ptr<folly::IOThreadPoolExecutor> executor =
       std::make_unique<folly::IOThreadPoolExecutor>(1);
   // auto hiveConnectorFactory = std::make_shared<hive::HiveConnectorFactory>();
   // registerConnectorFactory(hiveConnectorFactory);
   auto hiveConnector = getConnectorFactory("hive")->newConnector(
-      "hive-connector", nullptr, nullptr, executor.get());
+      "hive-connector", memConfig, nullptr, executor.get());
   registerConnector(hiveConnector);
   dwrf::registerDwrfReaderFactory();
   // Register Velox functions

diff --git a/cpp/velox/compute/VeloxPlanConverter.h b/cpp/velox/compute/VeloxPlanConverter.h
@@ -42,6 +42,7 @@
 #include "velox/connectors/hive/FileHandle.h"
 #include "velox/connectors/hive/HiveConnector.h"
 #include "velox/connectors/hive/HiveConnectorSplit.h"
+#include "velox/connectors/hive/storage_adapters/hdfs/HdfsFileSystem.h"
 #include "velox/core/Expressions.h"
 #include "velox/core/ITypedExpr.h"
 #include "velox/core/PlanNode.h"