Update build dependencies (#185)

Updated dependencies:

libhdfs3: Try find_package first
gflags: Allow static library
xsimd, gtest: Allow finding from system
Arrow, Parquet: Prefer external build if -D<package>_DIR=/path/to/arrow/install is defined
flex 2.5.13 -> 2.6.0 (2.6.0 removed register keyword)
Compat boost >= 1.79

Author: ccat3z

CMakeLists.txt

@@ -171,10 +171,15 @@ if(VELOX_ENABLE_S3)
 endif()
 
 if(VELOX_ENABLE_HDFS)
-  find_library(
-    LIBHDFS3
-    NAMES libhdfs3.so libhdfs3.dylib
-    HINTS "${CMAKE_SOURCE_DIR}/hawq/depends/libhdfs3/_build/src/" REQUIRED)
+  find_package(libhdfs3)
+  if(libhdfs3_FOUND AND TARGET HDFS::hdfs3)
+    set(LIBHDFS3 HDFS::hdfs3)
+  else()
+    find_library(
+      LIBHDFS3
+      NAMES libhdfs3.so libhdfs3.dylib
+      HINTS "${CMAKE_SOURCE_DIR}/hawq/depends/libhdfs3/_build/src/" REQUIRED)
+  endif()
   add_definitions(-DVELOX_ENABLE_HDFS3)
 endif()
 
@@ -329,7 +334,7 @@ resolve_dependency(Boost 1.66.0 COMPONENTS ${BOOST_INCLUDE_LIBRARIES})
 # for reference. find_package(range-v3)
 
 set_source(gflags)
-resolve_dependency(gflags COMPONENTS shared)
+resolve_dependency(gflags)
 if(NOT TARGET gflags::gflags)
   # This is a bit convoluted, but we want to be able to use gflags::gflags as a
   # target even when velox is built as a subproject which uses
@@ -469,7 +474,7 @@ if(CMAKE_HOST_SYSTEM_NAME MATCHES "Darwin")
   endif()
 endif()
 find_package(BISON 3.0.4 REQUIRED)
-find_package(FLEX 2.5.13 REQUIRED)
+find_package(FLEX 2.6.0 REQUIRED) # for cxx17
 
 include_directories(SYSTEM velox)
 include_directories(SYSTEM velox/external)
@@ -478,14 +483,17 @@ include_directories(SYSTEM velox/external/duckdb/tpch/dbgen/include)
 
 # these were previously vendored in third-party/
 if(NOT VELOX_DISABLE_GOOGLETEST)
-  set(gtest_SOURCE BUNDLED)
-  resolve_dependency(gtest)
-  set(VELOX_GTEST_INCUDE_DIR
-      "${gtest_SOURCE_DIR}/googletest/include"
-      PARENT_SCOPE)
+  set_source(GTest)
+  resolve_dependency(GTest)
+  foreach(tgt gtest gtest_main gmock gmock_main)
+      if (NOT TARGET ${tgt} AND TARGET GTest::${tgt})
+          add_library(${tgt} INTERFACE IMPORTED)
+          target_link_libraries(${tgt} INTERFACE GTest::${tgt})
+      endif() 
+  endforeach(tgt)
 endif()
 
-set(xsimd_SOURCE BUNDLED)
+set_source(xsimd)
 resolve_dependency(xsimd)
 
 include(CTest) # include after project() but before add_subdirectory()

third_party/CMakeLists.txt

@@ -24,6 +24,35 @@ if(VELOX_ENABLE_ARROW)
   else()
     set(THRIFT_SOURCE "BUNDLED")
   endif()
+
+  # Use external arrow & parquet only if <package>_DIR is defined
+  if(Thrift_FOUND AND DEFINED Arrow_DIR AND DEFINED Parquet_DIR)
+    find_package(Arrow PATHS "${Arrow_DIR}" NO_DEFAULT_PATH)
+    find_package(Parquet PATHS "${Parquet_DIR}" NO_DEFAULT_PATH)
+    if(Arrow_FOUND AND Parquet_FOUND)
+      add_library(arrow INTERFACE)
+      add_library(parquet INTERFACE)
+      add_library(thrift INTERFACE)
+
+      if(TARGET Arrow::arrow_static)
+        target_link_libraries(arrow INTERFACE Arrow::arrow_static)
+      else()
+        target_link_libraries(arrow INTERFACE Arrow::arrow_shared)
+      endif()
+
+      if(TARGET Parquet::parquet_static)
+        target_link_libraries(parquet INTERFACE Parquet::parquet_static)
+      else()
+        target_link_libraries(parquet INTERFACE Parquet::parquet_shared)
+      endif()
+
+      target_link_libraries(thrift INTERFACE thrift::thrift)
+
+      message(STATUS "Using external arrow")
+      return()
+    endif()
+  endif()
+
   set(ARROW_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep")
   set(ARROW_CMAKE_ARGS
       -DARROW_PARQUET=ON

velox/connectors/hive/storage_adapters/hdfs/tests/HdfsMiniCluster.cpp

@@ -72,7 +72,7 @@ HdfsMiniCluster::HdfsMiniCluster() {
         "Failed to find minicluster executable {}'", miniClusterExecutableName);
   }
   boost::filesystem::path hadoopHomeDirectory = exePath_;
-  hadoopHomeDirectory.remove_leaf().remove_leaf();
+  hadoopHomeDirectory.remove_filename().remove_filename();
   setupEnvironment(hadoopHomeDirectory.string());
 }
 

velox/type/UnscaledLongDecimal.h

@@ -217,7 +217,9 @@ struct UnscaledLongDecimal {
 
   static inline UnscaledLongDecimal convert(int256_t in, bool* overflow) {
     int128_t result;
+#ifndef INT128_MAX
     int128_t INT128_MAX = int128_t(int128_t(-1L)) >> 1;
+#endif
     constexpr int256_t UINT128_MASK = std::numeric_limits<uint128_t>::max();
 
     int256_t inAbs = abs(in);