dylib functionality + namespace wrapper and other review comments + s…

…uccessful mac & linux compilation

velox/common/CMakeLists.txt

@@ -15,6 +15,7 @@ add_subdirectory(base)
 add_subdirectory(caching)
 add_subdirectory(compression)
 add_subdirectory(config)
+add_subdirectory(dynamic_registry)
 add_subdirectory(encode)
 add_subdirectory(file)
 add_subdirectory(hyperloglog)

velox/common/dynamic_registry/CMakeLists.txt

@@ -0,0 +1,21 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+velox_add_library(velox_dynamic_library_loader DynamicLibraryLoader.cpp)
+
+velox_link_libraries(velox_dynamic_library_loader PRIVATE velox_exception)
+
+if(${VELOX_BUILD_TESTING})
+  add_subdirectory(tests)
+endif()

velox/common/dynamic_registry/DynamicLibraryLoader.cpp

@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <dlfcn.h>
+#include <iostream>
+#include "velox/common/base/Exceptions.h"
+
+namespace facebook::velox {
+
+static constexpr const char* kSymbolName = "registry";
+
+void loadDynamicLibrary(const char* fileName) {
+  // Try to dynamically load the shared library.
+  void* handler = dlopen(fileName, RTLD_NOW);
+
+  if (handler == nullptr) {
+    VELOX_USER_FAIL("Error while loading shared library: {}", dlerror());
+  }
+
+  // Lookup the symbol.
+  void* registrySymbol = dlsym(handler, kSymbolName);
+  auto loadUserLibrary = reinterpret_cast<void (*)()>(registrySymbol);
+  const char* error = dlerror();
+
+  if (error != nullptr) {
+    VELOX_USER_FAIL("Couldn't find Velox registry symbol: {}", error);
+  }
+  // Invoke the registry function.
+  loadUserLibrary();
+  LOG(INFO) << "Loaded: " << fileName;
+}
+
+} // namespace facebook::velox

velox/common/dynamic_registry/DynamicLibraryLoader.h

@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+namespace facebook::velox {
+
+/// Dynamically opens and registers functions defined in a shared library.
+///
+/// The library being linked needs to provide a function with the following
+/// signature:
+///
+/// void registry();
+///
+/// The registration function needs to be defined in the root-level namespace,
+/// and be enclosed by a extern "C" directive to prevent the compiler from
+/// mangling the symbol name.
+
+/// The function uses dlopen to load the shared library.
+/// It then searches for the "void registry()" C function which typically
+/// contains all the registration code for the UDFs defined in library. After
+/// locating the function it executes the registration bringing the
+/// user-defined Velox components such as function in the scope of the
+/// Velox runtime.
+///
+/// Loading a library twice can cause a components to be registered twice.
+/// This can fail for certain Velox components.
+
+void loadDynamicLibrary(const char* fileName);
+
+} // namespace facebook::velox

velox/common/dynamic_registry/DynamicUdf.h

@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// This header must be included in any dynamically loaded user-defined
+// function library. It enforces the correct function declaration with
+// the proper signature.
+#pragma once
+
+#include "velox/functions/Udf.h"
+
+extern "C" {
+// The "registry()" declaration and "check()" function ensure the correct
+// registry signature function is defined by the user.
+void registry();
+void check() {
+  registry();
+}
+}

velox/common/dynamic_registry/README.md

@@ -0,0 +1,34 @@
+# Dynamic Loading of Velox Extensions
+
+This generic utility adds extensibility features to load User Defined Functions (UDFs), connectors, or types without having to fork and build Velox, through the use of shared libraries.
+
+## Getting started
+1. Create a cpp file for your dynamic library
+For dynamically loaded function registration, the format followed is mirrored of that of built-in function registration with some noted differences. Using [MyDynamicTestFunction.cpp](tests/MyDynamicTestFunction.cpp) as an example, the function uses the extern "C" keyword to protect against name mangling. A registry() function call is also necessary here. 
+Make sure to also include the necessary header file:
+    ```
+    #include "velox/functions/DynamicUdf.h"
+    ```
+
+2. Register functions dynamically by creating .dylib (MacOS) or .so (Linux) shared libraries.
+These shared libraries may be made using CMakeLists like the following:
+
+    ```
+        add_library(name_of_dynamic_fn SHARED TestFunction.cpp)
+        target_link_libraries(name_of_dynamic_fn PRIVATE fmt::fmt glog::glog xsimd)
+        target_link_options(name_of_dynamic_fn PRIVATE "-Wl,-undefined,dynamic_lookup")
+    ```
+    Above, the `fmt::fmt` and `xsimd` libraries are required for all necessary symbols 
+    to be defined when loading the `TestFunction.cpp` dynamically. 
+    Additionally `glog::glog` is currently required on MacOs. 
+    The `target_link_options` allows for symbols to be resolved at runtime on MacOS. 
+
+    On Linux `glog::glog` and the `target_link_options` are optional.
+
+## Notes
+    In Velox, a function's signature is determined solely by its name and argument types.
+    Return type is not taken into account. As a result, if a function with an identical
+    signature is added but with a different return type, it will overwrite the existing function.
+
+    Function overloading is supported therefore multiple functions can share the same name as
+    long as they differ in the number or types of arguments.

velox/common/dynamic_registry/tests/CMakeLists.txt

@@ -0,0 +1,93 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# To test functions being added by dynamically linked libraries, we compile
+# `MyDynamicFunction.cpp` as a small .so library, and use the
+# VELOX_TEST_DYNAMIC_LIBRARY_PATH macro to locate the .so binary.
+
+add_library(velox_function_my_dynamic SHARED MyDynamicFunction.cpp)
+add_library(velox_overwrite_int_function_my_dynamic SHARED
+            MyDynamicIntFunctionOverwrite.cpp)
+add_library(velox_overwrite_varchar_function_my_dynamic SHARED
+            MyDynamicVarcharFunctionOverwrite.cpp)
+add_library(velox_function_err_my_dynamic SHARED MyDynamicErrFunction.cpp)
+add_library(velox_overload_int_function_my_dynamic SHARED
+            MyDynamicIntFunctionOverload.cpp)
+add_library(velox_overload_varchar_function_my_dynamic SHARED
+            MyDynamicVarcharFunctionOverload.cpp)
+
+set(CMAKE_DYLIB_TEST_LINK_LIBRARIES fmt::fmt glog::glog xsimd)
+
+target_link_libraries(
+  velox_function_my_dynamic
+  PRIVATE ${CMAKE_DYLIB_TEST_LINK_LIBRARIES})
+
+target_link_libraries(
+  velox_overwrite_int_function_my_dynamic
+  PRIVATE ${CMAKE_DYLIB_TEST_LINK_LIBRARIES})
+
+target_link_libraries(
+  velox_overwrite_varchar_function_my_dynamic
+  PRIVATE ${CMAKE_DYLIB_TEST_LINK_LIBRARIES})
+
+target_link_libraries(
+  velox_function_err_my_dynamic
+  PRIVATE ${CMAKE_DYLIB_TEST_LINK_LIBRARIES})
+
+target_link_libraries(
+  velox_overload_int_function_my_dynamic
+  PRIVATE ${CMAKE_DYLIB_TEST_LINK_LIBRARIES})
+
+target_link_libraries(
+  velox_overload_varchar_function_my_dynamic
+  PRIVATE ${CMAKE_DYLIB_TEST_LINK_LIBRARIES})
+
+set(COMMON_LIBRARY_LINK_OPTIONS "-Wl,-undefined,dynamic_lookup")
+
+target_link_options(velox_function_my_dynamic PRIVATE
+                    ${COMMON_LIBRARY_LINK_OPTIONS})
+target_link_options(velox_overwrite_int_function_my_dynamic PRIVATE
+                    ${COMMON_LIBRARY_LINK_OPTIONS})
+target_link_options(velox_overwrite_varchar_function_my_dynamic PRIVATE
+                    ${COMMON_LIBRARY_LINK_OPTIONS})
+target_link_options(velox_function_err_my_dynamic PRIVATE
+                    ${COMMON_LIBRARY_LINK_OPTIONS})
+target_link_options(velox_overload_int_function_my_dynamic PRIVATE
+                    ${COMMON_LIBRARY_LINK_OPTIONS})
+target_link_options(velox_overload_varchar_function_my_dynamic PRIVATE
+                    ${COMMON_LIBRARY_LINK_OPTIONS})
+
+# Here's the actual test which will dynamically load the library defined above.
+add_executable(velox_function_dynamic_link_test DynamicLinkTest.cpp)
+
+target_link_libraries(
+  velox_function_dynamic_link_test
+  velox_functions_test_lib
+  velox_dynamic_library_loader
+  velox_function_registry
+  xsimd
+  GTest::gmock
+  GTest::gtest
+  GTest::gtest_main)
+
+target_compile_definitions(
+  velox_function_dynamic_link_test
+  PRIVATE VELOX_TEST_DYNAMIC_LIBRARY_PATH="${CMAKE_CURRENT_BINARY_DIR}")
+target_compile_definitions(
+  velox_function_dynamic_link_test
+  PRIVATE
+    VELOX_TEST_DYNAMIC_LIBRARY_PATH_SUFFIX="${CMAKE_SHARED_LIBRARY_SUFFIX}")
+
+add_test(NAME velox_function_dynamic_link_test
+         COMMAND velox_function_dynamic_link_test)