dylib functionality + test + readme with updates from reviews

velox/common/CMakeLists.txt

@@ -15,6 +15,7 @@ add_subdirectory(base)
 add_subdirectory(caching)
 add_subdirectory(compression)
 add_subdirectory(config)
+add_subdirectory(dynamic_registry)
 add_subdirectory(encode)
 add_subdirectory(file)
 add_subdirectory(hyperloglog)

velox/common/dynamic_registry/CMakeLists.txt

@@ -0,0 +1,6 @@
+velox_add_library(velox_dynamic_function_loader DynamicLibraryLoader.cpp)
+velox_link_libraries(velox_dynamic_function_loader PRIVATE velox_exception)
+
+if(${VELOX_BUILD_TESTING})
+  add_subdirectory(tests)
+endif()

velox/common/dynamic_registry/DynamicLibraryLoader.cpp

@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <dlfcn.h>
+#include <iostream>
+#include "velox/common/base/Exceptions.h"
+
+namespace facebook::velox {
+
+static constexpr const char* kSymbolName = "registry";
+
+void loadDynamicLibrary(const char* fileName) {
+  // Try to dynamically load the shared library.
+  void* handler = dlopen(fileName, RTLD_NOW);
+
+  if (handler == nullptr) {
+    VELOX_USER_FAIL("Error while loading shared library: {}", dlerror());
+  }
+
+  // Lookup the symbol.
+  void* registrySymbol = dlsym(handler, kSymbolName);
+  auto registryItem = reinterpret_cast<void (*)()>(registrySymbol);
+  char* error = dlerror();
+
+  if (error != nullptr) {
+    VELOX_USER_FAIL("Couldn't find Velox registry symbol: {}", error);
+  }
+  // Invoke the registry function.
+  registryItem();
+  LOG(INFO) << "Loaded: " << fileName;
+}
+
+} // namespace facebook::velox

velox/common/dynamic_registry/DynamicLibraryLoader.h

@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+namespace facebook::velox {
+
+/// Dynamically opens and registers functions defined in a shared library (.so)
+///
+/// Given a shared library name (.so), this function will open it using dlopen,
+/// search for a "void registry()" C function containing the registration code
+/// for the functions defined in library, and execute it.
+///
+/// The library being linked needs to provide a function with the following
+/// signature:
+///
+///   void registry();
+///
+/// The registration function needs to be defined in the top-level namespace,
+/// and be enclosed by a extern "C" directive to prevent the compiler from
+/// mangling the symbol name.
+void loadDynamicLibrary(const char* fileName);
+
+} // namespace facebook::velox

velox/common/dynamic_registry/README.md

@@ -0,0 +1,15 @@
+# Velox: Dynamically Loading Registry Libraries in C++
+
+This generic utility adds extensibility features to load User Defined Functions (UDFs), connectors, or types without having to fork and build Velox, through the use of shared libraries.
+
+## Getting started
+1. Create a cpp file for your dynamic library
+For dynamically loaded function registration, the format followed is mirrored of that of built-in function registration with some noted differences. Using [MyDynamicTestFunction.cpp](tests/MyDynamicTestFunction.cpp) as an example, the function uses the extern "C" keyword to protect against name mangling. A registry() function call is also necessary here.
+
+2. Register functions dynamically by creating .dylib or .so shared libraries and dropping them in a plugin directory
+These shared libraries may be made using CMakeLists like the following:
+
+```
+add_library(name_of_dynamic_fn SHARED TestFunction.cpp)
+target_link_libraries(name_of_dynamic_fn PRIVATE xsimd fmt::fmt velox_expression)
+```

velox/common/dynamic_registry/tests/CMakeLists.txt

@@ -0,0 +1,44 @@
+# To test functions being added by dynamically linked libraries, we compile
+# `MyDynamicFunction.cpp` as a small .so library, and use the
+# VELOX_TEST_DYNAMIC_LIBRARY_PATH macro to locate the .so binary.
+
+add_library(velox_function_my_dynamic SHARED MyDynamicFunction.cpp)
+  target_link_libraries(
+    velox_function_my_dynamic xsimd fmt)
+
+add_library(velox_int_function_my_dynamic SHARED MyDynamicIntFunction.cpp)
+target_link_libraries(
+  velox_int_function_my_dynamic xsimd fmt)
+
+add_library(velox_str_function_my_dynamic SHARED MyDynamicStrFunction.cpp)
+target_link_libraries(
+  velox_str_function_my_dynamic xsimd fmt)
+
+add_library(velox_function_same_twice_my_dynamic SHARED MyDynamicSameTwiceFunction.cpp)
+  target_link_libraries(
+    velox_function_same_twice_my_dynamic xsimd fmt)
+
+add_library(velox_function_err_my_dynamic SHARED MyDynamicErrFunction.cpp)
+    target_link_libraries(
+      velox_function_err_my_dynamic xsimd fmt)
+
+# Here's the actual test which will dynamically load the library defined above.
+add_executable(velox_function_dynamic_link_test DynamicLinkTest.cpp)
+
+target_compile_definitions(velox_function_dynamic_link_test PRIVATE
+  VELOX_TEST_DYNAMIC_LIBRARY_PATH="${CMAKE_CURRENT_BINARY_DIR}")
+target_compile_definitions(velox_function_dynamic_link_test PRIVATE
+  VELOX_TEST_DYNAMIC_LIBRARY_PATH_SUFFIX="${CMAKE_SHARED_LIBRARY_SUFFIX}")
+
+add_test(NAME velox_function_dynamic_link_test
+         COMMAND velox_function_dynamic_link_test)
+
+target_link_libraries(
+  velox_function_dynamic_link_test
+  velox_functions_test_lib
+  velox_dynamic_function_loader
+  velox_function_registry
+  xsimd
+  GTest::gmock
+  GTest::gtest
+  GTest::gtest_main)

velox/common/dynamic_registry/tests/DynamicLinkTest.cpp

@@ -0,0 +1,168 @@
+/*
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+#include "velox/common/base/tests/GTestUtils.h"
+#include "velox/common/dynamic_registry/DynamicLibraryLoader.h"
+#include "velox/functions/FunctionRegistry.h"
+#include "velox/functions/prestosql/tests/utils/FunctionBaseTest.h"
+
+namespace facebook::velox::functions::test {
+
+class DynamicLinkTest : public FunctionBaseTest {};
+
+TEST_F(DynamicLinkTest, dynamicLoadOneFunc) {
+  const auto dynamicFunction = [&](std::optional<double> a) {
+    return evaluateOnce<int64_t>("dynamic_1()", a);
+  };
+
+  auto signaturesBefore = getFunctionSignatures().size();
+
+  // Function does not exist yet.
+  VELOX_ASSERT_THROW(
+      dynamicFunction(0), "Scalar function doesn't exist: dynamic_1.");
+
+  // Dynamically load the library.
+  std::string libraryPath = VELOX_TEST_DYNAMIC_LIBRARY_PATH;
+  libraryPath.append("/libvelox_function_my_dynamic")
+  .append(VELOX_TEST_DYNAMIC_LIBRARY_PATH_SUFFIX);
+
+  loadDynamicLibrary(libraryPath.data());
+
+  auto signaturesAfter = getFunctionSignatures().size();
+  EXPECT_EQ(signaturesAfter, signaturesBefore + 1);
+
+  // Make sure the function exists now.
+  EXPECT_EQ(123, dynamicFunction(0));
+}
+
+TEST_F(DynamicLinkTest, dynamicLoadSameFuncTwice) {
+  const auto dynamicFunction = [&](std::optional<double> a) {
+    return evaluateOnce<int64_t>("dynamic_2()", a);
+  };
+  auto signaturesBefore = getFunctionSignatures().size();
+
+  // Function does not exist yet.
+  VELOX_ASSERT_THROW(
+      dynamicFunction(0), "Scalar function doesn't exist: dynamic_2.");
+
+  // Dynamically load the library.
+  std::string libraryPath = VELOX_TEST_DYNAMIC_LIBRARY_PATH;
+  libraryPath.append("/libvelox_function_same_twice_my_dynamic")
+  .append(VELOX_TEST_DYNAMIC_LIBRARY_PATH_SUFFIX);
+
+  loadDynamicLibrary(libraryPath.data());
+
+  auto signaturesAfterFirst = getFunctionSignatures().size();
+  EXPECT_EQ(signaturesAfterFirst, signaturesBefore + 1);
+
+  // Make sure the function exists now.
+  EXPECT_EQ(123, dynamicFunction(0));
+  // load same shared library again
+  loadDynamicLibrary(libraryPath.data());
+  auto signaturesAfterSecond = getFunctionSignatures().size();
+  // should have no change from the second attempt
+  EXPECT_EQ(signaturesAfterSecond, signaturesAfterFirst);
+}
+
+TEST_F(DynamicLinkTest, dynamicLoadTwoOfTheSameName) {
+  const auto dynamicFunctionInt = [&](std::optional<double> a) {
+    return evaluateOnce<int64_t>("dynamic_3()", a);
+  };
+  const auto dynamicFunctionStr = [&](std::optional<std::string> a) {
+    return evaluateOnce<std::string>("dynamic_3()", a);
+  };
+
+  auto signaturesBefore = getFunctionSignatures().size();
+
+  // Function does not exist yet.
+  VELOX_ASSERT_THROW(
+      dynamicFunctionStr("0"), "Scalar function doesn't exist: dynamic_3.");
+
+  // Dynamically load the library.
+  std::string libraryPathStr = VELOX_TEST_DYNAMIC_LIBRARY_PATH;
+ libraryPathStr.append("/libvelox_str_function_my_dynamic")
+ .append(VELOX_TEST_DYNAMIC_LIBRARY_PATH_SUFFIX);
+
+  loadDynamicLibrary(libraryPathStr.data());
+
+  auto signaturesAfterFirst = getFunctionSignatures().size();
+  EXPECT_EQ(signaturesAfterFirst, signaturesBefore + 1);
+
+  // Make sure the function exists now.
+  EXPECT_EQ("123", dynamicFunctionStr("0"));
+
+  // Function does not exist yet.
+  VELOX_ASSERT_THROW(
+    dynamicFunctionInt(0), 
+    "Expression evaluation result is not of expected type: dynamic_3() -> CONSTANT vector of type VARCHAR");
+
+  // Dynamically load the library.
+  std::string libraryPathInt = VELOX_TEST_DYNAMIC_LIBRARY_PATH;
+  libraryPathInt.append("/libvelox_int_function_my_dynamic")
+  .append(VELOX_TEST_DYNAMIC_LIBRARY_PATH_SUFFIX);
+
+  loadDynamicLibrary(libraryPathInt.data());
+  // confirm the first function loaded got rewritten
+  VELOX_ASSERT_THROW(
+    dynamicFunctionStr("0"), 
+    "Expression evaluation result is not of expected type: dynamic_3() -> CONSTANT vector of type BIGINT");
+
+  // confirm the second function got loaded
+  EXPECT_EQ(123, dynamicFunctionInt(0));
+  auto signaturesAfterSecond = getFunctionSignatures().size();
+  EXPECT_EQ(signaturesAfterSecond, signaturesAfterFirst);
+
+}
+
+//TODO fix this testcase. currently not finding the case giving us an error
+TEST_F(DynamicLinkTest, dynamicLoadErrFunc) {
+  // This works
+  const auto dynamicFunction = [&](const std::optional<int64_t> a, std::optional<int64_t> b) {
+    return evaluateOnce<int64_t>("dynamic_4(c0)", a, b);
+  };
+  // This works too
+  const auto dynamicFunctionErr = [&](const std::optional<int64_t> a) {
+    return evaluateOnce<int64_t>("dynamic_4(c0)", a);
+  };
+
+  auto signaturesBefore = getFunctionSignatures().size();
+
+  // Function does not exist yet.
+  VELOX_ASSERT_THROW(
+      dynamicFunction(0,0), "Scalar function doesn't exist: dynamic_4.");
+
+  // Dynamically load the library.
+  std::string libraryPath = VELOX_TEST_DYNAMIC_LIBRARY_PATH;
+  libraryPath.append("/libvelox_function_err_my_dynamic")
+  .append(VELOX_TEST_DYNAMIC_LIBRARY_PATH_SUFFIX);
+
+  loadDynamicLibrary(libraryPath.data());
+
+  auto signaturesAfter = getFunctionSignatures().size();
+  EXPECT_EQ(signaturesAfter, signaturesBefore + 1);
+  auto x = dynamicFunction(0,0);
+  auto y = dynamicFunctionErr(0);
+  EXPECT_EQ(123, dynamicFunction(0,0));
+  EXPECT_EQ(123, dynamicFunctionErr(0));
+  // VELOX_ASSERT_THROW(
+  //   dynamicFunctionErr(0), 
+  //   "Expression evaluation result is not of expected type: dynamic_4() -> CONSTANT vector of type BIGINT");
+}
+
+} // namespace facebook::velox::functions::test

velox/common/dynamic_registry/tests/MyDynamicErrFunction.cpp

@@ -0,0 +1,31 @@
+#include "velox/functions/Udf.h"
+#include "velox/type/SimpleFunctionApi.h"
+
+
+// This file defines a mock function that will be dynamically linked and
+// registered. There are no restrictions as to how the function needs to be
+// defined, but the library (.so) needs to provide a `void registry()` C
+// function in the top-level namespace.
+//
+// (note the extern "C" directive to prevent the compiler from mangling the
+// symbol name).
+
+namespace facebook::velox::common::dynamicRegistry {
+
+template <typename T>
+struct Dynamic123Function {
+  FOLLY_ALWAYS_INLINE bool call(int64_t& result, int64_t arg) {
+    result = 123;
+    return true;
+  }
+};
+
+} // namespace facebook::velox::common::dynamicRegistry
+
+extern "C" {
+ void registry() {
+  facebook::velox::registerFunction<
+      facebook::velox::common::dynamicRegistry::Dynamic123Function,
+      int64_t, int64_t>({"dynamic_4"});
+}
+}

velox/common/dynamic_registry/tests/MyDynamicFunction.cpp

@@ -0,0 +1,30 @@
+#include "velox/functions/Udf.h"
+
+// This file defines a mock function that will be dynamically linked and
+// registered. There are no restrictions as to how the function needs to be
+// defined, but the library (.so) needs to provide a `void registry()` C
+// function in the top-level namespace.
+//
+// (note the extern "C" directive to prevent the compiler from mangling the
+// symbol name).
+
+namespace facebook::velox::common::dynamicRegistry {
+
+template <typename T>
+struct Dynamic123Function {
+  FOLLY_ALWAYS_INLINE bool call(int64_t& result) {
+    result = 123;
+    return true;
+  }
+};
+
+} // namespace facebook::velox::common::dynamicRegistry
+
+extern "C" {
+
+void registry() {
+  facebook::velox::registerFunction<
+      facebook::velox::common::dynamicRegistry::Dynamic123Function,
+      int64_t>({"dynamic_1"});
+}
+}