From 021d24e4d3b30283971aedb4701e9466d86cb792 Mon Sep 17 00:00:00 2001 From: Frank Liu Date: Mon, 3 Dec 2018 17:27:37 -0800 Subject: [PATCH] [MXNET-1224]: improve scala maven jni build and packing. --- .../assembly/linux-x86_64-cpu/pom.xml | 4 ++ .../src/main/assembly/assembly.xml | 6 ++ .../assembly/linux-x86_64-gpu/pom.xml | 4 ++ .../src/main/assembly/assembly.xml | 6 ++ .../osx-x86_64-cpu/main/assembly/assembly.xml | 30 --------- scala-package/assembly/osx-x86_64-cpu/pom.xml | 4 ++ .../src/main/assembly/assembly.xml | 6 ++ scala-package/core/pom.xml | 8 +++ .../mxnet/util/NativeLibraryLoader.scala | 55 ++++++++++------ scala-package/examples/pom.xml | 4 ++ scala-package/infer/pom.xml | 4 ++ .../init-native/linux-x86_64/pom.xml | 42 ++++++++++--- scala-package/init-native/osx-x86_64/pom.xml | 49 +++++++++++++-- scala-package/native/README.md | 63 +++++++++++++++++++ scala-package/native/linux-x86_64-cpu/pom.xml | 25 ++++---- scala-package/native/linux-x86_64-gpu/pom.xml | 25 ++++---- scala-package/native/osx-x86_64-cpu/pom.xml | 50 ++++++++++++--- scala-package/pom.xml | 2 + 18 files changed, 291 insertions(+), 96 deletions(-) delete mode 100644 scala-package/assembly/osx-x86_64-cpu/main/assembly/assembly.xml create mode 100644 scala-package/native/README.md diff --git a/scala-package/assembly/linux-x86_64-cpu/pom.xml b/scala-package/assembly/linux-x86_64-cpu/pom.xml index abefead175c7..1658f36e6bbd 100644 --- a/scala-package/assembly/linux-x86_64-cpu/pom.xml +++ b/scala-package/assembly/linux-x86_64-cpu/pom.xml @@ -14,6 +14,10 @@ MXNet Scala Package - Full Linux-x86_64 CPU-only jar + + ${project.parent.parent.basedir}/.. + + org.apache.mxnet diff --git a/scala-package/assembly/linux-x86_64-cpu/src/main/assembly/assembly.xml b/scala-package/assembly/linux-x86_64-cpu/src/main/assembly/assembly.xml index a574f8af25d9..f4c2017c8241 100644 --- a/scala-package/assembly/linux-x86_64-cpu/src/main/assembly/assembly.xml +++ b/scala-package/assembly/linux-x86_64-cpu/src/main/assembly/assembly.xml @@ -25,4 +25,10 @@ + + + ${MXNET_DIR}/lib/libmxnet.so + lib/native + + diff --git a/scala-package/assembly/linux-x86_64-gpu/pom.xml b/scala-package/assembly/linux-x86_64-gpu/pom.xml index 96ffa38c6af2..c80515e7b107 100644 --- a/scala-package/assembly/linux-x86_64-gpu/pom.xml +++ b/scala-package/assembly/linux-x86_64-gpu/pom.xml @@ -14,6 +14,10 @@ MXNet Scala Package - Full Linux-x86_64 GPU jar + + ${project.parent.parent.basedir}/.. + + org.apache.mxnet diff --git a/scala-package/assembly/linux-x86_64-gpu/src/main/assembly/assembly.xml b/scala-package/assembly/linux-x86_64-gpu/src/main/assembly/assembly.xml index 3a064bf9f2ce..2aca64bdf1a9 100644 --- a/scala-package/assembly/linux-x86_64-gpu/src/main/assembly/assembly.xml +++ b/scala-package/assembly/linux-x86_64-gpu/src/main/assembly/assembly.xml @@ -25,4 +25,10 @@ + + + ${MXNET_DIR}/lib/libmxnet.so + lib/native + + diff --git a/scala-package/assembly/osx-x86_64-cpu/main/assembly/assembly.xml b/scala-package/assembly/osx-x86_64-cpu/main/assembly/assembly.xml deleted file mode 100644 index fecafecad31e..000000000000 --- a/scala-package/assembly/osx-x86_64-cpu/main/assembly/assembly.xml +++ /dev/null @@ -1,30 +0,0 @@ - - full - - jar - - false - - - - *:*:jar - - / - true - true - runtime - - - lib/native - ${artifact.artifactId}${dashClassifier?}.${artifact.extension} - false - false - false - - *:*:dll:* - *:*:so:* - *:*:jnilib:* - - - - diff --git a/scala-package/assembly/osx-x86_64-cpu/pom.xml b/scala-package/assembly/osx-x86_64-cpu/pom.xml index 5c5733a9a4ce..62979a140fdc 100644 --- a/scala-package/assembly/osx-x86_64-cpu/pom.xml +++ b/scala-package/assembly/osx-x86_64-cpu/pom.xml @@ -14,6 +14,10 @@ MXNet Scala Package - Full OSX-x86_64 CPU-only jar + + ${project.parent.parent.basedir}/.. + + org.apache.mxnet diff --git a/scala-package/assembly/osx-x86_64-cpu/src/main/assembly/assembly.xml b/scala-package/assembly/osx-x86_64-cpu/src/main/assembly/assembly.xml index bdbd09f170c0..e9bc3728fcd0 100644 --- a/scala-package/assembly/osx-x86_64-cpu/src/main/assembly/assembly.xml +++ b/scala-package/assembly/osx-x86_64-cpu/src/main/assembly/assembly.xml @@ -25,4 +25,10 @@ + + + ${MXNET_DIR}/lib/libmxnet.so + lib/native + + diff --git a/scala-package/core/pom.xml b/scala-package/core/pom.xml index 484fbbd96790..976383f2e7d5 100644 --- a/scala-package/core/pom.xml +++ b/scala-package/core/pom.xml @@ -12,6 +12,7 @@ true + ${project.parent.basedir}/.. mxnet-core_2.11 @@ -77,6 +78,9 @@ -Djava.library.path=${project.parent.basedir}/native/${platform}/target \ -Dlog4j.configuration=file://${project.basedir}/src/test/resources/log4j.properties + + ${MXNET_DIR}/lib + @@ -88,6 +92,10 @@ -Djava.library.path=${project.parent.basedir}/native/${platform}/target ${skipTests} + always + + ${MXNET_DIR}/lib + diff --git a/scala-package/core/src/main/scala/org/apache/mxnet/util/NativeLibraryLoader.scala b/scala-package/core/src/main/scala/org/apache/mxnet/util/NativeLibraryLoader.scala index e94d320391fa..2ce893b478ed 100644 --- a/scala-package/core/src/main/scala/org/apache/mxnet/util/NativeLibraryLoader.scala +++ b/scala-package/core/src/main/scala/org/apache/mxnet/util/NativeLibraryLoader.scala @@ -85,12 +85,10 @@ private[mxnet] object NativeLibraryLoader { } logger.debug(s"Attempting to load $loadLibname") val libFileInJar = libPathInJar + loadLibname - val is: InputStream = getClass.getResourceAsStream(libFileInJar) - if (is == null) { - throw new UnsatisfiedLinkError(s"Couldn't find the resource $loadLibname") - } - logger.info(s"Loading $loadLibname from $libPathInJar copying to $libname") - loadLibraryFromStream(libname, is) + saveLibraryToTemp("libmxnet.so", "/lib/native/libmxnet.so") + val tempfile: File = saveLibraryToTemp(libname, libFileInJar) + + loadLibraryFromFile(libname, tempfile) } /** @@ -109,7 +107,7 @@ private[mxnet] object NativeLibraryLoader { @throws(classOf[IOException]) private def createTempFile(name: String): File = { - new File(_tempDir + File.separator + name) + new File(_tempDir, name) } /** @@ -117,11 +115,34 @@ private[mxnet] object NativeLibraryLoader { * and loads from there. * * @param libname name of the library (just used in constructing the library name) - * @param is InputStream pointing to the library + * @param tempfile File pointing to the library */ - private def loadLibraryFromStream(libname: String, is: InputStream) { + private def loadLibraryFromFile(libname: String, tempfile: File) { + try { + logger.debug("Loading library from {}", tempfile.getPath) + System.load(tempfile.getPath) + } catch { + case ule: UnsatisfiedLinkError => + logger.error("Couldn't load copied link file: {}", ule.toString) + throw ule + } + } + + /** + * Load a system library from a stream. Copies the library to a temp file + * and loads from there. + * + * @param libname name of the library (just used in constructing the library name) + * @param resource String resource path in the jar file + */ + private def saveLibraryToTemp(libname: String, resource: String): File = { try { - val tempfile: File = createTempFile(libname) + val is: InputStream = getClass.getResourceAsStream(resource) + if (is == null) { + throw new UnsatisfiedLinkError(s"Couldn't find the resource $resource") + } + + val tempfile: File = new File(_tempDir, libname) val os: OutputStream = new FileOutputStream(tempfile) logger.debug("tempfile.getPath() = {}", tempfile.getPath) val savedTime: Long = System.currentTimeMillis @@ -131,20 +152,14 @@ private[mxnet] object NativeLibraryLoader { os.write(buf, 0, len) len = is.read(buf) } - os.flush() - val lock: InputStream = new FileInputStream(tempfile) os.close() + is.close() val seconds: Double = (System.currentTimeMillis - savedTime).toDouble / 1e3 - logger.debug(s"Copying took $seconds seconds.") - logger.debug("Loading library from {}", tempfile.getPath) - System.load(tempfile.getPath) - lock.close() + logger.debug(s"Copying $libname took $seconds seconds.") + tempfile } catch { case io: IOException => - logger.error("Could not create the temp file: {}", io.toString) - case ule: UnsatisfiedLinkError => - logger.error("Couldn't load copied link file: {}", ule.toString) - throw ule + throw new UnsatisfiedLinkError(s"Could not create temp file for $libname") } } } diff --git a/scala-package/examples/pom.xml b/scala-package/examples/pom.xml index 8d3d156a0b18..bea7c5e422e2 100644 --- a/scala-package/examples/pom.xml +++ b/scala-package/examples/pom.xml @@ -15,6 +15,7 @@ true + ${project.parent.basedir}/.. @@ -137,6 +138,9 @@ -Djava.library.path=${project.parent.basedir}/native/${platform}/target \ -Dlog4j.configuration=file://${project.basedir}/src/test/resources/log4j.properties + + ${MXNET_DIR}/lib + diff --git a/scala-package/infer/pom.xml b/scala-package/infer/pom.xml index ac76cdd19f3b..fb5cf370a009 100644 --- a/scala-package/infer/pom.xml +++ b/scala-package/infer/pom.xml @@ -15,6 +15,7 @@ true + ${project.parent.basedir}/.. @@ -77,6 +78,9 @@ -Djava.library.path=${project.parent.basedir}/native/${platform}/target \ -Dlog4j.configuration=file://${project.basedir}/src/test/resources/log4j.properties + + ${MXNET_DIR}/lib + diff --git a/scala-package/init-native/linux-x86_64/pom.xml b/scala-package/init-native/linux-x86_64/pom.xml index b71d7cf71528..242f2f3d5626 100644 --- a/scala-package/init-native/linux-x86_64/pom.xml +++ b/scala-package/init-native/linux-x86_64/pom.xml @@ -16,6 +16,10 @@ so + + ${project.parent.parent.basedir}/.. + + org.apache.mxnet @@ -62,22 +66,24 @@ -std=c++0x - -I${project.basedir}/../../../include - ${all_includes} - ${cflags} + -I${MXNET_DIR}/include + -I${MXNET_DIR}/3rdparty/dmlc-core/include + -I${MXNET_DIR}/3rdparty/mshadow + -I${MXNET_DIR}/3rdparty/dlpack/include + -I${MXNET_DIR}/3rdparty/tvm/nnvm/include + -DMSHADOW_USE_MKL=0 -DMSHADOW_USE_CUDA=0 + -O3 -DNDEBUG=1 -fPIC -msse3 -mf16c + -Wall -Wsign-compare -Wno-unused-parameter -Wno-unknown-pragmas -Wno-unused-local-typedefs -shared - ${all_ldpaths} -Wl,--whole-archive - ${lddeps} - -Wl,--no-whole-archive + -Wl,--no-whole-archive -pthread -lm -fopenmp -lrt - ${ldflags} - -fopenmp + -Wl,-rpath=${dollar}ORIGIN -lmxnet -L${MXNET_DIR}/lib @@ -86,7 +92,6 @@ javah generate-sources - linux default ${project.build.directory}/custom-javah ${basedir} @@ -101,6 +106,25 @@ + + + org.codehaus.mojo + exec-maven-plugin + 1.6.0 + + + link-native-lib + generate-resources + + exec + + + ln + -sf ${MXNET_DIR}/lib/libmxnet.so ${project.build.directory}/libmxnet.so + + + + diff --git a/scala-package/init-native/osx-x86_64/pom.xml b/scala-package/init-native/osx-x86_64/pom.xml index b4a0b1d6584a..12f4d800eba4 100644 --- a/scala-package/init-native/osx-x86_64/pom.xml +++ b/scala-package/init-native/osx-x86_64/pom.xml @@ -16,6 +16,10 @@ jnilib + + ${project.parent.parent.basedir}/.. + + org.apache.mxnet @@ -62,8 +66,14 @@ -std=c++0x - -I${project.basedir}/../../../include - ${cflags} + -I${MXNET_DIR}/include + -I${MXNET_DIR}/3rdparty/dmlc-core/include + -I${MXNET_DIR}/3rdparty/mshadow + -I${MXNET_DIR}/3rdparty/dlpack/include + -I${MXNET_DIR}/3rdparty/tvm/nnvm/include + -DMSHADOW_USE_MKL=0 -DMSHADOW_USE_CUDA=0 + -g -O0 -fPIC -msse3 -mf16c + -Wall -Wsign-compare -Wno-unused-parameter -Wno-unknown-pragmas -Wno-unused-local-typedefs -shared @@ -72,11 +82,9 @@ -framework JavaVM -Wl,-exported_symbol,_Java_* -Wl,-x - ${lddeps} - -force_load ${project.basedir}/../../../lib/libmxnet.a - ${ldflags} + -lmxnet -L${MXNET_DIR}/lib @@ -85,7 +93,6 @@ javah generate-sources - darwin default ${project.build.directory}/custom-javah ${basedir} @@ -100,6 +107,36 @@ + + + org.codehaus.mojo + exec-maven-plugin + 1.6.0 + + + post-native-build + package + + exec + + + install_name_tool + -change lib/libmxnet.so @loader_path/libmxnet.so ${project.build.directory}/${artifactId}.jnilib + + + + link-native-lib + generate-resources + + exec + + + ln + -sf ${MXNET_DIR}/lib/libmxnet.so ${project.build.directory}/libmxnet.so + + + + diff --git a/scala-package/native/README.md b/scala-package/native/README.md new file mode 100644 index 000000000000..cb6dd3890dd2 --- /dev/null +++ b/scala-package/native/README.md @@ -0,0 +1,63 @@ +# MXNet Scala JNI + +MXNet Scala JNI is a thin wrapper layer of underlying libmxnet.so. + +## javah +JNI native code requires a header file that matches the java/scala interface, +this file is usually generated with javah. + +In our case, jni_helper_func.h is generated and will be used to compile native code. + + +## Linker options + +Scala JNI (libmxnet-scala.so/libmxnet-scala.jnilib) is dynamically linked to libmxnet.so. +MXNet Scala will trying to load libmxnet.so from system LD_LIBRARY_PATH first. +If it failed, the try to resolve libmxnet.so in the same location as libmxnet-scala.so file. + +### Linux +``` +-Wl,-rpath=$ORIGIN -lmxnet +``` +Above option will tell system to looking for libmxnet.so from the same location. + + +### Mac OSX +On Mac, we have to execute install_name_tool command to change library loading path: +```bash +install_name_tool -change lib/libmxnet.so @loader_path/libmxnet.so libmxnet-scala.jnilib +``` + +Other linker options: +* -shared : link as shared library +* -Wl,-install_name,libmxnet-scala.jnilib : avoid use build machine's absolute path +* -framework JavaVM : Stand jni options for mac +* -Wl,-exported_symbol,_Java_* : Stand jni options for mac +* -Wl,-x : Do not put non-global symbols in the output file's symbol table. + + +## Compiler flags + +Scala JNI code technically doesn't need on any of MXNet make flags, +however c_api.h header links to many other dependencies header file, +which requires us to add DMSHADOW_USE_MKL and DMSHADOW_USE_CUDA to compile the JNI code. +These flags are not actually used by JNI and won't impact Scala's behavior. + + +### Linux + +``` +-DMSHADOW_USE_MKL=0 +-DMSHADOW_USE_CUDA=0 +-O3 -DNDEBUG=1 -fPIC -msse3 -mf16c +-Wall -Wsign-compare -Wno-unused-parameter -Wno-unknown-pragmas -Wno-unused-local-typedefs +``` + +### Mac OSX + +``` +-DMSHADOW_USE_MKL=0 +-DMSHADOW_USE_CUDA=0 +-g -O0 -fPIC -msse3 -mf16c +-Wall -Wsign-compare -Wno-unused-parameter -Wno-unknown-pragmas -Wno-unused-local-typedefs +``` diff --git a/scala-package/native/linux-x86_64-cpu/pom.xml b/scala-package/native/linux-x86_64-cpu/pom.xml index 2415cf7d26db..7cfd01a4ef79 100644 --- a/scala-package/native/linux-x86_64-cpu/pom.xml +++ b/scala-package/native/linux-x86_64-cpu/pom.xml @@ -16,6 +16,10 @@ so + + ${project.parent.parent.basedir}/.. + + org.apache.mxnet @@ -62,22 +66,20 @@ -std=c++0x - -I${project.basedir}/../../../include - ${all_includes} - ${cflags} + -I${MXNET_DIR}/include + -I${MXNET_DIR}/3rdparty/dmlc-core/include + -I${MXNET_DIR}/3rdparty/mshadow + -I${MXNET_DIR}/3rdparty/dlpack/include + -I${MXNET_DIR}/3rdparty/tvm/nnvm/include + -DMSHADOW_USE_MKL=0 -DMSHADOW_USE_CUDA=0 + -O3 -DNDEBUG=1 -fPIC -msse3 -mf16c + -Wall -Wsign-compare -Wno-unused-parameter -Wno-unknown-pragmas -Wno-unused-local-typedefs -shared - - ${all_ldpaths} - -Wl,--whole-archive - ${lddeps} - -Wl,--no-whole-archive - - ${ldflags} - -fopenmp + -Wl,-rpath=${dollar}ORIGIN -lmxnet -L${MXNET_DIR}/lib @@ -86,7 +88,6 @@ javah generate-sources - linux default ${project.build.directory}/custom-javah ${basedir} diff --git a/scala-package/native/linux-x86_64-gpu/pom.xml b/scala-package/native/linux-x86_64-gpu/pom.xml index 0186217234bc..668f330b5ff9 100644 --- a/scala-package/native/linux-x86_64-gpu/pom.xml +++ b/scala-package/native/linux-x86_64-gpu/pom.xml @@ -16,6 +16,10 @@ so + + ${project.parent.parent.basedir}/.. + + org.apache.mxnet @@ -62,22 +66,20 @@ -std=c++0x - -I${project.basedir}/../../../include - ${all_includes} - ${cflags} + -I${MXNET_DIR}/include + -I${MXNET_DIR}/3rdparty/dmlc-core/include + -I${MXNET_DIR}/3rdparty/mshadow + -I${MXNET_DIR}/3rdparty/dlpack/include + -I${MXNET_DIR}/3rdparty/tvm/nnvm/include + -DMSHADOW_USE_MKL=0 -DMSHADOW_USE_CUDA=0 + -O3 -DNDEBUG=1 -fPIC -msse3 -mf16c + -Wall -Wsign-compare -Wno-unused-parameter -Wno-unknown-pragmas -Wno-unused-local-typedefs -shared - - ${all_ldpaths} - -Wl,--whole-archive - ${lddeps} - -Wl,--no-whole-archive - - ${ldflags} - -fopenmp + -Wl,-rpath=${dollar}ORIGIN -lmxnet -L${MXNET_DIR}/lib @@ -86,7 +88,6 @@ javah generate-sources - linux default ${project.build.directory}/custom-javah ${basedir} diff --git a/scala-package/native/osx-x86_64-cpu/pom.xml b/scala-package/native/osx-x86_64-cpu/pom.xml index 0ab7ca1dd0f0..425ca96815de 100644 --- a/scala-package/native/osx-x86_64-cpu/pom.xml +++ b/scala-package/native/osx-x86_64-cpu/pom.xml @@ -16,6 +16,10 @@ jnilib + + ${project.parent.parent.basedir}/.. + + org.apache.mxnet @@ -62,8 +66,14 @@ -std=c++0x - -I../../../include - ${cflags} + -I${MXNET_DIR}/include + -I${MXNET_DIR}/3rdparty/dmlc-core/include + -I${MXNET_DIR}/3rdparty/mshadow + -I${MXNET_DIR}/3rdparty/dlpack/include + -I${MXNET_DIR}/3rdparty/tvm/nnvm/include + -DMSHADOW_USE_MKL=0 -DMSHADOW_USE_CUDA=0 + -g -O0 -fPIC -msse3 -mf16c + -Wall -Wsign-compare -Wno-unused-parameter -Wno-unknown-pragmas -Wno-unused-local-typedefs -shared @@ -72,12 +82,9 @@ -framework JavaVM -Wl,-exported_symbol,_Java_* -Wl,-x - ${lddeps} - -force_load ${project.basedir}/../../../lib/libmxnet.a - -force_load ${project.basedir}/../../../3rdparty/tvm/nnvm/lib/libnnvm.a - ${ldflags} + -Wl,-install_name,libmxnet-scala.jnilib -lmxnet -L${MXNET_DIR}/lib @@ -86,7 +93,6 @@ javah generate-sources - darwin default ${project.build.directory}/custom-javah ${basedir} @@ -101,6 +107,36 @@ + + + org.codehaus.mojo + exec-maven-plugin + 1.6.0 + + + post-native-build + package + + exec + + + install_name_tool + -change lib/libmxnet.so @loader_path/libmxnet.so ${project.build.directory}/${artifactId}.jnilib + + + + link-native-lib + generate-resources + + exec + + + ln + -sf ${MXNET_DIR}/lib/libmxnet.so ${project.build.directory}/libmxnet.so + + + + diff --git a/scala-package/pom.xml b/scala-package/pom.xml index 151462cbcc68..6eb573bf3e23 100644 --- a/scala-package/pom.xml +++ b/scala-package/pom.xml @@ -39,6 +39,8 @@ 2.11.8 2.11 + g++ + $ pom