From 5632969919ab926743e78da1743cfc5d826cc0e6 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Thu, 6 Feb 2025 18:09:04 -0500 Subject: [PATCH 1/7] Update build gradle to separate nmslib / faiss generation Signed-off-by: Peter Zhu --- build.gradle | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 850d758f1..355ee80e1 100644 --- a/build.gradle +++ b/build.gradle @@ -347,13 +347,25 @@ task cmakeJniLib(type:Exec) { commandLine args } +task buildNmslib(type:Exec) { + workingDir 'jni' + environment('CC', 'gcc') + environment('CXX', 'g++') + environment('FC', 'gfortran') + commandLine 'make', 'opensearchknn_nmslib', '-j', "${nproc_count}" +} + task buildJniLib(type:Exec) { dependsOn cmakeJniLib workingDir 'jni' - commandLine 'make', 'opensearchknn_nmslib', 'opensearchknn_faiss', 'opensearchknn_common', '-j', "${nproc_count}" + environment('CC', 'gcc10-gcc') + environment('CXX', 'gcc10-gcc') + environment('FC', 'gcc10-gcc') + commandLine 'make', 'opensearchknn_faiss', 'opensearchknn_common', '-j', "${nproc_count}" } test { + dependsOn buildNmslib dependsOn buildJniLib systemProperty 'tests.security.manager', 'false' systemProperty "java.library.path", "$rootDir/jni/release" @@ -368,6 +380,7 @@ test { def _numNodes = findProperty('numNodes') as Integer ?: 1 integTest { if (integTestDependOnJniLib) { + dependsOn buildNmslib dependsOn buildJniLib } systemProperty 'tests.security.manager', 'false' @@ -468,6 +481,7 @@ task integTestRemote(type: RestIntegTestTask) { run { useCluster project.testClusters.integTest + dependsOn buildNmslib dependsOn buildJniLib doFirst { // There seems to be an issue when running multi node run or integ tasks with unicast_hosts From 787d49f5d7d5732d32df8f771f3ec145cbdaf21d Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Thu, 6 Feb 2025 21:09:52 -0500 Subject: [PATCH 2/7] Update scripts/build.sh to separate enable gcc switch Signed-off-by: Peter Zhu --- build.gradle | 14 +++++++------- scripts/build.sh | 10 +++++++++- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/build.gradle b/build.gradle index 355ee80e1..6651ca758 100644 --- a/build.gradle +++ b/build.gradle @@ -343,24 +343,24 @@ task cmakeJniLib(type:Exec) { args.add("-DBLAS_LIBRARIES=$rootDir\\src\\main\\resources\\windowsDependencies\\libopenblas.dll") args.add("-DLAPACK_LIBRARIES=$rootDir\\src\\main\\resources\\windowsDependencies\\libopenblas.dll") } + //if (System.getenv("LIBNAME") == "nmslib_linux_x64") { + // environment('CC', 'gcc10-gcc') + // environment('CXX', 'gcc10-g++') + // environment('FC', 'gcc10-gfortran') + //} commandLine args } task buildNmslib(type:Exec) { + dependsOn cmakeJniLib workingDir 'jni' - environment('CC', 'gcc') - environment('CXX', 'g++') - environment('FC', 'gfortran') - commandLine 'make', 'opensearchknn_nmslib', '-j', "${nproc_count}" + commandLine 'make', '-B', 'opensearchknn_nmslib', '-j', "${nproc_count}" } task buildJniLib(type:Exec) { dependsOn cmakeJniLib workingDir 'jni' - environment('CC', 'gcc10-gcc') - environment('CXX', 'gcc10-gcc') - environment('FC', 'gcc10-gcc') commandLine 'make', 'opensearchknn_faiss', 'opensearchknn_common', '-j', "${nproc_count}" } diff --git a/scripts/build.sh b/scripts/build.sh index 656915832..44ec81018 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -131,9 +131,14 @@ cd $work_dir ./gradlew :buildJniLib -Davx512.enabled=false -Davx512_spr.enabled=false -Davx2.enabled=false -Dbuild.lib.commit_patches=false -Dnproc.count=${NPROC_COUNT:-1} if [ "$PLATFORM" != "windows" ] && [ "$ARCHITECTURE" = "x64" ]; then + echo "Building k-NN library nmslib with gcc 10 on non-windows x64" + rm -rf jni/CMakeCache.txt jni/CMakeFiles + env CC=gcc10-gcc CXX=gcc10-g++ FC=gcc10-gfortran ./gradlew :buildNmslib -Dbuild.lib.commit_patches=false -Dbuild.lib.apply_patches=false --info + echo "Building k-NN library after enabling AVX2" # Skip applying patches as patches were applied already from previous :buildJniLib task # If we apply patches again, it fails with conflict + rm -rf jni/CMakeCache.txt jni/CMakeFiles ./gradlew :buildJniLib -Davx2.enabled=true -Davx512.enabled=false -Davx512_spr.enabled=false -Dbuild.lib.commit_patches=false -Dbuild.lib.apply_patches=false echo "Building k-NN library after enabling AVX512" @@ -141,6 +146,9 @@ if [ "$PLATFORM" != "windows" ] && [ "$ARCHITECTURE" = "x64" ]; then echo "Building k-NN library after enabling AVX512_SPR" ./gradlew :buildJniLib -Davx512_spr.enabled=true -Dbuild.lib.commit_patches=false -Dbuild.lib.apply_patches=false + +else + ./gradlew :buildNmslib -Dbuild.lib.commit_patches=false -Dbuild.lib.apply_patches=false --info fi ./gradlew publishPluginZipPublicationToZipStagingRepository -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER @@ -149,7 +157,7 @@ fi # Add lib to zip zipPath=$(find "$(pwd)/build/distributions" -path \*.zip) distributions="$(dirname "${zipPath}")" -mkdir $distributions/lib +mkdir -p $distributions/lib libPrefix="libopensearchknn" if [ "$PLATFORM" = "windows" ]; then libPrefix="opensearchknn" From 95d221da75bd5aed4d7ef9e5c130ff0d0b7a8aa6 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Thu, 6 Feb 2025 21:14:17 -0500 Subject: [PATCH 3/7] Remove test comments Signed-off-by: Peter Zhu --- build.gradle | 5 ----- 1 file changed, 5 deletions(-) diff --git a/build.gradle b/build.gradle index 6651ca758..57a330e7a 100644 --- a/build.gradle +++ b/build.gradle @@ -343,11 +343,6 @@ task cmakeJniLib(type:Exec) { args.add("-DBLAS_LIBRARIES=$rootDir\\src\\main\\resources\\windowsDependencies\\libopenblas.dll") args.add("-DLAPACK_LIBRARIES=$rootDir\\src\\main\\resources\\windowsDependencies\\libopenblas.dll") } - //if (System.getenv("LIBNAME") == "nmslib_linux_x64") { - // environment('CC', 'gcc10-gcc') - // environment('CXX', 'gcc10-g++') - // environment('FC', 'gcc10-gfortran') - //} commandLine args } From 5c60e646fe904e24210acc5ebb8cb3b435792a41 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Thu, 6 Feb 2025 21:15:12 -0500 Subject: [PATCH 4/7] Remove test comments Signed-off-by: Peter Zhu --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 57a330e7a..98b6ae038 100644 --- a/build.gradle +++ b/build.gradle @@ -350,7 +350,7 @@ task cmakeJniLib(type:Exec) { task buildNmslib(type:Exec) { dependsOn cmakeJniLib workingDir 'jni' - commandLine 'make', '-B', 'opensearchknn_nmslib', '-j', "${nproc_count}" + commandLine 'make', 'opensearchknn_nmslib', '-j', "${nproc_count}" } task buildJniLib(type:Exec) { From 21fd6a6ed003482cd20a5d4be654feb3cbda4f56 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Thu, 6 Feb 2025 21:17:13 -0500 Subject: [PATCH 5/7] Remove test comments Signed-off-by: Peter Zhu --- scripts/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/build.sh b/scripts/build.sh index 44ec81018..2a78fdbde 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -133,7 +133,7 @@ cd $work_dir if [ "$PLATFORM" != "windows" ] && [ "$ARCHITECTURE" = "x64" ]; then echo "Building k-NN library nmslib with gcc 10 on non-windows x64" rm -rf jni/CMakeCache.txt jni/CMakeFiles - env CC=gcc10-gcc CXX=gcc10-g++ FC=gcc10-gfortran ./gradlew :buildNmslib -Dbuild.lib.commit_patches=false -Dbuild.lib.apply_patches=false --info + env CC=gcc10-gcc CXX=gcc10-g++ FC=gcc10-gfortran ./gradlew :buildNmslib -Dbuild.lib.commit_patches=false -Dbuild.lib.apply_patches=false echo "Building k-NN library after enabling AVX2" # Skip applying patches as patches were applied already from previous :buildJniLib task @@ -148,7 +148,7 @@ if [ "$PLATFORM" != "windows" ] && [ "$ARCHITECTURE" = "x64" ]; then ./gradlew :buildJniLib -Davx512_spr.enabled=true -Dbuild.lib.commit_patches=false -Dbuild.lib.apply_patches=false else - ./gradlew :buildNmslib -Dbuild.lib.commit_patches=false -Dbuild.lib.apply_patches=false --info + ./gradlew :buildNmslib -Dbuild.lib.commit_patches=false -Dbuild.lib.apply_patches=false fi ./gradlew publishPluginZipPublicationToZipStagingRepository -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER From a67f9dee543feecc48a5e7f56ca690beebd5f2a3 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Thu, 6 Feb 2025 21:32:53 -0500 Subject: [PATCH 6/7] Updating restart and rolling upgrade bwc test bundle.gradle Signed-off-by: Peter Zhu --- qa/restart-upgrade/build.gradle | 1 + qa/rolling-upgrade/build.gradle | 1 + 2 files changed, 2 insertions(+) diff --git a/qa/restart-upgrade/build.gradle b/qa/restart-upgrade/build.gradle index b858af0ba..88c5c2e89 100644 --- a/qa/restart-upgrade/build.gradle +++ b/qa/restart-upgrade/build.gradle @@ -134,6 +134,7 @@ testClusters { // All nodes are upgraded to latest version and run the tests task testRestartUpgrade(type: StandaloneRestIntegTestTask) { dependsOn "testAgainstOldCluster" + dependsOn rootProject.tasks.buildNmslib dependsOn rootProject.tasks.buildJniLib dependsOn rootProject.tasks.assemble useCluster testClusters."${baseName}" diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle index cdfab0b96..9a9fec05a 100644 --- a/qa/rolling-upgrade/build.gradle +++ b/qa/rolling-upgrade/build.gradle @@ -52,6 +52,7 @@ task testAgainstOldCluster(type: StandaloneRestIntegTestTask) { // This results in a mixed cluster with 2 nodes on the old version and 1 upgraded node. task testAgainstOneThirdUpgradedCluster(type: StandaloneRestIntegTestTask) { useCluster testClusters."${baseName}" + dependsOn rootProject.tasks.buildNmslib dependsOn rootProject.tasks.buildJniLib dependsOn rootProject.tasks.assemble dependsOn "testAgainstOldCluster" From ffef4989967eddfdefa3f7ac7b59cbc344099742 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Thu, 6 Feb 2025 21:41:54 -0500 Subject: [PATCH 7/7] Enforce gcc10 for nmslib to compile and avx512_spr have no-op Signed-off-by: Peter Zhu --- .github/workflows/CI.yml | 5 +++++ .github/workflows/test_security.yml | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index ca39a5eb1..dbed97371 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -40,6 +40,11 @@ jobs: matrix: java: [11, 17, 21] + env: + CC: gcc10-gcc + CXX: gcc10-g++ + FC: gcc10-gfortran + name: Build and Test k-NN Plugin on Linux runs-on: ubuntu-latest needs: Get-CI-Image-Tag diff --git a/.github/workflows/test_security.yml b/.github/workflows/test_security.yml index 4cfbbf47c..0845c792a 100644 --- a/.github/workflows/test_security.yml +++ b/.github/workflows/test_security.yml @@ -39,6 +39,10 @@ jobs: strategy: matrix: java: [21] + env: + CC: gcc10-gcc + CXX: gcc10-g++ + FC: gcc10-gfortran name: Run Integration Tests on Linux runs-on: ubuntu-latest