Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update build.gradle and build.sh to separate x64 linux nmslib build with different gcc versions #2506

Merged
merged 7 commits into from
Feb 7, 2025
Merged

Update build.gradle and build.sh to separate x64 linux nmslib build with different gcc versions #2506

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
5632969
Update build gradle to separate nmslib / faiss generation
peterzhuamazon Feb 6, 2025
787d49f
Update scripts/build.sh to separate enable gcc switch
peterzhuamazon Feb 7, 2025
95d221d
Remove test comments
peterzhuamazon Feb 7, 2025
5c60e64
Remove test comments
peterzhuamazon Feb 7, 2025
21fd6a6
Remove test comments
peterzhuamazon Feb 7, 2025
a67f9de
Updating restart and rolling upgrade bwc test bundle.gradle
peterzhuamazon Feb 7, 2025
ffef498
Enforce gcc10 for nmslib to compile and avx512_spr have no-op
peterzhuamazon Feb 7, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 15 additions & 1 deletion build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -343,17 +343,29 @@ task cmakeJniLib(type:Exec) {
args.add("-DBLAS_LIBRARIES=$rootDir\\src\\main\\resources\\windowsDependencies\\libopenblas.dll")
args.add("-DLAPACK_LIBRARIES=$rootDir\\src\\main\\resources\\windowsDependencies\\libopenblas.dll")
}
//if (System.getenv("LIBNAME") == "nmslib_linux_x64") {
// environment('CC', 'gcc10-gcc')
// environment('CXX', 'gcc10-g++')
// environment('FC', 'gcc10-gfortran')
//}

commandLine args
}

task buildNmslib(type:Exec) {
dependsOn cmakeJniLib
workingDir 'jni'
commandLine 'make', '-B', 'opensearchknn_nmslib', '-j', "${nproc_count}"
}

task buildJniLib(type:Exec) {
dependsOn cmakeJniLib
workingDir 'jni'
commandLine 'make', 'opensearchknn_nmslib', 'opensearchknn_faiss', 'opensearchknn_common', '-j', "${nproc_count}"
commandLine 'make', 'opensearchknn_faiss', 'opensearchknn_common', '-j', "${nproc_count}"
}

test {
dependsOn buildNmslib
dependsOn buildJniLib
systemProperty 'tests.security.manager', 'false'
systemProperty "java.library.path", "$rootDir/jni/release"
Expand All @@ -368,6 +380,7 @@ test {
def _numNodes = findProperty('numNodes') as Integer ?: 1
integTest {
if (integTestDependOnJniLib) {
dependsOn buildNmslib
dependsOn buildJniLib
}
systemProperty 'tests.security.manager', 'false'
Expand Down Expand Up @@ -468,6 +481,7 @@ task integTestRemote(type: RestIntegTestTask) {

run {
useCluster project.testClusters.integTest
dependsOn buildNmslib
dependsOn buildJniLib
doFirst {
// There seems to be an issue when running multi node run or integ tasks with unicast_hosts
Expand Down
10 changes: 9 additions & 1 deletion scripts/build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,16 +131,24 @@ cd $work_dir
./gradlew :buildJniLib -Davx512.enabled=false -Davx512_spr.enabled=false -Davx2.enabled=false -Dbuild.lib.commit_patches=false -Dnproc.count=${NPROC_COUNT:-1}

if [ "$PLATFORM" != "windows" ] && [ "$ARCHITECTURE" = "x64" ]; then
echo "Building k-NN library nmslib with gcc 10 on non-windows x64"
rm -rf jni/CMakeCache.txt jni/CMakeFiles
env CC=gcc10-gcc CXX=gcc10-g++ FC=gcc10-gfortran ./gradlew :buildNmslib -Dbuild.lib.commit_patches=false -Dbuild.lib.apply_patches=false --info

echo "Building k-NN library after enabling AVX2"
# Skip applying patches as patches were applied already from previous :buildJniLib task
# If we apply patches again, it fails with conflict
rm -rf jni/CMakeCache.txt jni/CMakeFiles
./gradlew :buildJniLib -Davx2.enabled=true -Davx512.enabled=false -Davx512_spr.enabled=false -Dbuild.lib.commit_patches=false -Dbuild.lib.apply_patches=false

echo "Building k-NN library after enabling AVX512"
./gradlew :buildJniLib -Davx512.enabled=true -Davx512_spr.enabled=false -Dbuild.lib.commit_patches=false -Dbuild.lib.apply_patches=false

echo "Building k-NN library after enabling AVX512_SPR"
./gradlew :buildJniLib -Davx512_spr.enabled=true -Dbuild.lib.commit_patches=false -Dbuild.lib.apply_patches=false

else
./gradlew :buildNmslib -Dbuild.lib.commit_patches=false -Dbuild.lib.apply_patches=false --info
fi

./gradlew publishPluginZipPublicationToZipStagingRepository -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER
Expand All @@ -149,7 +157,7 @@ fi
# Add lib to zip
zipPath=$(find "$(pwd)/build/distributions" -path \*.zip)
distributions="$(dirname "${zipPath}")"
mkdir $distributions/lib
mkdir -p $distributions/lib
libPrefix="libopensearchknn"
if [ "$PLATFORM" = "windows" ]; then
libPrefix="opensearchknn"
Expand Down
Loading