Merge branch 'opensearch-project:main' into remote_index_build_service

.github/workflows/CI.yml

@@ -40,6 +40,11 @@ jobs:
       matrix:
         java: [21, 23]
 
+    env:
+       CC: gcc10-gcc
+       CXX: gcc10-g++
+       FC: gcc10-gfortran
+
     name: Build and Test k-NN Plugin on Linux
     runs-on: ubuntu-latest
     needs: Get-CI-Image-Tag

.github/workflows/backwards_compatibility_tests_workflow.yml

@@ -35,7 +35,7 @@ jobs:
       matrix:
         java: [ 21 ]
         os: [ubuntu-latest]
-        bwc_version : [ "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0", "2.9.0", "2.10.0", "2.11.0", "2.12.0", "2.13.0", "2.14.0", "2.15.0", "2.16.0", "2.17.0", "2.18.0", "2.19.0-SNAPSHOT"]
+        bwc_version : [ "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0", "2.9.0", "2.10.0", "2.11.0", "2.12.0", "2.13.0", "2.14.0", "2.15.0", "2.16.0", "2.17.0", "2.18.0","2.19.0-SNAPSHOT", "2.20.0-SNAPSHOT"]
         opensearch_version : [ "3.0.0-SNAPSHOT" ]
         exclude:
           - os: windows-latest
@@ -130,7 +130,7 @@ jobs:
       matrix:
         java: [ 21 ]
         os: [ubuntu-latest]
-        bwc_version: [ "2.19.0-SNAPSHOT" ]
+        bwc_version: [ "2.20.0-SNAPSHOT" ]
         opensearch_version: [ "3.0.0-SNAPSHOT" ]
 
     name: k-NN Rolling-Upgrade BWC Tests

.github/workflows/maven-publish.yml

@@ -32,4 +32,7 @@ jobs:
           export SONATYPE_PASSWORD=$(aws secretsmanager get-secret-value --secret-id maven-snapshots-password --query SecretString --output text)
           echo "::add-mask::$SONATYPE_USERNAME"
           echo "::add-mask::$SONATYPE_PASSWORD"
+          # For zip
           ./gradlew publishPluginZipPublicationToSnapshotsRepository
+          # For jar
+          ./gradlew publishNebulaPublicationToSnapshotsRepository

.github/workflows/test_security.yml

@@ -39,6 +39,10 @@ jobs:
     strategy:
       matrix:
         java: [21]
+    env:
+       CC: gcc10-gcc
+       CXX: gcc10-g++
+       FC: gcc10-gfortran
 
     name: Run Integration Tests on Linux
     runs-on: ubuntu-latest

CHANGELOG.md

@@ -13,6 +13,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 * Upgrade min JDK compatibility to JDK 21 [#2422](https://github.com/opensearch-project/k-NN/pull/2422)
 ### Documentation
 ### Maintenance
+* Update package name to fix compilation issue [#2513](https://github.com/opensearch-project/k-NN/pull/2513)
 ### Refactoring
 
 ## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.18...2.x)
@@ -22,6 +23,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add expand_nested_docs Parameter support to NMSLIB engine (#2331)[https://github.com/opensearch-project/k-NN/pull/2331]
 - Add a new build mode, `FAISS_OPT_LEVEL=avx512_spr`, which enables the use of advanced AVX-512 instructions introduced with Intel(R) Sapphire Rapids (#2404)[https://github.com/opensearch-project/k-NN/pull/2404]
 - Add cosine similarity support for faiss engine (#2376)[https://github.com/opensearch-project/k-NN/pull/2376]
+- Add derived source feature for vector fields (#2449)[https://github.com/opensearch-project/k-NN/pull/2449]
 ### Enhancements
 - Introduced a writing layer in native engines where relies on the writing interface to process IO. (#2241)[https://github.com/opensearch-project/k-NN/pull/2241]
 - Allow method parameter override for training based indices (#2290) https://github.com/opensearch-project/k-NN/pull/2290]
@@ -30,6 +32,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Use one formula to calculate cosine similarity (#2357)[https://github.com/opensearch-project/k-NN/pull/2357]
 - Add WithFieldName implementation to KNNQueryBuilder (#2398)[https://github.com/opensearch-project/k-NN/pull/2398]
 - Make the build work for M series MacOS without manual code changes and local JAVA_HOME config (#2397)[https://github.com/opensearch-project/k-NN/pull/2397]
+- Enabled concurrent graph creation for Lucene engine with index thread qty settings(#2480)[https://github.com/opensearch-project/k-NN/pull/2480]
 - Remove DocsWithFieldSet reference from NativeEngineFieldVectorsWriter (#2408)[https://github.com/opensearch-project/k-NN/pull/2408]
 ### Bug Fixes
 * Fixing the bug when a segment has no vector field present for disk based vector search (#2282)[https://github.com/opensearch-project/k-NN/pull/2282]

build.gradle

@@ -15,8 +15,8 @@ buildscript {
     ext {
         // build.version_qualifier parameter applies to knn plugin artifacts only. OpenSearch version must be set
         // explicitly as 'opensearch.version' property, for instance opensearch.version=2.0.0-rc1-SNAPSHOT
-        opensearch_version = System.getProperty("opensearch.version", "3.0.0-SNAPSHOT")
-        version_qualifier = System.getProperty("build.version_qualifier", "")
+        opensearch_version = System.getProperty("opensearch.version", "3.0.0-alpha1-SNAPSHOT")
+        version_qualifier = System.getProperty("build.version_qualifier", "alpha1")
         opensearch_group = "org.opensearch"
         isSnapshot = "true" == System.getProperty("build.snapshot", "true")
         avx2_enabled = System.getProperty("avx2.enabled", "true")
@@ -329,11 +329,52 @@ task windowsPatches(type:Exec) {
     commandLine 'cmd', '/c', "Powershell -File $rootDir\\scripts\\windowsScript.ps1"
 }
 
-task cmakeJniLib(type:Exec) {
-    workingDir 'jni'
+def findExecutable(String executableName, List<String> additionalPaths = []) {
+    // Print the task's environment before setting it
+    // Print PATH specifically
+    logger.lifecycle("\nSystem PATH:")
+    logger.lifecycle(System.getenv("PATH"))
+
+    def commonBasePaths = [
+            "/opt/homebrew/bin",
+            "/usr/local/bin",
+            "/usr/bin"
+    ]
+
+    // Start with just the executable name (will use system PATH)
+    def execPath = executableName
+
+    if (Os.isFamily(Os.FAMILY_MAC)) {
+        def searchPaths = []
+        // Add common paths
+        commonBasePaths.each { basePath ->
+            searchPaths.add("${basePath}/${executableName}")
+        }
+        // Add any additional specific paths
+        searchPaths.addAll(additionalPaths)
+
+        for (path in searchPaths) {
+            if (new File(path).exists()) {
+                logger.lifecycle("Found ${executableName} at: ${path}")
+                execPath = path
+                break
+            }
+        }
+    }
+
+    return execPath
+}
+
+tasks.register('cmakeJniLib', Exec) {
+    def cmakePath = findExecutable("cmake")
+    logger.lifecycle("Using cmake at: ${cmakePath}")
+    // Inherit the current environment
+    environment System.getenv()
+
     def args = []
-    args.add("cmake")
-    args.add(".")
+    args.add(cmakePath)
+    args.add("-S jni")
+    args.add("-B jni/build")
     args.add("-DKNN_PLUGIN_VERSION=${opensearch_version}")
     args.add("-DAVX2_ENABLED=${avx2_enabled}")
     args.add("-DAVX512_ENABLED=${avx512_enabled}")
@@ -343,7 +384,7 @@ task cmakeJniLib(type:Exec) {
     def javaHome = Jvm.current().getJavaHome()
     logger.lifecycle("Java home directory used by gradle: $javaHome")
     if (Os.isFamily(Os.FAMILY_MAC)) {
-        environment('JAVA_HOME',javaHome)
+        environment('JAVA_HOME', javaHome)
     }
     if (Os.isFamily(Os.FAMILY_WINDOWS)) {
         dependsOn windowsPatches
@@ -353,16 +394,47 @@ task cmakeJniLib(type:Exec) {
         args.add("-DLAPACK_LIBRARIES=$rootDir\\src\\main\\resources\\windowsDependencies\\libopenblas.dll")
     }
 
+    // Print the task's environment after setting it
+    logger.lifecycle("\nTask Environment PATH:")
+    logger.lifecycle(environment.get('PATH'))
+
+    // Print the command that will be executed
+    logger.lifecycle("CMake command: ${args.join(' ')}")
+    def outputStream = new ByteArrayOutputStream()
     commandLine args
+    standardOutput = outputStream
+}
+
+task buildNmslib(type:Exec) {
+    dependsOn cmakeJniLib
+    def makePath = findExecutable("make")
+    logger.lifecycle("Using make at: ${makePath}")
+    // Ensure makePath is treated as a String. If parsing to an int is required,
+    // handle it explicitly, though a path typically should not be parsed as an int.
+    if (makePath.isEmpty()) {
+        throw new GradleException("Make not found in PATH. Please install Make.")
+    }
+    def outputStream = new ByteArrayOutputStream()
+    commandLine makePath, '-Cjni/build', 'opensearchknn_nmslib', '-j', "${nproc_count}"
+    standardOutput = outputStream
 }
 
 task buildJniLib(type:Exec) {
     dependsOn cmakeJniLib
-    workingDir 'jni'
-    commandLine 'make', 'opensearchknn_nmslib', 'opensearchknn_faiss', 'opensearchknn_common', '-j', "${nproc_count}"
+    def makePath = findExecutable("make")
+    logger.lifecycle("Using make at: ${makePath}")
+    // Ensure makePath is treated as a String. If parsing to an int is required,
+    // handle it explicitly, though a path typically should not be parsed as an int.
+    if (makePath.isEmpty()) {
+        throw new GradleException("Make not found in PATH. Please install Make.")
+    }
+    def outputStream = new ByteArrayOutputStream()
+    commandLine makePath, '-Cjni/build', 'opensearchknn_faiss', 'opensearchknn_common', '-j', "${nproc_count}"
+    standardOutput = outputStream
 }
 
 test {
+    dependsOn buildNmslib
     dependsOn buildJniLib
     systemProperty 'tests.security.manager', 'false'
     systemProperty "java.library.path", "$rootDir/jni/release"
@@ -377,6 +449,7 @@ test {
 def _numNodes = findProperty('numNodes') as Integer ?: 1
 integTest {
     if (integTestDependOnJniLib) {
+        dependsOn buildNmslib
         dependsOn buildJniLib
     }
     systemProperty 'tests.security.manager', 'false'
@@ -399,6 +472,7 @@ integTest {
     systemProperty("https", is_https)
     systemProperty("user", user)
     systemProperty("password", password)
+    systemProperty("test.exhaustive", System.getProperty("test.exhaustive"))
 
     doFirst {
         // Tell the test JVM if the cluster JVM is running under a debugger so that tests can
@@ -462,6 +536,7 @@ task integTestRemote(type: RestIntegTestTask) {
     systemProperty 'cluster.number_of_nodes', "${_numNodes}"
 
     systemProperty 'tests.security.manager', 'false'
+    systemProperty("test.exhaustive", System.getProperty("test.exhaustive"))
 
     // Run tests with remote cluster only if rest case is defined
     if (System.getProperty("tests.rest.cluster") != null) {
@@ -473,6 +548,7 @@ task integTestRemote(type: RestIntegTestTask) {
 
 run {
     useCluster project.testClusters.integTest
+    dependsOn buildNmslib
     dependsOn buildJniLib
     doFirst {
         // There seems to be an issue when running multi node run or integ tasks with unicast_hosts

jni/cmake/init-faiss.cmake

@@ -111,10 +111,10 @@ endif()
 if(NOT DEFINED AVX512_SPR_ENABLED)
     # Check if the system is Intel(R) Sapphire Rapids or a newer-generation processor
     execute_process(COMMAND bash -c "lscpu | grep -q 'GenuineIntel' && lscpu | grep -i 'avx512_fp16' | grep -i 'avx512_bf16' | grep -i 'avx512_vpopcntdq'" OUTPUT_VARIABLE SPR_FLAGS OUTPUT_STRIP_TRAILING_WHITESPACE)
-    if (AND NOT "${SPR_FLAGS}" STREQUAL "")
-	set(AVX512_SPR_ENABLED true)
+    if (NOT "${SPR_FLAGS}" STREQUAL "")
+	      set(AVX512_SPR_ENABLED true)
     else()
-	set(AVX512_SPR_ENABLED false)
+	      set(AVX512_SPR_ENABLED false)
     endif()
 endif()
 

qa/restart-upgrade/build.gradle

@@ -135,6 +135,7 @@ testClusters {
 // All nodes are upgraded to latest version and run the tests
     task testRestartUpgrade(type: StandaloneRestIntegTestTask) {
         dependsOn "testAgainstOldCluster"
+        dependsOn rootProject.tasks.buildNmslib
         dependsOn rootProject.tasks.buildJniLib
         dependsOn rootProject.tasks.assemble
         useCluster testClusters."${baseName}"

qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ModelIT.java

@@ -53,7 +53,7 @@ public class ModelIT extends AbstractRestartUpgradeTestCase {
     private static final int DELAY_MILLI_SEC = 1000;
     private static final int MIN_NUM_OF_MODELS = 2;
     private static final int K = 5;
-    private static final int NUM_DOCS = 10;
+    private static final int NUM_DOCS = 1001;
     private static final int NUM_DOCS_TEST_MODEL_INDEX = 100;
     private static final int NUM_DOCS_TEST_MODEL_INDEX_DEFAULT = 100;
     private static final int NUM_DOCS_TEST_MODEL_INDEX_FOR_NON_KNN_INDEX = 100;

qa/rolling-upgrade/build.gradle

@@ -52,6 +52,7 @@ task testAgainstOldCluster(type: StandaloneRestIntegTestTask) {
 // This results in a mixed cluster with 2 nodes on the old version and 1 upgraded node.
 task testAgainstOneThirdUpgradedCluster(type: StandaloneRestIntegTestTask) {
     useCluster testClusters."${baseName}"
+    dependsOn rootProject.tasks.buildNmslib
     dependsOn rootProject.tasks.buildJniLib
     dependsOn rootProject.tasks.assemble
     dependsOn "testAgainstOldCluster"

scripts/build.sh

@@ -133,16 +133,24 @@ cd $work_dir
 ./gradlew :buildJniLib -Davx512.enabled=false -Davx512_spr.enabled=false -Davx2.enabled=false -Dbuild.lib.commit_patches=false -Dnproc.count=${NPROC_COUNT:-1}
 
 if [ "$PLATFORM" != "windows" ] && [ "$ARCHITECTURE" = "x64" ]; then
+  echo "Building k-NN library nmslib with gcc 10 on non-windows x64"
+  rm -rf jni/CMakeCache.txt jni/CMakeFiles
+  env CC=gcc10-gcc CXX=gcc10-g++ FC=gcc10-gfortran ./gradlew :buildNmslib -Dbuild.lib.commit_patches=false -Dbuild.lib.apply_patches=false
+
   echo "Building k-NN library after enabling AVX2"
   # Skip applying patches as patches were applied already from previous :buildJniLib task
   # If we apply patches again, it fails with conflict
+  rm -rf jni/CMakeCache.txt jni/CMakeFiles
   ./gradlew :buildJniLib -Davx2.enabled=true -Davx512.enabled=false -Davx512_spr.enabled=false -Dbuild.lib.commit_patches=false -Dbuild.lib.apply_patches=false
 
   echo "Building k-NN library after enabling AVX512"
   ./gradlew :buildJniLib -Davx512.enabled=true -Davx512_spr.enabled=false -Dbuild.lib.commit_patches=false -Dbuild.lib.apply_patches=false
 
   echo "Building k-NN library after enabling AVX512_SPR"
   ./gradlew :buildJniLib -Davx512_spr.enabled=true -Dbuild.lib.commit_patches=false -Dbuild.lib.apply_patches=false
+
+else
+  ./gradlew :buildNmslib -Dbuild.lib.commit_patches=false -Dbuild.lib.apply_patches=false
 fi
 
 ./gradlew publishPluginZipPublicationToZipStagingRepository -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER
@@ -151,7 +159,7 @@ fi
 # Add lib to zip
 zipPath=$(find "$(pwd)/build/distributions" -path \*.zip)
 distributions="$(dirname "${zipPath}")"
-mkdir $distributions/lib
+mkdir -p $distributions/lib
 libPrefix="libopensearchknn"
 if [ "$PLATFORM" = "windows" ]; then
     libPrefix="opensearchknn"

src/main/java/org/opensearch/knn/common/KNNConstants.java

@@ -161,4 +161,8 @@ public class KNNConstants {
 
     public static final String MODE_PARAMETER = "mode";
     public static final String COMPRESSION_LEVEL_PARAMETER = "compression_level";
+
+    public static final String DERIVED_VECTOR_FIELD_ATTRIBUTE_KEY = "knn-derived-source-enabled";
+    public static final String DERIVED_VECTOR_FIELD_ATTRIBUTE_TRUE_VALUE = "true";
+    public static final String DERIVED_VECTOR_FIELD_ATTRIBUTE_FALSE_VALUE = "false";
 }

src/main/java/org/opensearch/knn/index/KNNCircuitBreaker.java

@@ -13,7 +13,7 @@
 import org.opensearch.knn.plugin.transport.KNNStatsResponse;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.opensearch.client.Client;
+import org.opensearch.transport.client.Client;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.threadpool.ThreadPool;