Merge branch 'main' into extend-classpath-plugins

CHANGELOG-3.0.md

@@ -50,6 +50,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix 'org.apache.hc.core5.http.ParseException: Invalid protocol version' under JDK 16+ ([#4827](https://github.com/opensearch-project/OpenSearch/pull/4827))
 - Fix compression support for h2c protocol ([#4944](https://github.com/opensearch-project/OpenSearch/pull/4944))
 - Don't over-allocate in HeapBufferedAsyncEntityConsumer in order to consume the response ([#9993](https://github.com/opensearch-project/OpenSearch/pull/9993))
+- Fix swapped field formats in nodes API where `total_indexing_buffer_in_bytes` and `total_indexing_buffer` values were reversed ([#17070](https://github.com/opensearch-project/OpenSearch/pull/17070))
+
 
 ### Security
 

CHANGELOG.md

@@ -32,6 +32,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Support object fields in star-tree index([#16728](https://github.com/opensearch-project/OpenSearch/pull/16728/))
 - Support searching from doc_value using termQueryCaseInsensitive/termQuery in flat_object/keyword field([#16974](https://github.com/opensearch-project/OpenSearch/pull/16974/))
 - Added a new `time` field to replace the deprecated `getTime` field in `GetStats`. ([#17009](https://github.com/opensearch-project/OpenSearch/pull/17009))
+- Improve performance of the bitmap filtering([#16936](https://github.com/opensearch-project/OpenSearch/pull/16936/))
 - Enable testing for ExtensiblePlugins using classpath plugins ([#16908](https://github.com/opensearch-project/OpenSearch/pull/16908))
 
 ### Dependencies
@@ -67,7 +68,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `opentelemetry-semconv` from 1.27.0-alpha to 1.29.0-alpha ([#16700](https://github.com/opensearch-project/OpenSearch/pull/16700))
 - Bump `com.google.re2j:re2j` from 1.7 to 1.8 ([#17012](https://github.com/opensearch-project/OpenSearch/pull/17012))
 - Bump `com.squareup.okio:okio` from 3.9.1 to 3.10.2 ([#17060](https://github.com/opensearch-project/OpenSearch/pull/17060))
-- Bump `com.diffplug.spotless` from 6.25.0 to 7.0.2 ([#17058](https://github.com/opensearch-project/OpenSearch/pull/17058))
 
 ### Changed
 - Indexed IP field supports `terms_query` with more than 1025 IP masks [#16391](https://github.com/opensearch-project/OpenSearch/pull/16391)
@@ -76,6 +76,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Allow extended plugins to be optional ([#16909](https://github.com/opensearch-project/OpenSearch/pull/16909))
 - Use the correct type to widen the sort fields when merging top docs ([#16881](https://github.com/opensearch-project/OpenSearch/pull/16881))
 - Limit reader writer separation to remote store enabled clusters [#16760](https://github.com/opensearch-project/OpenSearch/pull/16760)
+- Optimize innerhits query performance [#16937](https://github.com/opensearch-project/OpenSearch/pull/16937)
 
 ### Deprecated
 - Performing update operation with default pipeline or final pipeline is deprecated ([#16712](https://github.com/opensearch-project/OpenSearch/pull/16712))

build.gradle

@@ -54,7 +54,7 @@ plugins {
   id 'lifecycle-base'
   id 'opensearch.docker-support'
   id 'opensearch.global-build-info'
-  id "com.diffplug.spotless" version "7.0.2" apply false
+  id "com.diffplug.spotless" version "6.25.0" apply false
   id "test-report-aggregation"
   id 'jacoco-report-aggregation'
 }

rest-api-spec/src/main/resources/rest-api-spec/test/nodes.info/50_nodes_total_indexing_buffer_format.yml

@@ -0,0 +1,21 @@
+---
+"Test total indexing buffer fields should have correct formats":
+  - skip:
+      version: " - 2.99.99"
+      reason: "this change is added in 3.0.0"
+      features: [arbitrary_key]
+
+  - do:
+      nodes.info: {}
+  - set:
+      nodes._arbitrary_key_: node_id
+
+  - do:
+      nodes.info:
+        human: true
+        filter_path: "nodes.*.total_indexing_buffer*"
+
+  - gte: { nodes.$node_id.total_indexing_buffer_in_bytes: 0 }
+
+  - match:
+      nodes.$node_id.total_indexing_buffer: /^\d+(\.\d+)?(b|kb|mb|gb|tb|pb)$/

server/src/main/java/org/opensearch/action/admin/cluster/node/info/NodesInfoResponse.java

@@ -98,7 +98,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.field("build_type", nodeInfo.getBuild().type().displayName());
             builder.field("build_hash", nodeInfo.getBuild().hash());
             if (nodeInfo.getTotalIndexingBuffer() != null) {
-                builder.humanReadableField("total_indexing_buffer", "total_indexing_buffer_in_bytes", nodeInfo.getTotalIndexingBuffer());
+                builder.humanReadableField("total_indexing_buffer_in_bytes", "total_indexing_buffer", nodeInfo.getTotalIndexingBuffer());
             }
 
             builder.startArray("roles");

server/src/main/java/org/opensearch/index/mapper/DocumentMapper.java

@@ -35,9 +35,7 @@
 import org.apache.lucene.document.StoredField;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.Query;
-import org.apache.lucene.search.ScoreMode;
-import org.apache.lucene.search.Scorer;
-import org.apache.lucene.search.Weight;
+import org.apache.lucene.util.BitSet;
 import org.apache.lucene.util.BytesRef;
 import org.opensearch.OpenSearchGenerationException;
 import org.opensearch.common.annotation.PublicApi;
@@ -53,6 +51,7 @@
 import org.opensearch.index.analysis.IndexAnalyzers;
 import org.opensearch.index.mapper.MapperService.MergeReason;
 import org.opensearch.index.mapper.MetadataFieldMapper.TypeParser;
+import org.opensearch.index.query.NestedQueryBuilder;
 import org.opensearch.search.internal.SearchContext;
 
 import java.io.IOException;
@@ -270,25 +269,15 @@ public ParsedDocument createNoopTombstoneDoc(String index, String reason) throws
      * Returns the best nested {@link ObjectMapper} instances that is in the scope of the specified nested docId.
      */
     public ObjectMapper findNestedObjectMapper(int nestedDocId, SearchContext sc, LeafReaderContext context) throws IOException {
+        if (sc instanceof NestedQueryBuilder.NestedInnerHitSubContext) {
+            ObjectMapper objectMapper = ((NestedQueryBuilder.NestedInnerHitSubContext) sc).getChildObjectMapper();
+            assert objectMappers().containsKey(objectMapper.fullPath());
+            assert containSubDocIdWithObjectMapper(nestedDocId, objectMapper, sc, context);
+            return objectMapper;
+        }
         ObjectMapper nestedObjectMapper = null;
         for (ObjectMapper objectMapper : objectMappers().values()) {
-            if (!objectMapper.nested().isNested()) {
-                continue;
-            }
-
-            Query filter = objectMapper.nestedTypeFilter();
-            if (filter == null) {
-                continue;
-            }
-            // We can pass down 'null' as acceptedDocs, because nestedDocId is a doc to be fetched and
-            // therefore is guaranteed to be a live doc.
-            final Weight nestedWeight = filter.createWeight(sc.searcher(), ScoreMode.COMPLETE_NO_SCORES, 1f);
-            Scorer scorer = nestedWeight.scorer(context);
-            if (scorer == null) {
-                continue;
-            }
-
-            if (scorer.iterator().advance(nestedDocId) == nestedDocId) {
+            if (containSubDocIdWithObjectMapper(nestedDocId, objectMapper, sc, context)) {
                 if (nestedObjectMapper == null) {
                     nestedObjectMapper = objectMapper;
                 } else {
@@ -301,6 +290,25 @@ public ObjectMapper findNestedObjectMapper(int nestedDocId, SearchContext sc, Le
         return nestedObjectMapper;
     }
 
+    private boolean containSubDocIdWithObjectMapper(int nestedDocId, ObjectMapper objectMapper, SearchContext sc, LeafReaderContext context)
+        throws IOException {
+        if (!objectMapper.nested().isNested()) {
+            return false;
+        }
+        Query filter = objectMapper.nestedTypeFilter();
+        if (filter == null) {
+            return false;
+        }
+        // We can pass down 'null' as acceptedDocs, because nestedDocId is a doc to be fetched and
+        // therefore is guaranteed to be a live doc.
+        BitSet nestedDocIds = sc.bitsetFilterCache().getBitSetProducer(filter).getBitSet(context);
+        if (nestedDocIds != null && nestedDocIds.get(nestedDocId)) {
+            return true;
+        } else {
+            return false;
+        }
+    }
+
     public DocumentMapper merge(Mapping mapping, MergeReason reason) {
         Mapping merged = this.mapping.merge(mapping, reason);
         return new DocumentMapper(mapperService, merged);

server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java

@@ -46,10 +46,8 @@
 import org.apache.lucene.sandbox.document.HalfFloatPoint;
 import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.IndexOrDocValuesQuery;
-import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;
-import org.apache.lucene.search.PointInSetQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.NumericUtils;
@@ -73,6 +71,7 @@
 import org.opensearch.search.DocValueFormat;
 import org.opensearch.search.lookup.SearchLookup;
 import org.opensearch.search.query.BitmapDocValuesQuery;
+import org.opensearch.search.query.BitmapIndexQuery;
 
 import java.io.IOException;
 import java.math.BigInteger;
@@ -81,7 +80,6 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -888,10 +886,10 @@ public Query bitmapQuery(String field, BytesArray bitmapArray, boolean isSearcha
                 }
 
                 if (isSearchable && hasDocValues) {
-                    return new IndexOrDocValuesQuery(bitmapIndexQuery(field, bitmap), new BitmapDocValuesQuery(field, bitmap));
+                    return new IndexOrDocValuesQuery(new BitmapIndexQuery(field, bitmap), new BitmapDocValuesQuery(field, bitmap));
                 }
                 if (isSearchable) {
-                    return bitmapIndexQuery(field, bitmap);
+                    return new BitmapIndexQuery(field, bitmap);
                 }
                 return new BitmapDocValuesQuery(field, bitmap);
             }
@@ -1507,40 +1505,6 @@ public static Query unsignedLongRangeQuery(
             }
             return builder.apply(l, u);
         }
-
-        static PointInSetQuery bitmapIndexQuery(String field, RoaringBitmap bitmap) {
-            final BytesRef encoded = new BytesRef(new byte[Integer.BYTES]);
-            return new PointInSetQuery(field, 1, Integer.BYTES, new PointInSetQuery.Stream() {
-
-                final Iterator<Integer> iterator = bitmap.iterator();
-
-                @Override
-                public BytesRef next() {
-                    int value;
-                    if (iterator.hasNext()) {
-                        value = iterator.next();
-                    } else {
-                        return null;
-                    }
-                    IntPoint.encodeDimension(value, encoded.bytes, 0);
-                    return encoded;
-                }
-            }) {
-                @Override
-                public Query rewrite(IndexSearcher indexSearcher) throws IOException {
-                    if (bitmap.isEmpty()) {
-                        return new MatchNoDocsQuery();
-                    }
-                    return super.rewrite(indexSearcher);
-                }
-
-                @Override
-                protected String toString(byte[] value) {
-                    assert value.length == Integer.BYTES;
-                    return Integer.toString(IntPoint.decodeDimension(value, 0));
-                }
-            };
-        }
     }
 
     /**

server/src/main/java/org/opensearch/index/query/NestedQueryBuilder.java

@@ -438,7 +438,7 @@ protected void doBuild(SearchContext parentSearchContext, InnerHitsContext inner
      *
      * @opensearch.internal
      */
-    static final class NestedInnerHitSubContext extends InnerHitsContext.InnerHitSubContext {
+    public static final class NestedInnerHitSubContext extends InnerHitsContext.InnerHitSubContext {
 
         private final ObjectMapper parentObjectMapper;
         private final ObjectMapper childObjectMapper;
@@ -507,6 +507,10 @@ public TopDocsAndMaxScore topDocs(SearchHit hit) throws IOException {
                 return new TopDocsAndMaxScore(td, maxScore);
             }
         }
+
+        public ObjectMapper getChildObjectMapper() {
+            return childObjectMapper;
+        }
     }
 
     @Override

server/src/main/java/org/opensearch/search/fetch/FetchPhase.java

@@ -38,10 +38,7 @@
 import org.apache.lucene.index.ReaderUtil;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.Query;
-import org.apache.lucene.search.ScoreMode;
-import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.TotalHits;
-import org.apache.lucene.search.Weight;
 import org.apache.lucene.util.BitSet;
 import org.opensearch.common.CheckedBiConsumer;
 import org.opensearch.common.annotation.PublicApi;
@@ -55,7 +52,6 @@
 import org.opensearch.core.common.text.Text;
 import org.opensearch.core.tasks.TaskCancelledException;
 import org.opensearch.core.xcontent.MediaType;
-import org.opensearch.index.IndexSettings;
 import org.opensearch.index.fieldvisitor.CustomFieldsVisitor;
 import org.opensearch.index.fieldvisitor.FieldsVisitor;
 import org.opensearch.index.mapper.DocumentMapper;
@@ -501,7 +497,6 @@ private SearchHit.NestedIdentity getInternalNestedIdentity(
         ObjectMapper current = nestedObjectMapper;
         String originalName = nestedObjectMapper.name();
         SearchHit.NestedIdentity nestedIdentity = null;
-        final IndexSettings indexSettings = context.getQueryShardContext().getIndexSettings();
         do {
             Query parentFilter;
             nestedParentObjectMapper = current.getParentObjectMapper(mapperService);
@@ -520,14 +515,13 @@ private SearchHit.NestedIdentity getInternalNestedIdentity(
                 current = nestedParentObjectMapper;
                 continue;
             }
-            final Weight childWeight = context.searcher()
-                .createWeight(context.searcher().rewrite(childFilter), ScoreMode.COMPLETE_NO_SCORES, 1f);
-            Scorer childScorer = childWeight.scorer(subReaderContext);
-            if (childScorer == null) {
+            BitSet childIter = context.bitsetFilterCache()
+                .getBitSetProducer(context.searcher().rewrite(childFilter))
+                .getBitSet(subReaderContext);
+            if (childIter == null) {
                 current = nestedParentObjectMapper;
                 continue;
             }
-            DocIdSetIterator childIter = childScorer.iterator();
 
             BitSet parentBits = context.bitsetFilterCache().getBitSetProducer(parentFilter).getBitSet(subReaderContext);
 
@@ -541,8 +535,8 @@ private SearchHit.NestedIdentity getInternalNestedIdentity(
              * that appear before him.
              */
             int previousParent = parentBits.prevSetBit(currentParent);
-            for (int docId = childIter.advance(previousParent + 1); docId < nestedSubDocId
-                && docId != DocIdSetIterator.NO_MORE_DOCS; docId = childIter.nextDoc()) {
+            for (int docId = childIter.nextSetBit(previousParent + 1); docId < nestedSubDocId
+                && docId != DocIdSetIterator.NO_MORE_DOCS; docId = childIter.nextSetBit(docId + 1)) {
                 offset++;
             }
             currentParent = nestedSubDocId;

server/src/main/java/org/opensearch/search/query/BitmapDocValuesQuery.java

@@ -30,6 +30,8 @@
 
 import org.roaringbitmap.RoaringBitmap;
 
+import static org.opensearch.search.query.BitmapIndexQuery.checkArgs;
+
 /**
  * Filter with bitmap
  * <p>
@@ -43,6 +45,7 @@ public class BitmapDocValuesQuery extends Query implements Accountable {
     final long max;
 
     public BitmapDocValuesQuery(String field, RoaringBitmap bitmap) {
+        checkArgs(field, bitmap);
         this.field = field;
         this.bitmap = bitmap;
         if (!bitmap.isEmpty()) {
@@ -111,8 +114,7 @@ public boolean isCacheable(LeafReaderContext ctx) {
 
     @Override
     public String toString(String field) {
-        // bitmap may contain high cardinality, so choose to not show the actual values in it
-        return field + " cardinality: " + bitmap.getLongCardinality();
+        return "BitmapDocValuesQuery(field=" + this.field + ")";
     }
 
     @Override
@@ -139,8 +141,8 @@ public int hashCode() {
 
     @Override
     public long ramBytesUsed() {
-        return RamUsageEstimator.shallowSizeOfInstance(BitmapDocValuesQuery.class) + RamUsageEstimator.sizeOfObject(field)
-            + RamUsageEstimator.sizeOfObject(bitmap);
+        return RamUsageEstimator.shallowSizeOfInstance(BitmapIndexQuery.class) + RamUsageEstimator.sizeOf(field) + bitmap
+            .getLongSizeInBytes();
     }
 
     @Override