Skip to content

Commit 4fd6877

Browse files
luyunchengluyuncheng
authored
Replace the deprecated IndexReader APIs with new storedFields() & termVectors() (opensearch-project#7792)
* 1. Remove calling deprecated document api Signed-off-by: luyuncheng <[email protected]> * 1. Remove calling deprecated document api 2. Fixed some calling and Tests Signed-off-by: luyuncheng <[email protected]> * 1. Remove calling deprecated document api 2. Fixed some calling and Tests 3. Spotless java Signed-off-by: luyuncheng <[email protected]> * add changelog Signed-off-by: luyuncheng <[email protected]> * add changelog Signed-off-by: luyuncheng <[email protected]> * merge main into branch Signed-off-by: luyuncheng <[email protected]> * merge main into branch update CHANGELOG.md Signed-off-by: luyuncheng <[email protected]> --------- Signed-off-by: luyuncheng <[email protected]>
1 parent e1a4125 commit 4fd6877

26 files changed

+92
-61
lines changed

‎CHANGELOG.md

+1
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
4646
- Change http code on create index API with bad input raising NotXContentException from 500 to 400 ([#4773](https://github.com/opensearch-project/OpenSearch/pull/4773))
4747
- Improve summary error message for invalid setting updates ([#4792](https://github.com/opensearch-project/OpenSearch/pull/4792))
4848
- Remote Segment Store Repository setting moved from `index.remote_store.repository` to `index.remote_store.segment.repository` and `cluster.remote_store.repository` to `cluster.remote_store.segment.repository` respectively for Index and Cluster level settings ([#8719](https://github.com/opensearch-project/OpenSearch/pull/8719))
49+
- Replace the deprecated IndexReader APIs with new storedFields() & termVectors() ([#7792](https://github.com/opensearch-project/OpenSearch/pull/7792))
4950

5051
### Deprecated
5152

‎server/src/main/java/org/opensearch/common/lucene/search/XMoreLikeThis.java

+6-2
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,9 @@
5656
import org.apache.lucene.index.IndexReader;
5757
import org.apache.lucene.index.IndexableField;
5858
import org.apache.lucene.index.PostingsEnum;
59+
import org.apache.lucene.index.StoredFields;
5960
import org.apache.lucene.index.Term;
61+
import org.apache.lucene.index.TermVectors;
6062
import org.apache.lucene.index.Terms;
6163
import org.apache.lucene.index.TermsEnum;
6264
import org.apache.lucene.search.BooleanClause;
@@ -808,8 +810,10 @@ public String describeParams() {
808810
*/
809811
private PriorityQueue<ScoreTerm> retrieveTerms(int docNum) throws IOException {
810812
Map<String, Int> termFreqMap = new HashMap<>();
813+
final TermVectors termVectors = ir.termVectors();
814+
final StoredFields storedFields = ir.storedFields();
811815
for (String fieldName : fieldNames) {
812-
final Fields vectors = ir.getTermVectors(docNum);
816+
final Fields vectors = termVectors.get(docNum);
813817
final Terms vector;
814818
if (vectors != null) {
815819
vector = vectors.terms(fieldName);
@@ -819,7 +823,7 @@ private PriorityQueue<ScoreTerm> retrieveTerms(int docNum) throws IOException {
819823

820824
// field does not store term vector info
821825
if (vector == null) {
822-
Document d = ir.document(docNum);
826+
Document d = storedFields.document(docNum);
823827
IndexableField fields[] = d.getFields(fieldName);
824828
for (IndexableField field : fields) {
825829
final String stringValue = field.stringValue();

‎server/src/main/java/org/opensearch/gateway/PersistedClusterStateService.java

+3-3
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
import org.apache.lucene.index.IndexWriterConfig;
4646
import org.apache.lucene.index.LeafReaderContext;
4747
import org.apache.lucene.index.SerialMergeScheduler;
48+
import org.apache.lucene.index.StoredFields;
4849
import org.apache.lucene.index.Term;
4950
import org.apache.lucene.search.DocIdSetIterator;
5051
import org.apache.lucene.search.IndexSearcher;
@@ -507,12 +508,11 @@ private static void consumeFromType(IndexSearcher indexSearcher, String type, Ch
507508
final Bits liveDocs = leafReaderContext.reader().getLiveDocs();
508509
final IntPredicate isLiveDoc = liveDocs == null ? i -> true : liveDocs::get;
509510
final DocIdSetIterator docIdSetIterator = scorer.iterator();
511+
final StoredFields storedFields = leafReaderContext.reader().storedFields();
510512
while (docIdSetIterator.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
511513
if (isLiveDoc.test(docIdSetIterator.docID())) {
512514
logger.trace("processing doc {}", docIdSetIterator.docID());
513-
bytesRefConsumer.accept(
514-
leafReaderContext.reader().document(docIdSetIterator.docID()).getBinaryValue(DATA_FIELD_NAME)
515-
);
515+
bytesRefConsumer.accept(storedFields.document(docIdSetIterator.docID()).getBinaryValue(DATA_FIELD_NAME));
516516
}
517517
}
518518
}

‎server/src/main/java/org/opensearch/index/engine/InternalEngine.java

+3-1
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949
import org.apache.lucene.index.ShuffleForcedMergePolicy;
5050
import org.apache.lucene.index.SoftDeletesRetentionMergePolicy;
5151
import org.apache.lucene.index.StandardDirectoryReader;
52+
import org.apache.lucene.index.StoredFields;
5253
import org.apache.lucene.index.Term;
5354
import org.apache.lucene.search.BooleanClause;
5455
import org.apache.lucene.search.BooleanQuery;
@@ -2889,14 +2890,15 @@ private void restoreVersionMapAndCheckpointTracker(DirectoryReader directoryRead
28892890
final CombinedDocValues dv = new CombinedDocValues(leaf.reader());
28902891
final IdOnlyFieldVisitor idFieldVisitor = new IdOnlyFieldVisitor();
28912892
final DocIdSetIterator iterator = scorer.iterator();
2893+
final StoredFields storedFields = leaf.reader().storedFields();
28922894
int docId;
28932895
while ((docId = iterator.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
28942896
final long primaryTerm = dv.docPrimaryTerm(docId);
28952897
final long seqNo = dv.docSeqNo(docId);
28962898
localCheckpointTracker.markSeqNoAsProcessed(seqNo);
28972899
localCheckpointTracker.markSeqNoAsPersisted(seqNo);
28982900
idFieldVisitor.reset();
2899-
leaf.reader().document(docId, idFieldVisitor);
2901+
storedFields.document(docId, idFieldVisitor);
29002902
if (idFieldVisitor.getId() == null) {
29012903
assert dv.isTombstone(docId);
29022904
continue;

‎server/src/main/java/org/opensearch/index/engine/LuceneChangesSnapshot.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -289,7 +289,7 @@ private Translog.Operation readDocAsOp(int docIndex) throws IOException {
289289
? SourceFieldMapper.RECOVERY_SOURCE_NAME
290290
: SourceFieldMapper.NAME;
291291
final FieldsVisitor fields = new FieldsVisitor(true, sourceField);
292-
leaf.reader().document(segmentDocID, fields);
292+
leaf.reader().storedFields().document(segmentDocID, fields);
293293

294294
final Translog.Operation op;
295295
final boolean isTombstone = parallelArray.isTombStone[docIndex];

‎server/src/main/java/org/opensearch/index/engine/TranslogLeafReader.java

+23-18
Original file line numberDiff line numberDiff line change
@@ -221,28 +221,33 @@ public int maxDoc() {
221221

222222
@Override
223223
public void document(int docID, StoredFieldVisitor visitor) throws IOException {
224-
if (docID != 0) {
225-
throw new IllegalArgumentException("no such doc ID " + docID);
226-
}
227-
if (visitor.needsField(FAKE_SOURCE_FIELD) == StoredFieldVisitor.Status.YES) {
228-
assert operation.source().toBytesRef().offset == 0;
229-
assert operation.source().toBytesRef().length == operation.source().toBytesRef().bytes.length;
230-
visitor.binaryField(FAKE_SOURCE_FIELD, operation.source().toBytesRef().bytes);
231-
}
232-
if (operation.routing() != null && visitor.needsField(FAKE_ROUTING_FIELD) == StoredFieldVisitor.Status.YES) {
233-
visitor.stringField(FAKE_ROUTING_FIELD, operation.routing());
234-
}
235-
if (visitor.needsField(FAKE_ID_FIELD) == StoredFieldVisitor.Status.YES) {
236-
BytesRef bytesRef = Uid.encodeId(operation.id());
237-
final byte[] id = new byte[bytesRef.length];
238-
System.arraycopy(bytesRef.bytes, bytesRef.offset, id, 0, bytesRef.length);
239-
visitor.binaryField(FAKE_ID_FIELD, id);
240-
}
224+
storedFields().document(docID, visitor);
241225
}
242226

243227
@Override
244228
public StoredFields storedFields() throws IOException {
245-
throw new UnsupportedOperationException();
229+
return new StoredFields() {
230+
@Override
231+
public void document(int docID, StoredFieldVisitor visitor) throws IOException {
232+
if (docID != 0) {
233+
throw new IllegalArgumentException("no such doc ID " + docID);
234+
}
235+
if (visitor.needsField(FAKE_SOURCE_FIELD) == StoredFieldVisitor.Status.YES) {
236+
assert operation.source().toBytesRef().offset == 0;
237+
assert operation.source().toBytesRef().length == operation.source().toBytesRef().bytes.length;
238+
visitor.binaryField(FAKE_SOURCE_FIELD, operation.source().toBytesRef().bytes);
239+
}
240+
if (operation.routing() != null && visitor.needsField(FAKE_ROUTING_FIELD) == StoredFieldVisitor.Status.YES) {
241+
visitor.stringField(FAKE_ROUTING_FIELD, operation.routing());
242+
}
243+
if (visitor.needsField(FAKE_ID_FIELD) == StoredFieldVisitor.Status.YES) {
244+
BytesRef bytesRef = Uid.encodeId(operation.id());
245+
final byte[] id = new byte[bytesRef.length];
246+
System.arraycopy(bytesRef.bytes, bytesRef.offset, id, 0, bytesRef.length);
247+
visitor.binaryField(FAKE_ID_FIELD, id);
248+
}
249+
}
250+
};
246251
}
247252

248253
@Override

‎server/src/main/java/org/opensearch/index/get/ShardGetService.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -276,7 +276,7 @@ private GetResult innerGetLoadFromStoredFields(
276276
);
277277
if (fieldVisitor != null) {
278278
try {
279-
docIdAndVersion.reader.document(docIdAndVersion.docId, fieldVisitor);
279+
docIdAndVersion.reader.storedFields().document(docIdAndVersion.docId, fieldVisitor);
280280
} catch (IOException e) {
281281
throw new OpenSearchException("Failed to get id [" + id + "]", e);
282282
}

‎server/src/main/java/org/opensearch/index/shard/ShardSplittingQuery.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -286,7 +286,7 @@ public Status needsField(FieldInfo fieldInfo) throws IOException {
286286
boolean matches(int doc) throws IOException {
287287
routing = id = null;
288288
leftToVisit = 2;
289-
leafReader.document(doc, this);
289+
leafReader.storedFields().document(doc, this);
290290
assert id != null : "docID must not be null - we might have hit a nested document";
291291
int targetShardId = OperationRouting.generateShardId(indexMetadata, id, routing);
292292
return targetShardId != shardId;

‎server/src/main/java/org/opensearch/index/termvectors/TermVectorsService.java

+5-2
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
import org.apache.lucene.index.IndexableField;
4040
import org.apache.lucene.index.MultiTerms;
4141
import org.apache.lucene.index.Term;
42+
import org.apache.lucene.index.TermVectors;
4243
import org.apache.lucene.index.Terms;
4344
import org.apache.lucene.index.memory.MemoryIndex;
4445
import org.opensearch.OpenSearchException;
@@ -127,7 +128,8 @@ static TermVectorsResponse getTermVectors(IndexShard indexShard, TermVectorsRequ
127128
/* or from an existing document */
128129
else if (docIdAndVersion != null) {
129130
// fields with stored term vectors
130-
termVectorsByField = docIdAndVersion.reader.getTermVectors(docIdAndVersion.docId);
131+
TermVectors termVectors = docIdAndVersion.reader.termVectors();
132+
termVectorsByField = termVectors.get(docIdAndVersion.docId);
131133
Set<String> selectedFields = request.selectedFields();
132134
// generate tvs for fields where analyzer is overridden
133135
if (selectedFields == null && request.perFieldAnalyzer() != null) {
@@ -322,7 +324,8 @@ private static Fields generateTermVectors(
322324
}
323325
}
324326
/* and read vectors from it */
325-
return index.createSearcher().getIndexReader().getTermVectors(0);
327+
TermVectors termVectors = index.createSearcher().getIndexReader().termVectors();
328+
return termVectors.get(0);
326329
}
327330

328331
private static Fields generateTermVectorsFromDoc(IndexShard indexShard, TermVectorsRequest request) throws IOException {

‎server/src/main/java/org/opensearch/search/fetch/FetchPhase.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -160,7 +160,7 @@ public void execute(SearchContext context) {
160160
SequentialStoredFieldsLeafReader lf = (SequentialStoredFieldsLeafReader) currentReaderContext.reader();
161161
fieldReader = lf.getSequentialStoredFieldsReader()::document;
162162
} else {
163-
fieldReader = currentReaderContext.reader()::document;
163+
fieldReader = currentReaderContext.reader().storedFields()::document;
164164
}
165165
for (FetchSubPhaseProcessor processor : processors) {
166166
processor.setNextReader(currentReaderContext);

‎server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightUtils.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ public static List<Object> loadFieldValues(
7272
) throws IOException {
7373
if (forceSource == false && fieldType.isStored()) {
7474
CustomFieldsVisitor fieldVisitor = new CustomFieldsVisitor(singleton(fieldType.name()), false);
75-
hitContext.reader().document(hitContext.docId(), fieldVisitor);
75+
hitContext.reader().storedFields().document(hitContext.docId(), fieldVisitor);
7676
List<Object> textsToHighlight = fieldVisitor.fields().get(fieldType.name());
7777
return textsToHighlight != null ? textsToHighlight : Collections.emptyList();
7878
}

‎server/src/main/java/org/opensearch/search/lookup/LeafFieldsLookup.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -153,7 +153,7 @@ private FieldLookup loadFieldData(String name) {
153153
List<Object> values = new ArrayList<>(2);
154154
SingleFieldsVisitor visitor = new SingleFieldsVisitor(data.fieldType(), values);
155155
try {
156-
reader.document(docId, visitor);
156+
reader.storedFields().document(docId, visitor);
157157
} catch (IOException e) {
158158
throw new OpenSearchParseException("failed to load field [{}]", e, name);
159159
}

‎server/src/main/java/org/opensearch/search/lookup/SourceLookup.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -140,7 +140,7 @@ public void setSegmentAndDocument(LeafReaderContext context, int docId) {
140140
SequentialStoredFieldsLeafReader lf = (SequentialStoredFieldsLeafReader) context.reader();
141141
fieldReader = lf.getSequentialStoredFieldsReader()::document;
142142
} else {
143-
fieldReader = context.reader()::document;
143+
fieldReader = context.reader().storedFields()::document;
144144
}
145145
} catch (IOException e) {
146146
throw new UncheckedIOException(e);

‎server/src/test/java/org/opensearch/common/lucene/LuceneTests.java

+5-2
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
import org.apache.lucene.index.IndexCommit;
3636
import org.apache.lucene.index.IndexFormatTooOldException;
3737
import org.apache.lucene.index.StandardDirectoryReader;
38+
import org.apache.lucene.index.StoredFields;
3839
import org.apache.lucene.tests.analysis.MockAnalyzer;
3940
import org.apache.lucene.analysis.core.KeywordAnalyzer;
4041
import org.apache.lucene.document.Document;
@@ -565,12 +566,13 @@ public void testWrapAllDocsLive() throws Exception {
565566
}
566567
try (DirectoryReader unwrapped = DirectoryReader.open(writer)) {
567568
DirectoryReader reader = Lucene.wrapAllDocsLive(unwrapped);
569+
StoredFields storedFields = reader.storedFields();
568570
assertThat(reader.numDocs(), equalTo(liveDocs.size()));
569571
IndexSearcher searcher = new IndexSearcher(reader);
570572
Set<String> actualDocs = new HashSet<>();
571573
TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), Integer.MAX_VALUE);
572574
for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
573-
actualDocs.add(reader.document(scoreDoc.doc).get("id"));
575+
actualDocs.add(storedFields.document(scoreDoc.doc).get("id"));
574576
}
575577
assertThat(actualDocs, equalTo(liveDocs));
576578
}
@@ -609,13 +611,14 @@ public void testWrapLiveDocsNotExposeAbortedDocuments() throws Exception {
609611
}
610612
try (DirectoryReader unwrapped = DirectoryReader.open(writer)) {
611613
DirectoryReader reader = Lucene.wrapAllDocsLive(unwrapped);
614+
StoredFields storedFields = reader.storedFields();
612615
assertThat(reader.maxDoc(), equalTo(numDocs + abortedDocs));
613616
assertThat(reader.numDocs(), equalTo(liveDocs.size()));
614617
IndexSearcher searcher = new IndexSearcher(reader);
615618
List<String> actualDocs = new ArrayList<>();
616619
TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), Integer.MAX_VALUE);
617620
for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
618-
actualDocs.add(reader.document(scoreDoc.doc).get("id"));
621+
actualDocs.add(storedFields.document(scoreDoc.doc).get("id"));
619622
}
620623
assertThat(actualDocs, equalTo(liveDocs));
621624
}

‎server/src/test/java/org/opensearch/common/lucene/index/FreqTermsEnumTests.java

+3-1
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@
4343
import org.apache.lucene.index.IndexWriterConfig;
4444
import org.apache.lucene.index.IndexableField;
4545
import org.apache.lucene.index.NoMergePolicy;
46+
import org.apache.lucene.index.StoredFields;
4647
import org.apache.lucene.index.Term;
4748
import org.apache.lucene.search.TermInSetQuery;
4849
import org.apache.lucene.search.Query;
@@ -149,9 +150,10 @@ public void setUp() throws Exception {
149150

150151
// now go over each doc, build the relevant references and filter
151152
reader = DirectoryReader.open(iw);
153+
StoredFields storedFields = reader.storedFields();
152154
List<BytesRef> filterTerms = new ArrayList<>();
153155
for (int docId = 0; docId < reader.maxDoc(); docId++) {
154-
Document doc = reader.document(docId);
156+
Document doc = storedFields.document(docId);
155157
addFreqs(doc, referenceAll);
156158
if (!deletedIds.contains(doc.getField("id").stringValue())) {
157159
addFreqs(doc, referenceNotDeleted);

‎server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java

+5-2
Original file line numberDiff line numberDiff line change
@@ -2541,6 +2541,7 @@ class OpAndVersion {
25412541
final Term uidTerm = newUid(doc);
25422542
engine.index(indexForDoc(doc));
25432543
final BiFunction<String, Engine.SearcherScope, Engine.Searcher> searcherFactory = engine::acquireSearcher;
2544+
25442545
for (int i = 0; i < thread.length; i++) {
25452546
thread[i] = new Thread(() -> {
25462547
startGun.countDown();
@@ -2549,10 +2550,12 @@ class OpAndVersion {
25492550
} catch (InterruptedException e) {
25502551
throw new AssertionError(e);
25512552
}
2553+
25522554
for (int op = 0; op < opsPerThread; op++) {
25532555
try (Engine.GetResult get = engine.get(new Engine.Get(true, false, doc.id(), uidTerm), searcherFactory)) {
2556+
25542557
FieldsVisitor visitor = new FieldsVisitor(true);
2555-
get.docIdAndVersion().reader.document(get.docIdAndVersion().docId, visitor);
2558+
get.docIdAndVersion().reader.storedFields().document(get.docIdAndVersion().docId, visitor);
25562559
List<String> values = new ArrayList<>(Strings.commaDelimitedListToSet(visitor.source().utf8ToString()));
25572560
String removed = op % 3 == 0 && values.size() > 0 ? values.remove(0) : null;
25582561
String added = "v_" + idGenerator.incrementAndGet();
@@ -2608,7 +2611,7 @@ class OpAndVersion {
26082611

26092612
try (Engine.GetResult get = engine.get(new Engine.Get(true, false, doc.id(), uidTerm), searcherFactory)) {
26102613
FieldsVisitor visitor = new FieldsVisitor(true);
2611-
get.docIdAndVersion().reader.document(get.docIdAndVersion().docId, visitor);
2614+
get.docIdAndVersion().reader.storedFields().document(get.docIdAndVersion().docId, visitor);
26122615
List<String> values = Arrays.asList(Strings.commaDelimitedListToStringArray(visitor.source().utf8ToString()));
26132616
assertThat(currentValues, equalTo(new HashSet<>(values)));
26142617
}

0 commit comments

Comments
 (0)