diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java index 17729d7c57dde..f5692ad6637f6 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java @@ -142,8 +142,7 @@ public Builder(String name, boolean ignoreMalformedByDefault, boolean coerceByDe this.indexMode = indexMode; this.indexed = Parameter.indexParam(m -> toType(m).indexed, () -> { if (indexMode == IndexMode.TIME_SERIES) { - var metricType = getMetric().getValue(); - return metricType != TimeSeriesParams.MetricType.COUNTER && metricType != TimeSeriesParams.MetricType.GAUGE; + return false; } else { return true; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzer.java b/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzer.java index a9eb8d4f8ed66..937242ebc1c13 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzer.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzer.java @@ -244,6 +244,19 @@ private DV iterateDocValues( return dv; } + private DV iterateDocValues2( + int maxDocs, + CheckedSupplier dvReader, + CheckedConsumer valueAccessor + ) throws IOException { + DV dv = dvReader.get(); + for (int docId = dv.nextDoc(); docId < maxDocs; docId = dv.nextDoc()) { + cancellationChecker.logEvent(); + valueAccessor.accept(dv); + } + return dv; + } + void analyzeDocValues(SegmentReader reader, IndexDiskUsageStats stats) throws IOException { if (reader.getDocValuesReader() == null) { return; @@ -258,30 +271,34 @@ void analyzeDocValues(SegmentReader reader, IndexDiskUsageStats stats) throws IO cancellationChecker.checkForCancellation(); directory.resetBytesRead(); switch (dvType) { - case NUMERIC -> iterateDocValues(maxDocs, () -> docValuesReader.getNumeric(field), NumericDocValues::longValue); - case SORTED_NUMERIC -> iterateDocValues(maxDocs, () -> docValuesReader.getSortedNumeric(field), dv -> { + case NUMERIC -> iterateDocValues2(maxDocs, () -> docValuesReader.getNumeric(field), NumericDocValues::longValue); + case SORTED_NUMERIC -> iterateDocValues2(maxDocs, () -> docValuesReader.getSortedNumeric(field), dv -> { for (int i = 0; i < dv.docValueCount(); i++) { cancellationChecker.logEvent(); dv.nextValue(); } }); - case BINARY -> iterateDocValues(maxDocs, () -> docValuesReader.getBinary(field), BinaryDocValues::binaryValue); + case BINARY -> iterateDocValues2(maxDocs, () -> docValuesReader.getBinary(field), BinaryDocValues::binaryValue); case SORTED -> { - SortedDocValues sorted = iterateDocValues(maxDocs, () -> docValuesReader.getSorted(field), SortedDocValues::ordValue); + SortedDocValues sorted = iterateDocValues2(maxDocs, () -> docValuesReader.getSorted(field), SortedDocValues::ordValue); if (sorted.getValueCount() > 0) { - sorted.lookupOrd(0); - sorted.lookupOrd(sorted.getValueCount() - 1); + for (int ord = 0; ord < sorted.getValueCount(); ord++) { + cancellationChecker.logEvent(); + sorted.lookupOrd(ord); + } } } case SORTED_SET -> { - SortedSetDocValues sortedSet = iterateDocValues(maxDocs, () -> docValuesReader.getSortedSet(field), dv -> { + SortedSetDocValues sortedSet = iterateDocValues2(maxDocs, () -> docValuesReader.getSortedSet(field), dv -> { for (int i = 0; i < dv.docValueCount(); i++) { cancellationChecker.logEvent(); } }); if (sortedSet.getValueCount() > 0) { - sortedSet.lookupOrd(0); - sortedSet.lookupOrd(sortedSet.getValueCount() - 1); + for (long ord = 0; ord < sortedSet.getValueCount(); ord++) { + cancellationChecker.logEvent(); + sortedSet.lookupOrd(ord); + } } } default -> { @@ -289,6 +306,16 @@ void analyzeDocValues(SegmentReader reader, IndexDiskUsageStats stats) throws IO throw new IllegalStateException("Unknown docValues type [" + dvType + "]"); } } + switch (field.docValuesSkipIndexType()) { + case NONE -> {} + case RANGE -> { + var skipper = docValuesReader.getSkipper(field); + while(skipper.maxDocID(0) != DocIdSetIterator.NO_MORE_DOCS) { + skipper.advance(skipper.maxDocID(0) + 1); + } + } + default -> throw new IllegalStateException("Unknown skipper [" + field.docValuesSkipIndexType() + "]"); + } stats.addDocValues(field.name, directory.getBytesRead()); } } diff --git a/server/src/main/java/org/elasticsearch/common/lucene/uid/PerThreadIDVersionAndSeqNoLookup.java b/server/src/main/java/org/elasticsearch/common/lucene/uid/PerThreadIDVersionAndSeqNoLookup.java index 0c705f7a36021..4160560a051a5 100644 --- a/server/src/main/java/org/elasticsearch/common/lucene/uid/PerThreadIDVersionAndSeqNoLookup.java +++ b/server/src/main/java/org/elasticsearch/common/lucene/uid/PerThreadIDVersionAndSeqNoLookup.java @@ -10,6 +10,8 @@ package org.elasticsearch.common.lucene.uid; import org.apache.lucene.document.LongPoint; +import org.apache.lucene.index.DocValuesSkipIndexType; +import org.apache.lucene.index.DocValuesSkipper; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; @@ -24,6 +26,7 @@ import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.lucene.uid.VersionsAndSeqNoResolver.DocIdAndSeqNo; import org.elasticsearch.common.lucene.uid.VersionsAndSeqNoResolver.DocIdAndVersion; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.SeqNoFieldMapper; import org.elasticsearch.index.mapper.VersionFieldMapper; @@ -95,10 +98,19 @@ final class PerThreadIDVersionAndSeqNoLookup { // Also check for the existence of the timestamp field, because sometimes a segment can only contain tombstone documents, // which don't have any mapped fields (also not the timestamp field) and just some meta fields like _id, _seq_no etc. if (loadTimestampRange && reader.getFieldInfos().fieldInfo(DataStream.TIMESTAMP_FIELD_NAME) != null) { - PointValues tsPointValues = reader.getPointValues(DataStream.TIMESTAMP_FIELD_NAME); - assert tsPointValues != null : "no timestamp field for reader:" + reader + " and parent:" + reader.getContext().parent.reader(); - minTimestamp = LongPoint.decodeDimension(tsPointValues.getMinPackedValue(), 0); - maxTimestamp = LongPoint.decodeDimension(tsPointValues.getMaxPackedValue(), 0); + var fieldInfo = reader.getFieldInfos().fieldInfo(DataStream.TIMESTAMP_FIELD_NAME); + if (fieldInfo.docValuesSkipIndexType() == DocValuesSkipIndexType.RANGE) { + DocValuesSkipper skipper = reader.getDocValuesSkipper(DataStream.TIMESTAMP_FIELD_NAME); + assert skipper != null : "no skipper for reader:" + reader + " and parent:" + reader.getContext().parent.reader(); + minTimestamp = skipper.minValue(); + maxTimestamp = skipper.maxValue(); + } else { + PointValues tsPointValues = reader.getPointValues(DataStream.TIMESTAMP_FIELD_NAME); + assert tsPointValues != null + : "no timestamp field for reader:" + reader + " and parent:" + reader.getContext().parent.reader(); + minTimestamp = LongPoint.decodeDimension(tsPointValues.getMinPackedValue(), 0); + maxTimestamp = LongPoint.decodeDimension(tsPointValues.getMaxPackedValue(), 0); + } } else { minTimestamp = 0; maxTimestamp = Long.MAX_VALUE; diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 4c7bda1c52f17..350b67a97f46e 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -688,7 +688,7 @@ public boolean isES87TSDBCodecEnabled() { public static final FeatureFlag DOC_VALUES_SKIPPER = new FeatureFlag("doc_values_skipper"); public static final Setting USE_DOC_VALUES_SKIPPER = Setting.boolSetting( "index.mapping.use_doc_values_skipper", - IndexSettings.DOC_VALUES_SKIPPER.isEnabled(), + false, Property.IndexScope, Property.Final ); diff --git a/server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesProducer.java b/server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesProducer.java index a7560ce6f3caf..d71b44b187968 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesProducer.java +++ b/server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesProducer.java @@ -40,6 +40,7 @@ import org.apache.lucene.util.compress.LZ4; import org.apache.lucene.util.packed.DirectMonotonicReader; import org.apache.lucene.util.packed.PackedInts; +import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.core.IOUtils; import java.io.IOException; @@ -826,6 +827,7 @@ private void readFields(IndexInput meta, FieldInfos infos) throws IOException { if (info == null) { throw new CorruptIndexException("Invalid field number: " + fieldNumber, meta); } +// System.out.println("name: " + info.name); byte type = meta.readByte(); if (info.docValuesSkipIndexType() != DocValuesSkipIndexType.NONE) { skippers.put(info.name, readDocValueSkipperMeta(meta)); @@ -885,6 +887,14 @@ private static void readNumeric(IndexInput meta, NumericEntry entry) throws IOEx entry.valuesOffset = meta.readLong(); entry.valuesLength = meta.readLong(); } + +// System.out.printf( +// "docsWithFieldLength=%s, numValues=%d, indexLength=%s, valuesLength=%s\n", +// ByteSizeValue.ofBytes(entry.docsWithFieldLength), +// entry.numValues, +// ByteSizeValue.ofBytes(entry.indexLength), +// ByteSizeValue.ofBytes(entry.valuesLength) +// ); } private BinaryEntry readBinary(IndexInput meta) throws IOException { @@ -926,6 +936,14 @@ private static SortedNumericEntry readSortedNumeric(IndexInput meta, SortedNumer entry.addressesMeta = DirectMonotonicReader.loadMeta(meta, entry.numDocsWithField + 1, blockShift); entry.addressesLength = meta.readLong(); } +// System.out.printf( +// "docsWithFieldLength=%s, numValues=%d, indexLength=%s, valuesLength=%s, addressesLength=%s\n", +// ByteSizeValue.ofBytes(entry.docsWithFieldLength), +// entry.numValues, +// ByteSizeValue.ofBytes(entry.indexLength), +// ByteSizeValue.ofBytes(entry.valuesLength), +// ByteSizeValue.ofBytes(entry.addressesLength) +// ); return entry; } @@ -935,6 +953,17 @@ private SortedEntry readSorted(IndexInput meta) throws IOException { readNumeric(meta, entry.ordsEntry); entry.termsDictEntry = new TermsDictEntry(); readTermDict(meta, entry.termsDictEntry); +// System.out.printf( +// "docsWithFieldLength=%s, numValues=%d, indexLength=%s, valuesLength=%s, termsDictSize=%d, termsAddressesLength%s, termsIndexLength=%s, termsDataLength=%s\n", +// ByteSizeValue.ofBytes(entry.ordsEntry.valuesLength), +// entry.ordsEntry.numValues, +// ByteSizeValue.ofBytes(entry.ordsEntry.indexLength), +// ByteSizeValue.ofBytes(entry.ordsEntry.valuesLength), +// entry.termsDictEntry.termsDictSize, +// ByteSizeValue.ofBytes(entry.termsDictEntry.termsAddressesLength), +// ByteSizeValue.ofBytes(entry.termsDictEntry.termsIndexLength), +// ByteSizeValue.ofBytes(entry.termsDictEntry.termsDataLength) +// ); return entry; } @@ -944,6 +973,17 @@ private SortedSetEntry readSortedSet(IndexInput meta) throws IOException { switch (multiValued) { case 0: // singlevalued entry.singleValueEntry = readSorted(meta); +// System.out.printf( +// "docsWithFieldLength=%s, numValues=%d, indexLength=%s, valuesLength=%s, termsDictSize=%d, termsAddressesLength%s, termsIndexLength=%s, termsDataLength=%s\n", +// ByteSizeValue.ofBytes(entry.singleValueEntry.ordsEntry.docsWithFieldLength), +// entry.singleValueEntry.ordsEntry.numValues, +// ByteSizeValue.ofBytes(entry.singleValueEntry.ordsEntry.indexLength), +// ByteSizeValue.ofBytes(entry.singleValueEntry.ordsEntry.valuesLength), +// entry.singleValueEntry.termsDictEntry.termsDictSize, +// ByteSizeValue.ofBytes(entry.singleValueEntry.termsDictEntry.termsAddressesLength), +// ByteSizeValue.ofBytes(entry.singleValueEntry.termsDictEntry.termsIndexLength), +// ByteSizeValue.ofBytes(entry.singleValueEntry.termsDictEntry.termsDataLength) +// ); return entry; case 1: // multivalued break; @@ -954,6 +994,17 @@ private SortedSetEntry readSortedSet(IndexInput meta) throws IOException { readSortedNumeric(meta, entry.ordsEntry); entry.termsDictEntry = new TermsDictEntry(); readTermDict(meta, entry.termsDictEntry); +// System.out.printf( +// "docsWithFieldLength=%s, numValues=%d, indexLength=%s, valuesLength=%s, termsDictSize=%d, termsAddressesLength%s, termsIndexLength=%s, termsDataLength=%s\n", +// ByteSizeValue.ofBytes(entry.ordsEntry.valuesLength), +// entry.ordsEntry.numValues, +// ByteSizeValue.ofBytes(entry.ordsEntry.indexLength), +// ByteSizeValue.ofBytes(entry.ordsEntry.valuesLength), +// entry.termsDictEntry.termsDictSize, +// ByteSizeValue.ofBytes(entry.termsDictEntry.termsAddressesLength), +// ByteSizeValue.ofBytes(entry.termsDictEntry.termsIndexLength), +// ByteSizeValue.ofBytes(entry.termsDictEntry.termsDataLength) +// ); return entry; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index a4a6006b58fd5..f7f7e1db22e65 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -456,7 +456,7 @@ public DateFieldMapper build(MapperBuilderContext context) { c.getIndexSettings().getMode(), c.getIndexSettings().getIndexSortConfig(), c.indexVersionCreated(), - IndexSettings.USE_DOC_VALUES_SKIPPER.get(c.getSettings()) + true //IndexSettings.USE_DOC_VALUES_SKIPPER.get(c.getSettings()) ); }); @@ -471,7 +471,7 @@ public DateFieldMapper build(MapperBuilderContext context) { c.getIndexSettings().getMode(), c.getIndexSettings().getIndexSortConfig(), c.indexVersionCreated(), - IndexSettings.USE_DOC_VALUES_SKIPPER.get(c.getSettings()) + true //IndexSettings.USE_DOC_VALUES_SKIPPER.get(c.getSettings()) ); }); @@ -1055,7 +1055,7 @@ private static boolean shouldUseDocValuesSkipper( return indexCreatedVersion.onOrAfter(IndexVersions.TIMESTAMP_DOC_VALUES_SPARSE_INDEX) && useDocValuesSkipper && hasDocValues - && IndexMode.LOGSDB.equals(indexMode) + && (indexMode == IndexMode.LOGSDB || indexMode == IndexMode.TIME_SERIES) && indexSortConfig != null && indexSortConfig.hasSortOnField(fullFieldName) && DataStreamTimestampFieldMapper.DEFAULT_PATH.equals(fullFieldName); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index 8e6224e5720cb..effb7fbf853b3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -29,6 +29,7 @@ import org.elasticsearch.common.network.NetworkAddress; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Tuple; +import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.fielddata.FieldDataContext; @@ -73,7 +74,7 @@ private static IpFieldMapper toType(FieldMapper in) { public static final class Builder extends FieldMapper.DimensionBuilder { - private final Parameter indexed = Parameter.indexParam(m -> toType(m).indexed, true); + private final Parameter indexed; private final Parameter hasDocValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true); private final Parameter stored = Parameter.storeParam(m -> toType(m).stored, false); @@ -94,13 +95,15 @@ public static final class Builder extends FieldMapper.DimensionBuilder { private final IndexVersion indexCreatedVersion; private final ScriptCompiler scriptCompiler; private final SourceKeepMode indexSourceKeepMode; + private final IndexMode indexMode; public Builder( String name, ScriptCompiler scriptCompiler, boolean ignoreMalformedByDefault, IndexVersion indexCreatedVersion, - SourceKeepMode indexSourceKeepMode + SourceKeepMode indexSourceKeepMode, + IndexMode indexMode ) { super(name); this.scriptCompiler = Objects.requireNonNull(scriptCompiler); @@ -108,9 +111,16 @@ public Builder( this.indexCreatedVersion = indexCreatedVersion; this.ignoreMalformed = Parameter.boolParam("ignore_malformed", true, m -> toType(m).ignoreMalformed, ignoreMalformedByDefault); this.script.precludesParameters(nullValue, ignoreMalformed); + this.indexed = Parameter.indexParam(m -> toType(m).indexed, () -> { + if (indexMode == IndexMode.TIME_SERIES) { + return false; + } else { + return true; + } + }); addScriptValidation(script, indexed, hasDocValues); this.dimension = TimeSeriesParams.dimensionParam(m -> toType(m).dimension).addValidator(v -> { - if (v && (indexed.getValue() == false || hasDocValues.getValue() == false)) { + if (v && (indexed.getValue() == false && hasDocValues.getValue() == false)) { throw new IllegalArgumentException( "Field [" + TimeSeriesParams.TIME_SERIES_DIMENSION_PARAM @@ -123,6 +133,7 @@ public Builder( } }); this.indexSourceKeepMode = indexSourceKeepMode; + this.indexMode = indexMode; } Builder nullValue(String nullValue) { @@ -226,7 +237,14 @@ public IpFieldMapper build(MapperBuilderContext context) { public static final TypeParser PARSER = createTypeParserWithLegacySupport((n, c) -> { boolean ignoreMalformedByDefault = IGNORE_MALFORMED_SETTING.get(c.getSettings()); - return new Builder(n, c.scriptCompiler(), ignoreMalformedByDefault, c.indexVersionCreated(), c.getIndexSettings().sourceKeepMode()); + return new Builder( + n, + c.scriptCompiler(), + ignoreMalformedByDefault, + c.indexVersionCreated(), + c.getIndexSettings().sourceKeepMode(), + c.getIndexSettings().getMode() + ); }); public static final class IpFieldType extends SimpleMappedFieldType { @@ -523,6 +541,7 @@ public TermsEnum getTerms(IndexReader reader, String prefix, boolean caseInsensi private final ScriptCompiler scriptCompiler; private final SourceKeepMode indexSourceKeepMode; private final String offsetsFieldName; + private final IndexMode indexMode; private IpFieldMapper( String simpleName, @@ -548,6 +567,7 @@ private IpFieldMapper( this.storeIgnored = storeIgnored; this.indexSourceKeepMode = builder.indexSourceKeepMode; this.offsetsFieldName = offsetsFieldName; + this.indexMode = builder.indexMode; } @Override @@ -605,7 +625,8 @@ private void indexValue(DocumentParserContext context, InetAddress address) { context.doc().add(field); } if (hasDocValues) { - context.doc().add(new SortedSetDocValuesField(fieldType().name(), new BytesRef(InetAddressPoint.encode(address)))); +// context.doc().add(new SortedSetDocValuesField(fieldType().name(), new BytesRef(InetAddressPoint.encode(address)))); + context.doc().add(SortedSetDocValuesField.indexedField(fieldType().name(), new BytesRef(InetAddressPoint.encode(address)))); } else if (stored || indexed) { context.addToFieldNames(fieldType().name()); } @@ -626,9 +647,9 @@ protected void indexScriptValues( @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(leafName(), scriptCompiler, ignoreMalformedByDefault, indexCreatedVersion, indexSourceKeepMode).dimension( - dimension - ).init(this); + return new Builder(leafName(), scriptCompiler, ignoreMalformedByDefault, indexCreatedVersion, indexSourceKeepMode, indexMode) + .dimension(dimension) + .init(this); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index b7e8e13e16839..6a9b02399282a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -105,8 +105,9 @@ public static class Defaults { FieldType ft = new FieldType(); ft.setTokenized(false); ft.setOmitNorms(true); - ft.setIndexOptions(IndexOptions.DOCS); + ft.setIndexOptions(IndexOptions.NONE); ft.setDocValuesType(DocValuesType.SORTED_SET); + ft.setDocValuesSkipIndexType(DocValuesSkipIndexType.RANGE); FIELD_TYPE = freezeAndDeduplicateFieldType(ft); } @@ -159,7 +160,7 @@ private static KeywordFieldMapper toType(FieldMapper in) { public static final class Builder extends FieldMapper.DimensionBuilder { - private final Parameter indexed = Parameter.indexParam(m -> toType(m).indexed, true); + private final Parameter indexed; private final Parameter hasDocValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true); private final Parameter stored = Parameter.storeParam(m -> toType(m).fieldType.stored(), false); @@ -263,10 +264,11 @@ private Builder( null ).acceptsNull(); this.script.precludesParameters(nullValue); + this.indexed = Parameter.indexParam(m -> toType(m).indexed, () -> indexMode != IndexMode.TIME_SERIES); addScriptValidation(script, indexed, hasDocValues); this.dimension = TimeSeriesParams.dimensionParam(m -> toType(m).fieldType().isDimension()).addValidator(v -> { - if (v && (indexed.getValue() == false || hasDocValues.getValue() == false)) { + if (v && (indexed.getValue() == false && hasDocValues.getValue() == false)) { throw new IllegalArgumentException( "Field [" + TimeSeriesParams.TIME_SERIES_DIMENSION_PARAM diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 078faa25938f5..8343d9ec0bfe5 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -179,14 +179,13 @@ public Builder( this.indexMode = mode; this.indexed = Parameter.indexParam(m -> toType(m).indexed, () -> { if (indexMode == IndexMode.TIME_SERIES) { - var metricType = getMetric().getValue(); - return metricType != MetricType.COUNTER && metricType != MetricType.GAUGE; + return false; } else { return true; } }); this.dimension = TimeSeriesParams.dimensionParam(m -> toType(m).dimension).addValidator(v -> { - if (v && (indexed.getValue() == false || hasDocValues.getValue() == false)) { + if (v && (indexed.getValue() == false && hasDocValues.getValue() == false)) { throw new IllegalArgumentException( "Field [" + TimeSeriesParams.TIME_SERIES_DIMENSION_PARAM @@ -404,7 +403,8 @@ public void addFields(LuceneDocument document, String name, Number value, boolea document.add(new HalfFloatPoint(name, f)); } if (docValued) { - document.add(new SortedNumericDocValuesField(name, HalfFloatPoint.halfFloatToSortableShort(f))); +// document.add(new SortedNumericDocValuesField(name, HalfFloatPoint.halfFloatToSortableShort(f))); + document.add(SortedNumericDocValuesField.indexedField(name, HalfFloatPoint.halfFloatToSortableShort(f))); } if (stored) { document.add(new StoredField(name, f)); @@ -591,7 +591,8 @@ public void addFields(LuceneDocument document, String name, Number value, boolea if (indexed && docValued) { document.add(new FloatField(name, f, Field.Store.NO)); } else if (docValued) { - document.add(new SortedNumericDocValuesField(name, NumericUtils.floatToSortableInt(f))); +// document.add(new SortedNumericDocValuesField(name, NumericUtils.floatToSortableInt(f))); + document.add(SortedNumericDocValuesField.indexedField(name, NumericUtils.floatToSortableInt(f))); } else if (indexed) { document.add(new FloatPoint(name, f)); } @@ -746,7 +747,8 @@ public void addFields(LuceneDocument document, String name, Number value, boolea if (indexed && docValued) { document.add(new DoubleField(name, d, Field.Store.NO)); } else if (docValued) { - document.add(new SortedNumericDocValuesField(name, NumericUtils.doubleToSortableLong(d))); +// document.add(new SortedNumericDocValuesField(name, NumericUtils.doubleToSortableLong(d))); + document.add(SortedNumericDocValuesField.indexedField(name, NumericUtils.doubleToSortableLong(d))); } else if (indexed) { document.add(new DoublePoint(name, d)); } @@ -1185,7 +1187,8 @@ public void addFields(LuceneDocument document, String name, Number value, boolea if (indexed && docValued) { document.add(new IntField(name, i, Field.Store.NO)); } else if (docValued) { - document.add(new SortedNumericDocValuesField(name, i)); + document.add(SortedNumericDocValuesField.indexedField(name, i)); +// document.add(new SortedNumericDocValuesField(name, i)); } else if (indexed) { document.add(new IntPoint(name, i)); } @@ -1337,7 +1340,8 @@ public void addFields(LuceneDocument document, String name, Number value, boolea if (indexed && docValued) { document.add(new LongField(name, l, Field.Store.NO)); } else if (docValued) { - document.add(new SortedNumericDocValuesField(name, l)); +// document.add(new SortedNumericDocValuesField(name, l)); + document.add(SortedNumericDocValuesField.indexedField(name, l)); } else if (indexed) { document.add(new LongPoint(name, l)); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DiskUsageAnalysisTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DiskUsageAnalysisTests.java new file mode 100644 index 0000000000000..995d88a7cf77e --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/DiskUsageAnalysisTests.java @@ -0,0 +1,258 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper; + +import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite; + +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.codecs.lucene90.Lucene90CompoundFormat; +import org.apache.lucene.document.InetAddressPoint; +import org.apache.lucene.index.*; +import org.apache.lucene.store.ChecksumIndexInput; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.packed.GrowableWriter; +import org.apache.lucene.util.packed.PackedInts; +import org.elasticsearch.common.io.stream.ByteArrayStreamInput; +import org.elasticsearch.common.network.NetworkAddress; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.*; + +@TimeoutSuite(millis = 12 * 60 * 60 * 1000) +public class DiskUsageAnalysisTests extends ESTestCase { + + public void testOffsetAnalysis() throws Exception { + final ByteArrayStreamInput scratch = new ByteArrayStreamInput(); + String fullPath = "/Users/mvg/temp/offset_data_dire/data/indices/-58uS6EbSbewM7FgL6lw2Q/0/index"; + try (var directory = FSDirectory.open(Path.of(fullPath))) { + try (var reader = DirectoryReader.open(directory)) { + long sum = 0L; + int count = 0; + for (var leafReaderContext : reader.leaves()) { + var leafReader = leafReaderContext.reader(); + var sortedSetDocValues = leafReader.getSortedSetDocValues("tags"); + if (sortedSetDocValues == null) { + continue; + } + var sortedDocValues = leafReader.getSortedDocValues("tags.offsets"); + if (sortedDocValues == null) { + continue; + } + for (int docId = 0; docId < leafReader.maxDoc(); docId++) { + if (sortedSetDocValues.advanceExact(docId)) { + StringBuilder builder = new StringBuilder(); + for (int i = 0; i < sortedSetDocValues.docValueCount(); i++) { + long ord = sortedSetDocValues.nextOrd(); + builder.append(sortedSetDocValues.lookupOrd(ord).utf8ToString()).append(','); + } + // System.out.println("values: " + builder.toString()); + if (sortedDocValues.advanceExact(docId)) { + var encodedValue = sortedDocValues.lookupOrd(sortedDocValues.ordValue()); + sum += encodedValue.length; + count++; + scratch.reset(encodedValue.bytes, encodedValue.offset, encodedValue.length); + int[] offsetToOrd = FieldArrayContext.parseOffsetArray(scratch); + GrowableWriter writer; + var m = PackedInts.getMutable(offsetToOrd.length, 5, PackedInts.Format.PACKED); + for (int i = 0; i < offsetToOrd.length; i++) { + m.set(i, offsetToOrd[i]); + } + // System.out.println("offsets: " + Arrays.toString(offsetToOrd)); + } else { + System.out.println("no binary docvalues"); + } + } + } + } + System.out.println("count: " + count); + System.out.println("sum: " + sum); + System.out.println("avg: " + (sum / count)); + } + } + } + + public void testIgnoredSourceAnalysis() throws Exception { + Set fieldNames = new HashSet<>(); + var indices = Files.list(Path.of("/Users/mvg/temp/offset_data_dire/data/indices")).toList(); + for (Path indexDir : indices) { + Path fullPath = indexDir.resolve("0/index"); + System.out.println("index dir: " + fullPath.toAbsolutePath()); + System.out.println("fieldNames: " + fieldNames); + try (var directory = FSDirectory.open(fullPath)) { + try (var reader = DirectoryReader.open(directory)) { + for (var leafReaderContext : reader.leaves()) { + System.out.println("new segment"); + var leafReader = leafReaderContext.reader(); + var fieldInfo = leafReader.getFieldInfos().fieldInfo("_ignored_source"); + if (fieldInfo == null) { + continue; + } + + for (int docId = 0; docId < leafReader.maxDoc(); docId++) { + leafReader.storedFields().document(docId, new StoredFieldVisitor() { + @Override + public Status needsField(FieldInfo fieldInfo) throws IOException { + return fieldInfo.getName().equals("_ignored_source") ? Status.YES : Status.NO; + } + + @Override + public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException { + var namedValue = IgnoredSourceFieldMapper.decodeAsMap(value).nameValue(); + fieldNames.add(namedValue.getFieldName()); + } + }); + } + } + } + } + } + System.out.println("fieldNames: " + fieldNames); + } + + public void testES87TSDBDocValuesFormatAnalysis() throws Exception { + String fullPath = "/Users/mvg/temp/otel-experiments/data-dir/indices/0Fx-NNq2ROe1QLFoZ4UF4A/0/index"; + try (var directory = FSDirectory.open(Path.of(fullPath))) { + try (var reader = DirectoryReader.open(directory)) { + for (var leafReaderContext : reader.leaves()) { + var leafReader = leafReaderContext.reader(); + + for (var field : leafReader.getFieldInfos()) { + if (field.getDocValuesType() == DocValuesType.NUMERIC || field.getDocValuesType() == DocValuesType.SORTED_NUMERIC) { + var dv = DocValues.unwrapSingleton(leafReader.getSortedNumericDocValues(field.getName())); + if (dv == null) { + System.out.println("multi valued field: " + field.getName()); + } + } + } + + for (FieldInfo fieldInfo : leafReader.getFieldInfos()) { + if (fieldInfo.getName().startsWith("_")) { + continue; + } + var sortedNumericDocValues = DocValues.unwrapSingleton(leafReader.getSortedNumericDocValues(fieldInfo.getName())); + if (sortedNumericDocValues == null) { + continue; + } + + List counts = new ArrayList<>(); + int counter = 0; + long prev = -1; + long prevDiff = -1; + for (int docId = 0; docId < leafReader.maxDoc(); docId++) { + if (sortedNumericDocValues.advanceExact(docId)) { + long value = sortedNumericDocValues.longValue(); + if (prev == -1) { + prev = value; + continue; + } + + long diff = value - prev; + if (diff == prevDiff) { + counter++; + } else { + if (counter > 1) { + counts.add(counter); + } + counter = 0; + } + prev = value; + prevDiff = diff; + } + } + + counts.sort(Integer::compareTo); + if (counts.isEmpty()) { + System.out.println("no adjacent diff for field: " + fieldInfo.getName()); + } else { + System.out.printf( + "field name: %s, mean: %d, max= %d\n", + fieldInfo.getName(), + counts.get(Math.min(counts.size() / 2, counts.size() - 1)), + counts.getLast() + ); + } + } + } + } + } + } + + public void testCheckField() throws Exception { + String fullPath = "/Users/mvg/temp/otel-experiments/data-dir/indices/0Fx-NNq2ROe1QLFoZ4UF4A/0/index"; + try (var directory = FSDirectory.open(Path.of(fullPath))) { + try (var reader = DirectoryReader.open(directory)) { + for (var leafReaderContext : reader.leaves()) { + var leafReader = leafReaderContext.reader(); + + Set values = new HashSet<>(); + var sortedNumericDocValues = leafReader.getSortedSetDocValues("resource.attributes.host.ip"); + for (int docId = 0; docId < leafReader.maxDoc(); docId++) { + if (sortedNumericDocValues.advanceExact(docId)) { + for (int i = 0; i < sortedNumericDocValues.docValueCount(); i++) { + values.add(BytesRef.deepCopyOf(sortedNumericDocValues.lookupOrd(sortedNumericDocValues.nextOrd()))); + } + } + } + + for (BytesRef value : values) { + byte[] bytes = Arrays.copyOfRange(value.bytes, value.offset, value.offset + value.length); + String ip = NetworkAddress.format(InetAddressPoint.decode(bytes)); + System.out.println(ip); + } + } + } + } + } + + public void testReadCompounFileSize() throws Exception { + String fullPath = "/Users/mvg/temp/otel-experiments/data-dir/indices/KhZ-KxSdTLCcevb3pIjeGA/0/index"; + try (var directory = FSDirectory.open(Path.of(fullPath))) { + try (ChecksumIndexInput entriesStream = directory.openChecksumInput("_9c.cfe")) { + CodecUtil.checkHeader(entriesStream, "Lucene90CompoundEntries", 0, 0); + try { + CodecUtil.checkIndexHeaderID(entriesStream, new byte[0]); + } catch (Exception e) { + + } + CodecUtil.checkIndexHeaderSuffix(entriesStream, ""); +// CodecUtil.checkIndexHeader( +// entriesStream, +// "Lucene90CompoundEntries", +// 0, +// 0, +// new byte[0], +// ""); + + long sum = 0; + final int numEntries = entriesStream.readVInt(); + Map m = new HashMap<>(); + for (int i = 0; i < numEntries; i++) { + final String id = entriesStream.readString(); + long offset = entriesStream.readLong(); + long length = entriesStream.readLong(); + + + sum += length; + m.put(id, length); + } + m.entrySet().stream().sorted((o1, o2) -> Long.compare(o2.getValue(), o1.getValue())).forEach(entry -> { + System.out.println(entry.getKey() + " / " + ByteSizeValue.ofBytes(entry.getValue())); + }); + System.out.println("sum = " + ByteSizeValue.ofBytes(sum)); + } + } + } + +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldTypeTests.java index 7dec761691ce6..1c9e8e3f822b1 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldTypeTests.java @@ -354,7 +354,8 @@ public void testFetchSourceValue() throws IOException { ScriptCompiler.NONE, true, IndexVersion.current(), - Mapper.SourceKeepMode.NONE + Mapper.SourceKeepMode.NONE, + null ).build(MapperBuilderContext.root(false, false)).fieldType(); assertEquals(List.of("2001:db8::2:1"), fetchSourceValue(mapper, "2001:db8::2:1")); assertEquals(List.of("2001:db8::2:1"), fetchSourceValue(mapper, "2001:db8:0:0:0:0:2:1")); @@ -365,7 +366,8 @@ public void testFetchSourceValue() throws IOException { ScriptCompiler.NONE, true, IndexVersion.current(), - Mapper.SourceKeepMode.NONE + Mapper.SourceKeepMode.NONE, + null ).nullValue("2001:db8:0:0:0:0:2:7").build(MapperBuilderContext.root(false, false)).fieldType(); assertEquals(List.of("2001:db8::2:7"), fetchSourceValue(nullValueMapper, null)); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index 0f69d0d37143c..cc864a24c78be 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -337,7 +337,7 @@ public void append(LogEvent event) { Configurator.shutdown(context); })); - BootstrapForTesting.ensureInitialized(); +// BootstrapForTesting.ensureInitialized(); /* * We need to exclude time zones not supported by joda (like SystemV* timezones) diff --git a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java index 5a667df9ffbbf..bdde8aed9f2f8 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java +++ b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java @@ -345,7 +345,7 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio if (context.mappingLookup().isSourceSynthetic()) { // Remember which documents had value in source so that it can be correctly // reconstructed in synthetic source - context.doc().add(new SortedNumericDocValuesField(fieldType().name(), 1)); +// context.doc().add(new SortedNumericDocValuesField(fieldType().name(), 1)); } } diff --git a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java index d0f2de17cadd1..2457bc1cb5dbc 100644 --- a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java +++ b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.versionfield; -import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.index.FilteredTermsEnum; @@ -98,7 +97,7 @@ public static class Defaults { FieldType ft = new FieldType(); ft.setTokenized(false); ft.setOmitNorms(true); - ft.setIndexOptions(IndexOptions.DOCS); + ft.setIndexOptions(IndexOptions.NONE); FIELD_TYPE = freezeAndDeduplicateFieldType(ft); } } @@ -391,8 +390,8 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio EncodedVersion encoding = encodeVersion(versionString); BytesRef encodedVersion = encoding.bytesRef; - context.doc().add(new Field(fieldType().name(), encodedVersion, fieldType)); context.doc().add(new SortedSetDocValuesField(fieldType().name(), encodedVersion)); +// context.doc().add(SortedSetDocValuesField.indexedField(fieldType().name(), encodedVersion)); } @Override diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/metrics-otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/metrics-otel@mappings.yaml index cfc68889e8c9e..8000e11ab0ca9 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/metrics-otel@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/metrics-otel@mappings.yaml @@ -36,11 +36,13 @@ template: time_series_metric: gauge - counter_double: mapping: - type: double + type: scaled_float + scaling_factor: 1000 time_series_metric: counter - gauge_double: mapping: - type: double + type: scaled_float + scaling_factor: 1000 time_series_metric: gauge - summary: mapping: