Implement parseBytesRef for TimeSeriesRoutingHashFieldType (elastic…

…#113373)

This implements the `parseBytesRef` method for the `_ts_routing_hash` field so we
can parse the values generated by the companion `format` method.
We parse the values when fetching them from the source when the field is used
as a `sort` paired with `search_after`.

Before this change a sort by and search_after `_ts_routing_hash` would yield
an `UnsupportedOperationException`

(cherry picked from commit 4e5e870)
Signed-off-by: Andrei Dan <[email protected]>

docs/changelog/113373.yaml

@@ -0,0 +1,6 @@
+pr: 113373
+summary: Implement `parseBytesRef` for `TimeSeriesRoutingHashFieldType`
+area: TSDB
+type: bug
+issues:
+ - 112399

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml

@@ -65,6 +65,9 @@ setup:
 
 ---
 generates a consistent id:
+  - requires:
+        cluster_features: "tsdb.ts_routing_hash_doc_value_parse_byte_ref"
+        reason: _tsid routing hash doc value parsing has been fixed
   - do:
       bulk:
         refresh: true
@@ -152,6 +155,50 @@ generates a consistent id:
   - match: { hits.hits.8._source.@timestamp: 2021-04-28T18:52:04.467Z }
   - match: { hits.hits.8._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
 
+  - do:
+      search:
+        index: id_generation_test
+        body:
+          query:
+            match_all: {}
+          sort: ["@timestamp", "_ts_routing_hash"]
+          _source: true
+          search_after: [ "2021-04-28T18:50:03.142Z", "cn4exQ" ]
+          docvalue_fields: [_ts_routing_hash]
+
+  - match: {hits.total.value: 9}
+
+  - match: { hits.hits.0._id: cZZNs7B9sSWsyrL5AAABeRnRGTM }
+  - match: { hits.hits.0._source.@timestamp: 2021-04-28T18:50:04.467Z }
+  - match: { hits.hits.0._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
+
+  - match: { hits.hits.1._id: cn4excfoxSs_KdA5AAABeRnRYiY }
+  - match: { hits.hits.1._source.@timestamp: 2021-04-28T18:50:23.142Z }
+  - match: { hits.hits.1._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 }
+
+  - match: { hits.hits.2._id: cZZNs7B9sSWsyrL5AAABeRnRZ1M }
+  - match: { hits.hits.2._source.@timestamp: 2021-04-28T18:50:24.467Z }
+  - match: { hits.hits.2._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
+
+  - match: { hits.hits.3._id: cZZNs7B9sSWsyrL5AAABeRnRtXM }
+  - match: { hits.hits.3._source.@timestamp: 2021-04-28T18:50:44.467Z }
+  - match: { hits.hits.3._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
+
+  - match: { hits.hits.4._id: cn4excfoxSs_KdA5AAABeRnR11Y }
+  - match: { hits.hits.4._source.@timestamp: 2021-04-28T18:50:53.142Z }
+  - match: { hits.hits.4._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 }
+
+  - match: { hits.hits.5._id: cn4excfoxSs_KdA5AAABeRnR_mY }
+  - match: { hits.hits.5._source.@timestamp: 2021-04-28T18:51:03.142Z }
+  - match: { hits.hits.5._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 }
+
+  - match: { hits.hits.6._id: cZZNs7B9sSWsyrL5AAABeRnSA5M }
+  - match: { hits.hits.6._source.@timestamp: 2021-04-28T18:51:04.467Z }
+  - match: { hits.hits.6._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
+
+  - match: { hits.hits.7._id: cZZNs7B9sSWsyrL5AAABeRnS7fM }
+  - match: { hits.hits.7._source.@timestamp: 2021-04-28T18:52:04.467Z }
+  - match: { hits.hits.7._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
 ---
 index a new document on top of an old one:
   - do:

server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java

@@ -43,7 +43,8 @@ public Set<NodeFeature> getFeatures() {
             SourceFieldMapper.SYNTHETIC_SOURCE_COPY_TO_FIX,
             FlattenedFieldMapper.IGNORE_ABOVE_SUPPORT,
             IndexSettings.IGNORE_ABOVE_INDEX_LEVEL_SETTING,
-            SourceFieldMapper.SYNTHETIC_SOURCE_COPY_TO_INSIDE_OBJECTS_FIX
+            SourceFieldMapper.SYNTHETIC_SOURCE_COPY_TO_INSIDE_OBJECTS_FIX,
+            TimeSeriesRoutingHashFieldMapper.TS_ROUTING_HASH_FIELD_PARSES_BYTES_REF
         );
     }
 }

server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapper.java

@@ -14,6 +14,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.util.ByteUtils;
+import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.fielddata.FieldData;
@@ -45,6 +46,7 @@ public class TimeSeriesRoutingHashFieldMapper extends MetadataFieldMapper {
     public static final TimeSeriesRoutingHashFieldMapper INSTANCE = new TimeSeriesRoutingHashFieldMapper();
 
     public static final TypeParser PARSER = new FixedTypeParser(c -> c.getIndexSettings().getMode().timeSeriesRoutingHashFieldMapper());
+    static final NodeFeature TS_ROUTING_HASH_FIELD_PARSES_BYTES_REF = new NodeFeature("tsdb.ts_routing_hash_doc_value_parse_byte_ref");
 
     static final class TimeSeriesRoutingHashFieldType extends MappedFieldType {
 
@@ -64,6 +66,13 @@ public Object format(BytesRef value) {
                 return Uid.decodeId(value.bytes, value.offset, value.length);
             }
 
+            @Override
+            public BytesRef parseBytesRef(Object value) {
+                if (value instanceof BytesRef valueAsBytesRef) {
+                    return valueAsBytesRef;
+                }
+                return Uid.encodeId(value.toString());
+            }
         };
 
         private TimeSeriesRoutingHashFieldType() {

server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java

@@ -20,6 +20,7 @@
 import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.index.mapper.DateFieldMapper.Resolution;
 import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper.TimeSeriesIdBuilder;
+import org.elasticsearch.index.mapper.TimeSeriesRoutingHashFieldMapper;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
@@ -33,6 +34,8 @@
 import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.longEncode;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.is;
 
 public class DocValueFormatTests extends ESTestCase {
 
@@ -388,4 +391,14 @@ public void testParseTsid() throws IOException {
         Object tsidBase64 = Base64.getUrlEncoder().withoutPadding().encodeToString(expectedBytes);
         assertEquals(tsidFormat, tsidBase64);
     }
+
+    public void testFormatAndParseTsRoutingHash() throws IOException {
+        BytesRef tsRoutingHashInput = new BytesRef("cn4exQ");
+        DocValueFormat docValueFormat = TimeSeriesRoutingHashFieldMapper.INSTANCE.fieldType().docValueFormat(null, ZoneOffset.UTC);
+        Object formattedValue = docValueFormat.format(tsRoutingHashInput);
+        // the format method takes BytesRef as input and outputs a String
+        assertThat(formattedValue, instanceOf(String.class));
+        // the parse method will output the BytesRef input
+        assertThat(docValueFormat.parseBytesRef(formattedValue), is(tsRoutingHashInput));
+    }
 }