Fetch meta fields in FetchFieldsPhase using ValueFetcher

docs/reference/search/profile.asciidoc

@@ -197,6 +197,17 @@ The API returns the following result:
             "stored_fields": ["_id", "_routing", "_source"]
           },
           "children": [
+            {
+              "type" : "FetchFieldsPhase",
+              "description" : "",
+              "time_in_nanos" : 238762,
+              "breakdown" : {
+                "process_count" : 5,
+                "process" : 227914,
+                "next_reader" : 10848,
+                "next_reader_count" : 1
+              }
+            },
             {
               "type": "FetchSourcePhase",
               "description": "",
@@ -1043,6 +1054,17 @@ And here is the fetch profile:
             "stored_fields": ["_id", "_routing", "_source"]
           },
           "children": [
+            {
+              "type" : "FetchFieldsPhase",
+              "description" : "",
+              "time_in_nanos" : 238762,
+              "breakdown" : {
+                "process_count" : 5,
+                "process" : 227914,
+                "next_reader" : 10848,
+                "next_reader_count" : 1
+              }
+            },
             {
               "type": "FetchSourcePhase",
               "description": "",

modules/parent-join/src/yamlRestTest/resources/rest-api-spec/test/30_inner_hits.yml

@@ -120,8 +120,8 @@ teardown:
 ---
 profile fetch:
   - skip:
-      version: ' - 7.15.99'
-      reason:  fetch profiling implemented in 7.16.0
+      version: ' - 8.13.99'
+      reason: fetch fields and stored_fields using ValueFetcher
 
   - do:
       search:
@@ -141,15 +141,20 @@ profile fetch:
   - gt: { profile.shards.0.fetch.breakdown.load_stored_fields_count: 0 }
   - gt: { profile.shards.0.fetch.breakdown.load_stored_fields: 0 }
   - match: { profile.shards.0.fetch.debug.stored_fields: [_id, _routing, _source] }
-  - length: { profile.shards.0.fetch.children: 3 }
-  - match: { profile.shards.0.fetch.children.0.type: FetchSourcePhase }
+  - length: { profile.shards.0.fetch.children: 4 }
+  - match: { profile.shards.0.fetch.children.0.type: FetchFieldsPhase }
   - gt: { profile.shards.0.fetch.children.0.breakdown.next_reader_count: 0 }
   - gt: { profile.shards.0.fetch.children.0.breakdown.next_reader: 0 }
   - gt: { profile.shards.0.fetch.children.0.breakdown.next_reader_count: 0 }
   - gt: { profile.shards.0.fetch.children.0.breakdown.next_reader: 0 }
-  - match: { profile.shards.0.fetch.children.1.type: InnerHitsPhase }
+  - match: { profile.shards.0.fetch.children.1.type: FetchSourcePhase }
   - gt: { profile.shards.0.fetch.children.1.breakdown.next_reader_count: 0 }
   - gt: { profile.shards.0.fetch.children.1.breakdown.next_reader: 0 }
   - gt: { profile.shards.0.fetch.children.1.breakdown.next_reader_count: 0 }
   - gt: { profile.shards.0.fetch.children.1.breakdown.next_reader: 0 }
-  - match: { profile.shards.0.fetch.children.2.type: StoredFieldsPhase }
+  - match: { profile.shards.0.fetch.children.2.type: InnerHitsPhase }
+  - gt: { profile.shards.0.fetch.children.2.breakdown.next_reader_count: 0 }
+  - gt: { profile.shards.0.fetch.children.2.breakdown.next_reader: 0 }
+  - gt: { profile.shards.0.fetch.children.2.breakdown.next_reader_count: 0 }
+  - gt: { profile.shards.0.fetch.children.2.breakdown.next_reader: 0 }
+  - match: { profile.shards.0.fetch.children.3.type: StoredFieldsPhase }

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/get/120_stored_fields_ignored.yml

@@ -0,0 +1,159 @@
+---
+"_ignored field through get api using stored_fields":
+  - do:
+      indices.create:
+        index: test
+        body:
+          mappings:
+            properties:
+              keyword:
+                type: keyword
+                ignore_above: 5
+              ip:
+                type: ip
+                ignore_malformed: true
+              value:
+                type: long
+                ignore_malformed: true
+
+  - do:
+      index:
+        index:   test
+        id:      1
+        refresh: true
+        body:
+          keyword: foo
+          ip: 192.168.0.1
+          value: 23
+  - do:
+      index:
+        index:   test
+        id:      2
+        refresh: true
+        body:
+          keyword: foobar
+          ip: garbage
+          value: missing
+  - do:
+      index:
+        index:   test
+        id:      3
+        refresh: true
+        body:
+          keyword:
+            - foo
+            - bar
+            - foobar
+          ip:
+            - 10.10.1.1
+            - 192.8.1.2
+            - 199.199.300.999
+          value:
+            - 1
+            - 2
+            - ops
+
+  - do:
+      get:
+        index: test
+        id:    1
+
+  - match: {_index: "test"}
+  - match: {_id: "1"}
+  - match: {_version: 1}
+  - match: {found: true}
+  - match:
+      _source:
+        keyword: foo
+        ip: 192.168.0.1
+        value: 23
+
+  - is_false: fields
+
+  - do:
+      get:
+        index: test
+        id: 2
+  - match: { _index: "test" }
+  - match: { _id: "2" }
+  - match: { _version: 1 }
+  - match: { found: true }
+  - match:
+      _source:
+        ip: garbage
+        keyword: foobar
+        value: missing
+
+  - is_false: fields
+
+  - do:
+      get:
+        index: test
+        id: 3
+  - match: { _index: "test" }
+  - match: { _id: "3" }
+  - match: { _version: 1 }
+  - match: { found: true }
+  - match:
+      _source:
+        ip:
+          - 10.10.1.1
+          - 192.8.1.2
+          - 199.199.300.999
+        keyword:
+          - foo
+          - bar
+          - foobar
+        value:
+          - 1
+          - 2
+          - ops
+
+  - is_false: fields
+
+  - do:
+      get:
+        index: test
+        id: 1
+        stored_fields:
+          - _ignored
+
+  - match: { _index: "test" }
+  - match: { _id: "1" }
+  - match: { _version: 1 }
+  - match: { found: true }
+  - match: { _ignored: null}
+
+  - do:
+      get:
+        index: test
+        id: 2
+        stored_fields:
+          - _ignored
+
+  - match: { _index: "test" }
+  - match: { _id: "2" }
+  - match: { _version: 1 }
+  - match: { found: true }
+  - match:
+      _ignored:
+        - ip
+        - keyword
+        - value
+
+  - do:
+      get:
+        index: test
+        id: 3
+        stored_fields:
+          - _ignored
+
+  - match: { _index: "test" }
+  - match: { _id: "3" }
+  - match: { _version: 1 }
+  - match: { found: true }
+  - match:
+      _ignored:
+        - ip
+        - keyword
+        - value

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/370_profile.yml

@@ -22,8 +22,8 @@ setup:
 ---
 fetch fields:
   - skip:
-      version: ' - 8.5.99'
-      reason:  stored fields phase added in 8.6
+      version: ' - 8.13.99'
+      reason: fetch fields and stored_fields using ValueFetcher
 
   - do:
       search:
@@ -44,17 +44,21 @@ fetch fields:
   - match: { profile.shards.0.fetch.debug.stored_fields: [_id, _routing, _source] }
   - length: { profile.shards.0.fetch.children: 2 }
   - match: { profile.shards.0.fetch.children.0.type: FetchFieldsPhase }
-  - match: { profile.shards.0.fetch.children.1.type: StoredFieldsPhase }
   - gt: { profile.shards.0.fetch.children.0.breakdown.next_reader_count: 0 }
   - gt: { profile.shards.0.fetch.children.0.breakdown.next_reader: 0 }
   - gt: { profile.shards.0.fetch.children.0.breakdown.next_reader_count: 0 }
   - gt: { profile.shards.0.fetch.children.0.breakdown.next_reader: 0 }
+  - match: { profile.shards.0.fetch.children.1.type: StoredFieldsPhase }
+  - gt: { profile.shards.0.fetch.children.1.breakdown.next_reader_count: 0 }
+  - gt: { profile.shards.0.fetch.children.1.breakdown.next_reader: 0 }
+  - gt: { profile.shards.0.fetch.children.1.breakdown.next_reader_count: 0 }
+  - gt: { profile.shards.0.fetch.children.1.breakdown.next_reader: 0 }
 
 ---
 fetch source:
   - skip:
-      version: ' - 8.5.99'
-      reason:  stored fields phase added in 8.6
+      version: ' - 8.13.99'
+      reason: fetch fields and stored_fields using ValueFetcher
 
   - do:
       search:
@@ -71,20 +75,21 @@ fetch source:
   - gt: { profile.shards.0.fetch.breakdown.load_stored_fields_count: 0 }
   - gt: { profile.shards.0.fetch.breakdown.load_stored_fields: 0 }
   - match: { profile.shards.0.fetch.debug.stored_fields: [_id, _routing, _source] }
-  - length: { profile.shards.0.fetch.children: 2 }
-  - match: { profile.shards.0.fetch.children.0.type: FetchSourcePhase }
-  - gt: { profile.shards.0.fetch.children.0.breakdown.next_reader_count: 0 }
-  - gt: { profile.shards.0.fetch.children.0.breakdown.next_reader: 0 }
-  - gt: { profile.shards.0.fetch.children.0.breakdown.next_reader_count: 0 }
-  - gt: { profile.shards.0.fetch.children.0.breakdown.next_reader: 0 }
-  - match: { profile.shards.0.fetch.children.0.debug.fast_path: 1 }
-  - match: { profile.shards.0.fetch.children.1.type: StoredFieldsPhase }
+  - length: { profile.shards.0.fetch.children: 3 }
+  - match: { profile.shards.0.fetch.children.0.type: FetchFieldsPhase }
+  - match: { profile.shards.0.fetch.children.1.type: FetchSourcePhase }
+  - gt: { profile.shards.0.fetch.children.1.breakdown.next_reader_count: 0 }
+  - gt: { profile.shards.0.fetch.children.1.breakdown.next_reader: 0 }
+  - gt: { profile.shards.0.fetch.children.1.breakdown.next_reader_count: 0 }
+  - gt: { profile.shards.0.fetch.children.1.breakdown.next_reader: 0 }
+  - match: { profile.shards.0.fetch.children.1.debug.fast_path: 1 }
+  - match: { profile.shards.0.fetch.children.2.type: StoredFieldsPhase }
 
 ---
 fetch nested source:
   - skip:
-      version: ' - 8.5.99'
-      reason:  stored fields phase added in 8.6
+      version: ' - 8.13.99'
+      reason: fetch fields and stored_fields using ValueFetcher
 
   - do:
       indices.create:
@@ -135,24 +140,25 @@ fetch nested source:
   - gt: { profile.shards.0.fetch.breakdown.load_stored_fields_count: 0 }
   - gt: { profile.shards.0.fetch.breakdown.load_stored_fields: 0 }
   - match: { profile.shards.0.fetch.debug.stored_fields: [_id, _routing, _source] }
-  - length: { profile.shards.0.fetch.children: 3 }
-  - match: { profile.shards.0.fetch.children.0.type: FetchSourcePhase }
-  - gt: { profile.shards.0.fetch.children.0.breakdown.next_reader_count: 0 }
-  - gt: { profile.shards.0.fetch.children.0.breakdown.next_reader: 0 }
-  - gt: { profile.shards.0.fetch.children.0.breakdown.next_reader_count: 0 }
-  - gt: { profile.shards.0.fetch.children.0.breakdown.next_reader: 0 }
-  - match: { profile.shards.0.fetch.children.1.type: InnerHitsPhase }
+  - length: { profile.shards.0.fetch.children: 4 }
+  - match: { profile.shards.0.fetch.children.0.type: FetchFieldsPhase }
+  - match: { profile.shards.0.fetch.children.1.type: FetchSourcePhase }
   - gt: { profile.shards.0.fetch.children.1.breakdown.next_reader_count: 0 }
   - gt: { profile.shards.0.fetch.children.1.breakdown.next_reader: 0 }
   - gt: { profile.shards.0.fetch.children.1.breakdown.next_reader_count: 0 }
   - gt: { profile.shards.0.fetch.children.1.breakdown.next_reader: 0 }
-  - match: { profile.shards.0.fetch.children.2.type: StoredFieldsPhase }
+  - match: { profile.shards.0.fetch.children.2.type: InnerHitsPhase }
+  - gt: { profile.shards.0.fetch.children.2.breakdown.next_reader_count: 0 }
+  - gt: { profile.shards.0.fetch.children.2.breakdown.next_reader: 0 }
+  - gt: { profile.shards.0.fetch.children.2.breakdown.next_reader_count: 0 }
+  - gt: { profile.shards.0.fetch.children.2.breakdown.next_reader: 0 }
+  - match: { profile.shards.0.fetch.children.3.type: StoredFieldsPhase }
 
 ---
 disabling stored fields removes fetch sub phases:
   - skip:
       version: ' - 7.15.99'
-      reason:  fetch profiling implemented in 7.16.0
+      reason: fetch profiling implemented in 7.16.0
 
   - do:
       search: