Merge branch 'retrievers' into tests

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/SymbolicLinkPreservingTarFuncTest.groovy

@@ -8,6 +8,8 @@
 
 package org.elasticsearch.gradle.internal
 
+import spock.lang.Ignore
+
 import org.apache.commons.compress.archivers.tar.TarArchiveEntry
 import org.apache.commons.compress.archivers.tar.TarArchiveInputStream
 import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream
@@ -36,6 +38,11 @@ class SymbolicLinkPreservingTarFuncTest extends AbstractGradleFuncTest {
         final Path linkToRealFolder = archiveSourceRoot.resolve("link-to-real-folder");
         Files.createSymbolicLink(linkToRealFolder, Paths.get("./real-folder"));
 
+        final Path realFolder2 = testProjectDir.getRoot().toPath().resolve("real-folder2")
+        final Path realFolderSub = realFolder2.resolve("sub")
+        Files.createDirectory(realFolder2);
+        Files.createDirectory(realFolderSub);
+
         buildFile << """
 import org.elasticsearch.gradle.internal.SymbolicLinkPreservingTar
 
@@ -56,6 +63,12 @@ tasks.register("buildBZip2Tar", SymbolicLinkPreservingTar) { SymbolicLinkPreserv
   tar.compression = Compression.BZIP2
   tar.preserveFileTimestamps = ${preserverTimestamp}
   from fileTree("archiveRoot")
+
+  into('config') {
+    dirMode 0750
+    fileMode 0660
+    from "real-folder2"
+  }
 }
 """
         when:
@@ -118,14 +131,20 @@ tasks.register("buildTar", SymbolicLinkPreservingTar) { SymbolicLinkPreservingTa
                 if (entry.getName().equals("real-folder/")) {
                     assert entry.isDirectory()
                     realFolderEntry = true
-                } else if (entry.getName().equals("real-folder/file")) {
+                }  else if (entry.getName().equals("real-folder/file")) {
                     assert entry.isFile()
                     fileEntry = true
                 } else if (entry.getName().equals("real-folder/link-to-file")) {
                     assert entry.isSymbolicLink()
                     assert normalized(entry.getLinkName()) == "./file"
                     linkToFileEntry = true
-                } else if (entry.getName().equals("link-in-folder/")) {
+                } else if (entry.getName().equals("config/")) {
+                    assert entry.isDirectory()
+                    assert entry.getMode() == 16877
+                } else if (entry.getName().equals("config/sub/")) {
+                    assert entry.isDirectory()
+                    assert entry.getMode() == 16872
+                }else if (entry.getName().equals("link-in-folder/")) {
                     assert entry.isDirectory()
                     linkInFolderEntry = true
                 } else if (entry.getName().equals("link-in-folder/link-to-file")) {

docs/changelog/102428.yaml

@@ -0,0 +1,5 @@
+pr: 102428
+summary: "ESQL: Add option to drop null fields"
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/103399.yaml

@@ -0,0 +1,6 @@
+pr: 103399
+summary: "add validation on _id field when upsert new doc"
+area: Search
+type: bug
+issues:
+  - 102981

docs/changelog/103763.yaml

@@ -0,0 +1,6 @@
+pr: 103763
+summary: Ref count search response bytes
+area: Search
+type: enhancement
+issues:
+ - 102657

docs/changelog/104006.yaml

@@ -0,0 +1,5 @@
+pr: 104006
+summary: Add support for more than one `inner_hit` when searching nested vectors
+area: Vector Search
+type: enhancement
+issues: []

docs/changelog/104033.yaml

@@ -0,0 +1,5 @@
+pr: 104033
+summary: Add Query Users API
+area: Security
+type: enhancement
+issues: []

docs/changelog/104087.yaml

@@ -0,0 +1,13 @@
+pr: 104087
+summary: Deprecate machine learning on Intel macOS
+area: Machine Learning
+type: deprecation
+issues: []
+deprecation:
+  title: Deprecate machine learning on Intel macOS
+  area: Packaging
+  details: The machine learning plugin will be permanently disabled on macOS x86_64
+    in new minor versions released from December 2024 onwards.
+  impact: To continue to use machine learning functionality on macOS please switch to
+    an arm64 machine (Apple silicon). Alternatively, it will still be possible to run
+    Elasticsearch with machine learning enabled in a Docker container on macOS x86_64.

docs/changelog/104182.yaml

@@ -0,0 +1,5 @@
+pr: 104182
+summary: "Apm-data: fix `@custom` component templates"
+area: Data streams
+type: bug
+issues: []

docs/changelog/104269.yaml

@@ -0,0 +1,5 @@
+pr: 104269
+summary: "ESQL: Support loading shapes from source into WKB blocks"
+area: "ES|QL"
+type: enhancement
+issues: []

docs/changelog/104355.yaml

@@ -0,0 +1,5 @@
+pr: 104355
+summary: Prepare enrich plan to support multi clusters
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/104386.yaml

@@ -0,0 +1,6 @@
+pr: 104386
+summary: "X-pack/plugin/apm-data: add dynamic setting for enabling template registry"
+area: Data streams
+type: enhancement
+issues:
+ - 104385

docs/changelog/104396.yaml

@@ -0,0 +1,5 @@
+pr: 104396
+summary: Report current master in `PeerFinder`
+area: Cluster Coordination
+type: enhancement
+issues: []

docs/changelog/104406.yaml

@@ -0,0 +1,5 @@
+pr: 104406
+summary: Support patch transport version from 8.12
+area: Downsampling
+type: enhancement
+issues: []

docs/changelog/104407.yaml

@@ -0,0 +1,6 @@
+pr: 104407
+summary: Set read timeout for fetching IMDSv2 token
+area: Discovery-Plugins
+type: enhancement
+issues:
+ - 104244

docs/changelog/104418.yaml

@@ -0,0 +1,6 @@
+pr: 104418
+summary: Fix `routing_path` when template has multiple `path_match` and multi-fields
+area: TSDB
+type: bug
+issues:
+ - 104400

docs/changelog/104460.yaml

@@ -0,0 +1,5 @@
+pr: 104460
+summary: Dyamically adjust node metrics cache expire
+area: Search
+type: enhancement
+issues: []

docs/reference/esql/esql-query-api.asciidoc

@@ -43,6 +43,12 @@ or alias you search.
 (Optional, string) Separator for CSV results. Defaults to `,`. The API only
 supports this parameter for CSV responses.
 
+`drop_null_columns`::
+(Optional, boolean) Should columns that are entirely `null` be removed from
+the `columns` and `values` portion of the results? Defaults to `false`. If
+`true` the the response will include an extra section under the name
+`all_columns` which has the name of all columns.
+
 `format`::
 (Optional, string) Format for the response. For valid values, refer to
 <<esql-rest-format>>.
@@ -75,17 +81,12 @@ responses. See <<esql-rest-columnar>>.
 
 `columns`::
 (array of objects)
-Column headings for the search results. Each object is a column.
-+
-.Properties of `columns` objects
-[%collapsible%open]
-=====
-`name`::
-(string) Name of the column.
-
-`type`::
-(string) Data type for the column.
-=====
+Column `name` and `type` for each column returned in `values`. Each object is a single column.
+
+`all_columns`::
+(array of objects)
+Column `name` and `type` for each queried column. Each object is a single column. This is only
+returned if `drop_null_columns` is sent with the request.
 
 `rows`::
 (array of arrays)

docs/reference/esql/functions/types/mv_count.asciidoc

@@ -3,9 +3,11 @@
 v | result
 boolean | integer
 cartesian_point | integer
+cartesian_shape | integer
 datetime | integer
 double | integer
 geo_point | integer
+geo_shape | integer
 integer | integer
 ip | integer
 keyword | integer

docs/reference/esql/functions/types/mv_first.asciidoc

@@ -3,9 +3,11 @@
 v | result
 boolean | boolean
 cartesian_point | cartesian_point
+cartesian_shape | cartesian_shape
 datetime | datetime
 double | double
 geo_point | geo_point
+geo_shape | geo_shape
 integer | integer
 ip | ip
 keyword | keyword

docs/reference/esql/functions/types/mv_last.asciidoc

@@ -3,9 +3,11 @@
 v | result
 boolean | boolean
 cartesian_point | cartesian_point
+cartesian_shape | cartesian_shape
 datetime | datetime
 double | double
 geo_point | geo_point
+geo_shape | geo_shape
 integer | integer
 ip | ip
 keyword | keyword

docs/reference/esql/functions/types/to_string.asciidoc

@@ -3,9 +3,11 @@
 v | result
 boolean | keyword
 cartesian_point | keyword
+cartesian_shape | keyword
 datetime | keyword
 double | keyword
 geo_point | keyword
+geo_shape | keyword
 integer | keyword
 ip | keyword
 keyword | keyword