From 9cc75734d0a6fe1c82e57c6c900daa7c5ac0db1e Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Mon, 24 Feb 2025 08:26:32 -0800 Subject: [PATCH 01/12] fix stale data in synthetic source for string stored field (#123105) (#123277) Co-authored-by: jeffganmr <106223805+jeffganmr@users.noreply.github.com> --- docs/changelog/123105.yaml | 6 +++ .../mapper/StringStoredFieldFieldLoader.java | 2 +- .../StringStoredFieldFieldLoaderTests.java | 52 +++++++++++++++++++ 3 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 docs/changelog/123105.yaml create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/StringStoredFieldFieldLoaderTests.java diff --git a/docs/changelog/123105.yaml b/docs/changelog/123105.yaml new file mode 100644 index 0000000000000..f763b51436434 --- /dev/null +++ b/docs/changelog/123105.yaml @@ -0,0 +1,6 @@ +pr: 123105 +summary: fix stale data in synthetic source for string stored field +area: Mapping +type: bug +issues: + - 123110 diff --git a/server/src/main/java/org/elasticsearch/index/mapper/StringStoredFieldFieldLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/StringStoredFieldFieldLoader.java index c8a8dd4b4a898..c114a1619c7a9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/StringStoredFieldFieldLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/StringStoredFieldFieldLoader.java @@ -54,7 +54,7 @@ public final void write(XContentBuilder b) throws IOException { case 1: b.field(simpleName); write(b, values.get(0)); - return; + break; default: b.startArray(simpleName); for (Object value : values) { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/StringStoredFieldFieldLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/StringStoredFieldFieldLoaderTests.java new file mode 100644 index 0000000000000..dcd6c35ab1700 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/StringStoredFieldFieldLoaderTests.java @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentType; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public class StringStoredFieldFieldLoaderTests extends ESTestCase { + + public void testLoadStoredFieldAndReset() throws IOException { + var sfl = new StringStoredFieldFieldLoader("foo", "foo") { + @Override + protected void write(XContentBuilder b, Object value) throws IOException { + b.value((String) value); + } + }; + + var storedFieldLoaders = sfl.storedFieldLoaders().collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + storedFieldLoaders.get("foo").load(List.of("one")); + + var result = XContentBuilder.builder(XContentType.JSON.xContent()); + result.startObject(); + sfl.write(result); + result.endObject(); + + assertEquals(""" + {"foo":"one"}""", Strings.toString(result)); + + var empty = XContentBuilder.builder(XContentType.JSON.xContent()); + empty.startObject(); + // reset() should have been called after previous write + sfl.write(empty); + empty.endObject(); + + assertEquals("{}", Strings.toString(empty)); + } + +} From cc3c3870ecd91511f600a96612c30a3abbbb0129 Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 24 Feb 2025 16:33:42 +0000 Subject: [PATCH 02/12] Deduplicate allocation stats calls (#123267) (#123280) These things can be quite expensive and there's no need to recompute them in parallel across all management threads as done today. This commit adds a deduplicator to avoid redundant work. Backport of #123246 to `8.x` --- docs/changelog/123246.yaml | 5 ++ .../TransportGetAllocationStatsAction.java | 30 ++++++++--- .../allocation/AllocationStatsService.java | 3 ++ ...ransportGetAllocationStatsActionTests.java | 51 +++++++++++++++++++ 4 files changed, 82 insertions(+), 7 deletions(-) create mode 100644 docs/changelog/123246.yaml diff --git a/docs/changelog/123246.yaml b/docs/changelog/123246.yaml new file mode 100644 index 0000000000000..3477cf70ac89b --- /dev/null +++ b/docs/changelog/123246.yaml @@ -0,0 +1,5 @@ +pr: 123246 +summary: Deduplicate allocation stats calls +area: Allocation +type: bug +issues: [] diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsAction.java index 259a244bff919..a22173a4c5b89 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsAction.java @@ -13,9 +13,12 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionRunnable; import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.SingleResultDeduplicator; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequestParameters.Metric; import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.action.support.master.MasterNodeReadRequest; import org.elasticsearch.action.support.master.TransportMasterNodeReadAction; import org.elasticsearch.cluster.ClusterState; @@ -28,6 +31,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.features.FeatureService; @@ -47,7 +51,7 @@ public class TransportGetAllocationStatsAction extends TransportMasterNodeReadAc public static final ActionType TYPE = new ActionType<>("cluster:monitor/allocation/stats"); - private final AllocationStatsService allocationStatsService; + private final SingleResultDeduplicator> allocationStatsSupplier; private final DiskThresholdSettings diskThresholdSettings; private final FeatureService featureService; @@ -70,9 +74,15 @@ public TransportGetAllocationStatsAction( TransportGetAllocationStatsAction.Request::new, indexNameExpressionResolver, TransportGetAllocationStatsAction.Response::new, - threadPool.executor(ThreadPool.Names.MANAGEMENT) + // DIRECT is ok here because we fork the allocation stats computation onto a MANAGEMENT thread if needed, or else we return + // very cheaply. + EsExecutors.DIRECT_EXECUTOR_SERVICE + ); + final var managementExecutor = threadPool.executor(ThreadPool.Names.MANAGEMENT); + this.allocationStatsSupplier = new SingleResultDeduplicator<>( + threadPool.getThreadContext(), + l -> managementExecutor.execute(ActionRunnable.supply(l, allocationStatsService::stats)) ); - this.allocationStatsService = allocationStatsService; this.diskThresholdSettings = new DiskThresholdSettings(clusterService.getSettings(), clusterService.getClusterSettings()); this.featureService = featureService; } @@ -89,15 +99,21 @@ protected void doExecute(Task task, Request request, ActionListener li @Override protected void masterOperation(Task task, Request request, ClusterState state, ActionListener listener) throws Exception { - listener.onResponse( - new Response( - request.metrics().contains(Metric.ALLOCATIONS) ? allocationStatsService.stats() : Map.of(), + // NB we are still on a transport thread here - if adding more functionality here make sure to fork to a different pool + + final SubscribableListener> allocationStatsStep = request.metrics().contains(Metric.ALLOCATIONS) + ? SubscribableListener.newForked(allocationStatsSupplier::execute) + : SubscribableListener.newSucceeded(Map.of()); + + allocationStatsStep.andThenApply( + allocationStats -> new Response( + allocationStats, request.metrics().contains(Metric.FS) && featureService.clusterHasFeature(clusterService.state(), AllocationStatsFeatures.INCLUDE_DISK_THRESHOLD_SETTINGS) ? diskThresholdSettings : null ) - ); + ).addListener(listener); } @Override diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationStatsService.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationStatsService.java index 3651f560e6dde..fa4d60c83e5ef 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationStatsService.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationStatsService.java @@ -18,6 +18,7 @@ import org.elasticsearch.cluster.routing.allocation.allocator.ShardsAllocator; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.util.Maps; +import org.elasticsearch.transport.Transports; import java.util.Map; @@ -41,6 +42,8 @@ public AllocationStatsService( } public Map stats() { + assert Transports.assertNotTransportThread("too expensive for a transport worker"); + var state = clusterService.state(); var info = clusterInfoService.getClusterInfo(); var desiredBalance = desiredBalanceShardsAllocator != null ? desiredBalanceShardsAllocator.getDesiredBalance() : null; diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsActionTests.java index f3d8f8860ba83..4eed6cf0f62e2 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsActionTests.java @@ -32,8 +32,13 @@ import java.util.EnumSet; import java.util.Map; import java.util.Set; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import static org.hamcrest.Matchers.anEmptyMap; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.not; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; @@ -120,4 +125,50 @@ public void testReturnsOnlyRequestedStats() throws Exception { assertNull(response.getDiskThresholdSettings()); } } + + public void testDeduplicatesStatsComputations() throws InterruptedException { + final var requestCounter = new AtomicInteger(); + final var isExecuting = new AtomicBoolean(); + when(allocationStatsService.stats()).thenAnswer(invocation -> { + try { + assertTrue(isExecuting.compareAndSet(false, true)); + assertThat(Thread.currentThread().getName(), containsString("[management]")); + return Map.of(Integer.toString(requestCounter.incrementAndGet()), NodeAllocationStatsTests.randomNodeAllocationStats()); + } finally { + Thread.yield(); + assertTrue(isExecuting.compareAndSet(true, false)); + } + }); + + final var threads = new Thread[between(1, 5)]; + final var startBarrier = new CyclicBarrier(threads.length); + for (int i = 0; i < threads.length; i++) { + threads[i] = new Thread(() -> { + safeAwait(startBarrier); + + final var minRequestIndex = requestCounter.get(); + + final TransportGetAllocationStatsAction.Response response = safeAwait( + l -> action.masterOperation( + mock(Task.class), + new TransportGetAllocationStatsAction.Request( + TEST_REQUEST_TIMEOUT, + TaskId.EMPTY_TASK_ID, + EnumSet.of(Metric.ALLOCATIONS) + ), + ClusterState.EMPTY_STATE, + l + ) + ); + + final var requestIndex = Integer.valueOf(response.getNodeAllocationStats().keySet().iterator().next()); + assertThat(requestIndex, greaterThanOrEqualTo(minRequestIndex)); // did not get a stale result + }, "thread-" + i); + threads[i].start(); + } + + for (final var thread : threads) { + thread.join(); + } + } } From e12d7775e7d460456f5cb86e3f12b8c462865058 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 24 Feb 2025 11:41:09 -0500 Subject: [PATCH 03/12] ESQL: Add known issue for slow VALUES (#123222) Co-authored-by: Liam Thompson <32779855+leemthompo@users.noreply.github.com> --- docs/reference/release-notes/8.16.4.asciidoc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/reference/release-notes/8.16.4.asciidoc b/docs/reference/release-notes/8.16.4.asciidoc index 47fff79f1c3ec..795665a68a911 100644 --- a/docs/reference/release-notes/8.16.4.asciidoc +++ b/docs/reference/release-notes/8.16.4.asciidoc @@ -39,4 +39,10 @@ Snapshot/Restore:: Ingest Node:: * Improve memory aspects of enrich cache {es-pull}120256[#120256] (issues: {es-issue}96050[#96050], {es-issue}120021[#120021]) +[discrete] +[[known-issues-8.16.4]] +=== Known issues +ES|QL:: + +* The `VALUES` aggregate function can run for an extremely long time when collecting many groups. Processing hundreds of thousands of groups may take several minutes on a single thread, while processing millions of groups can take days. The function has O(n^2) complexity with respect to the number of groups. During execution, these operations will not respond to the tasks cancellation API. This issue has been fixed by {es-pull}#123073[#123073] and the fix is available in versions 8.16.5, 8.17.3, 8.18.0, and all releases after that. From 6af16455b937b8ca019e333a845029e165d9a923 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Tue, 25 Feb 2025 04:33:57 +1100 Subject: [PATCH 04/12] Mute org.elasticsearch.gradle.internal.InternalBwcGitPluginFuncTest current repository can be cloned #123297 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 181f2f6a94dbc..ca0532f376e73 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -304,6 +304,9 @@ tests: method: "builds distribution from branches via archives extractedAssemble [bwcDistVersion: 8.2.1, bwcProject: bugfix, expectedAssembleTaskName: extractedAssemble, #2]" issue: https://github.com/elastic/elasticsearch/issues/119871 +- class: org.elasticsearch.gradle.internal.InternalBwcGitPluginFuncTest + method: current repository can be cloned + issue: https://github.com/elastic/elasticsearch/issues/123297 # Examples: # From a45569c981b9b9ca660c44037ef4a17bdd211f01 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Tue, 25 Feb 2025 04:34:03 +1100 Subject: [PATCH 05/12] Mute org.elasticsearch.gradle.internal.InternalDistributionBwcSetupPluginFuncTest builds distribution from branches via archives extractedAssemble [bwcDistVersion: 8.4.0, bwcProject: minor, expectedAssembleTaskName: extractedAssemble, #0] #123219 --- muted-tests.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index ca0532f376e73..faba1ebdba7aa 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -307,6 +307,10 @@ tests: - class: org.elasticsearch.gradle.internal.InternalBwcGitPluginFuncTest method: current repository can be cloned issue: https://github.com/elastic/elasticsearch/issues/123297 +- class: org.elasticsearch.gradle.internal.InternalDistributionBwcSetupPluginFuncTest + method: "builds distribution from branches via archives extractedAssemble [bwcDistVersion: 8.4.0, bwcProject: minor, expectedAssembleTaskName: + extractedAssemble, #0]" + issue: https://github.com/elastic/elasticsearch/issues/123219 # Examples: # From b8f8723e6c3833befccb3d9f580467f2dc65a69b Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Mon, 24 Feb 2025 20:53:21 -0500 Subject: [PATCH 06/12] Register IngestGeoIpMetadata as a NamedXContent (#123079) (#123329) --- docs/changelog/123079.yaml | 5 + .../qa/full-cluster-restart/build.gradle | 10 +- .../ingest/geoip/FullClusterRestartIT.java | 138 ++++---------- .../qa/geoip-reindexed/build.gradle | 34 ++++ .../ingest/geoip/GeoIpReindexedIT.java | 169 ++++++++++++++++++ .../ingest/geoip/IngestGeoIpMetadata.java | 2 +- .../ingest/geoip/IngestGeoIpPlugin.java | 5 + 7 files changed, 245 insertions(+), 118 deletions(-) create mode 100644 docs/changelog/123079.yaml create mode 100644 modules/ingest-geoip/qa/geoip-reindexed/build.gradle create mode 100644 modules/ingest-geoip/qa/geoip-reindexed/src/javaRestTest/java/org/elasticsearch/ingest/geoip/GeoIpReindexedIT.java diff --git a/docs/changelog/123079.yaml b/docs/changelog/123079.yaml new file mode 100644 index 0000000000000..1817415bc8974 --- /dev/null +++ b/docs/changelog/123079.yaml @@ -0,0 +1,5 @@ +pr: 123079 +summary: Register `IngestGeoIpMetadata` as a NamedXContent +area: Ingest Node +type: bug +issues: [] diff --git a/modules/ingest-geoip/qa/full-cluster-restart/build.gradle b/modules/ingest-geoip/qa/full-cluster-restart/build.gradle index c6f4e1667957d..35e1514781b9f 100644 --- a/modules/ingest-geoip/qa/full-cluster-restart/build.gradle +++ b/modules/ingest-geoip/qa/full-cluster-restart/build.gradle @@ -7,9 +7,6 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -import org.elasticsearch.gradle.Version -import org.elasticsearch.gradle.VersionProperties -import org.elasticsearch.gradle.internal.info.BuildParams import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask apply plugin: 'elasticsearch.internal-java-rest-test' @@ -21,12 +18,7 @@ dependencies { javaRestTestImplementation(testArtifact(project(":qa:full-cluster-restart"), "javaRestTest")) } -assert Version.fromString(VersionProperties.getVersions().get("elasticsearch")).getMajor() == 8 : - "If we are targeting a branch other than 8, we should enable migration tests" - -// once we are ready to test migrations from 8.x to 9.x, we can set the compatible version to 8.0.0 -// see https://github.com/elastic/elasticsearch/pull/93666 -buildParams.bwcVersions.withWireCompatible(v -> v.before("7.0.0")) { bwcVersion, baseName -> +buildParams.bwcVersions.withWireCompatible(v -> v.onOrAfter("8.15.0")) { bwcVersion, baseName -> tasks.register(bwcTaskName(bwcVersion), StandaloneRestIntegTestTask) { usesBwcDistribution(bwcVersion) systemProperty("tests.old_cluster_version", bwcVersion) diff --git a/modules/ingest-geoip/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/ingest/geoip/FullClusterRestartIT.java b/modules/ingest-geoip/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/ingest/geoip/FullClusterRestartIT.java index cf66a978c6b5f..d9d79da418600 100644 --- a/modules/ingest-geoip/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/ingest/geoip/FullClusterRestartIT.java +++ b/modules/ingest-geoip/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/ingest/geoip/FullClusterRestartIT.java @@ -12,39 +12,39 @@ import com.carrotsearch.randomizedtesting.annotations.Name; -import org.apache.http.util.EntityUtils; import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.cluster.FeatureFlag; import org.elasticsearch.test.cluster.local.distribution.DistributionType; -import org.elasticsearch.test.rest.ObjectPath; import org.elasticsearch.upgrades.FullClusterRestartUpgradeStatus; import org.elasticsearch.upgrades.ParameterizedFullClusterRestartTestCase; import org.junit.ClassRule; import org.junit.rules.RuleChain; import org.junit.rules.TestRule; -import java.io.IOException; import java.util.List; import java.util.Map; -import java.util.Objects; +import java.util.Set; import java.util.concurrent.TimeUnit; -import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; public class FullClusterRestartIT extends ParameterizedFullClusterRestartTestCase { private static final boolean useFixture = Boolean.getBoolean("geoip_use_service") == false; - private static GeoIpHttpFixture fixture = new GeoIpHttpFixture(useFixture); + private static final GeoIpHttpFixture fixture = new GeoIpHttpFixture(useFixture); - private static ElasticsearchCluster cluster = ElasticsearchCluster.local() + private static final ElasticsearchCluster cluster = ElasticsearchCluster.local() .distribution(DistributionType.DEFAULT) .version(getOldClusterTestVersion()) .nodes(2) - .setting("indices.memory.shard_inactive_time", "60m") - .setting("xpack.security.enabled", "false") .setting("ingest.geoip.downloader.endpoint", () -> fixture.getAddress(), s -> useFixture) + .setting("xpack.security.enabled", "false") + // .setting("logger.org.elasticsearch.ingest.geoip", "TRACE") .feature(FeatureFlag.TIME_SERIES_MODE) .build(); @@ -60,110 +60,32 @@ protected ElasticsearchCluster getUpgradeCluster() { return cluster; } - public void testGeoIpSystemFeaturesMigration() throws Exception { + @SuppressWarnings("unchecked") + public void testGeoIpDatabaseConfigurations() throws Exception { if (isRunningAgainstOldCluster()) { - Request enableDownloader = new Request("PUT", "/_cluster/settings"); - enableDownloader.setJsonEntity(""" - {"persistent": {"ingest.geoip.downloader.enabled": true}} - """); - assertOK(client().performRequest(enableDownloader)); - - Request putPipeline = new Request("PUT", "/_ingest/pipeline/geoip"); - putPipeline.setJsonEntity(""" + Request putConfiguration = new Request("PUT", "_ingest/ip_location/database/my-database-1"); + putConfiguration.setJsonEntity(""" { - "description": "Add geoip info", - "processors": [{ - "geoip": { - "field": "ip", - "target_field": "geo", - "database_file": "GeoLite2-Country.mmdb" - } - }] + "name": "GeoIP2-Domain", + "maxmind": { + "account_id": "1234567" + } } """); - assertOK(client().performRequest(putPipeline)); - - // wait for the geo databases to all be loaded - assertBusy(() -> testDatabasesLoaded(), 30, TimeUnit.SECONDS); - - // the geoip index should be created - assertBusy(() -> testCatIndices(".geoip_databases")); - assertBusy(() -> testIndexGeoDoc()); - } else { - Request migrateSystemFeatures = new Request("POST", "/_migration/system_features"); - assertOK(client().performRequest(migrateSystemFeatures)); - - assertBusy(() -> testCatIndices(".geoip_databases-reindexed-for-8", "my-index-00001")); - assertBusy(() -> testIndexGeoDoc()); - - Request disableDownloader = new Request("PUT", "/_cluster/settings"); - disableDownloader.setJsonEntity(""" - {"persistent": {"ingest.geoip.downloader.enabled": false}} - """); - assertOK(client().performRequest(disableDownloader)); - - // the geoip index should be deleted - assertBusy(() -> testCatIndices("my-index-00001")); - - Request enableDownloader = new Request("PUT", "/_cluster/settings"); - enableDownloader.setJsonEntity(""" - {"persistent": {"ingest.geoip.downloader.enabled": true}} - """); - assertOK(client().performRequest(enableDownloader)); - - // wait for the geo databases to all be loaded - assertBusy(() -> testDatabasesLoaded(), 30, TimeUnit.SECONDS); - - // the geoip index should be recreated - assertBusy(() -> testCatIndices(".geoip_databases", "my-index-00001")); - assertBusy(() -> testIndexGeoDoc()); + assertOK(client().performRequest(putConfiguration)); } - } - - @SuppressWarnings("unchecked") - private void testDatabasesLoaded() throws IOException { - Request getTaskState = new Request("GET", "/_cluster/state"); - ObjectPath state = ObjectPath.createFromResponse(client().performRequest(getTaskState)); - - List tasks = state.evaluate("metadata.persistent_tasks.tasks"); - // Short-circuit to avoid using steams if the list is empty - if (tasks.isEmpty()) { - fail(); - } - Map databases = (Map) tasks.stream().map(task -> { - try { - return ObjectPath.evaluate(task, "task.geoip-downloader.state.databases"); - } catch (IOException e) { - return null; - } - }).filter(Objects::nonNull).findFirst().orElse(null); - - assertNotNull(databases); - - for (String name : List.of("GeoLite2-ASN.mmdb", "GeoLite2-City.mmdb", "GeoLite2-Country.mmdb")) { - Object database = databases.get(name); - assertNotNull(database); - assertNotNull(ObjectPath.evaluate(database, "md5")); - } - } - - private void testCatIndices(String... indexNames) throws IOException { - Request catIndices = new Request("GET", "_cat/indices/*?s=index&h=index&expand_wildcards=all"); - String response = EntityUtils.toString(client().performRequest(catIndices).getEntity()); - List indices = List.of(response.trim().split("\\s+")); - assertThat(indices, contains(indexNames)); - } - - private void testIndexGeoDoc() throws IOException { - Request putDoc = new Request("PUT", "/my-index-00001/_doc/my_id?pipeline=geoip"); - putDoc.setJsonEntity(""" - {"ip": "89.160.20.128"} - """); - assertOK(client().performRequest(putDoc)); - Request getDoc = new Request("GET", "/my-index-00001/_doc/my_id"); - ObjectPath doc = ObjectPath.createFromResponse(client().performRequest(getDoc)); - assertNull(doc.evaluate("_source.tags")); - assertEquals("Sweden", doc.evaluate("_source.geo.country_name")); + assertBusy(() -> { + Request getConfiguration = new Request("GET", "_ingest/ip_location/database/my-database-1"); + Response response = assertOK(client().performRequest(getConfiguration)); + Map map = responseAsMap(response); + assertThat(map.keySet(), equalTo(Set.of("databases"))); + List> databases = (List>) map.get("databases"); + assertThat(databases, hasSize(1)); + Map database = databases.get(0); + assertThat(database.get("id"), is("my-database-1")); + assertThat(database.get("version"), is(1)); + assertThat(database.get("database"), equalTo(Map.of("name", "GeoIP2-Domain", "maxmind", Map.of("account_id", "1234567")))); + }, 30, TimeUnit.SECONDS); } } diff --git a/modules/ingest-geoip/qa/geoip-reindexed/build.gradle b/modules/ingest-geoip/qa/geoip-reindexed/build.gradle new file mode 100644 index 0000000000000..c6f4e1667957d --- /dev/null +++ b/modules/ingest-geoip/qa/geoip-reindexed/build.gradle @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import org.elasticsearch.gradle.Version +import org.elasticsearch.gradle.VersionProperties +import org.elasticsearch.gradle.internal.info.BuildParams +import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask + +apply plugin: 'elasticsearch.internal-java-rest-test' +apply plugin: 'elasticsearch.bwc-test' + + +dependencies { + javaRestTestImplementation project(':test:fixtures:geoip-fixture') + javaRestTestImplementation(testArtifact(project(":qa:full-cluster-restart"), "javaRestTest")) +} + +assert Version.fromString(VersionProperties.getVersions().get("elasticsearch")).getMajor() == 8 : + "If we are targeting a branch other than 8, we should enable migration tests" + +// once we are ready to test migrations from 8.x to 9.x, we can set the compatible version to 8.0.0 +// see https://github.com/elastic/elasticsearch/pull/93666 +buildParams.bwcVersions.withWireCompatible(v -> v.before("7.0.0")) { bwcVersion, baseName -> + tasks.register(bwcTaskName(bwcVersion), StandaloneRestIntegTestTask) { + usesBwcDistribution(bwcVersion) + systemProperty("tests.old_cluster_version", bwcVersion) + } +} diff --git a/modules/ingest-geoip/qa/geoip-reindexed/src/javaRestTest/java/org/elasticsearch/ingest/geoip/GeoIpReindexedIT.java b/modules/ingest-geoip/qa/geoip-reindexed/src/javaRestTest/java/org/elasticsearch/ingest/geoip/GeoIpReindexedIT.java new file mode 100644 index 0000000000000..6313e486108c2 --- /dev/null +++ b/modules/ingest-geoip/qa/geoip-reindexed/src/javaRestTest/java/org/elasticsearch/ingest/geoip/GeoIpReindexedIT.java @@ -0,0 +1,169 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +package org.elasticsearch.ingest.geoip; + +import fixture.geoip.GeoIpHttpFixture; + +import com.carrotsearch.randomizedtesting.annotations.Name; + +import org.apache.http.util.EntityUtils; +import org.elasticsearch.client.Request; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.FeatureFlag; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.elasticsearch.test.rest.ObjectPath; +import org.elasticsearch.upgrades.FullClusterRestartUpgradeStatus; +import org.elasticsearch.upgrades.ParameterizedFullClusterRestartTestCase; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import org.junit.rules.TestRule; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.TimeUnit; + +import static org.hamcrest.Matchers.contains; + +public class GeoIpReindexedIT extends ParameterizedFullClusterRestartTestCase { + + private static final boolean useFixture = Boolean.getBoolean("geoip_use_service") == false; + + private static GeoIpHttpFixture fixture = new GeoIpHttpFixture(useFixture); + + private static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .version(getOldClusterTestVersion()) + .nodes(2) + .setting("indices.memory.shard_inactive_time", "60m") + .setting("xpack.security.enabled", "false") + .setting("ingest.geoip.downloader.endpoint", () -> fixture.getAddress(), s -> useFixture) + .feature(FeatureFlag.TIME_SERIES_MODE) + .build(); + + @ClassRule + public static TestRule ruleChain = RuleChain.outerRule(fixture).around(cluster); + + public GeoIpReindexedIT(@Name("cluster") FullClusterRestartUpgradeStatus upgradeStatus) { + super(upgradeStatus); + } + + @Override + protected ElasticsearchCluster getUpgradeCluster() { + return cluster; + } + + public void testGeoIpSystemFeaturesMigration() throws Exception { + if (isRunningAgainstOldCluster()) { + Request enableDownloader = new Request("PUT", "/_cluster/settings"); + enableDownloader.setJsonEntity(""" + {"persistent": {"ingest.geoip.downloader.enabled": true}} + """); + assertOK(client().performRequest(enableDownloader)); + + Request putPipeline = new Request("PUT", "/_ingest/pipeline/geoip"); + putPipeline.setJsonEntity(""" + { + "description": "Add geoip info", + "processors": [{ + "geoip": { + "field": "ip", + "target_field": "geo", + "database_file": "GeoLite2-Country.mmdb" + } + }] + } + """); + assertOK(client().performRequest(putPipeline)); + + // wait for the geo databases to all be loaded + assertBusy(() -> testDatabasesLoaded(), 30, TimeUnit.SECONDS); + + // the geoip index should be created + assertBusy(() -> testCatIndices(".geoip_databases")); + assertBusy(() -> testIndexGeoDoc()); + } else { + Request migrateSystemFeatures = new Request("POST", "/_migration/system_features"); + assertOK(client().performRequest(migrateSystemFeatures)); + + assertBusy(() -> testCatIndices(".geoip_databases-reindexed-for-8", "my-index-00001")); + assertBusy(() -> testIndexGeoDoc()); + + Request disableDownloader = new Request("PUT", "/_cluster/settings"); + disableDownloader.setJsonEntity(""" + {"persistent": {"ingest.geoip.downloader.enabled": false}} + """); + assertOK(client().performRequest(disableDownloader)); + + // the geoip index should be deleted + assertBusy(() -> testCatIndices("my-index-00001")); + + Request enableDownloader = new Request("PUT", "/_cluster/settings"); + enableDownloader.setJsonEntity(""" + {"persistent": {"ingest.geoip.downloader.enabled": true}} + """); + assertOK(client().performRequest(enableDownloader)); + + // wait for the geo databases to all be loaded + assertBusy(() -> testDatabasesLoaded(), 30, TimeUnit.SECONDS); + + // the geoip index should be recreated + assertBusy(() -> testCatIndices(".geoip_databases", "my-index-00001")); + assertBusy(() -> testIndexGeoDoc()); + } + } + + @SuppressWarnings("unchecked") + private void testDatabasesLoaded() throws IOException { + Request getTaskState = new Request("GET", "/_cluster/state"); + ObjectPath state = ObjectPath.createFromResponse(client().performRequest(getTaskState)); + + List tasks = state.evaluate("metadata.persistent_tasks.tasks"); + // Short-circuit to avoid using steams if the list is empty + if (tasks.isEmpty()) { + fail(); + } + Map databases = (Map) tasks.stream().map(task -> { + try { + return ObjectPath.evaluate(task, "task.geoip-downloader.state.databases"); + } catch (IOException e) { + return null; + } + }).filter(Objects::nonNull).findFirst().orElse(null); + + assertNotNull(databases); + + for (String name : List.of("GeoLite2-ASN.mmdb", "GeoLite2-City.mmdb", "GeoLite2-Country.mmdb")) { + Object database = databases.get(name); + assertNotNull(database); + assertNotNull(ObjectPath.evaluate(database, "md5")); + } + } + + private void testCatIndices(String... indexNames) throws IOException { + Request catIndices = new Request("GET", "_cat/indices/*?s=index&h=index&expand_wildcards=all"); + String response = EntityUtils.toString(client().performRequest(catIndices).getEntity()); + List indices = List.of(response.trim().split("\\s+")); + assertThat(indices, contains(indexNames)); + } + + private void testIndexGeoDoc() throws IOException { + Request putDoc = new Request("PUT", "/my-index-00001/_doc/my_id?pipeline=geoip"); + putDoc.setJsonEntity(""" + {"ip": "89.160.20.128"} + """); + assertOK(client().performRequest(putDoc)); + + Request getDoc = new Request("GET", "/my-index-00001/_doc/my_id"); + ObjectPath doc = ObjectPath.createFromResponse(client().performRequest(getDoc)); + assertNull(doc.evaluate("_source.tags")); + assertEquals("Sweden", doc.evaluate("_source.geo.country_name")); + } +} diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpMetadata.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpMetadata.java index b6e73f3f33f7c..c976c7f2c7aed 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpMetadata.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpMetadata.java @@ -45,7 +45,7 @@ public final class IngestGeoIpMetadata implements Metadata.Custom { @SuppressWarnings("unchecked") private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( - "ingest_geoip_metadata", + TYPE, a -> new IngestGeoIpMetadata( ((List) a[0]).stream().collect(Collectors.toMap((m) -> m.database().id(), Function.identity())) ) diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java index 3107f0bed55e8..2e0f3a11b3052 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java @@ -214,6 +214,11 @@ public List getRestHandlers( @Override public List getNamedXContent() { return List.of( + new NamedXContentRegistry.Entry( + Metadata.Custom.class, + new ParseField(IngestGeoIpMetadata.TYPE), + IngestGeoIpMetadata::fromXContent + ), new NamedXContentRegistry.Entry(PersistentTaskParams.class, new ParseField(GEOIP_DOWNLOADER), GeoIpTaskParams::fromXContent), new NamedXContentRegistry.Entry(PersistentTaskState.class, new ParseField(GEOIP_DOWNLOADER), GeoIpTaskState::fromXContent), new NamedXContentRegistry.Entry( From b45ee0b3c7cc00564eb5ee4da2d83b99ee695975 Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Tue, 25 Feb 2025 10:44:19 +0100 Subject: [PATCH 07/12] Move eclipse specific extention property into elasticsearch eclipse plugin (#123320) (#123344) --- .../internal/conventions/EclipseConventionPlugin.java | 10 ++++++++++ build.gradle | 9 --------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/EclipseConventionPlugin.java b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/EclipseConventionPlugin.java index 58b183fac3155..48465cb08cc79 100644 --- a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/EclipseConventionPlugin.java +++ b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/EclipseConventionPlugin.java @@ -15,6 +15,7 @@ import org.gradle.api.Plugin; import org.gradle.api.Project; import org.gradle.api.Transformer; +import org.gradle.api.invocation.Gradle; import org.gradle.api.plugins.JavaBasePlugin; import org.gradle.api.plugins.JavaPluginExtension; import org.gradle.api.tasks.Copy; @@ -38,6 +39,15 @@ public class EclipseConventionPlugin implements Plugin { @Override public void apply(Project project) { project.getPlugins().apply(EclipsePlugin.class); + Gradle gradle = project.getGradle(); + + boolean isEclipse = project.getProviders().systemProperty("eclipse.launcher").isPresent() || // Gradle launched from Eclipse + project.getProviders().systemProperty("eclipse.application").isPresent() || // Gradle launched from the Eclipse compiler server + gradle.getStartParameter().getTaskNames().contains("eclipse") || // Gradle launched from the command line to do eclipse stuff + gradle.getStartParameter().getTaskNames().contains("cleanEclipse"); + // for eclipse ide specific hacks... + project.getExtensions().add("isEclipse", isEclipse); + EclipseModel eclipseModel = project.getExtensions().getByType(EclipseModel.class); EclipseProject eclipseProject = eclipseModel.getProject(); diff --git a/build.gradle b/build.gradle index c68dfea0e8b93..4e0697f01d293 100644 --- a/build.gradle +++ b/build.gradle @@ -250,15 +250,6 @@ allprojects { } } - // injecting groovy property variables into all projects - project.ext { - // for ide hacks... - isEclipse = providers.systemProperty("eclipse.launcher").isPresent() || // Detects gradle launched from Eclipse's IDE - providers.systemProperty("eclipse.application").isPresent() || // Detects gradle launched from the Eclipse compiler server - gradle.startParameter.taskNames.contains('eclipse') || // Detects gradle launched from the command line to do eclipse stuff - gradle.startParameter.taskNames.contains('cleanEclipse') - } - ext.bwc_tests_enabled = bwc_tests_enabled // eclipse configuration From 19402e2c684ea5114f93f58477a90d1a2f480544 Mon Sep 17 00:00:00 2001 From: David Turner Date: Tue, 25 Feb 2025 19:59:14 +0000 Subject: [PATCH 08/12] Reduce licence checks in `LicensedWriteLoadForecaster` (#123369) (#123408) Rather than checking the license (updating the usage map) on every single shard, just do it once at the start of a computation that needs to forecast write loads. Backport of #123346 to 8.x Closes #123247 --- docs/changelog/123346.yaml | 6 ++ .../TransportGetDesiredBalanceAction.java | 1 + .../rollover/MetadataRolloverService.java | 1 + .../allocation/AllocationStatsService.java | 2 + .../allocation/WriteLoadForecaster.java | 5 ++ .../allocator/BalancedShardsAllocator.java | 5 ++ .../cluster/ESAllocationTestCase.java | 3 + .../WriteLoadForecasterIT.java | 2 + .../LicensedWriteLoadForecaster.java | 49 +++++++++++-- .../LicensedWriteLoadForecasterTests.java | 73 ++++++++++++++++++- 10 files changed, 139 insertions(+), 8 deletions(-) create mode 100644 docs/changelog/123346.yaml diff --git a/docs/changelog/123346.yaml b/docs/changelog/123346.yaml new file mode 100644 index 0000000000000..42c6fbf6931ad --- /dev/null +++ b/docs/changelog/123346.yaml @@ -0,0 +1,6 @@ +pr: 123346 +summary: Reduce license checks in `LicensedWriteLoadForecaster` +area: CRUD +type: bug +issues: + - 123247 diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceAction.java index 3731ca7de28ee..b9bd7dc4d718e 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceAction.java @@ -95,6 +95,7 @@ protected void masterOperation( return; } var clusterInfo = clusterInfoService.getClusterInfo(); + writeLoadForecaster.refreshLicense(); listener.onResponse( new DesiredBalanceResponse( desiredBalanceShardsAllocator.getStats(), diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java index 0c22a17bb1f6b..63a5a792db3c9 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java @@ -430,6 +430,7 @@ yield new DataStreamAutoShardingEvent( ); } + writeLoadForecaster.refreshLicense(); metadataBuilder = writeLoadForecaster.withWriteLoadForecastForWriteIndex(dataStreamName, metadataBuilder); metadataBuilder = withShardSizeForecastForWriteIndex(dataStreamName, metadataBuilder); diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationStatsService.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationStatsService.java index fa4d60c83e5ef..f64a63332a371 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationStatsService.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationStatsService.java @@ -44,6 +44,8 @@ public AllocationStatsService( public Map stats() { assert Transports.assertNotTransportThread("too expensive for a transport worker"); + writeLoadForecaster.refreshLicense(); + var state = clusterService.state(); var info = clusterInfoService.getClusterInfo(); var desiredBalance = desiredBalanceShardsAllocator != null ? desiredBalanceShardsAllocator.getDesiredBalance() : null; diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/WriteLoadForecaster.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/WriteLoadForecaster.java index e7ca51eee815e..7bebedd9fdde4 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/WriteLoadForecaster.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/WriteLoadForecaster.java @@ -21,6 +21,8 @@ public interface WriteLoadForecaster { OptionalDouble getForecastedWriteLoad(IndexMetadata indexMetadata); + void refreshLicense(); + class DefaultWriteLoadForecaster implements WriteLoadForecaster { @Override public Metadata.Builder withWriteLoadForecastForWriteIndex(String dataStreamName, Metadata.Builder metadata) { @@ -31,5 +33,8 @@ public Metadata.Builder withWriteLoadForecastForWriteIndex(String dataStreamName public OptionalDouble getForecastedWriteLoad(IndexMetadata indexMetadata) { return OptionalDouble.empty(); } + + @Override + public void refreshLicense() {} } } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java index 27087992f9d2b..e58b1ecb73372 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java @@ -170,6 +170,11 @@ private static float ensureValidThreshold(float threshold) { @Override public void allocate(RoutingAllocation allocation) { + if (allocation.metadata().indices().isEmpty() == false) { + // must not use licensed features when just starting up + writeLoadForecaster.refreshLicense(); + } + assert allocation.ignoreDisable() == false; if (allocation.routingNodes().size() == 0) { diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/ESAllocationTestCase.java b/test/framework/src/main/java/org/elasticsearch/cluster/ESAllocationTestCase.java index c8d66f389dab1..f3db9cb50313c 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/ESAllocationTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/ESAllocationTestCase.java @@ -86,6 +86,9 @@ public Metadata.Builder withWriteLoadForecastForWriteIndex(String dataStreamName public OptionalDouble getForecastedWriteLoad(IndexMetadata indexMetadata) { return indexMetadata.getForecastedWriteLoad(); } + + @Override + public void refreshLicense() {} }; public static MockAllocationService createAllocationService() { diff --git a/x-pack/plugin/write-load-forecaster/src/internalClusterTest/java/org/elasticsearch/xpack/writeloadforecaster/WriteLoadForecasterIT.java b/x-pack/plugin/write-load-forecaster/src/internalClusterTest/java/org/elasticsearch/xpack/writeloadforecaster/WriteLoadForecasterIT.java index b37b026b853e2..5c174d1bddef2 100644 --- a/x-pack/plugin/write-load-forecaster/src/internalClusterTest/java/org/elasticsearch/xpack/writeloadforecaster/WriteLoadForecasterIT.java +++ b/x-pack/plugin/write-load-forecaster/src/internalClusterTest/java/org/elasticsearch/xpack/writeloadforecaster/WriteLoadForecasterIT.java @@ -84,6 +84,7 @@ public void testWriteLoadForecastGetsPopulatedDuringRollovers() throws Exception assertAllPreviousForecastsAreClearedAfterRollover(dataStream, metadata); setHasValidLicense(false); + writeLoadForecaster.refreshLicense(); final OptionalDouble forecastedWriteLoadAfterLicenseChange = writeLoadForecaster.getForecastedWriteLoad(writeIndexMetadata); assertThat(forecastedWriteLoadAfterLicenseChange.isPresent(), is(equalTo(false))); @@ -131,6 +132,7 @@ public void testWriteLoadForecastIsOverriddenBySetting() throws Exception { assertAllPreviousForecastsAreClearedAfterRollover(dataStream, metadata); setHasValidLicense(false); + writeLoadForecaster.refreshLicense(); final OptionalDouble forecastedWriteLoadAfterLicenseChange = writeLoadForecaster.getForecastedWriteLoad(writeIndexMetadata); assertThat(forecastedWriteLoadAfterLicenseChange.isPresent(), is(equalTo(false))); diff --git a/x-pack/plugin/write-load-forecaster/src/main/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecaster.java b/x-pack/plugin/write-load-forecaster/src/main/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecaster.java index d4a85ce859b2b..45c5abdc61fd6 100644 --- a/x-pack/plugin/write-load-forecaster/src/main/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecaster.java +++ b/x-pack/plugin/write-load-forecaster/src/main/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecaster.java @@ -19,8 +19,12 @@ import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.Index; +import org.elasticsearch.logging.LogManager; +import org.elasticsearch.logging.Logger; import org.elasticsearch.threadpool.ThreadPool; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.VarHandle; import java.util.List; import java.util.Objects; import java.util.OptionalDouble; @@ -30,6 +34,9 @@ import static org.elasticsearch.xpack.writeloadforecaster.WriteLoadForecasterPlugin.OVERRIDE_WRITE_LOAD_FORECAST_SETTING; class LicensedWriteLoadForecaster implements WriteLoadForecaster { + + private static final Logger logger = LogManager.getLogger(LicensedWriteLoadForecaster.class); + public static final Setting MAX_INDEX_AGE_SETTING = Setting.timeSetting( "write_load_forecaster.max_index_age", TimeValue.timeValueDays(7), @@ -37,23 +44,26 @@ class LicensedWriteLoadForecaster implements WriteLoadForecaster { Setting.Property.NodeScope, Setting.Property.Dynamic ); - private final BooleanSupplier hasValidLicense; + private final BooleanSupplier hasValidLicenseSupplier; private final ThreadPool threadPool; private volatile TimeValue maxIndexAge; + @SuppressWarnings("unused") // modified via VH_HAS_VALID_LICENSE_FIELD + private volatile boolean hasValidLicense; + LicensedWriteLoadForecaster( - BooleanSupplier hasValidLicense, + BooleanSupplier hasValidLicenseSupplier, ThreadPool threadPool, Settings settings, ClusterSettings clusterSettings ) { - this(hasValidLicense, threadPool, MAX_INDEX_AGE_SETTING.get(settings)); + this(hasValidLicenseSupplier, threadPool, MAX_INDEX_AGE_SETTING.get(settings)); clusterSettings.addSettingsUpdateConsumer(MAX_INDEX_AGE_SETTING, this::setMaxIndexAgeSetting); } // exposed for tests only - LicensedWriteLoadForecaster(BooleanSupplier hasValidLicense, ThreadPool threadPool, TimeValue maxIndexAge) { - this.hasValidLicense = hasValidLicense; + LicensedWriteLoadForecaster(BooleanSupplier hasValidLicenseSupplier, ThreadPool threadPool, TimeValue maxIndexAge) { + this.hasValidLicenseSupplier = hasValidLicenseSupplier; this.threadPool = threadPool; this.maxIndexAge = maxIndexAge; } @@ -64,7 +74,7 @@ private void setMaxIndexAgeSetting(TimeValue updatedMaxIndexAge) { @Override public Metadata.Builder withWriteLoadForecastForWriteIndex(String dataStreamName, Metadata.Builder metadata) { - if (hasValidLicense.getAsBoolean() == false) { + if (hasValidLicense == false) { return metadata; } @@ -143,7 +153,7 @@ static OptionalDouble forecastIndexWriteLoad(List indicesWriteLo @Override @SuppressForbidden(reason = "This is the only place where IndexMetadata#getForecastedWriteLoad is allowed to be used") public OptionalDouble getForecastedWriteLoad(IndexMetadata indexMetadata) { - if (hasValidLicense.getAsBoolean() == false) { + if (hasValidLicense == false) { return OptionalDouble.empty(); } @@ -154,4 +164,29 @@ public OptionalDouble getForecastedWriteLoad(IndexMetadata indexMetadata) { return indexMetadata.getForecastedWriteLoad(); } + + /** + * Used to atomically {@code getAndSet()} the {@link #hasValidLicense} field. This is better than an + * {@link java.util.concurrent.atomic.AtomicBoolean} because it takes one less pointer dereference on each read. + */ + private static final VarHandle VH_HAS_VALID_LICENSE_FIELD; + + static { + try { + VH_HAS_VALID_LICENSE_FIELD = MethodHandles.lookup() + .in(LicensedWriteLoadForecaster.class) + .findVarHandle(LicensedWriteLoadForecaster.class, "hasValidLicense", boolean.class); + } catch (NoSuchFieldException | IllegalAccessException e) { + throw new RuntimeException(e); + } + } + + @Override + public void refreshLicense() { + final var newValue = hasValidLicenseSupplier.getAsBoolean(); + final var oldValue = (boolean) VH_HAS_VALID_LICENSE_FIELD.getAndSet(this, newValue); + if (newValue != oldValue) { + logger.info("license state changed, now [{}]", newValue ? "valid" : "not valid"); + } + } } diff --git a/x-pack/plugin/write-load-forecaster/src/test/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecasterTests.java b/x-pack/plugin/write-load-forecaster/src/test/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecasterTests.java index 790af0a201578..162e84b2562c5 100644 --- a/x-pack/plugin/write-load-forecaster/src/test/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecasterTests.java +++ b/x-pack/plugin/write-load-forecaster/src/test/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecasterTests.java @@ -7,6 +7,8 @@ package org.elasticsearch.xpack.writeloadforecaster; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.core.LogEvent; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexMetadataStats; @@ -19,6 +21,7 @@ import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockLog; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; import org.junit.After; @@ -30,9 +33,12 @@ import java.util.OptionalDouble; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import static org.elasticsearch.xpack.writeloadforecaster.LicensedWriteLoadForecaster.forecastIndexWriteLoad; import static org.hamcrest.Matchers.closeTo; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.is; @@ -53,7 +59,13 @@ public void tearDownThreadPool() { public void testWriteLoadForecastIsAddedToWriteIndex() { final TimeValue maxIndexAge = TimeValue.timeValueDays(7); final AtomicBoolean hasValidLicense = new AtomicBoolean(true); - final WriteLoadForecaster writeLoadForecaster = new LicensedWriteLoadForecaster(hasValidLicense::get, threadPool, maxIndexAge); + final AtomicInteger licenseCheckCount = new AtomicInteger(); + final WriteLoadForecaster writeLoadForecaster = new LicensedWriteLoadForecaster(() -> { + licenseCheckCount.incrementAndGet(); + return hasValidLicense.get(); + }, threadPool, maxIndexAge); + + writeLoadForecaster.refreshLicense(); final Metadata.Builder metadataBuilder = Metadata.builder(); final String dataStreamName = "logs-es"; @@ -95,8 +107,12 @@ public void testWriteLoadForecastIsAddedToWriteIndex() { assertThat(forecastedWriteLoad.isPresent(), is(true)); assertThat(forecastedWriteLoad.getAsDouble(), is(greaterThan(0.0))); + assertThat(licenseCheckCount.get(), equalTo(1)); hasValidLicense.set(false); + writeLoadForecaster.refreshLicense(); + assertThat(licenseCheckCount.get(), equalTo(2)); + final OptionalDouble forecastedWriteLoadAfterLicenseChange = writeLoadForecaster.getForecastedWriteLoad(writeIndex); assertThat(forecastedWriteLoadAfterLicenseChange.isPresent(), is(false)); } @@ -136,6 +152,7 @@ public void testUptimeIsUsedToWeightWriteLoad() { metadataBuilder.put(dataStream); final WriteLoadForecaster writeLoadForecaster = new LicensedWriteLoadForecaster(() -> true, threadPool, maxIndexAge); + writeLoadForecaster.refreshLicense(); final Metadata.Builder updatedMetadataBuilder = writeLoadForecaster.withWriteLoadForecastForWriteIndex( dataStream.getName(), @@ -154,6 +171,7 @@ public void testForecastedWriteLoadIsOverriddenBySetting() { final TimeValue maxIndexAge = TimeValue.timeValueDays(7); final AtomicBoolean hasValidLicense = new AtomicBoolean(true); final WriteLoadForecaster writeLoadForecaster = new LicensedWriteLoadForecaster(hasValidLicense::get, threadPool, maxIndexAge); + writeLoadForecaster.refreshLicense(); final Metadata.Builder metadataBuilder = Metadata.builder(); final String dataStreamName = "logs-es"; @@ -197,6 +215,7 @@ public void testForecastedWriteLoadIsOverriddenBySetting() { assertThat(forecastedWriteLoad.getAsDouble(), is(equalTo(0.6))); hasValidLicense.set(false); + writeLoadForecaster.refreshLicense(); final OptionalDouble forecastedWriteLoadAfterLicenseChange = writeLoadForecaster.getForecastedWriteLoad(writeIndex); assertThat(forecastedWriteLoadAfterLicenseChange.isPresent(), is(false)); @@ -327,4 +346,56 @@ private DataStream createDataStream(String name, List backingIndices) { .setIndexMode(IndexMode.STANDARD) .build(); } + + public void testLicenseStateLogging() { + + final var seenMessages = new ArrayList(); + + final var collectingLoggingAssertion = new MockLog.SeenEventExpectation( + "seen event", + LicensedWriteLoadForecaster.class.getCanonicalName(), + Level.INFO, + "*" + ) { + @Override + public boolean innerMatch(LogEvent event) { + final var message = event.getMessage().getFormattedMessage(); + if (message.startsWith("license state changed, now [")) { + seenMessages.add(message); + return true; + } + + return false; + } + }; + + MockLog.assertThatLogger(() -> { + final var hasValidLicense = new AtomicBoolean(); + final var writeLoadForecaster = new LicensedWriteLoadForecaster(hasValidLicense::get, threadPool, randomTimeValue()); + assertThat(seenMessages, empty()); + writeLoadForecaster.refreshLicense(); + assertThat(seenMessages, empty()); + + hasValidLicense.set(true); + writeLoadForecaster.refreshLicense(); + assertThat(seenMessages, contains("license state changed, now [valid]")); + writeLoadForecaster.refreshLicense(); + assertThat(seenMessages, contains("license state changed, now [valid]")); + + hasValidLicense.set(false); + writeLoadForecaster.refreshLicense(); + assertThat(seenMessages, contains("license state changed, now [valid]", "license state changed, now [not valid]")); + + hasValidLicense.set(true); + ESTestCase.startInParallel(between(1, 10), ignored -> writeLoadForecaster.refreshLicense()); + assertThat( + seenMessages, + contains( + "license state changed, now [valid]", + "license state changed, now [not valid]", + "license state changed, now [valid]" + ) + ); + }, LicensedWriteLoadForecaster.class, collectingLoggingAssertion); + } } From c8c16d056614e68949b6c4c75caa305e629deccf Mon Sep 17 00:00:00 2001 From: Niels Bauman <33722607+nielsbauman@users.noreply.github.com> Date: Wed, 26 Feb 2025 12:49:14 +0100 Subject: [PATCH 09/12] [8.16] Fix failing test(s) in `TimeSeriesDataStreamsIT` (#123378) (#123475) * Fix failing test(s) in `TimeSeriesDataStreamsIT` (#123378) When these tests were run around midnight, the use of `DataStream#getDefaultBackingIndexName` could result in a potential mismatch in the generated index name and the one that the cluster actually created. Instead, we need to obtain the backing index and extract the desired index name from there. Fixes #123086 Relates #123376 (cherry picked from commit f0f0eeb07797aed0cc2b7ae4351a5adf8fc50756) # Conflicts: # x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesDataStreamsIT.java * Fix compilation --- .../xpack/ilm/TimeSeriesDataStreamsIT.java | 48 +++++++++---------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesDataStreamsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesDataStreamsIT.java index 28f97adec8814..88fcef854d2d5 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesDataStreamsIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesDataStreamsIT.java @@ -9,7 +9,6 @@ import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; -import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.xcontent.XContentHelper; @@ -80,20 +79,15 @@ public void testRolloverAction() throws Exception { indexDocument(client(), dataStream, true); - assertBusy(() -> assertTrue(indexExists(DataStream.getDefaultBackingIndexName(dataStream, 2)))); - assertBusy( - () -> assertTrue( - Boolean.parseBoolean( - (String) getIndexSettingsAsMap(DataStream.getDefaultBackingIndexName(dataStream, 2)).get("index.hidden") - ) - ) - ); - assertBusy( - () -> assertThat( - getStepKeyForIndex(client(), DataStream.getDefaultBackingIndexName(dataStream, 1)), - equalTo(PhaseCompleteStep.finalStep("hot").getKey()) - ) - ); + assertBusy(() -> { + final var backingIndices = getBackingIndices(client(), dataStream); + assertEquals(2, backingIndices.size()); + assertTrue(Boolean.parseBoolean((String) getIndexSettingsAsMap(backingIndices.get(1)).get("index.hidden"))); + }); + assertBusy(() -> { + final var backingIndices = getBackingIndices(client(), dataStream); + assertEquals(PhaseCompleteStep.finalStep("hot").getKey(), getStepKeyForIndex(client(), backingIndices.get(0))); + }); } public void testRolloverIsSkippedOnManualDataStreamRollover() throws Exception { @@ -103,7 +97,7 @@ public void testRolloverIsSkippedOnManualDataStreamRollover() throws Exception { indexDocument(client(), dataStream, true); - String firstGenerationIndex = DataStream.getDefaultBackingIndexName(dataStream, 1); + String firstGenerationIndex = getBackingIndices(client(), dataStream).get(0); assertBusy( () -> assertThat(getStepKeyForIndex(client(), firstGenerationIndex).name(), equalTo(WaitForRolloverReadyStep.NAME)), 30, @@ -111,7 +105,10 @@ public void testRolloverIsSkippedOnManualDataStreamRollover() throws Exception { ); rolloverMaxOneDocCondition(client(), dataStream); - assertBusy(() -> assertThat(indexExists(DataStream.getDefaultBackingIndexName(dataStream, 2)), is(true)), 30, TimeUnit.SECONDS); + assertBusy(() -> { + final var backingIndices = getBackingIndices(client(), dataStream); + assertEquals(2, backingIndices.size()); + }, 30, TimeUnit.SECONDS); // even though the first index doesn't have 2 documents to fulfill the rollover condition, it should complete the rollover action // because it's not the write index anymore @@ -128,7 +125,7 @@ public void testShrinkActionInPolicyWithoutHotPhase() throws Exception { createComposableTemplate(client(), template, dataStream + "*", getTemplate(policyName)); indexDocument(client(), dataStream, true); - String backingIndexName = DataStream.getDefaultBackingIndexName(dataStream, 1); + String backingIndexName = getBackingIndices(client(), dataStream).get(0); assertBusy( () -> assertThat( "original index must wait in the " + CheckNotDataStreamWriteIndexStep.NAME + " until it is not the write index anymore", @@ -142,8 +139,11 @@ public void testShrinkActionInPolicyWithoutHotPhase() throws Exception { // Manual rollover the original index such that it's not the write index in the data stream anymore rolloverMaxOneDocCondition(client(), dataStream); // Wait for rollover to happen - String rolloverIndex = DataStream.getDefaultBackingIndexName(dataStream, 2); - assertBusy(() -> assertTrue("the rollover action created the rollover index", indexExists(rolloverIndex)), 30, TimeUnit.SECONDS); + assertBusy( + () -> assertEquals("the rollover action created the rollover index", 2, getBackingIndices(client(), dataStream).size()), + 30, + TimeUnit.SECONDS + ); String shrunkenIndex = waitAndGetShrinkIndexName(client(), backingIndexName); assertBusy(() -> assertTrue(indexExists(shrunkenIndex)), 30, TimeUnit.SECONDS); @@ -159,7 +159,7 @@ public void testSearchableSnapshotAction() throws Exception { createComposableTemplate(client(), template, dataStream + "*", getTemplate(policyName)); indexDocument(client(), dataStream, true); - String backingIndexName = DataStream.getDefaultBackingIndexName(dataStream, 1); + String backingIndexName = getBackingIndices(client(), dataStream).get(0); String restoredIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + backingIndexName; assertBusy( @@ -190,7 +190,7 @@ public void testReadOnlyAction() throws Exception { createComposableTemplate(client(), template, dataStream + "*", getTemplate(policyName)); indexDocument(client(), dataStream, true); - String backingIndexName = DataStream.getDefaultBackingIndexName(dataStream, 1); + String backingIndexName = getBackingIndices(client(), dataStream).get(0); assertBusy( () -> assertThat( "index must wait in the " + CheckNotDataStreamWriteIndexStep.NAME + " until it is not the write index anymore", @@ -220,7 +220,7 @@ public void testFreezeAction() throws Exception { createComposableTemplate(client(), template, dataStream + "*", getTemplate(policyName)); indexDocument(client(), dataStream, true); - String backingIndexName = DataStream.getDefaultBackingIndexName(dataStream, 1); + String backingIndexName = getBackingIndices(client(), dataStream).get(0); assertBusy( () -> assertThat( "index must wait in the " + CheckNotDataStreamWriteIndexStep.NAME + " until it is not the write index anymore", @@ -249,7 +249,7 @@ public void checkForceMergeAction(String codec) throws Exception { createComposableTemplate(client(), template, dataStream + "*", getTemplate(policyName)); indexDocument(client(), dataStream, true); - String backingIndexName = DataStream.getDefaultBackingIndexName(dataStream, 1); + String backingIndexName = getBackingIndices(client(), dataStream).get(0); assertBusy( () -> assertThat( "index must wait in the " + CheckNotDataStreamWriteIndexStep.NAME + " until it is not the write index anymore", From ae2d91893869f70ee9f7bddbb60283045aa7cb36 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 26 Feb 2025 13:01:58 +0100 Subject: [PATCH 10/12] [8.16] Upgrade Netty to `4.1.118.Final` (#122371) (#123481) * Upgrade Netty to `4.1.118.Final` (#122371) This PR upgrades Netty to `4.1.118.Final`. * Bring back old netty versions for hdfs test fixture --- build-tools-internal/version.properties | 2 +- gradle/verification-metadata.xml | 84 +++++++++---------- .../plugin-metadata/plugin-security.policy | 2 + .../plugin-metadata/plugin-security.policy | 3 +- .../bootstrap/test-framework.policy | 3 +- .../plugin-metadata/plugin-security.policy | 3 +- 6 files changed, 51 insertions(+), 46 deletions(-) diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties index d3ca046adefd8..5b8d80f0cde77 100644 --- a/build-tools-internal/version.properties +++ b/build-tools-internal/version.properties @@ -14,7 +14,7 @@ log4j = 2.19.0 slf4j = 2.0.6 ecsLogging = 1.2.0 jna = 5.12.1 -netty = 4.1.115.Final +netty = 4.1.118.Final commons_lang3 = 3.9 google_oauth_client = 1.34.1 diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index f21cbb812d608..7292be3d2c49e 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -1389,9 +1389,9 @@ - - - + + + @@ -1399,9 +1399,9 @@ - - - + + + @@ -1409,29 +1409,29 @@ - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + @@ -1439,9 +1439,9 @@ - - - + + + @@ -1449,14 +1449,14 @@ - - - + + + - - - + + + @@ -1464,14 +1464,14 @@ - - - + + + - - - + + + @@ -1479,9 +1479,9 @@ - - - + + + @@ -1489,9 +1489,9 @@ - - - + + + diff --git a/modules/repository-azure/src/main/plugin-metadata/plugin-security.policy b/modules/repository-azure/src/main/plugin-metadata/plugin-security.policy index 8a7c623597376..3aeeb6bde3914 100644 --- a/modules/repository-azure/src/main/plugin-metadata/plugin-security.policy +++ b/modules/repository-azure/src/main/plugin-metadata/plugin-security.policy @@ -12,6 +12,8 @@ grant { permission java.net.SocketPermission "*", "connect"; // io.netty.util.concurrent.GlobalEventExecutor.startThread permission java.lang.RuntimePermission "setContextClassLoader"; + // io.netty.util.concurrent.GlobalEventExecutor.startThread + permission java.lang.RuntimePermission "getClassLoader"; // Used by jackson bean deserialization permission java.lang.RuntimePermission "accessDeclaredMembers"; permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; diff --git a/modules/transport-netty4/src/main/plugin-metadata/plugin-security.policy b/modules/transport-netty4/src/main/plugin-metadata/plugin-security.policy index ed278af96d926..dbf8e728c1606 100644 --- a/modules/transport-netty4/src/main/plugin-metadata/plugin-security.policy +++ b/modules/transport-netty4/src/main/plugin-metadata/plugin-security.policy @@ -14,8 +14,9 @@ grant codeBase "${codebase.netty-common}" { // netty makes and accepts socket connections permission java.net.SocketPermission "*", "accept,connect"; - // Netty sets custom classloader for some of its internal threads + // Netty gets and sets classloaders for some of its internal threads permission java.lang.RuntimePermission "setContextClassLoader"; + permission java.lang.RuntimePermission "getClassLoader"; }; grant codeBase "${codebase.netty-transport}" { diff --git a/server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy b/server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy index c122d26fb7f2f..e5d4e05c4b641 100644 --- a/server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy +++ b/server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy @@ -119,8 +119,9 @@ grant codeBase "${codebase.httpasyncclient}" { grant codeBase "${codebase.netty-common}" { // for reading the system-wide configuration for the backlog of established sockets permission java.io.FilePermission "/proc/sys/net/core/somaxconn", "read"; - // Netty sets custom classloader for some of its internal threads + // Netty gets and sets classloaders for some of its internal threads permission java.lang.RuntimePermission "setContextClassLoader"; + permission java.lang.RuntimePermission "getClassLoader"; permission java.net.SocketPermission "*", "accept,connect"; }; diff --git a/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy index d814dfbb1c117..b4791207a15bf 100644 --- a/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy +++ b/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy @@ -46,8 +46,9 @@ grant { grant codeBase "${codebase.netty-common}" { // for reading the system-wide configuration for the backlog of established sockets permission java.io.FilePermission "/proc/sys/net/core/somaxconn", "read"; - // Netty sets custom classloader for some of its internal threads + // Netty gets and sets classloaders for some of its internal threads permission java.lang.RuntimePermission "setContextClassLoader"; + permission java.lang.RuntimePermission "getClassLoader"; }; grant codeBase "${codebase.netty-transport}" { From cf7ccb19c4ea24852a26fd1ee9dabfaa2e2f2341 Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Wed, 26 Feb 2025 13:12:01 +0100 Subject: [PATCH 11/12] [Gradle] Fix build finished hooks on ci when using configuration cache (#116888) (#122740) Fixes two incompatibilities with Gradle configuration cache in our build scan build finished hook: referencing static methods from build script referencing gradle object from closure (cherry picked from commit 929d39820a657b423911ccfe9f974cffac5a9e34) --- .../groovy/elasticsearch.build-scan.gradle | 10 +++++----- .../gradle/internal/util/CiUtils.java | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+), 5 deletions(-) create mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/CiUtils.java diff --git a/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle b/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle index 9d1c1aa56e2db..8702f5a9bf0e9 100644 --- a/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle +++ b/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle @@ -12,10 +12,14 @@ import java.time.LocalDateTime; import org.elasticsearch.gradle.Architecture import org.elasticsearch.gradle.OS +import static org.elasticsearch.gradle.internal.util.CiUtils.safeName import java.lang.management.ManagementFactory import java.time.LocalDateTime +// Resolving this early to avoid issues with the build scan plugin in combination with the configuration cache usage +def taskNames = gradle.startParameter.taskNames.join(' ') + develocity { buildScan { @@ -112,7 +116,7 @@ develocity { // Add a build annotation // See: https://buildkite.com/docs/agent/v3/cli-annotate - def body = """
${System.getenv('BUILDKITE_LABEL')} :gradle: ${result.failures ? 'failed' : 'successful'} build: gradle ${gradle.startParameter.taskNames.join(' ')}
""" + def body = """
${System.getenv('BUILDKITE_LABEL')} :gradle: ${result.failures ? 'failed' : 'successful'} build: gradle ${taskNames}
""" def process = [ 'buildkite-agent', 'annotate', @@ -133,7 +137,3 @@ develocity { } } } - -static def safeName(String string) { - return string.replaceAll(/[^a-zA-Z0-9_\-\.]+/, ' ').trim().replaceAll(' ', '_').toLowerCase() -} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/CiUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/CiUtils.java new file mode 100644 index 0000000000000..1b019a6cbd3e6 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/CiUtils.java @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.util; + +public class CiUtils { + + static String safeName(String input) { + return input.replaceAll("[^a-zA-Z0-9_\\-\\.]+", " ").trim().replaceAll(" ", "_").toLowerCase(); + } + +} From e40319c7a04c64c5588cfa74db01654e9073916f Mon Sep 17 00:00:00 2001 From: Ioana Tagirta Date: Wed, 26 Feb 2025 17:26:57 +0100 Subject: [PATCH 12/12] Remove references to doc types in percolator docs (#123508) (#123529) --- docs/reference/query-dsl/percolate-query.asciidoc | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/reference/query-dsl/percolate-query.asciidoc b/docs/reference/query-dsl/percolate-query.asciidoc index 25b995eefc219..493d90cea93bd 100644 --- a/docs/reference/query-dsl/percolate-query.asciidoc +++ b/docs/reference/query-dsl/percolate-query.asciidoc @@ -133,7 +133,6 @@ The following parameters are required when percolating a document: This is an optional parameter. `document`:: The source of the document being percolated. `documents`:: Like the `document` parameter, but accepts multiple documents via a json array. -`document_type`:: The type / mapping of the document being percolated. This parameter is deprecated and will be removed in Elasticsearch 8.0. Instead of specifying the source of the document being percolated, the source can also be retrieved from an already stored document. The `percolate` query will then internally execute a get request to fetch that document. @@ -142,7 +141,6 @@ In that case the `document` parameter can be substituted with the following para [horizontal] `index`:: The index the document resides in. This is a required parameter. -`type`:: The type of the document to fetch. This parameter is deprecated and will be removed in Elasticsearch 8.0. `id`:: The id of the document to fetch. This is a required parameter. `routing`:: Optionally, routing to be used to fetch document to percolate. `preference`:: Optionally, preference to be used to fetch document to percolate.