From c225382f18c49e14fe8eacaaa4790d3a72f80018 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Fri, 7 Feb 2025 08:20:59 +1100 Subject: [PATCH 1/5] Mute org.elasticsearch.analysis.common.CommonAnalysisClientYamlTestSuiteIT test {yaml=analysis-common/40_token_filters/stemmer_override file access} #121625 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 9f95c0af69abc..ba67eb8c3577c 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -386,6 +386,9 @@ tests: - class: org.elasticsearch.xpack.searchablesnapshots.FrozenSearchableSnapshotsIntegTests method: testCreateAndRestorePartialSearchableSnapshot issue: https://github.com/elastic/elasticsearch/issues/121927 +- class: org.elasticsearch.analysis.common.CommonAnalysisClientYamlTestSuiteIT + method: test {yaml=analysis-common/40_token_filters/stemmer_override file access} + issue: https://github.com/elastic/elasticsearch/issues/121625 # Examples: # From c38ea536a0c146119c7ccd4054577e0cdabe5bae Mon Sep 17 00:00:00 2001 From: Pat Whelan Date: Thu, 6 Feb 2025 16:27:43 -0500 Subject: [PATCH 2/5] [ML] Parse mid-stream errors from OpenAI and EIS (#121806) When we are already parsing events, we can receive errors as the next event. OpenAI formats these as: ``` event: error data: ``` Elastic formats these as: ``` data: ``` Unified will consolidate them into the new error structure. --- ...eUnifiedChatCompletionResponseHandler.java | 33 ++++++++++- ...iUnifiedChatCompletionResponseHandler.java | 45 ++++++++++++++- .../OpenAiUnifiedStreamingProcessor.java | 28 +++++++++- ...icInferenceServiceErrorResponseEntity.java | 42 ++++++++------ .../elastic/ElasticInferenceServiceTests.java | 56 ++++++++++++++----- .../services/openai/OpenAiServiceTests.java | 51 ++++++++++++++--- 6 files changed, 213 insertions(+), 42 deletions(-) diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/elastic/ElasticInferenceServiceUnifiedChatCompletionResponseHandler.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/elastic/ElasticInferenceServiceUnifiedChatCompletionResponseHandler.java index db09317b7b790..a240035468b8a 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/elastic/ElasticInferenceServiceUnifiedChatCompletionResponseHandler.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/elastic/ElasticInferenceServiceUnifiedChatCompletionResponseHandler.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.inference.external.elastic; import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xpack.core.inference.results.StreamingUnifiedChatCompletionResults; import org.elasticsearch.xpack.core.inference.results.UnifiedChatCompletionException; import org.elasticsearch.xpack.inference.external.http.HttpResult; @@ -15,12 +16,15 @@ import org.elasticsearch.xpack.inference.external.http.retry.ResponseParser; import org.elasticsearch.xpack.inference.external.openai.OpenAiUnifiedStreamingProcessor; import org.elasticsearch.xpack.inference.external.request.Request; +import org.elasticsearch.xpack.inference.external.response.elastic.ElasticInferenceServiceErrorResponseEntity; import org.elasticsearch.xpack.inference.external.response.streaming.ServerSentEventParser; import org.elasticsearch.xpack.inference.external.response.streaming.ServerSentEventProcessor; import java.util.Locale; import java.util.concurrent.Flow; +import static org.elasticsearch.core.Strings.format; + public class ElasticInferenceServiceUnifiedChatCompletionResponseHandler extends ElasticInferenceServiceResponseHandler { public ElasticInferenceServiceUnifiedChatCompletionResponseHandler(String requestType, ResponseParser parseFunction) { super(requestType, parseFunction, true); @@ -29,7 +33,8 @@ public ElasticInferenceServiceUnifiedChatCompletionResponseHandler(String reques @Override public InferenceServiceResults parseResult(Request request, Flow.Publisher flow) { var serverSentEventProcessor = new ServerSentEventProcessor(new ServerSentEventParser()); - var openAiProcessor = new OpenAiUnifiedStreamingProcessor(); // EIS uses the unified API spec + // EIS uses the unified API spec + var openAiProcessor = new OpenAiUnifiedStreamingProcessor((m, e) -> buildMidStreamError(request, m, e)); flow.subscribe(serverSentEventProcessor); serverSentEventProcessor.subscribe(openAiProcessor); @@ -52,4 +57,30 @@ protected Exception buildError(String message, Request request, HttpResult resul return super.buildError(message, request, result, errorResponse); } } + + private static Exception buildMidStreamError(Request request, String message, Exception e) { + var errorResponse = ElasticInferenceServiceErrorResponseEntity.fromString(message); + if (errorResponse.errorStructureFound()) { + return new UnifiedChatCompletionException( + RestStatus.INTERNAL_SERVER_ERROR, + format( + "%s for request from inference entity id [%s]. Error message: [%s]", + SERVER_ERROR_OBJECT, + request.getInferenceEntityId(), + errorResponse.getErrorMessage() + ), + "error", + "stream_error" + ); + } else if (e != null) { + return UnifiedChatCompletionException.fromThrowable(e); + } else { + return new UnifiedChatCompletionException( + RestStatus.INTERNAL_SERVER_ERROR, + format("%s for request from inference entity id [%s]", SERVER_ERROR_OBJECT, request.getInferenceEntityId()), + "error", + "stream_error" + ); + } + } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiUnifiedChatCompletionResponseHandler.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiUnifiedChatCompletionResponseHandler.java index 2901b449f8a6e..b2096253bdeb7 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiUnifiedChatCompletionResponseHandler.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiUnifiedChatCompletionResponseHandler.java @@ -9,6 +9,7 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.XContentFactory; @@ -29,6 +30,8 @@ import java.util.Optional; import java.util.concurrent.Flow; +import static org.elasticsearch.core.Strings.format; + public class OpenAiUnifiedChatCompletionResponseHandler extends OpenAiChatCompletionResponseHandler { public OpenAiUnifiedChatCompletionResponseHandler(String requestType, ResponseParser parseFunction) { super(requestType, parseFunction, OpenAiErrorResponse::fromResponse); @@ -37,7 +40,7 @@ public OpenAiUnifiedChatCompletionResponseHandler(String requestType, ResponsePa @Override public InferenceServiceResults parseResult(Request request, Flow.Publisher flow) { var serverSentEventProcessor = new ServerSentEventProcessor(new ServerSentEventParser()); - var openAiProcessor = new OpenAiUnifiedStreamingProcessor(); + var openAiProcessor = new OpenAiUnifiedStreamingProcessor((m, e) -> buildMidStreamError(request, m, e)); flow.subscribe(serverSentEventProcessor); serverSentEventProcessor.subscribe(openAiProcessor); @@ -64,6 +67,33 @@ protected Exception buildError(String message, Request request, HttpResult resul } } + private static Exception buildMidStreamError(Request request, String message, Exception e) { + var errorResponse = OpenAiErrorResponse.fromString(message); + if (errorResponse instanceof OpenAiErrorResponse oer) { + return new UnifiedChatCompletionException( + RestStatus.INTERNAL_SERVER_ERROR, + format( + "%s for request from inference entity id [%s]. Error message: [%s]", + SERVER_ERROR_OBJECT, + request.getInferenceEntityId(), + errorResponse.getErrorMessage() + ), + oer.type(), + oer.code(), + oer.param() + ); + } else if (e != null) { + return UnifiedChatCompletionException.fromThrowable(e); + } else { + return new UnifiedChatCompletionException( + RestStatus.INTERNAL_SERVER_ERROR, + format("%s for request from inference entity id [%s]", SERVER_ERROR_OBJECT, request.getInferenceEntityId()), + errorResponse != null ? errorResponse.getClass().getSimpleName() : "unknown", + "stream_error" + ); + } + } + private static class OpenAiErrorResponse extends ErrorResponse { private static final ConstructingObjectParser, Void> ERROR_PARSER = new ConstructingObjectParser<>( "open_ai_error", @@ -103,6 +133,19 @@ private static ErrorResponse fromResponse(HttpResult response) { return ErrorResponse.UNDEFINED_ERROR; } + private static ErrorResponse fromString(String response) { + try ( + XContentParser parser = XContentFactory.xContent(XContentType.JSON) + .createParser(XContentParserConfiguration.EMPTY, response) + ) { + return ERROR_PARSER.apply(parser, null).orElse(ErrorResponse.UNDEFINED_ERROR); + } catch (Exception e) { + // swallow the error + } + + return ErrorResponse.UNDEFINED_ERROR; + } + @Nullable private final String code; @Nullable diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiUnifiedStreamingProcessor.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiUnifiedStreamingProcessor.java index 599d71df3dcfa..bfd4456279a8a 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiUnifiedStreamingProcessor.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiUnifiedStreamingProcessor.java @@ -20,6 +20,7 @@ import org.elasticsearch.xpack.core.inference.results.StreamingUnifiedChatCompletionResults; import org.elasticsearch.xpack.inference.common.DelegatingProcessor; import org.elasticsearch.xpack.inference.external.response.streaming.ServerSentEvent; +import org.elasticsearch.xpack.inference.external.response.streaming.ServerSentEventField; import java.io.IOException; import java.util.ArrayDeque; @@ -28,6 +29,7 @@ import java.util.Iterator; import java.util.List; import java.util.concurrent.LinkedBlockingDeque; +import java.util.function.BiFunction; import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; import static org.elasticsearch.xpack.inference.external.response.XContentUtils.moveToFirstToken; @@ -57,7 +59,13 @@ public class OpenAiUnifiedStreamingProcessor extends DelegatingProcessor errorParser; private final Deque buffer = new LinkedBlockingDeque<>(); + private volatile boolean previousEventWasError = false; + + public OpenAiUnifiedStreamingProcessor(BiFunction errorParser) { + this.errorParser = errorParser; + } @Override protected void upstreamRequest(long n) { @@ -71,7 +79,25 @@ protected void upstreamRequest(long n) { @Override protected void next(Deque item) throws Exception { var parserConfig = XContentParserConfiguration.EMPTY.withDeprecationHandler(LoggingDeprecationHandler.INSTANCE); - var results = parseEvent(item, OpenAiUnifiedStreamingProcessor::parse, parserConfig, logger); + + var results = new ArrayDeque(item.size()); + for (var event : item) { + if (ServerSentEventField.EVENT == event.name() && "error".equals(event.value())) { + previousEventWasError = true; + } else if (ServerSentEventField.DATA == event.name() && event.hasValue()) { + if (previousEventWasError) { + throw errorParser.apply(event.value(), null); + } + + try { + var delta = parse(parserConfig, event); + delta.forEachRemaining(results::offer); + } catch (Exception e) { + logger.warn("Failed to parse event from inference provider: {}", event); + throw errorParser.apply(event.value(), e); + } + } + } if (results.isEmpty()) { upstream().request(1); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/elastic/ElasticInferenceServiceErrorResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/elastic/ElasticInferenceServiceErrorResponseEntity.java index 696be7b2acdd2..29b0903901694 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/elastic/ElasticInferenceServiceErrorResponseEntity.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/elastic/ElasticInferenceServiceErrorResponseEntity.java @@ -9,6 +9,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.CheckedSupplier; import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentParserConfiguration; @@ -16,6 +17,18 @@ import org.elasticsearch.xpack.inference.external.http.HttpResult; import org.elasticsearch.xpack.inference.external.http.retry.ErrorResponse; +import java.io.IOException; + +/** + * An example error response would look like + * + * + * { + * "error": "some error" + * } + * + * + */ public class ElasticInferenceServiceErrorResponseEntity extends ErrorResponse { private static final Logger logger = LogManager.getLogger(ElasticInferenceServiceErrorResponseEntity.class); @@ -24,24 +37,18 @@ private ElasticInferenceServiceErrorResponseEntity(String errorMessage) { super(errorMessage); } - /** - * An example error response would look like - * - * - * { - * "error": "some error" - * } - * - * - * @param response The error response - * @return An error entity if the response is JSON with the above structure - * or {@link ErrorResponse#UNDEFINED_ERROR} if the error field wasn't found - */ public static ErrorResponse fromResponse(HttpResult response) { - try ( - XContentParser jsonParser = XContentFactory.xContent(XContentType.JSON) - .createParser(XContentParserConfiguration.EMPTY, response.body()) - ) { + return fromParser( + () -> XContentFactory.xContent(XContentType.JSON).createParser(XContentParserConfiguration.EMPTY, response.body()) + ); + } + + public static ErrorResponse fromString(String response) { + return fromParser(() -> XContentFactory.xContent(XContentType.JSON).createParser(XContentParserConfiguration.EMPTY, response)); + } + + private static ErrorResponse fromParser(CheckedSupplier jsonParserFactory) { + try (XContentParser jsonParser = jsonParserFactory.get()) { var responseMap = jsonParser.map(); var error = (String) responseMap.get("error"); if (error != null) { @@ -50,7 +57,6 @@ public static ErrorResponse fromResponse(HttpResult response) { } catch (Exception e) { logger.debug("Failed to parse error response", e); } - return ErrorResponse.UNDEFINED_ERROR; } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceTests.java index fd02ef4ed2326..8c70520617dc6 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceTests.java @@ -970,14 +970,51 @@ public void testDefaultConfigs_Returns_DefaultChatCompletion_V1_WhenTaskTypeIsCo } public void testUnifiedCompletionError() throws Exception { + testUnifiedStreamError(404, """ + { + "error": "The model `rainbow-sprinkles` does not exist or you do not have access to it." + }""", """ + {\ + "error":{\ + "code":"not_found",\ + "message":"Received an unsuccessful status code for request from inference entity id [id] status \ + [404]. Error message: [The model `rainbow-sprinkles` does not exist or you do not have access to it.]",\ + "type":"error"\ + }}"""); + } + + public void testUnifiedCompletionErrorMidStream() throws Exception { + testUnifiedStreamError(200, """ + data: { "error": "some error" } + + """, """ + {\ + "error":{\ + "code":"stream_error",\ + "message":"Received an error response for request from inference entity id [id]. Error message: [some error]",\ + "type":"error"\ + }}"""); + } + + public void testUnifiedCompletionMalformedError() throws Exception { + testUnifiedStreamError(200, """ + data: { i am not json } + + """, """ + {\ + "error":{\ + "code":"bad_request",\ + "message":"[1:3] Unexpected character ('i' (code 105)): was expecting double-quote to start field name\\n\ + at [Source: (String)\\"{ i am not json }\\"; line: 1, column: 3]",\ + "type":"x_content_parse_exception"\ + }}"""); + } + + private void testUnifiedStreamError(int responseCode, String responseJson, String expectedJson) throws Exception { var eisGatewayUrl = getUrl(webServer); var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); try (var service = createService(senderFactory, eisGatewayUrl)) { - var responseJson = """ - { - "error": "The model `rainbow-sprinkles` does not exist or you do not have access to it." - }"""; - webServer.enqueue(new MockResponse().setResponseCode(404).setBody(responseJson)); + webServer.enqueue(new MockResponse().setResponseCode(responseCode).setBody(responseJson)); var model = new ElasticInferenceServiceCompletionModel( "id", TaskType.COMPLETION, @@ -1012,14 +1049,7 @@ public void testUnifiedCompletionError() throws Exception { }); var json = XContentHelper.convertToJson(BytesReference.bytes(builder), false, builder.contentType()); - assertThat(json, is(""" - {\ - "error":{\ - "code":"not_found",\ - "message":"Received an unsuccessful status code for request from inference entity id [id] status \ - [404]. Error message: [The model `rainbow-sprinkles` does not exist or you do not have access to it.]",\ - "type":"error"\ - }}""")); + assertThat(json, is(expectedJson)); } }); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java index 34539042c1f0b..1ff3e7e62823f 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java @@ -1078,7 +1078,17 @@ public void testUnifiedCompletionError() throws Exception { } }"""; webServer.enqueue(new MockResponse().setResponseCode(404).setBody(responseJson)); + testStreamError(""" + {\ + "error":{\ + "code":"model_not_found",\ + "message":"Received an unsuccessful status code for request from inference entity id [id] status \ + [404]. Error message: [The model `gpt-4awero` does not exist or you do not have access to it.]",\ + "type":"invalid_request_error"\ + }}"""); + } + private void testStreamError(String expectedResponse) throws Exception { var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); try (var service = new OpenAiService(senderFactory, createWithEmptySettings(threadPool))) { var model = OpenAiChatCompletionModelTests.createChatCompletionModel(getUrl(webServer), "org", "secret", "model", "user"); @@ -1107,19 +1117,44 @@ public void testUnifiedCompletionError() throws Exception { }); var json = XContentHelper.convertToJson(BytesReference.bytes(builder), false, builder.contentType()); - assertThat(json, is(""" - {\ - "error":{\ - "code":"model_not_found",\ - "message":"Received an unsuccessful status code for request from inference entity id [id] status \ - [404]. Error message: [The model `gpt-4awero` does not exist or you do not have access to it.]",\ - "type":"invalid_request_error"\ - }}""")); + assertThat(json, is(expectedResponse)); } }); } } + public void testMidStreamUnifiedCompletionError() throws Exception { + String responseJson = """ + event: error + data: { "error": { "message": "Timed out waiting for more data", "type": "timeout" } } + + """; + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); + testStreamError(""" + {\ + "error":{\ + "message":"Received an error response for request from inference entity id [id]. Error message: \ + [Timed out waiting for more data]",\ + "type":"timeout"\ + }}"""); + } + + public void testUnifiedCompletionMalformedError() throws Exception { + String responseJson = """ + data: { invalid json } + + """; + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); + testStreamError(""" + {\ + "error":{\ + "code":"bad_request",\ + "message":"[1:3] Unexpected character ('i' (code 105)): was expecting double-quote to start field name\\n\ + at [Source: (String)\\"{ invalid json }\\"; line: 1, column: 3]",\ + "type":"x_content_parse_exception"\ + }}"""); + } + public void testInfer_StreamRequest() throws Exception { String responseJson = """ data: {\ From a6a0e2d29dc0375b7375c7ccf4a9f6a6ff9a1b7d Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Fri, 7 Feb 2025 08:39:40 +1100 Subject: [PATCH 3/5] Mute org.elasticsearch.test.rest.ClientYamlTestSuiteIT test {yaml=update/100_synthetic_source/stored text} #121964 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index ba67eb8c3577c..55dc8f8104f7d 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -389,6 +389,9 @@ tests: - class: org.elasticsearch.analysis.common.CommonAnalysisClientYamlTestSuiteIT method: test {yaml=analysis-common/40_token_filters/stemmer_override file access} issue: https://github.com/elastic/elasticsearch/issues/121625 +- class: org.elasticsearch.test.rest.ClientYamlTestSuiteIT + method: test {yaml=update/100_synthetic_source/stored text} + issue: https://github.com/elastic/elasticsearch/issues/121964 # Examples: # From bcca97d47d9317d4b0a7455c652961080d062865 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Fri, 7 Feb 2025 08:39:48 +1100 Subject: [PATCH 4/5] Mute org.elasticsearch.test.rest.ClientYamlTestSuiteIT test {yaml=update/100_synthetic_source/keyword} #121965 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 55dc8f8104f7d..af561d3200e98 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -392,6 +392,9 @@ tests: - class: org.elasticsearch.test.rest.ClientYamlTestSuiteIT method: test {yaml=update/100_synthetic_source/stored text} issue: https://github.com/elastic/elasticsearch/issues/121964 +- class: org.elasticsearch.test.rest.ClientYamlTestSuiteIT + method: test {yaml=update/100_synthetic_source/keyword} + issue: https://github.com/elastic/elasticsearch/issues/121965 # Examples: # From e24489f1420370a7638dd761527f9af0ea45dc6e Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Fri, 7 Feb 2025 08:40:19 +1100 Subject: [PATCH 5/5] Mute org.elasticsearch.xpack.esql.plugin.DataNodeRequestSenderTests testDoNotRetryOnRequestLevelFailure #121966 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index af561d3200e98..6bc7c5008b0c4 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -395,6 +395,9 @@ tests: - class: org.elasticsearch.test.rest.ClientYamlTestSuiteIT method: test {yaml=update/100_synthetic_source/keyword} issue: https://github.com/elastic/elasticsearch/issues/121965 +- class: org.elasticsearch.xpack.esql.plugin.DataNodeRequestSenderTests + method: testDoNotRetryOnRequestLevelFailure + issue: https://github.com/elastic/elasticsearch/issues/121966 # Examples: #