From 0f93fb307fa67ce0912177520ff6af7ff91befe8 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Wed, 18 Jan 2023 10:36:46 -0600 Subject: [PATCH 1/3] Adding to the documentation and tests for the _none pipeline --- docs/reference/rest-api/common-parms.asciidoc | 3 +- .../rest-api-spec/test/ingest/70_bulk.yml | 39 +++++++++++++ .../ingest/IngestServiceTests.java | 55 +++++++++++++++++++ 3 files changed, 96 insertions(+), 1 deletion(-) diff --git a/docs/reference/rest-api/common-parms.asciidoc b/docs/reference/rest-api/common-parms.asciidoc index 92141b9d1952e..8bb08ac494ee8 100644 --- a/docs/reference/rest-api/common-parms.asciidoc +++ b/docs/reference/rest-api/common-parms.asciidoc @@ -713,7 +713,8 @@ end::payloads[] tag::pipeline[] `pipeline`:: -(Optional, string) ID of the pipeline to use to preprocess incoming documents. +(Optional, string) ID of the pipeline to use to preprocess incoming documents. The special pipeline +name _none indicates no ingest pipeline will run, other than the final pipeline if it exists. end::pipeline[] tag::pages-processed[] diff --git a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/70_bulk.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/70_bulk.yml index d7738987de18b..c71e13429f72d 100644 --- a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/70_bulk.yml +++ b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/70_bulk.yml @@ -145,6 +145,45 @@ teardown: - is_false: _source.field1 - match: {_source.field2: value2} +--- +"Test bulk request with _none request pipeline and default pipeline": + + - do: + bulk: + pipeline: pipeline1 + body: + - index: + _index: test_index + _id: test_id1 + - f1: v1 + - index: + _index: test_index + _id: test_id2 + pipeline: _none + - f1: v2 + - gte: { ingest_took: 0 } + + - do: + cluster.state: {} + # Get master node id + - set: { master_node: master } + + - do: + get: + index: test_index + id: test_id1 + + - match: {_source.field1: value1} + - is_false: _source.field2 + + - do: + get: + index: test_index + id: test_id2 + + - is_false: _source.field1 + - is_false: _source.field2 + --- "Update with pipeline": - skip: diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java index 1f1dc3d11c029..84041e9887174 100644 --- a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java @@ -2029,6 +2029,61 @@ public void testPutPipelineWithVersionedUpdateIncrementsVersion() throws Excepti assertThat(updatedConfig.getVersion(), equalTo(existingVersion + 1)); } + public void testResolvePipelinesWithNonePipeline() { + // _none request pipeline: + { + Metadata metadata = Metadata.builder().build(); + IndexRequest indexRequest = new IndexRequest("idx").setPipeline(IngestService.NOOP_PIPELINE_NAME); + boolean result = IngestService.resolvePipelines(indexRequest, indexRequest, metadata); + assertThat(result, is(false)); + assertThat(indexRequest.isPipelineResolved(), is(true)); + assertThat(indexRequest.getPipeline(), equalTo(IngestService.NOOP_PIPELINE_NAME)); + } + + // _none default pipeline: + { + IndexMetadata.Builder builder = IndexMetadata.builder("idx") + .settings(settings(Version.CURRENT).put(IndexSettings.DEFAULT_PIPELINE.getKey(), IngestService.NOOP_PIPELINE_NAME)) + .numberOfShards(1) + .numberOfReplicas(0); + Metadata metadata = Metadata.builder().put(builder).build(); + IndexRequest indexRequest = new IndexRequest("idx"); + boolean result = IngestService.resolvePipelines(indexRequest, indexRequest, metadata); + assertThat(result, is(false)); + assertThat(indexRequest.isPipelineResolved(), is(true)); + assertThat(indexRequest.getPipeline(), equalTo(IngestService.NOOP_PIPELINE_NAME)); + } + + // _none request pipeline with default pipeline: + { + IndexMetadata.Builder builder = IndexMetadata.builder("idx") + .settings(settings(Version.CURRENT).put(IndexSettings.DEFAULT_PIPELINE.getKey(), "default-pipeline")) + .numberOfShards(1) + .numberOfReplicas(0); + Metadata metadata = Metadata.builder().put(builder).build(); + IndexRequest indexRequest = new IndexRequest("idx").setPipeline(IngestService.NOOP_PIPELINE_NAME); + boolean result = IngestService.resolvePipelines(indexRequest, indexRequest, metadata); + assertThat(result, is(false)); + assertThat(indexRequest.isPipelineResolved(), is(true)); + assertThat(indexRequest.getPipeline(), equalTo(IngestService.NOOP_PIPELINE_NAME)); + } + + // _none request pipeline with final pipeline: + { + IndexMetadata.Builder builder = IndexMetadata.builder("idx") + .settings(settings(Version.CURRENT).put(IndexSettings.FINAL_PIPELINE.getKey(), "final-pipeline")) + .numberOfShards(1) + .numberOfReplicas(0); + Metadata metadata = Metadata.builder().put(builder).build(); + IndexRequest indexRequest = new IndexRequest("idx").setPipeline(IngestService.NOOP_PIPELINE_NAME); + boolean result = IngestService.resolvePipelines(indexRequest, indexRequest, metadata); + assertThat(result, is(true)); + assertThat(indexRequest.isPipelineResolved(), is(true)); + assertThat(indexRequest.getPipeline(), equalTo(IngestService.NOOP_PIPELINE_NAME)); + assertThat(indexRequest.getFinalPipeline(), equalTo("final-pipeline")); + } + } + private static Tuple randomMapEntry() { return tuple(randomAlphaOfLength(5), randomObject()); } From e3d0d019fe8a4caab593d270beea326022263520 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Wed, 18 Jan 2023 12:35:42 -0600 Subject: [PATCH 2/3] Update docs/reference/rest-api/common-parms.asciidoc Co-authored-by: David Kilfoyle <41695641+kilfoyle@users.noreply.github.com> --- docs/reference/rest-api/common-parms.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/rest-api/common-parms.asciidoc b/docs/reference/rest-api/common-parms.asciidoc index 8bb08ac494ee8..575e708b26f75 100644 --- a/docs/reference/rest-api/common-parms.asciidoc +++ b/docs/reference/rest-api/common-parms.asciidoc @@ -714,7 +714,7 @@ end::payloads[] tag::pipeline[] `pipeline`:: (Optional, string) ID of the pipeline to use to preprocess incoming documents. The special pipeline -name _none indicates no ingest pipeline will run, other than the final pipeline if it exists. +name `_none` indicates no ingest pipeline will run, other than the final pipeline if it exists. end::pipeline[] tag::pages-processed[] From 612a6901e337c73df9ea6d3b6793bf8a064a1844 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Wed, 18 Jan 2023 16:41:02 -0600 Subject: [PATCH 3/3] code review feedback; making it clear that _none applies to default and final pipelines --- docs/reference/index-modules.asciidoc | 4 +-- docs/reference/rest-api/common-parms.asciidoc | 6 ++-- .../ingest/IngestServiceTests.java | 29 +++++++++++++++++++ 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/docs/reference/index-modules.asciidoc b/docs/reference/index-modules.asciidoc index e258d8a74cc25..d5fc1fef7f338 100644 --- a/docs/reference/index-modules.asciidoc +++ b/docs/reference/index-modules.asciidoc @@ -320,7 +320,7 @@ Defaults to `*`, which matches all fields eligible for Default <> for the index. Index requests will fail if the default pipeline is set and the pipeline does not exist. The default may be overridden using the `pipeline` parameter. The special pipeline name `_none` indicates - no ingest pipeline should be run. + no default ingest pipeline will run. [[index-final-pipeline]] `index.final_pipeline`:: @@ -328,7 +328,7 @@ Final <> for the index. Indexing requests will fail if the final pipeline is set and the pipeline does not exist. The final pipeline always runs after the request pipeline (if specified) and the default pipeline (if it exists). The special pipeline name `_none` -indicates no ingest pipeline will run. +indicates no final ingest pipeline will run. + NOTE: You can't use a final pipeline to change the `_index` field. If the pipeline attempts to change the `_index` field, the indexing request will fail. diff --git a/docs/reference/rest-api/common-parms.asciidoc b/docs/reference/rest-api/common-parms.asciidoc index 8bb08ac494ee8..ca2d55c543336 100644 --- a/docs/reference/rest-api/common-parms.asciidoc +++ b/docs/reference/rest-api/common-parms.asciidoc @@ -713,8 +713,10 @@ end::payloads[] tag::pipeline[] `pipeline`:: -(Optional, string) ID of the pipeline to use to preprocess incoming documents. The special pipeline -name _none indicates no ingest pipeline will run, other than the final pipeline if it exists. +(Optional, string) ID of the pipeline to use to preprocess incoming documents. If the index has a +default ingest pipeline specified, then setting the value to `_none` disables the default ingest +pipeline for this request. If a final pipeline is configured it will always run, regardless of the +value of this parameter. end::pipeline[] tag::pages-processed[] diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java index 84041e9887174..59fc993801427 100644 --- a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java @@ -2054,6 +2054,20 @@ public void testResolvePipelinesWithNonePipeline() { assertThat(indexRequest.getPipeline(), equalTo(IngestService.NOOP_PIPELINE_NAME)); } + // _none default pipeline with request pipeline: + { + IndexMetadata.Builder builder = IndexMetadata.builder("idx") + .settings(settings(Version.CURRENT).put(IndexSettings.DEFAULT_PIPELINE.getKey(), IngestService.NOOP_PIPELINE_NAME)) + .numberOfShards(1) + .numberOfReplicas(0); + Metadata metadata = Metadata.builder().put(builder).build(); + IndexRequest indexRequest = new IndexRequest("idx").setPipeline("pipeline1"); + boolean result = IngestService.resolvePipelines(indexRequest, indexRequest, metadata); + assertThat(result, is(true)); + assertThat(indexRequest.isPipelineResolved(), is(true)); + assertThat(indexRequest.getPipeline(), equalTo("pipeline1")); + } + // _none request pipeline with default pipeline: { IndexMetadata.Builder builder = IndexMetadata.builder("idx") @@ -2082,6 +2096,21 @@ public void testResolvePipelinesWithNonePipeline() { assertThat(indexRequest.getPipeline(), equalTo(IngestService.NOOP_PIPELINE_NAME)); assertThat(indexRequest.getFinalPipeline(), equalTo("final-pipeline")); } + + // _none final pipeline: + { + IndexMetadata.Builder builder = IndexMetadata.builder("idx") + .settings(settings(Version.CURRENT).put(IndexSettings.FINAL_PIPELINE.getKey(), IngestService.NOOP_PIPELINE_NAME)) + .numberOfShards(1) + .numberOfReplicas(0); + Metadata metadata = Metadata.builder().put(builder).build(); + IndexRequest indexRequest = new IndexRequest("idx").setPipeline("pipeline1"); + boolean result = IngestService.resolvePipelines(indexRequest, indexRequest, metadata); + assertThat(result, is(true)); + assertThat(indexRequest.isPipelineResolved(), is(true)); + assertThat(indexRequest.getPipeline(), equalTo("pipeline1")); + assertThat(indexRequest.getFinalPipeline(), equalTo(IngestService.NOOP_PIPELINE_NAME)); + } } private static Tuple randomMapEntry() {